2023-09-23T22:07:03.104 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T22:07:03.104 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882 --verbose --name teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/orig.config.yaml 2023-09-23T22:07:03.163 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T22:07:03.247 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7401882' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 1860 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi052.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi151.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T22:07:03.247 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T22:07:03.248 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T22:07:03.248 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T22:07:03.249 INFO:teuthology.task.internal:Checking packages... 2023-09-23T22:07:03.270 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T22:07:03.270 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:07:03.270 INFO:teuthology.packaging:ref: None 2023-09-23T22:07:03.270 INFO:teuthology.packaging:tag: None 2023-09-23T22:07:03.270 INFO:teuthology.packaging:branch: reef 2023-09-23T22:07:03.270 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:07:03.271 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T22:07:03.484 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-462.gceea2059 2023-09-23T22:07:03.485 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T22:07:03.490 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T22:07:03.490 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T22:07:03.497 INFO:teuthology.task.internal:Saving configuration 2023-09-23T22:07:03.517 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T22:07:03.523 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T22:07:03.547 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi052.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:01:58.000280', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T22:07:03.571 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:01:57.997706', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T22:07:03.571 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T22:07:03.578 INFO:teuthology.task.internal:roles: ubuntu@smithi052.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T22:07:03.578 INFO:teuthology.task.internal:roles: ubuntu@smithi151.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T22:07:03.578 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T22:07:03.646 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb3fa302af0>, signals=[15]) 2023-09-23T22:07:03.647 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T22:07:03.652 INFO:teuthology.task.internal:Opening connections... 2023-09-23T22:07:03.653 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi052.front.sepia.ceph.com 2023-09-23T22:07:03.654 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi052.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:07:03.723 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi151.front.sepia.ceph.com 2023-09-23T22:07:03.723 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:07:03.787 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T22:07:03.794 DEBUG:teuthology.orchestra.run.smithi052:> uname -m 2023-09-23T22:07:03.816 INFO:teuthology.orchestra.run.smithi052.stdout:x86_64 2023-09-23T22:07:03.816 DEBUG:teuthology.orchestra.run.smithi052:> cat /etc/os-release 2023-09-23T22:07:03.876 INFO:teuthology.orchestra.run.smithi052.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:ID="rhel" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:ID_LIKE="fedora" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:VERSION_ID="8.6" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:PLATFORM_ID="platform:el8" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:ANSI_COLOR="0;31" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T22:07:03.877 INFO:teuthology.orchestra.run.smithi052.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T22:07:03.878 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:07:03.878 INFO:teuthology.orchestra.run.smithi052.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T22:07:03.878 INFO:teuthology.orchestra.run.smithi052.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T22:07:03.878 INFO:teuthology.orchestra.run.smithi052.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T22:07:03.878 INFO:teuthology.orchestra.run.smithi052.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T22:07:03.878 INFO:teuthology.lock.ops:Updating smithi052.front.sepia.ceph.com on lock server 2023-09-23T22:07:03.901 DEBUG:teuthology.orchestra.run.smithi151:> uname -m 2023-09-23T22:07:03.921 INFO:teuthology.orchestra.run.smithi151.stdout:x86_64 2023-09-23T22:07:03.922 DEBUG:teuthology.orchestra.run.smithi151:> cat /etc/os-release 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:ID="rhel" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:ID_LIKE="fedora" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION_ID="8.6" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:PLATFORM_ID="platform:el8" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:ANSI_COLOR="0;31" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T22:07:03.981 INFO:teuthology.orchestra.run.smithi151.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T22:07:03.982 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T22:07:03.982 INFO:teuthology.lock.ops:Updating smithi151.front.sepia.ceph.com on lock server 2023-09-23T22:07:04.001 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T22:07:04.010 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T22:07:04.015 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T22:07:04.015 DEBUG:teuthology.orchestra.run.smithi052:> test '!' -e /home/ubuntu/cephtest 2023-09-23T22:07:04.018 DEBUG:teuthology.orchestra.run.smithi151:> test '!' -e /home/ubuntu/cephtest 2023-09-23T22:07:04.039 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T22:07:04.045 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T22:07:04.045 DEBUG:teuthology.orchestra.run.smithi052:> test -z $(ls -A /var/lib/ceph) 2023-09-23T22:07:04.077 DEBUG:teuthology.orchestra.run.smithi151:> test -z $(ls -A /var/lib/ceph) 2023-09-23T22:07:04.106 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T22:07:04.163 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T22:07:04.179 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T22:07:04.179 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T22:07:04.179 DEBUG:teuthology.orchestra.run.smithi052:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T22:07:04.180 DEBUG:teuthology.orchestra.run.smithi151:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T22:07:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:07:04.199 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2023-09-23T22:07:04.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:07:04.200 DEBUG:teuthology.orchestra.run.smithi052:> uname -r 2023-09-23T22:07:04.259 INFO:teuthology.orchestra.run.smithi151.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:04.259 INFO:teuthology.task.kernel:Running kernel on smithi151: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:04.259 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum install -y kernel 2023-09-23T22:07:04.260 INFO:teuthology.orchestra.run.smithi052.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:04.261 INFO:teuthology.task.kernel:Running kernel on smithi052: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:04.261 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum install -y kernel 2023-09-23T22:07:05.448 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:07:05.448 INFO:teuthology.orchestra.run.smithi151.stdout:Unable to read consumer identity 2023-09-23T22:07:05.463 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:07:05.463 INFO:teuthology.orchestra.run.smithi052.stdout:Unable to read consumer identity 2023-09-23T22:07:06.890 INFO:teuthology.orchestra.run.smithi151.stdout:Copr repo for python3-asyncssh owned by ceph 18 kB/s | 3.5 kB 00:00 2023-09-23T22:07:06.922 INFO:teuthology.orchestra.run.smithi052.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-23T22:07:08.131 INFO:teuthology.orchestra.run.smithi052.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-09-23T22:07:08.849 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 7.7 MB/s | 16 MB 00:02 2023-09-23T22:07:11.823 INFO:teuthology.orchestra.run.smithi052.stdout:lab-extras 351 kB/s | 24 kB 00:00 2023-09-23T22:07:12.324 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 444 kB/s | 24 kB 00:00 2023-09-23T22:07:12.559 INFO:teuthology.orchestra.run.smithi052.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T22:07:12.613 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:07:12.614 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:07:12.615 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:07:12.734 DEBUG:teuthology.orchestra.run.smithi052:> echo no | sudo yum reinstall kernel || true 2023-09-23T22:07:13.035 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T22:07:13.085 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:07:13.086 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:07:13.086 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:07:13.125 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:07:13.125 INFO:teuthology.orchestra.run.smithi052.stdout:Unable to read consumer identity 2023-09-23T22:07:13.212 DEBUG:teuthology.orchestra.run.smithi151:> echo no | sudo yum reinstall kernel || true 2023-09-23T22:07:13.399 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 10:07:11 PM UTC. 2023-09-23T22:07:13.442 INFO:teuthology.orchestra.run.smithi052.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:07:13.469 INFO:teuthology.orchestra.run.smithi052.stderr:Error: No packages marked for reinstall. 2023-09-23T22:07:13.517 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum reinstall -y kernel || true 2023-09-23T22:07:13.540 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:07:13.541 INFO:teuthology.orchestra.run.smithi151.stdout:Unable to read consumer identity 2023-09-23T22:07:13.803 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 10:07:12 PM UTC. 2023-09-23T22:07:13.844 INFO:teuthology.orchestra.run.smithi151.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:07:13.868 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:07:13.869 INFO:teuthology.orchestra.run.smithi052.stdout:Unable to read consumer identity 2023-09-23T22:07:13.894 INFO:teuthology.orchestra.run.smithi151.stderr:Error: No packages marked for reinstall. 2023-09-23T22:07:13.940 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum reinstall -y kernel || true 2023-09-23T22:07:14.138 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:00:03 ago on Sat 23 Sep 2023 10:07:11 PM UTC. 2023-09-23T22:07:14.183 INFO:teuthology.orchestra.run.smithi052.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:07:14.203 INFO:teuthology.orchestra.run.smithi052.stderr:Error: No packages marked for reinstall. 2023-09-23T22:07:14.255 DEBUG:teuthology.orchestra.run.smithi052:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T22:07:14.271 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:07:14.271 INFO:teuthology.orchestra.run.smithi151.stdout:Unable to read consumer identity 2023-09-23T22:07:14.338 INFO:teuthology.orchestra.run.smithi052.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:14.338 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:14.339 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T22:07:14.339 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T22:07:14.339 DEBUG:teuthology.orchestra.run.smithi052:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T22:07:14.366 INFO:teuthology.orchestra.run.smithi052.stdout:ttyS1 2023-09-23T22:07:14.377 DEBUG:teuthology.parallel:result is None 2023-09-23T22:07:14.529 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 10:07:12 PM UTC. 2023-09-23T22:07:14.571 INFO:teuthology.orchestra.run.smithi151.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T22:07:14.592 INFO:teuthology.orchestra.run.smithi151.stderr:Error: No packages marked for reinstall. 2023-09-23T22:07:14.638 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T22:07:14.719 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:14.720 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T22:07:14.720 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T22:07:14.720 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T22:07:14.720 DEBUG:teuthology.orchestra.run.smithi151:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T22:07:14.759 INFO:teuthology.orchestra.run.smithi151.stdout:ttyS1 2023-09-23T22:07:14.771 DEBUG:teuthology.parallel:result is None 2023-09-23T22:07:14.771 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T22:07:14.778 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T22:07:14.778 DEBUG:teuthology.orchestra.run.smithi052:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T22:07:14.780 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T22:07:14.828 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T22:07:14.835 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T22:07:14.841 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T22:07:14.841 DEBUG:teuthology.orchestra.run.smithi052:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T22:07:14.843 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T22:07:14.890 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T22:07:14.897 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T22:07:14.898 DEBUG:teuthology.orchestra.run.smithi052:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T22:07:14.902 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T22:07:14.938 INFO:teuthology.orchestra.run.smithi052.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:07:14.949 INFO:teuthology.orchestra.run.smithi052.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:07:14.976 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:07:14.986 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T22:07:14.988 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T22:07:14.995 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T22:07:14.995 DEBUG:teuthology.orchestra.run.smithi052:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T22:07:14.997 DEBUG:teuthology.orchestra.run.smithi151:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T22:07:15.024 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T22:07:15.032 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T22:07:15.033 DEBUG:teuthology.orchestra.run.smithi052:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T22:07:15.066 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T22:07:15.083 DEBUG:teuthology.orchestra.run.smithi052:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:07:15.168 DEBUG:teuthology.orchestra.run.smithi052:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:07:15.226 DEBUG:teuthology.orchestra.run.smithi052:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:07:15.311 DEBUG:teuthology.orchestra.run.smithi052:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:07:15.354 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:07:15.355 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T22:07:15.445 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:07:15.491 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T22:07:15.536 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:07:15.621 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T22:07:15.662 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:07:15.663 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T22:07:15.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo service rsyslog restart 2023-09-23T22:07:15.741 DEBUG:teuthology.orchestra.run.smithi151:> sudo service rsyslog restart 2023-09-23T22:07:15.801 INFO:teuthology.orchestra.run.smithi052.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T22:07:15.850 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T22:07:16.256 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T22:07:16.264 INFO:teuthology.task.internal:Starting timer... 2023-09-23T22:07:16.265 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T22:07:16.274 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T22:07:16.282 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T22:07:16.282 DEBUG:teuthology.orchestra.run.smithi052:> sudo service auditd rotate 2023-09-23T22:07:16.355 INFO:teuthology.orchestra.run.smithi052.stdout:Rotating logs: 2023-09-23T22:07:16.357 DEBUG:teuthology.orchestra.run.smithi151:> sudo service auditd rotate 2023-09-23T22:07:16.404 INFO:teuthology.orchestra.run.smithi151.stdout:Rotating logs: 2023-09-23T22:07:16.406 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T22:07:16.406 DEBUG:teuthology.orchestra.run.smithi052:> /usr/sbin/getenforce 2023-09-23T22:07:16.424 INFO:teuthology.orchestra.run.smithi052.stdout:Permissive 2023-09-23T22:07:16.424 DEBUG:teuthology.orchestra.run.smithi151:> /usr/sbin/getenforce 2023-09-23T22:07:16.446 INFO:teuthology.orchestra.run.smithi151.stdout:Permissive 2023-09-23T22:07:16.446 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi052.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi151.front.sepia.ceph.com': 'permissive'} 2023-09-23T22:07:16.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T22:07:16.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:07:16.492 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T22:07:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:07:16.519 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T22:07:16.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo /usr/sbin/setenforce permissive 2023-09-23T22:07:16.562 DEBUG:teuthology.orchestra.run.smithi151:> sudo /usr/sbin/setenforce permissive 2023-09-23T22:07:16.589 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T22:07:16.599 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T22:07:16.604 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T22:07:16.713 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T22:07:16.727 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T22:07:16.728 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi052.front.sepia.ceph.com,smithi151.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T22:16:19.273 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi052.front.sepia.ceph.com'), Remote(name='ubuntu@smithi151.front.sepia.ceph.com')] 2023-09-23T22:16:19.296 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi052.front.sepia.ceph.com' 2023-09-23T22:16:19.297 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi052.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:16:19.367 DEBUG:teuthology.orchestra.run.smithi052:> true 2023-09-23T22:16:19.436 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi052.front.sepia.ceph.com' 2023-09-23T22:16:19.436 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-09-23T22:16:19.437 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T22:16:19.505 DEBUG:teuthology.orchestra.run.smithi151:> true 2023-09-23T22:16:19.571 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2023-09-23T22:16:19.572 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T22:16:19.582 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T22:16:19.582 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T22:16:19.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T22:16:19.585 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T22:16:19.585 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T22:16:19.618 INFO:teuthology.orchestra.run.smithi052.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T22:16:19.634 INFO:teuthology.orchestra.run.smithi052.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T22:16:19.644 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T22:16:19.658 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T22:16:19.710 INFO:teuthology.orchestra.run.smithi052.stderr:sudo: ntpd: command not found 2023-09-23T22:16:19.723 INFO:teuthology.orchestra.run.smithi052.stdout:506 Cannot talk to daemon 2023-09-23T22:16:19.732 INFO:teuthology.orchestra.run.smithi151.stderr:sudo: ntpd: command not found 2023-09-23T22:16:19.739 INFO:teuthology.orchestra.run.smithi052.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T22:16:19.743 INFO:teuthology.orchestra.run.smithi151.stdout:506 Cannot talk to daemon 2023-09-23T22:16:19.753 INFO:teuthology.orchestra.run.smithi052.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T22:16:19.758 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T22:16:19.771 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T22:16:19.825 INFO:teuthology.orchestra.run.smithi052.stderr:bash: ntpq: command not found 2023-09-23T22:16:19.828 INFO:teuthology.orchestra.run.smithi052.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T22:16:19.828 INFO:teuthology.orchestra.run.smithi052.stdout:=============================================================================== 2023-09-23T22:16:19.828 INFO:teuthology.orchestra.run.smithi052.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.828 INFO:teuthology.orchestra.run.smithi052.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.828 INFO:teuthology.orchestra.run.smithi052.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.829 INFO:teuthology.orchestra.run.smithi052.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.840 INFO:teuthology.orchestra.run.smithi151.stderr:bash: ntpq: command not found 2023-09-23T22:16:19.842 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T22:16:19.843 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2023-09-23T22:16:19.843 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.843 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.843 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.843 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T22:16:19.844 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T22:16:19.856 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T22:16:19.856 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi052.front.sepia.ceph.com 2023-09-23T22:16:19.856 DEBUG:teuthology.orchestra.run.smithi052:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T22:16:19.857 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi151.front.sepia.ceph.com 2023-09-23T22:16:19.857 DEBUG:teuthology.orchestra.run.smithi151:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T22:16:20.253 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:16:20.260 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:16:21.088 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 10:14:29 PM UTC. 2023-09-23T22:16:21.119 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:23 ago on Sat 23 Sep 2023 10:14:57 PM UTC. 2023-09-23T22:16:22.963 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout:Resetting modules: 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout: container-tools 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:16:22.964 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:16:23.096 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout: Package Architecture Version Repository Size 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout:Resetting modules: 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout: container-tools 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:16:23.097 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:16:23.098 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:16:23.932 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 10:14:29 PM UTC. 2023-09-23T22:16:24.097 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:26 ago on Sat 23 Sep 2023 10:14:57 PM UTC. 2023-09-23T22:16:26.488 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:16:26.699 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:16:26.938 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:16:27.160 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:16:27.723 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:58 ago on Sat 23 Sep 2023 10:14:29 PM UTC. 2023-09-23T22:16:27.966 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:30 ago on Sat 23 Sep 2023 10:14:57 PM UTC. 2023-09-23T22:16:29.564 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout:========================================================================================================================== 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout:========================================================================================================================== 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout:Installing group/module packages: 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T22:16:29.659 INFO:teuthology.orchestra.run.smithi151.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout:Downgrading: 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T22:16:29.660 INFO:teuthology.orchestra.run.smithi151.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout:Installing module profiles: 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: container-tools/common 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout:Enabling module streams: 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: container-tools 3.0 2023-09-23T22:16:29.661 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout:========================================================================================================================== 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout:Install 7 Packages 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout:Downgrade 12 Packages 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 51 M 2023-09-23T22:16:29.662 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:16:29.738 INFO:teuthology.orchestra.run.smithi151.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 702 kB/s | 53 kB 00:00 2023-09-23T22:16:29.773 INFO:teuthology.orchestra.run.smithi151.stdout:(2/19): containers-common-1.2.4-1.module+el8.6. 831 kB/s | 91 kB 00:00 2023-09-23T22:16:29.797 INFO:teuthology.orchestra.run.smithi151.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 381 kB/s | 51 kB 00:00 2023-09-23T22:16:29.823 INFO:teuthology.orchestra.run.smithi151.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.3 MB/s | 69 kB 00:00 2023-09-23T22:16:29.848 INFO:teuthology.orchestra.run.smithi151.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.6 MB/s | 511 kB 00:00 2023-09-23T22:16:29.873 INFO:teuthology.orchestra.run.smithi151.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.4 MB/s | 72 kB 00:00 2023-09-23T22:16:29.915 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:16:29.948 INFO:teuthology.orchestra.run.smithi151.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 19 MB/s | 2.9 MB 00:00 2023-09-23T22:16:29.974 INFO:teuthology.orchestra.run.smithi151.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 557 kB/s | 56 kB 00:00 2023-09-23T22:16:30.015 INFO:teuthology.orchestra.run.smithi052.stdout:========================================================================================================================== 2023-09-23T22:16:30.015 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout:========================================================================================================================== 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout:Installing group/module packages: 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T22:16:30.016 INFO:teuthology.orchestra.run.smithi052.stdout:Downgrading: 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T22:16:30.017 INFO:teuthology.orchestra.run.smithi052.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout:Installing module profiles: 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout: container-tools/common 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout:Enabling module streams: 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout: container-tools 3.0 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout:========================================================================================================================== 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout:Install 7 Packages 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout:Downgrade 12 Packages 2023-09-23T22:16:30.018 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:16:30.019 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 51 M 2023-09-23T22:16:30.019 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:16:30.027 INFO:teuthology.orchestra.run.smithi151.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 649 kB/s | 51 kB 00:00 2023-09-23T22:16:30.088 INFO:teuthology.orchestra.run.smithi052.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 1.3 MB/s | 91 kB 00:00 2023-09-23T22:16:30.104 INFO:teuthology.orchestra.run.smithi052.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 625 kB/s | 53 kB 00:00 2023-09-23T22:16:30.108 INFO:teuthology.orchestra.run.smithi151.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 3.9 MB/s | 322 kB 00:00 2023-09-23T22:16:30.121 INFO:teuthology.orchestra.run.smithi052.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 505 kB/s | 51 kB 00:00 2023-09-23T22:16:30.163 INFO:teuthology.orchestra.run.smithi052.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 6.6 MB/s | 511 kB 00:00 2023-09-23T22:16:30.166 INFO:teuthology.orchestra.run.smithi151.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 702 kB/s | 41 kB 00:00 2023-09-23T22:16:30.180 INFO:teuthology.orchestra.run.smithi052.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.2 MB/s | 69 kB 00:00 2023-09-23T22:16:30.206 INFO:teuthology.orchestra.run.smithi052.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.7 MB/s | 72 kB 00:00 2023-09-23T22:16:30.231 INFO:teuthology.orchestra.run.smithi052.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 2.2 MB/s | 56 kB 00:00 2023-09-23T22:16:30.256 INFO:teuthology.orchestra.run.smithi052.stdout:(8/19): conmon-2.0.26-1.module+el8.6.0+14874+64 2.0 MB/s | 51 kB 00:00 2023-09-23T22:16:30.300 INFO:teuthology.orchestra.run.smithi151.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 24 MB/s | 11 MB 00:00 2023-09-23T22:16:30.314 INFO:teuthology.orchestra.run.smithi052.stdout:(9/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 15 MB/s | 2.9 MB 00:00 2023-09-23T22:16:30.357 INFO:teuthology.orchestra.run.smithi052.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 7.5 MB/s | 322 kB 00:00 2023-09-23T22:16:30.367 INFO:teuthology.orchestra.run.smithi151.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 11 MB/s | 2.2 MB 00:00 2023-09-23T22:16:30.398 INFO:teuthology.orchestra.run.smithi052.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 984 kB/s | 41 kB 00:00 2023-09-23T22:16:30.490 INFO:teuthology.orchestra.run.smithi052.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 24 MB/s | 2.2 MB 00:00 2023-09-23T22:16:30.525 INFO:teuthology.orchestra.run.smithi151.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 28 MB/s | 6.4 MB 00:00 2023-09-23T22:16:30.633 INFO:teuthology.orchestra.run.smithi052.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 24 MB/s | 11 MB 00:00 2023-09-23T22:16:30.776 INFO:teuthology.orchestra.run.smithi151.stdout:(15/19): containernetworking-plugins-0.9.1-1.mo 24 MB/s | 19 MB 00:00 2023-09-23T22:16:30.799 INFO:teuthology.orchestra.run.smithi052.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 21 MB/s | 6.4 MB 00:00 2023-09-23T22:16:30.826 INFO:teuthology.orchestra.run.smithi151.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 3.6 MB/s | 1.1 MB 00:00 2023-09-23T22:16:30.866 INFO:teuthology.orchestra.run.smithi052.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 16 MB/s | 1.1 MB 00:00 2023-09-23T22:16:30.926 INFO:teuthology.orchestra.run.smithi151.stdout:(17/19): buildah-1.19.9-3.module+el8.6.0+14874+ 12 MB/s | 6.8 MB 00:00 2023-09-23T22:16:30.950 INFO:teuthology.orchestra.run.smithi052.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 21 MB/s | 6.8 MB 00:00 2023-09-23T22:16:30.951 INFO:teuthology.orchestra.run.smithi151.stdout:(18/19): crun-0.18-3.module+el8.6.0+14874+64436 1.0 MB/s | 185 kB 00:00 2023-09-23T22:16:30.975 INFO:teuthology.orchestra.run.smithi052.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.7 MB/s | 185 kB 00:00 2023-09-23T22:16:30.976 INFO:teuthology.orchestra.run.smithi151.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 337 kB/s | 51 kB 00:00 2023-09-23T22:16:30.977 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:16:30.977 INFO:teuthology.orchestra.run.smithi151.stdout:Total 38 MB/s | 51 MB 00:01 2023-09-23T22:16:30.992 INFO:teuthology.orchestra.run.smithi052.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.2 MB/s | 51 kB 00:00 2023-09-23T22:16:31.201 INFO:teuthology.orchestra.run.smithi052.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 20 MB/s | 19 MB 00:00 2023-09-23T22:16:31.202 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:16:31.202 INFO:teuthology.orchestra.run.smithi052.stdout:Total 43 MB/s | 51 MB 00:01 2023-09-23T22:16:31.453 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:16:31.496 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:16:31.496 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:16:31.694 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:16:31.739 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:16:31.740 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:16:31.939 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:16:31.943 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:16:32.191 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:16:32.195 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:16:33.090 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:16:33.139 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T22:16:33.315 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:16:33.324 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:16:33.347 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:16:33.362 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:16:33.417 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T22:16:33.534 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:16:33.613 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:16:33.622 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T22:16:33.642 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:16:33.821 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:16:50.487 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:16:52.250 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T22:16:52.474 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T22:16:52.712 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T22:16:52.861 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T22:16:53.054 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T22:16:53.055 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T22:16:53.055 INFO:teuthology.orchestra.run.smithi151.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T22:16:53.055 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:16:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T22:16:54.170 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T22:16:54.496 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T22:16:54.684 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T22:16:54.896 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T22:16:55.067 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T22:16:55.262 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T22:16:55.263 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T22:16:55.263 INFO:teuthology.orchestra.run.smithi052.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T22:16:55.263 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:16:55.629 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T22:16:55.897 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T22:16:56.213 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T22:16:56.244 INFO:teuthology.orchestra.run.smithi151.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T22:16:56.444 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T22:16:56.502 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T22:16:57.238 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T22:16:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T22:16:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T22:16:58.194 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T22:16:58.234 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T22:16:58.410 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T22:16:58.595 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T22:16:58.622 INFO:teuthology.orchestra.run.smithi052.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T22:16:58.701 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T22:16:58.701 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:16:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T22:16:59.073 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:16:59.238 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T22:16:59.238 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:16:59.387 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:16:59.512 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:16:59.649 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T22:16:59.721 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T22:16:59.887 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T22:17:00.072 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T22:17:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T22:17:00.328 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T22:17:00.396 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T22:17:00.526 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T22:17:00.527 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:17:00.626 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T22:17:00.686 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:17:00.827 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T22:17:01.151 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T22:17:01.152 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:17:01.209 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T22:17:01.457 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T22:17:01.597 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:17:01.597 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T22:17:01.597 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T22:17:01.597 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T22:17:01.598 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T22:17:01.599 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T22:17:01.600 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T22:17:01.600 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T22:17:01.600 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T22:17:01.663 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T22:17:01.663 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:17:01.827 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T22:17:01.977 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:17:02.144 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T22:17:02.302 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T22:17:02.505 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T22:17:02.626 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T22:17:02.751 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T22:17:02.911 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T22:17:02.912 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:17:03.068 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:17:03.072 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T22:17:03.073 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:33 ago on Sat 23 Sep 2023 10:14:29 PM UTC. 2023-09-23T22:17:03.619 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T22:17:04.027 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T22:17:04.027 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T22:17:04.027 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T22:17:04.027 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T22:17:04.027 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T22:17:04.028 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T22:17:04.029 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T22:17:04.030 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T22:17:04.030 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T22:17:04.030 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T22:17:04.030 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T22:17:05.478 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:17:05.494 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T22:17:05.494 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:08 ago on Sat 23 Sep 2023 10:14:57 PM UTC. 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout:Downgraded: 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.671 INFO:teuthology.orchestra.run.smithi151.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.672 INFO:teuthology.orchestra.run.smithi151.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.673 INFO:teuthology.orchestra.run.smithi151.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:05.673 INFO:teuthology.orchestra.run.smithi151.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:05.673 INFO:teuthology.orchestra.run.smithi151.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T22:17:05.673 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:17:05.673 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:17:05.865 DEBUG:teuthology.parallel:result is None 2023-09-23T22:17:07.993 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout:Downgraded: 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.205 INFO:teuthology.orchestra.run.smithi052.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.206 INFO:teuthology.orchestra.run.smithi052.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T22:17:08.207 INFO:teuthology.orchestra.run.smithi052.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T22:17:08.207 INFO:teuthology.orchestra.run.smithi052.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T22:17:08.207 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:17:08.207 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:17:08.414 DEBUG:teuthology.parallel:result is None 2023-09-23T22:17:08.414 INFO:teuthology.run_tasks:Running task install... 2023-09-23T22:17:08.422 DEBUG:teuthology.task.install:project ceph 2023-09-23T22:17:08.423 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T22:17:08.423 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T22:17:08.423 INFO:teuthology.task.install:Using flavor: default 2023-09-23T22:17:08.429 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T22:17:08.429 INFO:teuthology.task.install:extra packages: [] 2023-09-23T22:17:08.429 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T22:17:08.429 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:17:08.430 INFO:teuthology.packaging:ref: None 2023-09-23T22:17:08.430 INFO:teuthology.packaging:tag: None 2023-09-23T22:17:08.430 INFO:teuthology.packaging:branch: pacific 2023-09-23T22:17:08.430 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:17:08.430 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-09-23T22:17:08.432 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T22:17:08.432 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:17:08.432 INFO:teuthology.packaging:ref: None 2023-09-23T22:17:08.432 INFO:teuthology.packaging:tag: None 2023-09-23T22:17:08.432 INFO:teuthology.packaging:branch: pacific 2023-09-23T22:17:08.432 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:17:08.432 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-09-23T22:17:08.645 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/ 2023-09-23T22:17:08.646 INFO:teuthology.task.install.rpm:Package version is 16.2.14-37.g2e1f1015 2023-09-23T22:17:08.660 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/ 2023-09-23T22:17:08.660 INFO:teuthology.task.install.rpm:Package version is 16.2.14-37.g2e1f1015 2023-09-23T22:17:08.825 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T22:17:08.825 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:17:08.825 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T22:17:08.827 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/pacific/2e1f10153ef2437dea8ce9168df827af9bb64fd7/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T22:17:08.827 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:17:08.828 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T22:17:08.856 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T22:17:08.857 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:17:08.857 INFO:teuthology.packaging:ref: None 2023-09-23T22:17:08.857 INFO:teuthology.packaging:tag: None 2023-09-23T22:17:08.857 INFO:teuthology.packaging:branch: pacific 2023-09-23T22:17:08.857 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:17:08.857 DEBUG:teuthology.orchestra.run.smithi052:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T22:17:08.860 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T22:17:08.860 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:17:08.860 INFO:teuthology.packaging:ref: None 2023-09-23T22:17:08.860 INFO:teuthology.packaging:tag: None 2023-09-23T22:17:08.860 INFO:teuthology.packaging:branch: pacific 2023-09-23T22:17:08.860 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:17:08.860 DEBUG:teuthology.orchestra.run.smithi151:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T22:17:08.925 DEBUG:teuthology.orchestra.run.smithi052:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T22:17:08.930 DEBUG:teuthology.orchestra.run.smithi151:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T22:17:09.004 DEBUG:teuthology.orchestra.run.smithi052:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T22:17:09.011 DEBUG:teuthology.orchestra.run.smithi151:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T22:17:09.072 INFO:teuthology.orchestra.run.smithi052.stdout:check_obsoletes = 1 2023-09-23T22:17:09.074 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum clean all 2023-09-23T22:17:09.079 INFO:teuthology.orchestra.run.smithi151.stdout:check_obsoletes = 1 2023-09-23T22:17:09.080 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum clean all 2023-09-23T22:17:09.455 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:17:09.468 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:17:09.782 INFO:teuthology.orchestra.run.smithi052.stdout:44 files removed 2023-09-23T22:17:09.786 INFO:teuthology.orchestra.run.smithi151.stdout:44 files removed 2023-09-23T22:17:09.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-radosgw 2023-09-23T22:17:09.832 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-radosgw 2023-09-23T22:17:10.167 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:17:10.181 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:17:10.953 INFO:teuthology.orchestra.run.smithi052.stdout:ceph packages for x86_64 232 kB/s | 76 kB 00:00 2023-09-23T22:17:11.060 INFO:teuthology.orchestra.run.smithi151.stdout:ceph packages for x86_64 174 kB/s | 76 kB 00:00 2023-09-23T22:17:11.279 INFO:teuthology.orchestra.run.smithi052.stdout:ceph noarch packages 52 kB/s | 15 kB 00:00 2023-09-23T22:17:11.354 INFO:teuthology.orchestra.run.smithi151.stdout:ceph noarch packages 58 kB/s | 15 kB 00:00 2023-09-23T22:17:11.655 INFO:teuthology.orchestra.run.smithi151.stdout:ceph source packages 5.1 kB/s | 1.4 kB 00:00 2023-09-23T22:17:11.663 INFO:teuthology.orchestra.run.smithi052.stdout:ceph source packages 4.0 kB/s | 1.4 kB 00:00 2023-09-23T22:17:11.806 INFO:teuthology.orchestra.run.smithi151.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-09-23T22:17:11.813 INFO:teuthology.orchestra.run.smithi052.stdout:Copr repo for python3-asyncssh owned by ceph 27 kB/s | 3.5 kB 00:00 2023-09-23T22:17:12.583 INFO:teuthology.orchestra.run.smithi052.stdout:Extra Packages for Enterprise Linux 21 MB/s | 16 MB 00:00 2023-09-23T22:17:14.092 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 7.0 MB/s | 16 MB 00:02 2023-09-23T22:17:16.306 INFO:teuthology.orchestra.run.smithi052.stdout:lab-extras 436 kB/s | 24 kB 00:00 2023-09-23T22:17:17.274 INFO:teuthology.orchestra.run.smithi052.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 59 MB/s | 53 MB 00:00 2023-09-23T22:17:17.581 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 426 kB/s | 24 kB 00:00 2023-09-23T22:17:18.492 INFO:teuthology.orchestra.run.smithi151.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-09-23T22:17:26.017 INFO:teuthology.orchestra.run.smithi052.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-09-23T22:17:26.443 INFO:teuthology.orchestra.run.smithi151.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 56 MB/s | 53 MB 00:00 2023-09-23T22:17:37.495 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:17:37.497 INFO:teuthology.orchestra.run.smithi151.stdout:=================================================================================================== 2023-09-23T22:17:37.497 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:17:37.497 INFO:teuthology.orchestra.run.smithi151.stdout:=================================================================================================== 2023-09-23T22:17:37.497 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-radosgw x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 12 M 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout:Upgrading: 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: librados2 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 3.7 M 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: librbd1 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 4.2 M 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-base x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 6.1 M 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-common x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 24 M 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-selinux x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 26 k 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T22:17:37.498 INFO:teuthology.orchestra.run.smithi151.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs2 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 824 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: libradosstriper1 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 494 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: librgw2 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 3.8 M 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T22:17:37.499 INFO:teuthology.orchestra.run.smithi151.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-argparse x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 45 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-common x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 99 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cephfs x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 215 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rados x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 390 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rbd x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 369 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rgw x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 114 k 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:17:37.500 INFO:teuthology.orchestra.run.smithi151.stdout:=================================================================================================== 2023-09-23T22:17:37.501 INFO:teuthology.orchestra.run.smithi151.stdout:Install 21 Packages 2023-09-23T22:17:37.501 INFO:teuthology.orchestra.run.smithi151.stdout:Upgrade 2 Packages 2023-09-23T22:17:37.501 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:17:37.501 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 57 M 2023-09-23T22:17:37.502 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:17:37.789 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:17:37.792 INFO:teuthology.orchestra.run.smithi052.stdout:=================================================================================================== 2023-09-23T22:17:37.792 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:17:37.792 INFO:teuthology.orchestra.run.smithi052.stdout:=================================================================================================== 2023-09-23T22:17:37.792 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:17:37.792 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-radosgw x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 12 M 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout:Upgrading: 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: librados2 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 3.7 M 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: librbd1 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 4.2 M 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-base x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 6.1 M 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-common x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 24 M 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-selinux x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 26 k 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-23T22:17:37.793 INFO:teuthology.orchestra.run.smithi052.stdout: libcephfs2 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 824 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: libradosstriper1 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 494 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: librgw2 x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 3.8 M 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: python3-ceph-argparse x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 45 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: python3-ceph-common x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 99 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cephfs x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 215 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T22:17:37.794 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rados x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 390 k 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rbd x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 369 k 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rgw x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 114 k 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout:=================================================================================================== 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout:Install 21 Packages 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout:Upgrade 2 Packages 2023-09-23T22:17:37.795 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:17:37.796 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 57 M 2023-09-23T22:17:37.796 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:17:39.710 INFO:teuthology.orchestra.run.smithi151.stdout:(1/23): ceph-base-16.2.14-37.g2e1f1015.el8.x86_ 3.3 MB/s | 6.1 MB 00:01 2023-09-23T22:17:39.818 INFO:teuthology.orchestra.run.smithi151.stdout:(2/23): ceph-selinux-16.2.14-37.g2e1f1015.el8.x 237 kB/s | 26 kB 00:00 2023-09-23T22:17:40.034 INFO:teuthology.orchestra.run.smithi052.stdout:(1/23): ceph-base-16.2.14-37.g2e1f1015.el8.x86_ 2.9 MB/s | 6.1 MB 00:02 2023-09-23T22:17:40.069 INFO:teuthology.orchestra.run.smithi151.stdout:(3/23): libcephfs2-16.2.14-37.g2e1f1015.el8.x86 3.2 MB/s | 824 kB 00:00 2023-09-23T22:17:40.126 INFO:teuthology.orchestra.run.smithi052.stdout:(2/23): ceph-selinux-16.2.14-37.g2e1f1015.el8.x 280 kB/s | 26 kB 00:00 2023-09-23T22:17:40.268 INFO:teuthology.orchestra.run.smithi151.stdout:(4/23): libradosstriper1-16.2.14-37.g2e1f1015.e 2.4 MB/s | 494 kB 00:00 2023-09-23T22:17:40.493 INFO:teuthology.orchestra.run.smithi052.stdout:(3/23): libcephfs2-16.2.14-37.g2e1f1015.el8.x86 2.2 MB/s | 824 kB 00:00 2023-09-23T22:17:40.552 INFO:teuthology.orchestra.run.smithi151.stdout:(5/23): ceph-radosgw-16.2.14-37.g2e1f1015.el8.x 4.3 MB/s | 12 MB 00:02 2023-09-23T22:17:40.619 INFO:teuthology.orchestra.run.smithi151.stdout:(6/23): python3-ceph-argparse-16.2.14-37.g2e1f1 679 kB/s | 45 kB 00:00 2023-09-23T22:17:40.678 INFO:teuthology.orchestra.run.smithi151.stdout:(7/23): python3-ceph-common-16.2.14-37.g2e1f101 1.7 MB/s | 99 kB 00:00 2023-09-23T22:17:40.744 INFO:teuthology.orchestra.run.smithi052.stdout:(4/23): ceph-common-16.2.14-37.g2e1f1015.el8.x8 8.6 MB/s | 24 MB 00:02 2023-09-23T22:17:40.769 INFO:teuthology.orchestra.run.smithi151.stdout:(8/23): python3-cephfs-16.2.14-37.g2e1f1015.el8 2.3 MB/s | 215 kB 00:00 2023-09-23T22:17:40.777 INFO:teuthology.orchestra.run.smithi052.stdout:(5/23): libradosstriper1-16.2.14-37.g2e1f1015.e 1.7 MB/s | 494 kB 00:00 2023-09-23T22:17:40.870 INFO:teuthology.orchestra.run.smithi151.stdout:(9/23): python3-rados-16.2.14-37.g2e1f1015.el8. 3.8 MB/s | 390 kB 00:00 2023-09-23T22:17:40.878 INFO:teuthology.orchestra.run.smithi052.stdout:(6/23): python3-ceph-argparse-16.2.14-37.g2e1f1 451 kB/s | 45 kB 00:00 2023-09-23T22:17:40.953 INFO:teuthology.orchestra.run.smithi151.stdout:(10/23): python3-rbd-16.2.14-37.g2e1f1015.el8.x 4.3 MB/s | 369 kB 00:00 2023-09-23T22:17:40.978 INFO:teuthology.orchestra.run.smithi052.stdout:(7/23): python3-ceph-common-16.2.14-37.g2e1f101 987 kB/s | 99 kB 00:00 2023-09-23T22:17:41.054 INFO:teuthology.orchestra.run.smithi151.stdout:(11/23): python3-rgw-16.2.14-37.g2e1f1015.el8.x 1.1 MB/s | 114 kB 00:00 2023-09-23T22:17:41.103 INFO:teuthology.orchestra.run.smithi052.stdout:(8/23): python3-cephfs-16.2.14-37.g2e1f1015.el8 1.7 MB/s | 215 kB 00:00 2023-09-23T22:17:41.137 INFO:teuthology.orchestra.run.smithi151.stdout:(12/23): librgw2-16.2.14-37.g2e1f1015.el8.x86_6 4.3 MB/s | 3.8 MB 00:00 2023-09-23T22:17:41.162 INFO:teuthology.orchestra.run.smithi052.stdout:(9/23): librgw2-16.2.14-37.g2e1f1015.el8.x86_64 9.0 MB/s | 3.8 MB 00:00 2023-09-23T22:17:41.221 INFO:teuthology.orchestra.run.smithi052.stdout:(10/23): python3-rbd-16.2.14-37.g2e1f1015.el8.x 6.2 MB/s | 369 kB 00:00 2023-09-23T22:17:41.338 INFO:teuthology.orchestra.run.smithi052.stdout:(11/23): ceph-radosgw-16.2.14-37.g2e1f1015.el8. 3.4 MB/s | 12 MB 00:03 2023-09-23T22:17:41.363 INFO:teuthology.orchestra.run.smithi052.stdout:(12/23): python3-rados-16.2.14-37.g2e1f1015.el8 1.5 MB/s | 390 kB 00:00 2023-09-23T22:17:41.388 INFO:teuthology.orchestra.run.smithi052.stdout:(13/23): python3-rgw-16.2.14-37.g2e1f1015.el8.x 685 kB/s | 114 kB 00:00 2023-09-23T22:17:41.488 INFO:teuthology.orchestra.run.smithi052.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 589 kB/s | 59 kB 00:00 2023-09-23T22:17:41.555 INFO:teuthology.orchestra.run.smithi052.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.6 MB/s | 306 kB 00:00 2023-09-23T22:17:41.572 INFO:teuthology.orchestra.run.smithi052.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 898 kB/s | 75 kB 00:00 2023-09-23T22:17:41.598 INFO:teuthology.orchestra.run.smithi052.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 773 kB/s | 181 kB 00:00 2023-09-23T22:17:41.612 INFO:teuthology.orchestra.run.smithi151.stdout:(13/23): gperftools-libs-2.7-9.el8.x86_64.rpm 548 kB/s | 306 kB 00:00 2023-09-23T22:17:41.637 INFO:teuthology.orchestra.run.smithi151.stdout:(14/23): leveldb-1.22-1.el8.x86_64.rpm 362 kB/s | 181 kB 00:00 2023-09-23T22:17:41.664 INFO:teuthology.orchestra.run.smithi052.stdout:(18/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 430 kB/s | 47 kB 00:00 2023-09-23T22:17:41.690 INFO:teuthology.orchestra.run.smithi052.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 589 kB/s | 44 kB 00:00 2023-09-23T22:17:41.715 INFO:teuthology.orchestra.run.smithi052.stdout:(20/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.8 MB/s | 259 kB 00:00 2023-09-23T22:17:41.730 INFO:teuthology.orchestra.run.smithi151.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 504 kB/s | 59 kB 00:00 2023-09-23T22:17:41.748 INFO:teuthology.orchestra.run.smithi052.stdout:(21/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.1 MB/s | 354 kB 00:00 2023-09-23T22:17:41.780 INFO:teuthology.orchestra.run.smithi151.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 528 kB/s | 75 kB 00:00 2023-09-23T22:17:41.805 INFO:teuthology.orchestra.run.smithi151.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.4 MB/s | 259 kB 00:00 2023-09-23T22:17:41.847 INFO:teuthology.orchestra.run.smithi151.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 657 kB/s | 44 kB 00:00 2023-09-23T22:17:41.872 INFO:teuthology.orchestra.run.smithi151.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.6 MB/s | 354 kB 00:00 2023-09-23T22:17:41.898 INFO:teuthology.orchestra.run.smithi151.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 933 kB/s | 47 kB 00:00 2023-09-23T22:17:42.049 INFO:teuthology.orchestra.run.smithi052.stdout:(22/23): librbd1-16.2.14-37.g2e1f1015.el8.x86_6 13 MB/s | 4.2 MB 00:00 2023-09-23T22:17:42.457 INFO:teuthology.orchestra.run.smithi052.stdout:(23/23): librados2-16.2.14-37.g2e1f1015.el8.x86 4.9 MB/s | 3.7 MB 00:00 2023-09-23T22:17:42.459 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:17:42.460 INFO:teuthology.orchestra.run.smithi052.stdout:Total 12 MB/s | 57 MB 00:04 2023-09-23T22:17:42.507 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:17:42.558 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:17:42.559 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:17:42.690 INFO:teuthology.orchestra.run.smithi151.stdout:(21/23): ceph-common-16.2.14-37.g2e1f1015.el8.x 5.0 MB/s | 24 MB 00:04 2023-09-23T22:17:42.765 INFO:teuthology.orchestra.run.smithi151.stdout:(22/23): librados2-16.2.14-37.g2e1f1015.el8.x86 4.2 MB/s | 3.7 MB 00:00 2023-09-23T22:17:42.905 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:17:42.909 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:17:43.032 INFO:teuthology.orchestra.run.smithi151.stdout:(23/23): librbd1-16.2.14-37.g2e1f1015.el8.x86_6 3.7 MB/s | 4.2 MB 00:01 2023-09-23T22:17:43.034 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:17:43.034 INFO:teuthology.orchestra.run.smithi151.stdout:Total 10 MB/s | 57 MB 00:05 2023-09-23T22:17:43.080 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:17:43.126 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:17:43.126 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:17:43.450 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:17:43.454 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:17:43.947 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:17:44.103 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:17:44.493 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:17:44.525 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:17:44.678 INFO:teuthology.orchestra.run.smithi052.stdout: Upgrading : librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 2/25 2023-09-23T22:17:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:17:44.776 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 2/25 2023-09-23T22:17:44.951 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 3/25 2023-09-23T22:17:45.100 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T22:17:45.164 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T22:17:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: Upgrading : librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 2/25 2023-09-23T22:17:45.337 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 5/25 2023-09-23T22:17:45.355 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 2/25 2023-09-23T22:17:45.551 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 3/25 2023-09-23T22:17:45.763 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 5/25 2023-09-23T22:17:45.793 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T22:17:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: Upgrading : librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 6/25 2023-09-23T22:17:45.979 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 5/25 2023-09-23T22:17:46.045 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 6/25 2023-09-23T22:17:46.211 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T22:17:46.273 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T22:17:46.388 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 5/25 2023-09-23T22:17:46.603 INFO:teuthology.orchestra.run.smithi151.stdout: Upgrading : librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 6/25 2023-09-23T22:17:46.670 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 6/25 2023-09-23T22:17:46.762 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T22:17:46.928 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 9/25 2023-09-23T22:17:46.932 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-09-23T22:17:47.013 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 9/25 2023-09-23T22:17:47.193 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-23T22:17:47.319 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-ceph-argparse-2:16.2.14-37.g2e1f1015.el8.x 11/25 2023-09-23T22:17:47.499 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 12/25 2023-09-23T22:17:47.603 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-23T22:17:47.866 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 13/25 2023-09-23T22:17:47.981 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-23T22:17:48.065 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 14/25 2023-09-23T22:17:48.170 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 9/25 2023-09-23T22:17:48.228 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 15/25 2023-09-23T22:17:48.252 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 9/25 2023-09-23T22:17:48.300 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 15/25 2023-09-23T22:17:48.442 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-23T22:17:48.447 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T22:17:48.566 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-ceph-argparse-2:16.2.14-37.g2e1f1015.el8.x 11/25 2023-09-23T22:17:48.668 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T22:17:48.706 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 12/25 2023-09-23T22:17:48.888 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T22:17:48.903 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 13/25 2023-09-23T22:17:49.094 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 14/25 2023-09-23T22:17:49.129 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-ceph-common-2:16.2.14-37.g2e1f1015.el8.x86 19/25 2023-09-23T22:17:49.236 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 15/25 2023-09-23T22:17:49.298 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 15/25 2023-09-23T22:17:49.395 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T22:17:49.587 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T22:17:49.839 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T22:17:49.970 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-ceph-common-2:16.2.14-37.g2e1f1015.el8.x86 19/25 2023-09-23T22:17:53.011 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 20/25 2023-09-23T22:17:53.228 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 20/25 2023-09-23T22:17:53.671 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 20/25 2023-09-23T22:17:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 20/25 2023-09-23T22:17:53.967 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 20/25 2023-09-23T22:17:54.464 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 21/25 2023-09-23T22:17:54.553 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 21/25 2023-09-23T22:17:54.647 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 20/25 2023-09-23T22:17:54.677 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 22/25 2023-09-23T22:17:54.937 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 21/25 2023-09-23T22:17:55.027 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 21/25 2023-09-23T22:17:55.136 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 22/25 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 22/25 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /sys 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /proc 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /dev 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /run 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /mnt 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /var/tmp 2023-09-23T22:18:13.509 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /home 2023-09-23T22:18:13.510 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /tmp 2023-09-23T22:18:13.510 INFO:teuthology.orchestra.run.smithi052.stdout:skipping the directory /dev 2023-09-23T22:18:13.510 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:13.541 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 22/25 2023-09-23T22:18:13.541 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /sys 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /proc 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /dev 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /run 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /mnt 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /var/tmp 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /home 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /tmp 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /dev 2023-09-23T22:18:13.542 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:14.718 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 23/25 2023-09-23T22:18:14.736 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 23/25 2023-09-23T22:18:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 23/25 2023-09-23T22:18:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:18:14.816 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 23/25 2023-09-23T22:18:14.816 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:18:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:18:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:18:15.018 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T22:18:15.018 INFO:teuthology.orchestra.run.smithi052.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 1/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 2/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 3/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 4/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 5/25 2023-09-23T22:18:15.645 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 6/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 7/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-ceph-argparse-2:16.2.14-37.g2e1f1015.el8.x 8/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-ceph-common-2:16.2.14-37.g2e1f1015.el8.x86 9/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 10/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 11/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 12/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 13/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-09-23T22:18:15.646 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 22/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T22:18:15.647 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 24/25 2023-09-23T22:18:15.649 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:18:15.649 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 1/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 2/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 3/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 4/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 5/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 6/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 7/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-ceph-argparse-2:16.2.14-37.g2e1f1015.el8.x 8/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-ceph-common-2:16.2.14-37.g2e1f1015.el8.x86 9/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 10/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 11/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 12/25 2023-09-23T22:18:15.650 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 13/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 22/25 2023-09-23T22:18:15.651 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T22:18:15.652 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 24/25 2023-09-23T22:18:16.999 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:18:17.000 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:00:51 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:18:17.029 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T22:18:17.029 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:51 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:18:19.493 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:18:19.636 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:18:19.681 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:19.681 INFO:teuthology.orchestra.run.smithi151.stdout:Upgraded: 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: leveldb-1.22-1.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.682 INFO:teuthology.orchestra.run.smithi151.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-argparse-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T22:18:19.683 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.684 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.684 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.684 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:19.684 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:18:19.842 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-test 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout:Upgraded: 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout: librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout: librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.845 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-radosgw-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-selinux-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: leveldb-1.22-1.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: libradosstriper1-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: librgw2-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T22:18:19.846 INFO:teuthology.orchestra.run.smithi052.stdout: python3-ceph-argparse-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: python3-ceph-common-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:19.847 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:18:20.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-test 2023-09-23T22:18:20.192 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:18:20.376 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:18:21.033 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:55 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:18:21.209 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:00:56 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:18:22.401 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout:=========================================================================================== 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout:=========================================================================================== 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-test x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 52 M 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout: libcephsqlite x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 174 k 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T22:18:22.402 INFO:teuthology.orchestra.run.smithi151.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout:=========================================================================================== 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout:Install 6 Packages 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:22.403 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 53 M 2023-09-23T22:18:22.404 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 195 M 2023-09-23T22:18:22.404 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:18:22.450 INFO:teuthology.orchestra.run.smithi151.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.5 MB/s | 70 kB 00:00 2023-09-23T22:18:22.526 INFO:teuthology.orchestra.run.smithi151.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T22:18:22.576 INFO:teuthology.orchestra.run.smithi151.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-09-23T22:18:22.626 INFO:teuthology.orchestra.run.smithi151.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.0 MB/s | 202 kB 00:00 2023-09-23T22:18:22.702 INFO:teuthology.orchestra.run.smithi151.stdout:(5/6): libcephsqlite-16.2.14-37.g2e1f1015.el8.x 584 kB/s | 174 kB 00:00 2023-09-23T22:18:22.722 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:18:22.723 INFO:teuthology.orchestra.run.smithi052.stdout:=========================================================================================== 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout:=========================================================================================== 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-test x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 52 M 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: libcephsqlite x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 174 k 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T22:18:22.724 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:22.725 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:18:22.725 INFO:teuthology.orchestra.run.smithi052.stdout:=========================================================================================== 2023-09-23T22:18:22.725 INFO:teuthology.orchestra.run.smithi052.stdout:Install 6 Packages 2023-09-23T22:18:22.725 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:22.726 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 53 M 2023-09-23T22:18:22.726 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 195 M 2023-09-23T22:18:22.726 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:18:22.756 INFO:teuthology.orchestra.run.smithi052.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.3 MB/s | 70 kB 00:00 2023-09-23T22:18:22.823 INFO:teuthology.orchestra.run.smithi052.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T22:18:22.865 INFO:teuthology.orchestra.run.smithi052.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.5 MB/s | 187 kB 00:00 2023-09-23T22:18:22.916 INFO:teuthology.orchestra.run.smithi052.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 3.9 MB/s | 202 kB 00:00 2023-09-23T22:18:23.041 INFO:teuthology.orchestra.run.smithi052.stdout:(5/6): libcephsqlite-16.2.14-37.g2e1f1015.el8.x 552 kB/s | 174 kB 00:00 2023-09-23T22:18:26.468 INFO:teuthology.orchestra.run.smithi151.stdout:(6/6): ceph-test-16.2.14-37.g2e1f1015.el8.x86_6 13 MB/s | 52 MB 00:04 2023-09-23T22:18:26.470 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:18:26.470 INFO:teuthology.orchestra.run.smithi151.stdout:Total 13 MB/s | 53 MB 00:04 2023-09-23T22:18:26.507 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:18:26.529 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:18:26.529 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:18:26.804 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:18:26.807 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:18:27.271 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:18:27.299 INFO:teuthology.orchestra.run.smithi052.stdout:(6/6): ceph-test-16.2.14-37.g2e1f1015.el8.x86_6 11 MB/s | 52 MB 00:04 2023-09-23T22:18:27.301 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:18:27.301 INFO:teuthology.orchestra.run.smithi052.stdout:Total 11 MB/s | 53 MB 00:04 2023-09-23T22:18:27.340 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:18:27.362 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:18:27.362 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:18:27.397 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:18:27.488 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:18:27.651 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:18:27.655 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:18:27.685 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T22:18:27.848 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T22:18:28.036 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T22:18:28.135 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:18:28.189 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 5/6 2023-09-23T22:18:28.306 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:18:28.390 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T22:18:28.588 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T22:18:28.741 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T22:18:28.888 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T22:18:29.039 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 5/6 2023-09-23T22:18:32.750 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 5/6 2023-09-23T22:18:33.035 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 6/6 2023-09-23T22:18:33.252 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 6/6 2023-09-23T22:18:33.252 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 1/6 2023-09-23T22:18:33.252 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 2/6 2023-09-23T22:18:33.252 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T22:18:33.252 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T22:18:33.252 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T22:18:33.907 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 5/6 2023-09-23T22:18:34.170 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 6/6 2023-09-23T22:18:34.363 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T22:18:34.364 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:08 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:18:34.392 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 6/6 2023-09-23T22:18:34.392 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 1/6 2023-09-23T22:18:34.393 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 2/6 2023-09-23T22:18:34.393 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T22:18:34.393 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T22:18:34.393 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T22:18:35.504 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T22:18:35.504 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:10 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:18:36.654 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: jq-1.6-3.el8.x86_64 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T22:18:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T22:18:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T22:18:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:36.838 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:18:36.964 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph 2023-09-23T22:18:37.301 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:18:38.036 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:18:38.102 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:12 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:18:38.238 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-test-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: jq-1.6-3.el8.x86_64 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: libcephsqlite-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T22:18:38.239 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:38.240 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:18:38.367 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph 2023-09-23T22:18:38.720 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:18:39.542 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:18:39.545 INFO:teuthology.orchestra.run.smithi151.stdout:====================================================================================================== 2023-09-23T22:18:39.545 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:18:39.545 INFO:teuthology.orchestra.run.smithi151.stdout:====================================================================================================== 2023-09-23T22:18:39.545 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: ceph x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 6.4 k 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mds x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 2.4 M 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 1.6 M 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-modules-core noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 244 k 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mon x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 5.0 M 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-osd x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 18 M 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T22:18:39.546 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T22:18:39.547 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T22:18:39.548 INFO:teuthology.orchestra.run.smithi151.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T22:18:39.549 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:39.549 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:18:39.549 INFO:teuthology.orchestra.run.smithi151.stdout:====================================================================================================== 2023-09-23T22:18:39.549 INFO:teuthology.orchestra.run.smithi151.stdout:Install 32 Packages 2023-09-23T22:18:39.549 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:39.550 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 31 M 2023-09-23T22:18:39.550 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 101 M 2023-09-23T22:18:39.550 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:18:39.567 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:14 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:18:40.101 INFO:teuthology.orchestra.run.smithi151.stdout:(1/32): ceph-16.2.14-37.g2e1f1015.el8.x86_64.rp 23 kB/s | 6.4 kB 00:00 2023-09-23T22:18:40.168 INFO:teuthology.orchestra.run.smithi151.stdout:(2/32): ceph-mds-16.2.14-37.g2e1f1015.el8.x86_6 7.0 MB/s | 2.4 MB 00:00 2023-09-23T22:18:40.218 INFO:teuthology.orchestra.run.smithi151.stdout:(3/32): ceph-mgr-16.2.14-37.g2e1f1015.el8.x86_6 4.0 MB/s | 1.6 MB 00:00 2023-09-23T22:18:40.310 INFO:teuthology.orchestra.run.smithi151.stdout:(4/32): ceph-mgr-modules-core-16.2.14-37.g2e1f1 2.6 MB/s | 244 kB 00:00 2023-09-23T22:18:40.587 INFO:teuthology.orchestra.run.smithi151.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 157 kB/s | 44 kB 00:00 2023-09-23T22:18:40.727 INFO:teuthology.orchestra.run.smithi151.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.3 MB/s | 185 kB 00:00 2023-09-23T22:18:40.794 INFO:teuthology.orchestra.run.smithi151.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.5 MB/s | 173 kB 00:00 2023-09-23T22:18:41.056 INFO:teuthology.orchestra.run.smithi151.stdout:(8/32): ceph-osd-16.2.14-37.g2e1f1015.el8.x86_6 21 MB/s | 18 MB 00:00 2023-09-23T22:18:41.069 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:18:41.074 INFO:teuthology.orchestra.run.smithi052.stdout:====================================================================================================== 2023-09-23T22:18:41.074 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:18:41.074 INFO:teuthology.orchestra.run.smithi052.stdout:====================================================================================================== 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: ceph x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 6.4 k 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mds x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 2.4 M 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 1.6 M 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-modules-core noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 244 k 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mon x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 5.0 M 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-osd x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 18 M 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T22:18:41.075 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T22:18:41.076 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T22:18:41.077 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:41.078 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:18:41.078 INFO:teuthology.orchestra.run.smithi052.stdout:====================================================================================================== 2023-09-23T22:18:41.078 INFO:teuthology.orchestra.run.smithi052.stdout:Install 32 Packages 2023-09-23T22:18:41.078 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:41.078 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 31 M 2023-09-23T22:18:41.078 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 101 M 2023-09-23T22:18:41.079 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:18:41.087 INFO:teuthology.orchestra.run.smithi151.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 1.3 MB/s | 384 kB 00:00 2023-09-23T22:18:41.170 INFO:teuthology.orchestra.run.smithi151.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 211 kB/s | 18 kB 00:00 2023-09-23T22:18:41.195 INFO:teuthology.orchestra.run.smithi151.stdout:(11/32): python3-jaraco-6.2-6.el8.noarch.rpm 78 kB/s | 11 kB 00:00 2023-09-23T22:18:41.226 INFO:teuthology.orchestra.run.smithi052.stdout:(1/32): ceph-16.2.14-37.g2e1f1015.el8.x86_64.rp 57 kB/s | 6.4 kB 00:00 2023-09-23T22:18:41.246 INFO:teuthology.orchestra.run.smithi151.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 648 kB/s | 49 kB 00:00 2023-09-23T22:18:41.321 INFO:teuthology.orchestra.run.smithi151.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 469 kB/s | 59 kB 00:00 2023-09-23T22:18:41.346 INFO:teuthology.orchestra.run.smithi151.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.8 MB/s | 283 kB 00:00 2023-09-23T22:18:41.397 INFO:teuthology.orchestra.run.smithi151.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 207 kB/s | 16 kB 00:00 2023-09-23T22:18:41.422 INFO:teuthology.orchestra.run.smithi151.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 249 kB/s | 19 kB 00:00 2023-09-23T22:18:41.481 INFO:teuthology.orchestra.run.smithi151.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 286 kB/s | 24 kB 00:00 2023-09-23T22:18:41.485 INFO:teuthology.orchestra.run.smithi052.stdout:(2/32): ceph-mds-16.2.14-37.g2e1f1015.el8.x86_6 6.6 MB/s | 2.4 MB 00:00 2023-09-23T22:18:41.506 INFO:teuthology.orchestra.run.smithi151.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 350 kB/s | 29 kB 00:00 2023-09-23T22:18:41.526 INFO:teuthology.orchestra.run.smithi052.stdout:(3/32): ceph-mgr-16.2.14-37.g2e1f1015.el8.x86_6 3.9 MB/s | 1.6 MB 00:00 2023-09-23T22:18:41.556 INFO:teuthology.orchestra.run.smithi151.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 365 kB/s | 27 kB 00:00 2023-09-23T22:18:41.592 INFO:teuthology.orchestra.run.smithi151.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 3.0 MB/s | 255 kB 00:00 2023-09-23T22:18:41.636 INFO:teuthology.orchestra.run.smithi052.stdout:(4/32): ceph-mgr-modules-core-16.2.14-37.g2e1f1 2.2 MB/s | 244 kB 00:00 2023-09-23T22:18:41.665 INFO:teuthology.orchestra.run.smithi151.stdout:(21/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.1 MB/s | 85 kB 00:00 2023-09-23T22:18:41.723 INFO:teuthology.orchestra.run.smithi151.stdout:(22/32): python3-zc-lockfile-2.0-2.el8.noarch.r 394 kB/s | 23 kB 00:00 2023-09-23T22:18:41.727 INFO:teuthology.orchestra.run.smithi052.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 476 kB/s | 44 kB 00:00 2023-09-23T22:18:41.748 INFO:teuthology.orchestra.run.smithi151.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.3 MB/s | 251 kB 00:00 2023-09-23T22:18:41.802 INFO:teuthology.orchestra.run.smithi052.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.4 MB/s | 185 kB 00:00 2023-09-23T22:18:41.807 INFO:teuthology.orchestra.run.smithi151.stdout:(24/32): python3-pytz-2017.2-9.el8.noarch.rpm 649 kB/s | 54 kB 00:00 2023-09-23T22:18:41.824 INFO:teuthology.orchestra.run.smithi151.stdout:(25/32): python3-markupsafe-0.23-19.el8.x86_64. 559 kB/s | 39 kB 00:00 2023-09-23T22:18:41.848 INFO:teuthology.orchestra.run.smithi052.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 4.1 MB/s | 173 kB 00:00 2023-09-23T22:18:41.858 INFO:teuthology.orchestra.run.smithi151.stdout:(26/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.1 MB/s | 157 kB 00:00 2023-09-23T22:18:41.892 INFO:teuthology.orchestra.run.smithi151.stdout:(27/32): python3-werkzeug-0.12.2-4.el8.noarch.r 6.7 MB/s | 457 kB 00:00 2023-09-23T22:18:41.912 INFO:teuthology.orchestra.run.smithi052.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 5.7 MB/s | 384 kB 00:00 2023-09-23T22:18:41.917 INFO:teuthology.orchestra.run.smithi151.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.7 MB/s | 103 kB 00:00 2023-09-23T22:18:41.948 INFO:teuthology.orchestra.run.smithi052.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 334 kB/s | 11 kB 00:00 2023-09-23T22:18:41.987 INFO:teuthology.orchestra.run.smithi052.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 421 kB/s | 18 kB 00:00 2023-09-23T22:18:42.025 INFO:teuthology.orchestra.run.smithi151.stdout:(29/32): ceph-mon-16.2.14-37.g2e1f1015.el8.x86_ 2.6 MB/s | 5.0 MB 00:01 2023-09-23T22:18:42.029 INFO:teuthology.orchestra.run.smithi052.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 1.1 MB/s | 49 kB 00:00 2023-09-23T22:18:42.050 INFO:teuthology.orchestra.run.smithi151.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 687 kB/s | 109 kB 00:00 2023-09-23T22:18:42.084 INFO:teuthology.orchestra.run.smithi151.stdout:(31/32): python3-cryptography-3.2.1-5.el8.x86_6 3.3 MB/s | 559 kB 00:00 2023-09-23T22:18:42.104 INFO:teuthology.orchestra.run.smithi052.stdout:(12/32): ceph-mon-16.2.14-37.g2e1f1015.el8.x86_ 5.7 MB/s | 5.0 MB 00:00 2023-09-23T22:18:42.109 INFO:teuthology.orchestra.run.smithi151.stdout:(32/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.8 MB/s | 238 kB 00:00 2023-09-23T22:18:42.111 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:18:42.112 INFO:teuthology.orchestra.run.smithi151.stdout:Total 12 MB/s | 31 MB 00:02 2023-09-23T22:18:42.130 INFO:teuthology.orchestra.run.smithi052.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 585 kB/s | 59 kB 00:00 2023-09-23T22:18:42.173 INFO:teuthology.orchestra.run.smithi052.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 375 kB/s | 16 kB 00:00 2023-09-23T22:18:42.204 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:18:42.213 INFO:teuthology.orchestra.run.smithi052.stdout:(15/32): python3-simplegeneric-0.8.1-17.el8.noa 450 kB/s | 19 kB 00:00 2023-09-23T22:18:42.242 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:18:42.242 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:18:42.247 INFO:teuthology.orchestra.run.smithi052.stdout:(16/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.9 MB/s | 283 kB 00:00 2023-09-23T22:18:42.273 INFO:teuthology.orchestra.run.smithi052.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 406 kB/s | 24 kB 00:00 2023-09-23T22:18:42.298 INFO:teuthology.orchestra.run.smithi052.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 582 kB/s | 29 kB 00:00 2023-09-23T22:18:42.323 INFO:teuthology.orchestra.run.smithi052.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 549 kB/s | 27 kB 00:00 2023-09-23T22:18:42.357 INFO:teuthology.orchestra.run.smithi052.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 4.3 MB/s | 255 kB 00:00 2023-09-23T22:18:42.382 INFO:teuthology.orchestra.run.smithi052.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 4.2 MB/s | 251 kB 00:00 2023-09-23T22:18:42.408 INFO:teuthology.orchestra.run.smithi052.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.7 MB/s | 85 kB 00:00 2023-09-23T22:18:42.433 INFO:teuthology.orchestra.run.smithi052.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 457 kB/s | 23 kB 00:00 2023-09-23T22:18:42.458 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:18:42.462 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:18:42.666 INFO:teuthology.orchestra.run.smithi052.stdout:(24/32): ceph-osd-16.2.14-37.g2e1f1015.el8.x86_ 16 MB/s | 18 MB 00:01 2023-09-23T22:18:42.691 INFO:teuthology.orchestra.run.smithi052.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 385 kB/s | 109 kB 00:00 2023-09-23T22:18:42.742 INFO:teuthology.orchestra.run.smithi052.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 1.8 MB/s | 559 kB 00:00 2023-09-23T22:18:42.767 INFO:teuthology.orchestra.run.smithi052.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 811 kB/s | 54 kB 00:00 2023-09-23T22:18:42.792 INFO:teuthology.orchestra.run.smithi052.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 776 kB/s | 39 kB 00:00 2023-09-23T22:18:42.818 INFO:teuthology.orchestra.run.smithi052.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.5 MB/s | 238 kB 00:00 2023-09-23T22:18:42.846 INFO:teuthology.orchestra.run.smithi052.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-23T22:18:42.868 INFO:teuthology.orchestra.run.smithi052.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-23T22:18:42.893 INFO:teuthology.orchestra.run.smithi052.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.4 MB/s | 457 kB 00:00 2023-09-23T22:18:42.896 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:18:42.896 INFO:teuthology.orchestra.run.smithi052.stdout:Total 17 MB/s | 31 MB 00:01 2023-09-23T22:18:42.991 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:18:43.032 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:18:43.032 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:18:43.260 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:18:43.264 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:18:43.639 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:18:43.824 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T22:18:44.081 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T22:18:44.296 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-09-23T22:18:44.371 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:18:44.589 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T22:18:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-09-23T22:18:44.860 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T22:18:45.005 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-09-23T22:18:45.083 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T22:18:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-09-23T22:18:45.302 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T22:18:45.344 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-09-23T22:18:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T22:18:45.645 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-09-23T22:18:45.743 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T22:18:45.822 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-09-23T22:18:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T22:18:46.038 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-09-23T22:18:46.293 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-09-23T22:18:46.324 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T22:18:46.469 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-09-23T22:18:46.616 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T22:18:46.680 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T22:18:46.770 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T22:18:46.851 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T22:18:46.942 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T22:18:47.065 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T22:18:47.138 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T22:18:47.220 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T22:18:47.364 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T22:18:47.388 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T22:18:47.533 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T22:18:47.553 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T22:18:47.750 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T22:18:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T22:18:47.908 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T22:18:47.964 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T22:18:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T22:18:48.158 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T22:18:48.227 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T22:18:48.716 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T22:18:48.870 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T22:18:49.075 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T22:18:49.235 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T22:18:49.351 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T22:18:49.522 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T22:18:49.555 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T22:18:49.778 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T22:18:49.903 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T22:18:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T22:18:50.214 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T22:18:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T22:18:50.531 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-37.g2e1f1015.el8.n 27/32 2023-09-23T22:18:50.613 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T22:18:50.696 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 28/32 2023-09-23T22:18:50.917 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-37.g2e1f1015.el8.n 27/32 2023-09-23T22:18:51.073 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 28/32 2023-09-23T22:18:52.181 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 28/32 2023-09-23T22:18:52.360 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 29/32 2023-09-23T22:18:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 28/32 2023-09-23T22:18:52.785 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 29/32 2023-09-23T22:18:52.828 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 29/32 2023-09-23T22:18:52.944 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 30/32 2023-09-23T22:18:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 30/32 2023-09-23T22:18:53.280 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 29/32 2023-09-23T22:18:53.360 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 31/32 2023-09-23T22:18:53.378 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 31/32 2023-09-23T22:18:53.428 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 30/32 2023-09-23T22:18:53.508 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 32/32 2023-09-23T22:18:53.658 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 30/32 2023-09-23T22:18:53.803 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 31/32 2023-09-23T22:18:53.824 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 31/32 2023-09-23T22:18:53.872 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 32/32 2023-09-23T22:18:53.872 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 1/32 2023-09-23T22:18:53.872 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 2/32 2023-09-23T22:18:53.872 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 3/32 2023-09-23T22:18:53.872 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 4/32 2023-09-23T22:18:53.872 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 5/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-37.g2e1f1015.el8.n 6/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T22:18:53.873 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-09-23T22:18:53.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-09-23T22:18:53.875 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-09-23T22:18:53.875 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-09-23T22:18:53.875 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-09-23T22:18:53.968 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 32/32 2023-09-23T22:18:54.351 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 32/32 2023-09-23T22:18:54.351 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 1/32 2023-09-23T22:18:54.351 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 2/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 3/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 4/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 5/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-37.g2e1f1015.el8.n 6/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T22:18:54.352 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T22:18:54.353 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T22:18:54.354 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T22:18:55.499 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-09-23T22:18:55.499 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:18:55.858 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T22:18:55.858 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:30 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:18:57.949 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:18:58.139 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:58.139 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-modules-core-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T22:18:58.140 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T22:18:58.141 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T22:18:58.142 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:18:58.143 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:18:58.305 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-base 2023-09-23T22:18:58.382 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:18:58.587 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:58.587 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mds-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-modules-core-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mon-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-osd-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T22:18:58.588 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T22:18:58.589 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:18:58.590 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:18:58.652 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:18:58.750 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-base 2023-09-23T22:18:59.121 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:18:59.474 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:18:59.939 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:00.733 INFO:teuthology.orchestra.run.smithi151.stdout:Package ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:19:00.841 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:00.842 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:19:00.842 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:00.952 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install cephadm 2023-09-23T22:19:01.297 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:01.356 INFO:teuthology.orchestra.run.smithi052.stdout:Package ceph-base-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:19:01.475 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:01.476 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:19:01.477 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:19:01.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install cephadm 2023-09-23T22:19:01.949 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:02.128 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:36 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:02.819 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:37 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:03.483 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:03.484 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:19:03.484 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout: cephadm noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 84 k 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 84 k 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 348 k 2023-09-23T22:19:03.485 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:19:03.647 INFO:teuthology.orchestra.run.smithi151.stdout:cephadm-16.2.14-37.g2e1f1015.el8.noarch.rpm 522 kB/s | 84 kB 00:00 2023-09-23T22:19:03.647 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:03.647 INFO:teuthology.orchestra.run.smithi151.stdout:Total 518 kB/s | 84 kB 00:00 2023-09-23T22:19:03.648 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:19:03.654 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:19:03.655 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:19:03.660 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:19:03.664 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:19:03.704 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:19:04.335 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout: cephadm noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 84 k 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:04.336 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:19:04.337 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:19:04.337 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:19:04.337 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:04.337 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 84 k 2023-09-23T22:19:04.337 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 348 k 2023-09-23T22:19:04.337 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:19:04.939 INFO:teuthology.orchestra.run.smithi052.stdout:cephadm-16.2.14-37.g2e1f1015.el8.noarch.rpm 140 kB/s | 84 kB 00:00 2023-09-23T22:19:04.939 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:04.940 INFO:teuthology.orchestra.run.smithi052.stdout:Total 140 kB/s | 84 kB 00:00 2023-09-23T22:19:04.940 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:19:04.949 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:19:04.949 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:19:04.952 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:19:04.956 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:19:05.385 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:19:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:06.999 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:07.592 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:07.766 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:08.085 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:08.085 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:42 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:08.863 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/1 2023-09-23T22:19:08.864 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:10.401 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:19:10.582 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:10.582 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:19:10.582 INFO:teuthology.orchestra.run.smithi151.stdout: cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:10.583 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:10.583 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:10.710 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T22:19:11.072 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:11.326 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:19:11.523 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:11.523 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:19:11.523 INFO:teuthology.orchestra.run.smithi052.stdout: cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:11.523 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:11.523 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:19:11.646 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T22:19:11.875 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:45 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:12.029 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:12.876 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:13.320 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:13.321 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:19:13.321 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2023-09-23T22:19:13.321 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 161 k 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 161 k 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 470 k 2023-09-23T22:19:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:19:13.510 INFO:teuthology.orchestra.run.smithi151.stdout:ceph-immutable-object-cache-16.2.14-37.g2e1f101 860 kB/s | 161 kB 00:00 2023-09-23T22:19:13.510 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:13.510 INFO:teuthology.orchestra.run.smithi151.stdout:Total 855 kB/s | 161 kB 00:00 2023-09-23T22:19:13.511 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:19:13.521 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:19:13.521 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:19:13.572 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:19:13.576 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:19:13.685 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:19:13.915 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.e 1/1 2023-09-23T22:19:14.256 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.e 1/1 2023-09-23T22:19:14.405 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repo Size 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 161 k 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:19:14.406 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:14.407 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 161 k 2023-09-23T22:19:14.407 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 470 k 2023-09-23T22:19:14.407 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:19:14.732 INFO:teuthology.orchestra.run.smithi052.stdout:ceph-immutable-object-cache-16.2.14-37.g2e1f101 497 kB/s | 161 kB 00:00 2023-09-23T22:19:14.732 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:14.732 INFO:teuthology.orchestra.run.smithi052.stdout:Total 495 kB/s | 161 kB 00:00 2023-09-23T22:19:14.733 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:19:14.745 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:19:14.745 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:19:14.800 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:19:14.804 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:19:14.920 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:19:15.109 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.e 1/1 2023-09-23T22:19:15.325 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.e 1/1 2023-09-23T22:19:15.325 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:49 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:15.472 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.e 1/1 2023-09-23T22:19:16.542 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.e 1/1 2023-09-23T22:19:16.542 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:17.763 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:19:17.945 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:17.945 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:19:17.945 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:19:17.945 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:17.946 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:18.072 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr 2023-09-23T22:19:18.420 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:19.247 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:19:19.284 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:53 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:19.447 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:19.447 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:19:19.448 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-immutable-object-cache-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:19:19.448 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:19.448 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:19:19.570 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-mgr 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:20.536 INFO:teuthology.orchestra.run.smithi151.stdout:Package ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:19:20.647 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:20.648 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:19:20.648 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:20.760 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T22:19:20.826 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:55 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:21.105 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:21.995 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:55 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:22.270 INFO:teuthology.orchestra.run.smithi052.stdout:Package ceph-mgr-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:19:22.391 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:22.392 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:19:22.393 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:19:22.508 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T22:19:22.877 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:23.367 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================================== 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================================== 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-dashboard noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 3.1 M 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-grafana-dashboards noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 24 k 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-prometheus-alerts noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 15 k 2023-09-23T22:19:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================================== 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout:Install 6 Packages 2023-09-23T22:19:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:23.370 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 3.4 M 2023-09-23T22:19:23.370 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 22 M 2023-09-23T22:19:23.370 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:19:23.800 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:01:58 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:23.831 INFO:teuthology.orchestra.run.smithi151.stdout:(1/6): ceph-grafana-dashboards-16.2.14-37.g2e1f 170 kB/s | 24 kB 00:00 2023-09-23T22:19:23.923 INFO:teuthology.orchestra.run.smithi151.stdout:(2/6): ceph-prometheus-alerts-16.2.14-37.g2e1f1 63 kB/s | 15 kB 00:00 2023-09-23T22:19:23.965 INFO:teuthology.orchestra.run.smithi151.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 252 kB/s | 34 kB 00:00 2023-09-23T22:19:24.040 INFO:teuthology.orchestra.run.smithi151.stdout:(4/6): ceph-mgr-dashboard-16.2.14-37.g2e1f1015. 8.9 MB/s | 3.1 MB 00:00 2023-09-23T22:19:24.066 INFO:teuthology.orchestra.run.smithi151.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 426 kB/s | 43 kB 00:00 2023-09-23T22:19:24.133 INFO:teuthology.orchestra.run.smithi151.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 940 kB/s | 196 kB 00:00 2023-09-23T22:19:24.134 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:24.135 INFO:teuthology.orchestra.run.smithi151.stdout:Total 4.5 MB/s | 3.4 MB 00:00 2023-09-23T22:19:24.146 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:19:24.156 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:19:24.156 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:19:24.224 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:19:24.228 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:19:24.545 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:19:24.706 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T22:19:24.886 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T22:19:25.099 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T22:19:25.220 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-37.g2e1f1015.el8.no 4/6 2023-09-23T22:19:25.344 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================================== 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================================== 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-dashboard noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 3.1 M 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-grafana-dashboards noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 24 k 2023-09-23T22:19:25.345 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-prometheus-alerts noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 15 k 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================================== 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout:Install 6 Packages 2023-09-23T22:19:25.346 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:25.347 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 3.4 M 2023-09-23T22:19:25.347 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 22 M 2023-09-23T22:19:25.347 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:19:25.624 INFO:teuthology.orchestra.run.smithi052.stdout:(1/6): ceph-prometheus-alerts-16.2.14-37.g2e1f1 106 kB/s | 15 kB 00:00 2023-09-23T22:19:25.707 INFO:teuthology.orchestra.run.smithi052.stdout:(2/6): ceph-grafana-dashboards-16.2.14-37.g2e1f 109 kB/s | 24 kB 00:00 2023-09-23T22:19:25.849 INFO:teuthology.orchestra.run.smithi052.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 149 kB/s | 34 kB 00:00 2023-09-23T22:19:25.891 INFO:teuthology.orchestra.run.smithi052.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 1.0 MB/s | 43 kB 00:00 2023-09-23T22:19:26.025 INFO:teuthology.orchestra.run.smithi052.stdout:(5/6): ceph-mgr-dashboard-16.2.14-37.g2e1f1015. 5.8 MB/s | 3.1 MB 00:00 2023-09-23T22:19:26.050 INFO:teuthology.orchestra.run.smithi052.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 574 kB/s | 196 kB 00:00 2023-09-23T22:19:26.053 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:26.053 INFO:teuthology.orchestra.run.smithi052.stdout:Total 4.9 MB/s | 3.4 MB 00:00 2023-09-23T22:19:26.065 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:19:26.077 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:19:26.077 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:19:26.157 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:19:26.159 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:19:26.238 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-37.g2e1f1015.el8.n 5/6 2023-09-23T22:19:26.449 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:19:26.526 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:19:26.625 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T22:19:26.746 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:19:26.747 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-37.g2e1f1015.el8.n 1/6 2023-09-23T22:19:26.747 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 2/6 2023-09-23T22:19:26.747 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-37.g2e1f1015.el8.no 3/6 2023-09-23T22:19:26.747 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T22:19:26.747 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T22:19:26.807 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T22:19:26.985 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T22:19:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-37.g2e1f1015.el8.no 4/6 2023-09-23T22:19:27.804 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T22:19:27.804 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:01 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:28.681 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-37.g2e1f1015.el8.n 5/6 2023-09-23T22:19:28.919 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:19:29.156 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:19:29.157 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-37.g2e1f1015.el8.n 1/6 2023-09-23T22:19:29.157 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 2/6 2023-09-23T22:19:29.157 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-37.g2e1f1015.el8.no 3/6 2023-09-23T22:19:29.157 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T22:19:29.157 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T22:19:30.165 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:19:30.212 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T22:19:30.212 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:05 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-grafana-dashboards-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-prometheus-alerts-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T22:19:30.348 INFO:teuthology.orchestra.run.smithi151.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T22:19:30.349 INFO:teuthology.orchestra.run.smithi151.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T22:19:30.349 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:30.349 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:30.467 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T22:19:30.815 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:31.633 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:05 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:32.735 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:19:32.936 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:32.936 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:19:32.936 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-grafana-dashboards-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:32.936 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-dashboard-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:32.937 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-prometheus-alerts-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:32.937 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T22:19:32.937 INFO:teuthology.orchestra.run.smithi052.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T22:19:32.937 INFO:teuthology.orchestra.run.smithi052.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T22:19:32.937 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:32.937 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:19:33.077 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================================================= 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================================================= 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 8.6 M 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:19:33.079 INFO:teuthology.orchestra.run.smithi151.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================================================= 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout:Install 8 Packages 2023-09-23T22:19:33.080 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:33.081 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 37 M 2023-09-23T22:19:33.081 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 210 M 2023-09-23T22:19:33.081 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:19:33.168 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T22:19:33.269 INFO:teuthology.orchestra.run.smithi151.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 20 MB/s | 3.7 MB 00:00 2023-09-23T22:19:33.329 INFO:teuthology.orchestra.run.smithi151.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 3.8 MB/s | 225 kB 00:00 2023-09-23T22:19:33.537 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:33.553 INFO:teuthology.orchestra.run.smithi151.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 29 MB/s | 14 MB 00:00 2023-09-23T22:19:33.653 INFO:teuthology.orchestra.run.smithi151.stdout:(4/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.9 MB 00:00 2023-09-23T22:19:33.762 INFO:teuthology.orchestra.run.smithi151.stdout:(5/8): ceph-mgr-diskprediction-local-16.2.14-37 13 MB/s | 8.6 MB 00:00 2023-09-23T22:19:33.795 INFO:teuthology.orchestra.run.smithi151.stdout:(6/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 4.4 MB/s | 644 kB 00:00 2023-09-23T22:19:33.887 INFO:teuthology.orchestra.run.smithi151.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 14 MB/s | 4.7 MB 00:00 2023-09-23T22:19:33.912 INFO:teuthology.orchestra.run.smithi151.stdout:(8/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.1 MB/s | 171 kB 00:00 2023-09-23T22:19:33.913 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:33.914 INFO:teuthology.orchestra.run.smithi151.stdout:Total 44 MB/s | 37 MB 00:00 2023-09-23T22:19:34.149 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:19:34.168 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:19:34.168 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:19:34.425 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:09 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:34.468 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:19:34.472 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:19:35.018 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:19:35.153 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:19:35.281 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:19:35.437 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:19:35.958 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout:======================================================================================================================= 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout:======================================================================================================================= 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 8.6 M 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T22:19:35.960 INFO:teuthology.orchestra.run.smithi052.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout:======================================================================================================================= 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout:Install 8 Packages 2023-09-23T22:19:35.961 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:35.962 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 37 M 2023-09-23T22:19:35.962 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 210 M 2023-09-23T22:19:35.962 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:19:36.037 INFO:teuthology.orchestra.run.smithi052.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 8.4 MB/s | 644 kB 00:00 2023-09-23T22:19:36.053 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:19:36.062 INFO:teuthology.orchestra.run.smithi052.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.7 MB/s | 171 kB 00:00 2023-09-23T22:19:36.189 INFO:teuthology.orchestra.run.smithi052.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 30 MB/s | 3.7 MB 00:00 2023-09-23T22:19:36.211 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:19:36.230 INFO:teuthology.orchestra.run.smithi052.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-09-23T22:19:36.380 INFO:teuthology.orchestra.run.smithi052.stdout:(5/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 32 MB/s | 4.9 MB 00:00 2023-09-23T22:19:36.531 INFO:teuthology.orchestra.run.smithi052.stdout:(6/8): python3-scipy-1.0.0-21.module+el8.5.0+10 28 MB/s | 14 MB 00:00 2023-09-23T22:19:36.639 INFO:teuthology.orchestra.run.smithi052.stdout:(7/8): ceph-mgr-diskprediction-local-16.2.14-37 13 MB/s | 8.6 MB 00:00 2023-09-23T22:19:36.764 INFO:teuthology.orchestra.run.smithi052.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2023-09-23T22:19:36.765 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:36.765 INFO:teuthology.orchestra.run.smithi052.stdout:Total 46 MB/s | 37 MB 00:00 2023-09-23T22:19:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:19:37.009 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:19:37.029 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:19:37.029 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:19:37.044 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:19:37.348 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:19:37.352 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:19:37.748 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:19:37.960 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:19:38.430 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:19:38.537 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:19:38.571 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:19:39.413 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:19:39.622 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:19:40.259 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T22:19:40.430 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:19:40.496 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:19:41.155 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T22:19:41.454 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:19:41.672 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:19:41.796 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T22:19:41.994 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015 8/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015 8/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015 1/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-09-23T22:19:42.250 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-09-23T22:19:43.327 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-09-23T22:19:43.327 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:17 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:19:44.900 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T22:19:45.054 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015 8/8 2023-09-23T22:19:45.312 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015 8/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015 1/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T22:19:45.313 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T22:19:45.697 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T22:19:45.886 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T22:19:45.887 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T22:19:45.887 INFO:teuthology.orchestra.run.smithi151.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T22:19:45.887 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:45.887 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:46.012 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-rook 2023-09-23T22:19:46.363 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:46.444 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T22:19:46.444 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:21 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:47.182 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:21 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:48.534 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:19:48.536 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================================= 2023-09-23T22:19:48.536 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:19:48.536 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================================= 2023-09-23T22:19:48.536 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:19:48.536 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-rook noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 27 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:19:48.537 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================================= 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout:Install 13 Packages 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:48.538 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 1.9 M 2023-09-23T22:19:48.539 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 24 M 2023-09-23T22:19:48.539 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:19:48.683 INFO:teuthology.orchestra.run.smithi151.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 159 kB/s | 16 kB 00:00 2023-09-23T22:19:48.708 INFO:teuthology.orchestra.run.smithi151.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 259 kB/s | 33 kB 00:00 2023-09-23T22:19:48.775 INFO:teuthology.orchestra.run.smithi151.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 888 kB/s | 81 kB 00:00 2023-09-23T22:19:48.842 INFO:teuthology.orchestra.run.smithi151.stdout:(4/13): ceph-mgr-rook-16.2.14-37.g2e1f1015.el8. 105 kB/s | 27 kB 00:00 2023-09-23T22:19:48.867 INFO:teuthology.orchestra.run.smithi151.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 700 kB/s | 64 kB 00:00 2023-09-23T22:19:48.907 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:19:48.931 INFO:teuthology.orchestra.run.smithi151.stdout:(6/13): python3-kubernetes-11.0.0-6.el8.noarch. 5.2 MB/s | 1.2 MB 00:00 2023-09-23T22:19:48.960 INFO:teuthology.orchestra.run.smithi151.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 522 kB/s | 61 kB 00:00 2023-09-23T22:19:48.993 INFO:teuthology.orchestra.run.smithi151.stdout:(8/13): python3-pyasn1-modules-0.3.7-6.el8.noar 877 kB/s | 110 kB 00:00 2023-09-23T22:19:49.027 INFO:teuthology.orchestra.run.smithi151.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 295 kB/s | 27 kB 00:00 2023-09-23T22:19:49.052 INFO:teuthology.orchestra.run.smithi151.stdout:(10/13): python3-jsonpointer-1.10-11.el8.noarch 374 kB/s | 20 kB 00:00 2023-09-23T22:19:49.077 INFO:teuthology.orchestra.run.smithi151.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.1 MB/s | 126 kB 00:00 2023-09-23T22:19:49.094 INFO:teuthology.orchestra.run.smithi151.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 633 kB/s | 43 kB 00:00 2023-09-23T22:19:49.109 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-diskprediction-local-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T22:19:49.110 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:49.111 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:19:49.119 INFO:teuthology.orchestra.run.smithi151.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 2.3 MB/s | 155 kB 00:00 2023-09-23T22:19:49.120 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:49.120 INFO:teuthology.orchestra.run.smithi151.stdout:Total 3.3 MB/s | 1.9 MB 00:00 2023-09-23T22:19:49.178 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:19:49.193 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:19:49.193 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:19:49.243 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-mgr-rook 2023-09-23T22:19:49.262 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:19:49.266 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:19:49.612 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:49.726 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:19:49.925 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T22:19:50.145 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T22:19:50.353 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T22:19:50.571 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-09-23T22:19:50.727 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:25 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:50.740 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-09-23T22:19:50.891 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-09-23T22:19:51.102 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-09-23T22:19:51.315 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T22:19:51.506 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T22:19:51.695 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T22:19:52.271 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:19:52.273 INFO:teuthology.orchestra.run.smithi052.stdout:======================================================================================================= 2023-09-23T22:19:52.273 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:19:52.273 INFO:teuthology.orchestra.run.smithi052.stdout:======================================================================================================= 2023-09-23T22:19:52.273 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:19:52.273 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-rook noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 27 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T22:19:52.274 INFO:teuthology.orchestra.run.smithi052.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout:======================================================================================================= 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout:Install 13 Packages 2023-09-23T22:19:52.275 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:19:52.276 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 1.9 M 2023-09-23T22:19:52.276 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 24 M 2023-09-23T22:19:52.276 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:19:52.487 INFO:teuthology.orchestra.run.smithi052.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 107 kB/s | 16 kB 00:00 2023-09-23T22:19:52.512 INFO:teuthology.orchestra.run.smithi052.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 185 kB/s | 33 kB 00:00 2023-09-23T22:19:52.562 INFO:teuthology.orchestra.run.smithi052.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.1 MB/s | 81 kB 00:00 2023-09-23T22:19:52.587 INFO:teuthology.orchestra.run.smithi052.stdout:(4/13): ceph-mgr-rook-16.2.14-37.g2e1f1015.el8. 108 kB/s | 27 kB 00:00 2023-09-23T22:19:52.621 INFO:teuthology.orchestra.run.smithi052.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 1.1 MB/s | 64 kB 00:00 2023-09-23T22:19:52.688 INFO:teuthology.orchestra.run.smithi052.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 640 kB/s | 43 kB 00:00 2023-09-23T22:19:52.730 INFO:teuthology.orchestra.run.smithi052.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 3.7 MB/s | 155 kB 00:00 2023-09-23T22:19:52.765 INFO:teuthology.orchestra.run.smithi052.stdout:(8/13): python3-kubernetes-11.0.0-6.el8.noarch. 4.6 MB/s | 1.2 MB 00:00 2023-09-23T22:19:52.776 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T22:19:52.789 INFO:teuthology.orchestra.run.smithi052.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.8 MB/s | 110 kB 00:00 2023-09-23T22:19:52.822 INFO:teuthology.orchestra.run.smithi052.stdout:(10/13): python3-websocket-client-0.56.0-5.el8. 261 kB/s | 61 kB 00:00 2023-09-23T22:19:52.848 INFO:teuthology.orchestra.run.smithi052.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 316 kB/s | 27 kB 00:00 2023-09-23T22:19:52.881 INFO:teuthology.orchestra.run.smithi052.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.5 MB/s | 126 kB 00:00 2023-09-23T22:19:52.906 INFO:teuthology.orchestra.run.smithi052.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 239 kB/s | 20 kB 00:00 2023-09-23T22:19:52.909 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:19:52.909 INFO:teuthology.orchestra.run.smithi052.stdout:Total 3.0 MB/s | 1.9 MB 00:00 2023-09-23T22:19:52.970 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:19:52.983 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T22:19:52.986 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:19:52.986 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:19:53.063 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:19:53.067 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:19:53.125 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 13/13 2023-09-23T22:19:53.368 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 13/13 2023-09-23T22:19:53.368 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 1/13 2023-09-23T22:19:53.368 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T22:19:53.368 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T22:19:53.368 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T22:19:53.368 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-23T22:19:53.369 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-09-23T22:19:53.533 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:19:53.731 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T22:19:53.984 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T22:19:54.160 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T22:19:54.311 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T22:19:54.478 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-09-23T22:19:54.479 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:28 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:19:54.496 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T22:19:54.735 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T22:19:54.925 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T22:19:55.128 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T22:19:55.311 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T22:19:55.506 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T22:19:56.643 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T22:19:56.886 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T22:19:57.019 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 13/13 2023-09-23T22:19:57.193 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:19:57.275 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 13/13 2023-09-23T22:19:57.275 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 1/13 2023-09-23T22:19:57.275 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T22:19:57.276 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T22:19:57.376 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:57.376 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T22:19:57.377 INFO:teuthology.orchestra.run.smithi151.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T22:19:57.378 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T22:19:57.378 INFO:teuthology.orchestra.run.smithi151.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T22:19:57.378 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:19:57.378 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:19:57.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T22:19:57.867 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:19:58.429 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T22:19:58.429 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:33 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:19:58.686 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:32 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:00.068 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:00.069 INFO:teuthology.orchestra.run.smithi151.stdout:============================================================================================== 2023-09-23T22:20:00.069 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:20:00.069 INFO:teuthology.orchestra.run.smithi151.stdout:============================================================================================== 2023-09-23T22:20:00.069 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:20:00.069 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-cephadm noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 116 k 2023-09-23T22:20:00.069 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout:============================================================================================== 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout:Install 6 Packages 2023-09-23T22:20:00.070 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:00.071 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 5.8 M 2023-09-23T22:20:00.071 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 24 M 2023-09-23T22:20:00.071 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:20:00.386 INFO:teuthology.orchestra.run.smithi151.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 77 kB/s | 21 kB 00:00 2023-09-23T22:20:00.445 INFO:teuthology.orchestra.run.smithi151.stdout:(2/6): ceph-mgr-cephadm-16.2.14-37.g2e1f1015.el 347 kB/s | 116 kB 00:00 2023-09-23T22:20:00.528 INFO:teuthology.orchestra.run.smithi151.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 6.3 MB/s | 538 kB 00:00 2023-09-23T22:20:00.553 INFO:teuthology.orchestra.run.smithi151.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 292 kB/s | 49 kB 00:00 2023-09-23T22:20:00.670 INFO:teuthology.orchestra.run.smithi151.stdout:(5/6): python3-babel-2.5.1-7.el8.noarch.rpm 34 MB/s | 4.8 MB 00:00 2023-09-23T22:20:00.695 INFO:teuthology.orchestra.run.smithi151.stdout:(6/6): python3-execnet-1.7.1-1.el8.noarch.rpm 607 kB/s | 355 kB 00:00 2023-09-23T22:20:00.697 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:00.697 INFO:teuthology.orchestra.run.smithi151.stdout:Total 9.3 MB/s | 5.8 MB 00:00 2023-09-23T22:20:00.749 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:20:00.759 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:20:00.759 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:20:00.836 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:20:00.840 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:20:01.264 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:20:01.466 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:01.466 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:20:01.466 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-rook-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:20:01.469 INFO:teuthology.orchestra.run.smithi052.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T22:20:01.469 INFO:teuthology.orchestra.run.smithi052.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T22:20:01.470 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:01.471 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:01.606 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T22:20:01.837 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:20:01.984 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:02.218 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-09-23T22:20:02.472 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-09-23T22:20:02.715 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-09-23T22:20:02.886 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:37 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:02.942 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-09-23T22:20:03.206 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-09-23T22:20:03.416 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:20:03.654 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:20:03.654 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/6 2023-09-23T22:20:03.654 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-09-23T22:20:03.654 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-09-23T22:20:03.654 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-09-23T22:20:03.654 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-23T22:20:04.412 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:04.413 INFO:teuthology.orchestra.run.smithi052.stdout:============================================================================================== 2023-09-23T22:20:04.413 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:20:04.413 INFO:teuthology.orchestra.run.smithi052.stdout:============================================================================================== 2023-09-23T22:20:04.413 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:20:04.413 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-cephadm noarch 2:16.2.14-37.g2e1f1015.el8 ceph-noarch 116 k 2023-09-23T22:20:04.413 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout:============================================================================================== 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout:Install 6 Packages 2023-09-23T22:20:04.414 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:04.415 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 5.8 M 2023-09-23T22:20:04.415 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 24 M 2023-09-23T22:20:04.415 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:20:04.738 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-09-23T22:20:04.738 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:38 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:04.766 INFO:teuthology.orchestra.run.smithi052.stdout:(1/6): ceph-mgr-cephadm-16.2.14-37.g2e1f1015.el 637 kB/s | 116 kB 00:00 2023-09-23T22:20:04.792 INFO:teuthology.orchestra.run.smithi052.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 102 kB/s | 21 kB 00:00 2023-09-23T22:20:04.867 INFO:teuthology.orchestra.run.smithi052.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 7.0 MB/s | 538 kB 00:00 2023-09-23T22:20:04.884 INFO:teuthology.orchestra.run.smithi052.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 417 kB/s | 49 kB 00:00 2023-09-23T22:20:04.993 INFO:teuthology.orchestra.run.smithi052.stdout:(5/6): python3-babel-2.5.1-7.el8.noarch.rpm 38 MB/s | 4.8 MB 00:00 2023-09-23T22:20:05.018 INFO:teuthology.orchestra.run.smithi052.stdout:(6/6): python3-execnet-1.7.1-1.el8.noarch.rpm 820 kB/s | 355 kB 00:00 2023-09-23T22:20:05.020 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:05.020 INFO:teuthology.orchestra.run.smithi052.stdout:Total 9.6 MB/s | 5.8 MB 00:00 2023-09-23T22:20:05.075 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:20:05.086 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:20:05.086 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:20:05.171 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:20:05.175 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:20:06.203 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:20:06.597 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-09-23T22:20:06.825 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-09-23T22:20:07.094 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-09-23T22:20:07.315 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-09-23T22:20:07.489 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:20:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-09-23T22:20:07.678 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:07.679 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:07.735 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:20:07.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-fuse 2023-09-23T22:20:07.986 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 6/6 2023-09-23T22:20:07.986 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 1/6 2023-09-23T22:20:07.986 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-09-23T22:20:07.986 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-09-23T22:20:07.986 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-09-23T22:20:07.986 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-23T22:20:08.170 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:09.085 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-09-23T22:20:09.085 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:44 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:09.179 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:43 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:10.557 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:10.558 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:10.558 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2023-09-23T22:20:10.558 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:10.558 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:20:10.558 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-fuse 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout: x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 910 k 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout:Install 2 Packages 2023-09-23T22:20:10.559 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:10.560 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 993 k 2023-09-23T22:20:10.560 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 2.8 M 2023-09-23T22:20:10.560 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:20:11.165 INFO:teuthology.orchestra.run.smithi151.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 137 kB/s | 83 kB 00:00 2023-09-23T22:20:11.478 INFO:teuthology.orchestra.run.smithi151.stdout:(2/2): ceph-fuse-16.2.14-37.g2e1f1015.el8.x86_6 991 kB/s | 910 kB 00:00 2023-09-23T22:20:11.479 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:11.480 INFO:teuthology.orchestra.run.smithi151.stdout:Total 1.1 MB/s | 993 kB 00:00 2023-09-23T22:20:11.492 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:20:11.514 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:20:11.514 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:20:11.591 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:20:11.595 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:20:11.680 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:20:11.754 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:20:11.886 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:11.886 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:20:11.886 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-mgr-cephadm-2:16.2.14-37.g2e1f1015.el8.noarch 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:11.887 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:12.034 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install ceph-fuse 2023-09-23T22:20:12.071 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T22:20:12.229 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 2/2 2023-09-23T22:20:12.411 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 2/2 2023-09-23T22:20:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/2 2023-09-23T22:20:13.287 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:48 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:13.642 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T22:20:13.643 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:47 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:14.792 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:14.793 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:14.793 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repository Size 2023-09-23T22:20:14.793 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:14.793 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-fuse 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout: x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 910 k 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout:Installing dependencies: 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout:Install 2 Packages 2023-09-23T22:20:14.794 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:14.795 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 993 k 2023-09-23T22:20:14.795 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 2.8 M 2023-09-23T22:20:14.795 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:20:14.860 INFO:teuthology.orchestra.run.smithi052.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.2 MB/s | 83 kB 00:00 2023-09-23T22:20:15.890 INFO:teuthology.orchestra.run.smithi052.stdout:(2/2): ceph-fuse-16.2.14-37.g2e1f1015.el8.x86_6 831 kB/s | 910 kB 00:01 2023-09-23T22:20:15.891 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:15.892 INFO:teuthology.orchestra.run.smithi052.stdout:Total 905 kB/s | 993 kB 00:01 2023-09-23T22:20:15.906 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:20:15.930 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:20:15.930 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:20:16.016 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:20:16.020 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:20:16.292 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:20:16.475 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:16.475 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:20:16.475 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T22:20:16.475 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:16.475 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:16.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install librados-devel 2023-09-23T22:20:16.636 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:20:16.913 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T22:20:16.966 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:17.066 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 2/2 2023-09-23T22:20:17.443 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 2/2 2023-09-23T22:20:17.443 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/2 2023-09-23T22:20:17.835 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:51 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:18.547 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T22:20:18.547 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:53 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:19.199 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout: librados-devel x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 131 k 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:19.200 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:20:19.201 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:19.201 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:20:19.201 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:19.201 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 131 k 2023-09-23T22:20:19.201 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 432 k 2023-09-23T22:20:19.201 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:20:19.378 INFO:teuthology.orchestra.run.smithi151.stdout:librados-devel-16.2.14-37.g2e1f1015.el8.x86_64. 743 kB/s | 131 kB 00:00 2023-09-23T22:20:19.378 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:19.379 INFO:teuthology.orchestra.run.smithi151.stdout:Total 737 kB/s | 131 kB 00:00 2023-09-23T22:20:19.379 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:20:19.384 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:20:19.384 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:20:19.438 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:20:19.442 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:20:19.564 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:20:19.752 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:20.005 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:21.074 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:21.074 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:55 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:21.097 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:20:21.298 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:21.298 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:20:21.298 INFO:teuthology.orchestra.run.smithi052.stdout: ceph-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T22:20:21.298 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:21.299 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:21.422 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install librados-devel 2023-09-23T22:20:21.800 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:22.752 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:02:57 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:23.655 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:20:23.838 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:23.838 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:20:23.838 INFO:teuthology.orchestra.run.smithi151.stdout: librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:20:23.838 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:23.838 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:23.972 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install libcephfs2 2023-09-23T22:20:24.248 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:24.248 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:24.248 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repo Size 2023-09-23T22:20:24.248 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:24.248 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout: librados-devel x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 131 k 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:24.249 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 131 k 2023-09-23T22:20:24.250 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 432 k 2023-09-23T22:20:24.250 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:20:24.334 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:24.426 INFO:teuthology.orchestra.run.smithi052.stdout:librados-devel-16.2.14-37.g2e1f1015.el8.x86_64. 740 kB/s | 131 kB 00:00 2023-09-23T22:20:24.427 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:24.427 INFO:teuthology.orchestra.run.smithi052.stdout:Total 735 kB/s | 131 kB 00:00 2023-09-23T22:20:24.427 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:20:24.433 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:20:24.434 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:20:24.492 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:20:24.497 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:20:24.618 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:20:24.814 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:25.056 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:25.326 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:02:59 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:26.168 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:26.168 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:01 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:26.595 INFO:teuthology.orchestra.run.smithi151.stdout:Package libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:26.706 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:26.707 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:26.707 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:26.875 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install libcephfs-devel 2023-09-23T22:20:27.236 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:28.680 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:02 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:29.561 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:20:29.761 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:29.761 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:20:29.761 INFO:teuthology.orchestra.run.smithi052.stdout: librados-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:20:29.761 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:29.761 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:29.889 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install libcephfs2 2023-09-23T22:20:30.066 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs-devel x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 25 k 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:20:30.067 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:30.068 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 25 k 2023-09-23T22:20:30.068 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 107 k 2023-09-23T22:20:30.068 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:20:30.204 INFO:teuthology.orchestra.run.smithi151.stdout:libcephfs-devel-16.2.14-37.g2e1f1015.el8.x86_64 187 kB/s | 25 kB 00:00 2023-09-23T22:20:30.205 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:30.205 INFO:teuthology.orchestra.run.smithi151.stdout:Total 185 kB/s | 25 kB 00:00 2023-09-23T22:20:30.205 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:20:30.209 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:20:30.209 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:20:30.211 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:20:30.215 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:20:30.265 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:30.269 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:20:30.434 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:30.655 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:31.155 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:06 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:31.680 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:31.681 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:05 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:32.505 INFO:teuthology.orchestra.run.smithi052.stdout:Package libcephfs2-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:32.628 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:32.630 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:20:32.630 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:32.732 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install libcephfs-devel 2023-09-23T22:20:33.108 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:34.741 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:09 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:35.539 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:20:35.720 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:35.720 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:20:35.721 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:20:35.721 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:35.721 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:35.885 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install librados2 2023-09-23T22:20:36.241 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:36.283 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repo Size 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout: libcephfs-devel x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 25 k 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:20:36.284 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:36.285 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 25 k 2023-09-23T22:20:36.285 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 107 k 2023-09-23T22:20:36.285 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:20:36.491 INFO:teuthology.orchestra.run.smithi052.stdout:libcephfs-devel-16.2.14-37.g2e1f1015.el8.x86_64 124 kB/s | 25 kB 00:00 2023-09-23T22:20:36.492 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:36.492 INFO:teuthology.orchestra.run.smithi052.stdout:Total 123 kB/s | 25 kB 00:00 2023-09-23T22:20:36.492 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:20:36.496 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:20:36.496 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:20:36.498 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:20:36.502 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:20:36.550 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:20:36.729 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:37.152 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:11 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:38.002 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:38.002 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:13 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:38.411 INFO:teuthology.orchestra.run.smithi151.stdout:Package librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:38.518 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:38.520 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:38.520 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:38.631 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install librbd1 2023-09-23T22:20:38.984 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:39.931 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:13 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:41.206 INFO:teuthology.orchestra.run.smithi151.stdout:Package librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:41.314 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:41.315 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:41.315 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:41.378 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:20:41.426 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-rados 2023-09-23T22:20:41.577 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:41.577 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:20:41.577 INFO:teuthology.orchestra.run.smithi052.stdout: libcephfs-devel-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:20:41.577 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:20:41.577 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:41.704 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install librados2 2023-09-23T22:20:41.779 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:42.078 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:42.934 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:16 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:43.569 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:18 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:44.193 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:44.301 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:44.302 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:44.302 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:44.408 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-rgw 2023-09-23T22:20:44.763 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:44.976 INFO:teuthology.orchestra.run.smithi052.stdout:Package librados2-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:45.095 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:45.096 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:20:45.096 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:45.203 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install librbd1 2023-09-23T22:20:45.582 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:46.295 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:20 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:47.116 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:47.635 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:47.743 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:47.745 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:47.745 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:47.856 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-cephfs 2023-09-23T22:20:48.210 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:48.525 INFO:teuthology.orchestra.run.smithi052.stdout:Package librbd1-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:48.642 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:48.644 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:20:48.644 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:48.747 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install python3-rados 2023-09-23T22:20:49.125 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:49.205 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:23 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:50.168 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:50.453 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:50.561 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:50.562 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:50.562 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:50.670 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-rbd 2023-09-23T22:20:51.027 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:51.583 INFO:teuthology.orchestra.run.smithi052.stdout:Package python3-rados-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:51.704 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:51.705 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:20:51.705 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:51.835 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install python3-rgw 2023-09-23T22:20:52.109 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:26 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:52.214 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:53.159 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:53.433 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:53.543 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:53.544 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2023-09-23T22:20:53.544 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:20:53.658 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install rbd-fuse 2023-09-23T22:20:54.012 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:54.573 INFO:teuthology.orchestra.run.smithi052.stdout:Package python3-rgw-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:54.694 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:54.695 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:20:54.695 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:54.811 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install python3-cephfs 2023-09-23T22:20:54.839 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:55.190 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:56.116 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:31 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:20:56.266 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-fuse x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 88 k 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:20:56.267 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:20:56.268 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:20:56.268 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 88 k 2023-09-23T22:20:56.268 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 240 k 2023-09-23T22:20:56.268 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:20:56.594 INFO:teuthology.orchestra.run.smithi151.stdout:rbd-fuse-16.2.14-37.g2e1f1015.el8.x86_64.rpm 272 kB/s | 88 kB 00:00 2023-09-23T22:20:56.594 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:20:56.594 INFO:teuthology.orchestra.run.smithi151.stdout:Total 271 kB/s | 88 kB 00:00 2023-09-23T22:20:56.595 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:20:56.604 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:20:56.604 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:20:56.658 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:20:56.661 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:20:56.769 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:20:56.954 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:57.191 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:57.537 INFO:teuthology.orchestra.run.smithi052.stdout:Package python3-cephfs-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:20:57.657 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:20:57.659 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:20:57.659 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:20:57.771 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install python3-rbd 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:20:58.269 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:20:58.269 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:32 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:20:59.015 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:34 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:00.416 INFO:teuthology.orchestra.run.smithi052.stdout:Package python3-rbd-2:16.2.14-37.g2e1f1015.el8.x86_64 is already installed. 2023-09-23T22:21:00.535 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:21:00.537 INFO:teuthology.orchestra.run.smithi052.stdout:Nothing to do. 2023-09-23T22:21:00.537 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:21:00.642 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install rbd-fuse 2023-09-23T22:21:00.951 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:21:01.016 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:21:01.132 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:01.132 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:21:01.132 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:21:01.132 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:01.132 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:21:01.260 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install rbd-mirror 2023-09-23T22:21:01.616 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:21:01.982 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:36 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:03.219 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:37 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:21:03.510 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:21:03.511 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:03.511 INFO:teuthology.orchestra.run.smithi052.stdout: Package Architecture Version Repository Size 2023-09-23T22:21:03.511 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:03.511 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout: rbd-fuse x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 88 k 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 88 k 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 240 k 2023-09-23T22:21:03.512 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:21:03.889 INFO:teuthology.orchestra.run.smithi052.stdout:rbd-fuse-16.2.14-37.g2e1f1015.el8.x86_64.rpm 235 kB/s | 88 kB 00:00 2023-09-23T22:21:03.889 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:21:03.889 INFO:teuthology.orchestra.run.smithi052.stdout:Total 234 kB/s | 88 kB 00:00 2023-09-23T22:21:03.890 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:21:03.901 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:21:03.901 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:21:03.961 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:21:03.965 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:21:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:21:04.250 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:04.492 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:04.582 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-mirror x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 3.7 M 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:04.583 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:21:04.584 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:21:04.584 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:21:04.584 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:04.584 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 3.7 M 2023-09-23T22:21:04.584 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 15 M 2023-09-23T22:21:04.584 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:21:04.966 INFO:teuthology.orchestra.run.smithi151.stdout:rbd-mirror-16.2.14-37.g2e1f1015.el8.x86_64.rpm 9.6 MB/s | 3.7 MB 00:00 2023-09-23T22:21:04.967 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:21:04.967 INFO:teuthology.orchestra.run.smithi151.stdout:Total 9.6 MB/s | 3.7 MB 00:00 2023-09-23T22:21:04.967 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:21:04.979 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:21:04.979 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:21:05.047 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:21:05.051 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:21:05.500 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:21:05.601 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:05.601 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:40 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:05.656 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:06.020 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:07.097 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:07.098 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:41 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:21:08.221 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:21:08.420 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:08.421 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:21:08.421 INFO:teuthology.orchestra.run.smithi052.stdout: rbd-fuse-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:21:08.421 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:08.421 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:21:08.548 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install rbd-mirror 2023-09-23T22:21:08.925 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:21:09.843 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:44 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:10.196 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:21:10.380 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:10.380 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:21:10.380 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:21:10.380 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:10.381 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:21:10.507 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install rbd-nbd 2023-09-23T22:21:10.864 INFO:teuthology.orchestra.run.smithi151.stdout:Updating Subscription Management repositories. 2023-09-23T22:21:11.381 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:21:11.382 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:11.382 INFO:teuthology.orchestra.run.smithi052.stdout: Package Arch Version Repo Size 2023-09-23T22:21:11.382 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:11.382 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:21:11.382 INFO:teuthology.orchestra.run.smithi052.stdout: rbd-mirror x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 3.7 M 2023-09-23T22:21:11.383 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:11.383 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:21:11.383 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:11.383 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:21:11.383 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:11.383 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 3.7 M 2023-09-23T22:21:11.384 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 15 M 2023-09-23T22:21:11.384 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:21:11.700 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:45 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:21:11.742 INFO:teuthology.orchestra.run.smithi052.stdout:rbd-mirror-16.2.14-37.g2e1f1015.el8.x86_64.rpm 10 MB/s | 3.7 MB 00:00 2023-09-23T22:21:11.743 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:21:11.743 INFO:teuthology.orchestra.run.smithi052.stdout:Total 10 MB/s | 3.7 MB 00:00 2023-09-23T22:21:11.743 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:21:11.756 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:21:11.756 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:21:11.828 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:21:11.832 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:21:12.314 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:21:12.444 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:12.812 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:13.066 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2023-09-23T22:21:13.067 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:21:13.067 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2023-09-23T22:21:13.067 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:21:13.067 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2023-09-23T22:21:13.067 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-nbd x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 173 k 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 173 k 2023-09-23T22:21:13.068 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 515 k 2023-09-23T22:21:13.069 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2023-09-23T22:21:13.257 INFO:teuthology.orchestra.run.smithi151.stdout:rbd-nbd-16.2.14-37.g2e1f1015.el8.x86_64.rpm 921 kB/s | 173 kB 00:00 2023-09-23T22:21:13.257 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:21:13.257 INFO:teuthology.orchestra.run.smithi151.stdout:Total 915 kB/s | 173 kB 00:00 2023-09-23T22:21:13.258 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2023-09-23T22:21:13.268 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2023-09-23T22:21:13.268 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2023-09-23T22:21:13.322 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2023-09-23T22:21:13.325 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2023-09-23T22:21:13.443 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2023-09-23T22:21:13.594 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:13.841 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:13.916 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:13.916 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:48 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:14.920 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:03:48 ago on Sat 23 Sep 2023 10:17:26 PM UTC. 2023-09-23T22:21:16.937 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:21:17.136 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:17.136 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:21:17.136 INFO:teuthology.orchestra.run.smithi052.stdout: rbd-mirror-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:21:17.136 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:17.136 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:21:17.259 DEBUG:teuthology.orchestra.run.smithi052:> sudo yum -y install rbd-nbd 2023-09-23T22:21:17.505 INFO:teuthology.orchestra.run.smithi151.stdout:Installed products updated. 2023-09-23T22:21:17.639 INFO:teuthology.orchestra.run.smithi052.stdout:Updating Subscription Management repositories. 2023-09-23T22:21:17.690 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:17.691 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2023-09-23T22:21:17.691 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:21:17.691 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:21:17.691 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2023-09-23T22:21:17.811 DEBUG:teuthology.parallel:result is None 2023-09-23T22:21:18.539 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:53 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:20.085 INFO:teuthology.orchestra.run.smithi052.stdout:Dependencies resolved. 2023-09-23T22:21:20.086 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:20.086 INFO:teuthology.orchestra.run.smithi052.stdout: Package Architecture Version Repository Size 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:Installing: 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout: rbd-nbd x86_64 2:16.2.14-37.g2e1f1015.el8 ceph 173 k 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction Summary 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:================================================================================ 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:Install 1 Package 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:Total download size: 173 k 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:Installed size: 515 k 2023-09-23T22:21:20.087 INFO:teuthology.orchestra.run.smithi052.stdout:Downloading Packages: 2023-09-23T22:21:20.270 INFO:teuthology.orchestra.run.smithi052.stdout:rbd-nbd-16.2.14-37.g2e1f1015.el8.x86_64.rpm 943 kB/s | 173 kB 00:00 2023-09-23T22:21:20.271 INFO:teuthology.orchestra.run.smithi052.stdout:-------------------------------------------------------------------------------- 2023-09-23T22:21:20.271 INFO:teuthology.orchestra.run.smithi052.stdout:Total 937 kB/s | 173 kB 00:00 2023-09-23T22:21:20.272 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction check 2023-09-23T22:21:20.283 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction check succeeded. 2023-09-23T22:21:20.284 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction test 2023-09-23T22:21:20.342 INFO:teuthology.orchestra.run.smithi052.stdout:Transaction test succeeded. 2023-09-23T22:21:20.346 INFO:teuthology.orchestra.run.smithi052.stdout:Running transaction 2023-09-23T22:21:20.464 INFO:teuthology.orchestra.run.smithi052.stdout: Preparing : 1/1 2023-09-23T22:21:20.640 INFO:teuthology.orchestra.run.smithi052.stdout: Installing : rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:20.888 INFO:teuthology.orchestra.run.smithi052.stdout: Running scriptlet: rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: Verifying : rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 1/1 2023-09-23T22:21:21.999 INFO:teuthology.orchestra.run.smithi052.stdout:Last metadata expiration check: 0:03:56 ago on Sat 23 Sep 2023 10:17:25 PM UTC. 2023-09-23T22:21:24.532 INFO:teuthology.orchestra.run.smithi052.stdout:Installed products updated. 2023-09-23T22:21:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:24.736 INFO:teuthology.orchestra.run.smithi052.stdout:Installed: 2023-09-23T22:21:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: rbd-nbd-2:16.2.14-37.g2e1f1015.el8.x86_64 2023-09-23T22:21:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:21:24.736 INFO:teuthology.orchestra.run.smithi052.stdout:Complete! 2023-09-23T22:21:24.876 DEBUG:teuthology.parallel:result is None 2023-09-23T22:21:24.876 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:21:24.876 INFO:teuthology.packaging:ref: None 2023-09-23T22:21:24.876 INFO:teuthology.packaging:tag: None 2023-09-23T22:21:24.876 INFO:teuthology.packaging:branch: pacific 2023-09-23T22:21:24.876 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:21:24.877 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-09-23T22:21:25.115 DEBUG:teuthology.orchestra.run.smithi052:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T22:21:25.138 INFO:teuthology.orchestra.run.smithi052.stdout:16.2.14-37.g2e1f1015.el8 2023-09-23T22:21:25.138 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-37.g2e1f1015.el8 2023-09-23T22:21:25.139 INFO:teuthology.task.install:The correct ceph version 16.2.14-37.g2e1f1015 is installed. 2023-09-23T22:21:25.139 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T22:21:25.139 INFO:teuthology.packaging:ref: None 2023-09-23T22:21:25.139 INFO:teuthology.packaging:tag: None 2023-09-23T22:21:25.139 INFO:teuthology.packaging:branch: pacific 2023-09-23T22:21:25.139 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:21:25.139 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-09-23T22:21:25.607 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T22:21:25.634 INFO:teuthology.orchestra.run.smithi151.stdout:16.2.14-37.g2e1f1015.el8 2023-09-23T22:21:25.634 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-37.g2e1f1015.el8 2023-09-23T22:21:25.634 INFO:teuthology.task.install:The correct ceph version 16.2.14-37.g2e1f1015 is installed. 2023-09-23T22:21:25.635 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T22:21:25.635 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:21:25.635 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T22:21:25.666 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:21:25.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T22:21:25.708 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T22:21:25.709 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:21:25.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T22:21:25.738 DEBUG:teuthology.orchestra.run.smithi052:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T22:21:25.807 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:21:25.807 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T22:21:25.840 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T22:21:25.910 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T22:21:25.910 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:21:25.910 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T22:21:25.939 DEBUG:teuthology.orchestra.run.smithi052:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T22:21:26.007 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:21:26.008 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T22:21:26.037 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T22:21:26.106 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T22:21:26.107 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:21:26.107 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T22:21:26.135 DEBUG:teuthology.orchestra.run.smithi052:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T22:21:26.203 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:21:26.203 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T22:21:26.232 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T22:21:26.300 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T22:21:26.368 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T22:21:26.368 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-09-23T22:21:26.369 INFO:tasks.cephadm:Cluster fsid is 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:21:26.369 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T22:21:26.369 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.52', 'mon.c': '[v2:172.21.15.52:3301,v1:172.21.15.52:6790]', 'mon.b': '172.21.15.151'} 2023-09-23T22:21:26.369 INFO:tasks.cephadm:First mon is mon.a on smithi052 2023-09-23T22:21:26.369 INFO:tasks.cephadm:First mgr is y 2023-09-23T22:21:26.369 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T22:21:26.370 DEBUG:teuthology.orchestra.run.smithi052:> sudo hostname $(hostname -s) 2023-09-23T22:21:26.398 DEBUG:teuthology.orchestra.run.smithi151:> sudo hostname $(hostname -s) 2023-09-23T22:21:26.427 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-09-23T22:21:26.427 DEBUG:teuthology.orchestra.run.smithi052:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T22:21:26.809 INFO:teuthology.orchestra.run.smithi052.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Sep 23 22:21 /home/ubuntu/cephtest/cephadm 2023-09-23T22:21:26.810 DEBUG:teuthology.orchestra.run.smithi151:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T22:21:26.970 INFO:teuthology.orchestra.run.smithi151.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Sep 23 22:21 /home/ubuntu/cephtest/cephadm 2023-09-23T22:21:26.970 DEBUG:teuthology.orchestra.run.smithi052:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T22:21:26.992 DEBUG:teuthology.orchestra.run.smithi151:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T22:21:27.024 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-09-23T22:21:27.024 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-09-23T22:21:27.035 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-09-23T22:21:28.265 INFO:teuthology.orchestra.run.smithi151.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T22:21:28.304 INFO:teuthology.orchestra.run.smithi052.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T22:22:25.664 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:22:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph_version": "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)", 2023-09-23T22:22:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "image_id": "1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e", 2023-09-23T22:22:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "repo_digests": [ 2023-09-23T22:22:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b" 2023-09-23T22:22:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:22:25.665 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph_version": "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)", 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout: "image_id": "1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e", 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout: "repo_digests": [ 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:396076557f19bac7d00d097a6db258821cfaec622cd332710d7957388aedf72b" 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:22:26.622 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:22:26.645 DEBUG:teuthology.orchestra.run.smithi052:> sudo mkdir -p /etc/ceph 2023-09-23T22:22:26.679 DEBUG:teuthology.orchestra.run.smithi151:> sudo mkdir -p /etc/ceph 2023-09-23T22:22:26.715 DEBUG:teuthology.orchestra.run.smithi052:> sudo chmod 777 /etc/ceph 2023-09-23T22:22:26.747 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 777 /etc/ceph 2023-09-23T22:22:26.784 INFO:tasks.cephadm:Writing seed config... 2023-09-23T22:22:26.785 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T22:22:26.785 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T22:22:26.786 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T22:22:26.787 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:22:26.787 DEBUG:teuthology.orchestra.run.smithi052:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T22:22:26.806 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 891acd3a-5a5f-11ee-8db4-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T22:22:26.806 DEBUG:teuthology.orchestra.run.smithi052:mon.a> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T22:22:26.848 DEBUG:teuthology.orchestra.run.smithi052:mgr.y> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T22:22:26.876 INFO:journalctl@ceph.mon.a.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:22:26.877 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T22:22:26.877 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.52 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:22:26.904 INFO:journalctl@ceph.mgr.y.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:22:27.089 INFO:teuthology.orchestra.run.smithi052.stderr:-------------------------------------------------------------------------------- 2023-09-23T22:22:27.089 INFO:teuthology.orchestra.run.smithi052.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '891acd3a-5a5f-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.52', '--skip-admin-label'] 2023-09-23T22:22:27.137 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: 3.0.1 2023-09-23T22:22:27.153 INFO:teuthology.orchestra.run.smithi052.stderr:Verifying podman|docker is present... 2023-09-23T22:22:27.198 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: 3.0.1 2023-09-23T22:22:27.214 INFO:teuthology.orchestra.run.smithi052.stderr:Verifying lvm2 is present... 2023-09-23T22:22:27.214 INFO:teuthology.orchestra.run.smithi052.stderr:Verifying time synchronization is in place... 2023-09-23T22:22:27.221 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T22:22:27.226 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: inactive 2023-09-23T22:22:27.233 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: enabled 2023-09-23T22:22:27.238 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: active 2023-09-23T22:22:27.239 INFO:teuthology.orchestra.run.smithi052.stderr:Unit chronyd.service is enabled and running 2023-09-23T22:22:27.239 INFO:teuthology.orchestra.run.smithi052.stderr:Repeating the final host check... 2023-09-23T22:22:27.282 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: 3.0.1 2023-09-23T22:22:27.305 INFO:teuthology.orchestra.run.smithi052.stderr:podman (/bin/podman) version 3.0.1 is present 2023-09-23T22:22:27.305 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl is present 2023-09-23T22:22:27.305 INFO:teuthology.orchestra.run.smithi052.stderr:lvcreate is present 2023-09-23T22:22:27.311 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T22:22:27.316 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: inactive 2023-09-23T22:22:27.322 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: enabled 2023-09-23T22:22:27.328 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: active 2023-09-23T22:22:27.329 INFO:teuthology.orchestra.run.smithi052.stderr:Unit chronyd.service is enabled and running 2023-09-23T22:22:27.329 INFO:teuthology.orchestra.run.smithi052.stderr:Host looks OK 2023-09-23T22:22:27.329 INFO:teuthology.orchestra.run.smithi052.stderr:Cluster fsid: 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:27.329 INFO:teuthology.orchestra.run.smithi052.stderr:Acquiring lock 139740546324968 on /run/cephadm/891acd3a-5a5f-11ee-8db4-212e2dc638e7.lock 2023-09-23T22:22:27.329 INFO:teuthology.orchestra.run.smithi052.stderr:Lock 139740546324968 acquired on /run/cephadm/891acd3a-5a5f-11ee-8db4-212e2dc638e7.lock 2023-09-23T22:22:27.330 INFO:teuthology.orchestra.run.smithi052.stderr:Verifying IP 172.21.15.52 port 3300 ... 2023-09-23T22:22:27.330 INFO:teuthology.orchestra.run.smithi052.stderr:Verifying IP 172.21.15.52 port 6789 ... 2023-09-23T22:22:27.330 INFO:teuthology.orchestra.run.smithi052.stderr:Base mon IP(s) is [172.21.15.52:3300, 172.21.15.52:6789], mon addrv is [v2:172.21.15.52:3300,v1:172.21.15.52:6789] 2023-09-23T22:22:27.333 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-23T22:22:27.333 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.52 metric 100 2023-09-23T22:22:27.336 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T22:22:27.336 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T22:22:27.336 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1790sec hoplimit 64 pref medium 2023-09-23T22:22:27.338 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T22:22:27.339 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-09-23T22:22:27.339 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-09-23T22:22:27.339 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T22:22:27.339 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:15de/64 scope link 2023-09-23T22:22:27.339 INFO:teuthology.orchestra.run.smithi052.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-09-23T22:22:27.340 INFO:teuthology.orchestra.run.smithi052.stderr:Mon IP `172.21.15.52` is in CIDR network `172.21.0.0/20` 2023-09-23T22:22:27.340 INFO:teuthology.orchestra.run.smithi052.stderr:Mon IP `172.21.15.52` is in CIDR network `172.21.0.0/20` 2023-09-23T22:22:27.340 INFO:teuthology.orchestra.run.smithi052.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T22:22:27.340 INFO:teuthology.orchestra.run.smithi052.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T22:22:27.341 INFO:teuthology.orchestra.run.smithi052.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T22:22:27.414 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Trying to pull quay.ceph.io/ceph-ci/ceph:pacific... 2023-09-23T22:22:27.699 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Getting image source signatures 2023-09-23T22:22:27.699 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Copying blob sha256:a58e4a2ea0ce9498ff39d84e580b065b5fe05a6105444a6d1d4efa5fa821bc2f 2023-09-23T22:22:27.700 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T22:22:27.700 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Copying config sha256:1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e 2023-09-23T22:22:27.763 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Writing manifest to image destination 2023-09-23T22:22:27.763 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: Storing signatures 2023-09-23T22:22:27.918 INFO:teuthology.orchestra.run.smithi052.stderr:/bin/podman: 1e6a756174a8dbc0d5ced5aeb4272e8f04dc2045a9196dffb3df976d1caca13e 2023-09-23T22:22:28.392 INFO:teuthology.orchestra.run.smithi052.stderr:ceph: ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable) 2023-09-23T22:22:28.856 INFO:teuthology.orchestra.run.smithi052.stderr:Ceph version: ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable) 2023-09-23T22:22:28.857 INFO:teuthology.orchestra.run.smithi052.stderr:Extracting ceph user uid/gid from container image... 2023-09-23T22:22:29.207 INFO:teuthology.orchestra.run.smithi052.stderr:stat: 167 167 2023-09-23T22:22:29.562 INFO:teuthology.orchestra.run.smithi052.stderr:Creating initial keys... 2023-09-23T22:22:29.906 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-authtool: AQAlZQ9lel5TNRAA9Rj2H2k2AMWN8Kvcw5/3Kg== 2023-09-23T22:22:30.689 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-authtool: AQAmZQ9lBvGvKBAA7yJlZKLz2g1WOJ44wpn3xQ== 2023-09-23T22:22:31.548 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-authtool: AQAnZQ9laAM+IBAAO+4Sw2gC1KlVRlEV2PTWaA== 2023-09-23T22:22:31.990 INFO:teuthology.orchestra.run.smithi052.stderr:Creating initial monmap... 2023-09-23T22:22:32.339 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T22:22:32.340 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:32.340 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T22:22:32.782 INFO:teuthology.orchestra.run.smithi052.stderr:monmaptool for a [v2:172.21.15.52:3300,v1:172.21.15.52:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T22:22:32.782 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/monmaptool: set fsid to 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:32.782 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T22:22:32.782 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:22:32.782 INFO:teuthology.orchestra.run.smithi052.stderr:Creating mon... 2023-09-23T22:22:33.203 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.173+0000 7f8b26a71880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 1 imported monmap: 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: epoch 0 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: last_changed 2023-09-23T22:22:32.333200+0000 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: created 2023-09-23T22:22:32.333200+0000 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] mon.a 2023-09-23T22:22:33.204 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 0 /usr/bin/ceph-mon: set fsid to 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: DB SUMMARY 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.205 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.create_if_missing: 1 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.env: 0x55c26d65e080 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.fs: Posix File System 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.info_log: 0x55c26e6e9aa0 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.statistics: (nil) 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.use_fsync: 0 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:22:33.206 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.db_log_dir: 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:22:33.207 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.write_buffer_manager: 0x55c26e6f1890 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:22:33.208 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.unordered_write: 0 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.row_cache: None 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.wal_filter: None 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:22:33.210 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:22:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.max_open_files: -1 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Compression algorithms supported: 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kZSTD supported: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T22:22:33.212 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.174+0000 7f8b26a71880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.merge_operator: 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_filter: None 2023-09-23T22:22:33.213 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c26e5fbd20) 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: index_type: 0 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-09-23T22:22:33.214 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: checksum: 1 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_cache: 0x55c26e634d10 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-09-23T22:22:33.215 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: format_version: 2 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-09-23T22:22:33.216 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: block_align: 0 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression: NoCompression 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.num_levels: 7 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:22:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:22:33.218 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:22:33.219 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:22:33.220 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:22:33.221 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.ttl: 2592000 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.175+0000 7f8b26a71880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.176+0000 7f8b26a71880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.222 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.176+0000 7f8b26a71880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.177+0000 7f8b26a71880 4 rocksdb: DB pointer 0x55c26e6ff800 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.177+0000 7f8b0f8fa700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.177+0000 7f8b0f8fa700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:22:33.223 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:33.224 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:33.225 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:22:33.226 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.178+0000 7f8b26a71880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.178+0000 7f8b26a71880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-09-23T22:22:33.227 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph-mon: debug 2023-09-23T22:22:33.178+0000 7f8b26a71880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-09-23T22:22:33.658 INFO:teuthology.orchestra.run.smithi052.stderr:create mon.a on 2023-09-23T22:22:33.672 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T22:22:33.770 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T22:22:33.870 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7.target. 2023-09-23T22:22:33.870 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7.target. 2023-09-23T22:22:34.097 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Failed to reset failed state of unit ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T22:22:34.104 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Created symlink /etc/systemd/system/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7.target.wants/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@.service. 2023-09-23T22:22:34.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 systemd[1]: Starting Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:22:34.745 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: disabled 2023-09-23T22:22:34.753 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: inactive 2023-09-23T22:22:34.753 INFO:teuthology.orchestra.run.smithi052.stderr:firewalld.service is not enabled 2023-09-23T22:22:34.753 INFO:teuthology.orchestra.run.smithi052.stderr:Not possible to enable service . firewalld.service is not available 2023-09-23T22:22:34.754 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for mon to start... 2023-09-23T22:22:34.754 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for mon... 2023-09-23T22:22:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 bash[99193]: 2261d6e5dee94f1006936decb5c62122217000f56ff41d2ec97d5cbf28b20eb0 2023-09-23T22:22:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 systemd[1]: Started Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:22:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 conmon[99235]: cluster 2023-09-23T22:22:34.777668+0000 mon.a (mon.0) 0 : 2023-09-23T22:22:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 conmon[99235]: [INF] mkfs 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 conmon[99235]: cluster 2023-09-23T22 2023-09-23T22:22:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:34 smithi052 conmon[99235]: :22:34.761101+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T22:22:35.428 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: cluster: 2023-09-23T22:22:35.428 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: id: 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:35.428 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: services: 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.650229s) 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mgr: no daemons active 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: data: 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: pgs: 2023-09-23T22:22:35.429 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:36.083 INFO:teuthology.orchestra.run.smithi052.stderr:mon is available 2023-09-23T22:22:36.083 INFO:teuthology.orchestra.run.smithi052.stderr:Assimilating anything we can from ceph.conf... 2023-09-23T22:22:36.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:35 smithi052 conmon[99235]: cluster 2023-09-23T22:22:34.776513+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T22:22:36.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:35 smithi052 conmon[99235]: cluster 2023-09-23T22:22:34.776738+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0]} removed_ranks: {} 2023-09-23T22:22:36.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:35 smithi052 conmon[99235]: cluster 2023-09-23T22:22:34.779418+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-09-23T22:22:36.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:35 smithi052 conmon[99235]: cluster 2023-09-23T22:22:34.781445+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-09-23T22:22:36.096 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:35 smithi052 conmon[99235]: cluster 2023-09-23T22:22:34.781841+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-09-23T22:22:36.096 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:35 smithi052 conmon[99235]: audit 2023-09-23T22:22:35.426692+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.52:0/1259411830' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T22:22:36.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: [global] 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: fsid = 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: log_to_journald = false 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.52:3300,v1:172.21.15.52:6789] 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: [mgr] 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:36.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: [osd] 2023-09-23T22:22:36.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-09-23T22:22:36.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-09-23T22:22:36.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:36 smithi052 conmon[99235]: audit 2023-09-23T22:22:36.710299+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.52:0/2154579474' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T22:22:36.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:36 smithi052 conmon[99235]: audit 2023-09-23T22:22:36.713058+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.52:0/2154579474' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T22:22:37.357 INFO:teuthology.orchestra.run.smithi052.stderr:Generating new minimal ceph.conf... 2023-09-23T22:22:38.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:38 smithi052 conmon[99235]: audit 2023-09-23T22:22: 2023-09-23T22:22:38.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:38 smithi052 conmon[99235]: 37.965398+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.52:0/4271792885' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:22:38.549 INFO:teuthology.orchestra.run.smithi052.stderr:Restarting the monitor... 2023-09-23T22:22:38.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:38 smithi052 systemd[1]: Stopping Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:22:38.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:38 smithi052 bash[99740]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-mon.a found: no such container 2023-09-23T22:22:39.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:38 smithi052 conmon[99235]: debug 2023-09-23T22:22:38.847+0000 7fb58dd91700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T22:22:39.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:38 smithi052 conmon[99235]: debug 2023-09-23T22:22:38.847+0000 7fb58dd91700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T22:22:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:39 smithi052 bash[99856]: 2261d6e5dee94f1006936decb5c62122217000f56ff41d2ec97d5cbf28b20eb0 2023-09-23T22:22:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:39 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T22:22:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:39 smithi052 systemd[1]: Stopped Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:22:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:39 smithi052 systemd[1]: Starting Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:22:40.068 INFO:teuthology.orchestra.run.smithi052.stderr:Setting mon public_network to 172.21.0.0/20 2023-09-23T22:22:40.313 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.049+0000 7f1a6d555880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:22:40.313 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.049+0000 7f1a6d555880 0 ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable), process ceph-mon, pid 7 2023-09-23T22:22:40.313 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.049+0000 7f1a6d555880 0 pidfile_write: ignore empty --pid-file 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 0 load: jerasure load: lrc load: isa 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: DB SUMMARY 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: CURRENT file: CURRENT 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86679 ; 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.create_if_missing: 0 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T22:22:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.env: 0x55a4dfb79080 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.fs: Posix File System 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.info_log: 0x55a4e1505a80 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.statistics: (nil) 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.use_fsync: 0 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:22:40.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:22:40.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.db_log_dir: 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:22:40.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.write_buffer_manager: 0x55a4e2200330 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:22:40.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.unordered_write: 0 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.row_cache: None 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.wal_filter: None 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:22:40.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:22:40.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_open_files: -1 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Compression algorithms supported: 2023-09-23T22:22:40.323 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kZSTD supported: 0 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-09-23T22:22:40.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.merge_operator: 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_filter: None 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a4e14afd18) 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cache_index_and_filter_blocks: 1 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:22:40.325 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: pin_top_level_index_and_filter: 1 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: index_type: 0 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: data_block_index_type: 0 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: index_shortening: 1 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: hash_index_allow_collision: 1 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: checksum: 1 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: no_block_cache: 0 2023-09-23T22:22:40.326 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_cache: 0x55a4e14e8f10 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_cache_name: BinnedLRUCache 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_cache_options: 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: capacity : 536870912 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: num_shard_bits : 4 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: strict_capacity_limit : 0 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: high_pri_pool_ratio: 0.000 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_cache_compressed: (nil) 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: persistent_cache: (nil) 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_size: 4096 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_size_deviation: 10 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_restart_interval: 16 2023-09-23T22:22:40.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: index_block_restart_interval: 1 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: metadata_block_size: 4096 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: partition_filters: 0 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: use_delta_encoding: 1 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: whole_key_filtering: 1 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: verify_compression: 0 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: read_amp_bytes_per_bit: 0 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: format_version: 2 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: enable_index_compression: 1 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: block_align: 0 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression: NoCompression 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.num_levels: 7 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:22:40.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:22:40.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:22:40.331 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:22:40.331 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:22:40.331 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 bash[100183]: da0d22fd61dec586ac9139c0da8f0a6ab794d365934165843b368ff4eb3a51a6 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:22:40.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:22:40.333 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.054+0000 7f1a6d555880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:22:40.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.ttl: 2592000 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507760056710, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.055+0000 7f1a6d555880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-09-23T22:22:40.335 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.056+0000 7f1a6d555880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.056+0000 7f1a6d555880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507760057567, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83279, "table_properties": {"data_size": 81552, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75873, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695507760, "oldest_key_time": 3, "file_creation_time": 0}} 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.056+0000 7f1a6d555880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.056+0000 7f1a6d555880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507760057961, "job": 1, "event": "recovery_finished"} 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.057+0000 7f1a6d555880 4 rocksdb: DB pointer 0x55a4e15b3800 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.057+0000 7f1a5435d700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T22:22:40.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.057+0000 7f1a5435d700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T22:22:40.337 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** DB Stats ** 2023-09-23T22:22:40.337 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:22:40.337 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** Compaction Stats [default] ** 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:40.338 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: L0 2/0 82.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Sum 2/0 82.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** Compaction Stats [default] ** 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:40.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Cumulative compaction: 0.00 GB write, 28.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Interval compaction: 0.00 GB write, 28.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** Compaction Stats [default] ** 2023-09-23T22:22:40.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: L0 2/0 82.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Sum 2/0 82.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** Compaction Stats [default] ** 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 140.3 0.00 0.00 1 0.001 0 0 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:40.341 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Cumulative compaction: 0.00 GB write, 28.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.058+0000 7f1a6d555880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] at bind addrs [v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 1 mon.a@-1(???) e1 preinit fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:40.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 0 mon.a@-1(???).mds e1 new map 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 0 mon.a@-1(???).mds e1 print_map 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: e1 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: legacy client fscid: -1 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: No filesystems configured 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:22:40.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: audit 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: 22:37.965398+0000 mon.a (mon 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: .0) 10 : audit [DBG] from='client.? 172.21.15.52:0/4271792885' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: debug 2023-09-23T22:22:40.059+0000 7f1a6d555880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 systemd[1]: Started Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cluster 2023-09-23T22:22:40.075880+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cluster 2023-09-23T22:22:40.075951+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0]} removed_ranks: {} 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cluster 2023-09-23T22:22:40.089540+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cluster 2023-09-23T22:22:40.089601+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-09-23T22:22:40.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:40 smithi052 conmon[100224]: cluster 2023-09-23T22:22:40.090019+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-09-23T22:22:41.165 INFO:teuthology.orchestra.run.smithi052.stderr:Wrote config to /etc/ceph/ceph.conf 2023-09-23T22:22:41.166 INFO:teuthology.orchestra.run.smithi052.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:22:41.166 INFO:teuthology.orchestra.run.smithi052.stderr:Creating mgr... 2023-09-23T22:22:41.166 INFO:teuthology.orchestra.run.smithi052.stderr:Verifying port 9283 ... 2023-09-23T22:22:41.298 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Failed to reset failed state of unit ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T22:22:41.304 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: Created symlink /etc/systemd/system/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7.target.wants/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@.service. 2023-09-23T22:22:41.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:41 smithi052 systemd[1]: Starting Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:22:41.946 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:41 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:22:41.946 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:41 smithi052 conmon[100224]: 22:22:40.638690+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.52:0/870921498' entity='client.admin' 2023-09-23T22:22:41.961 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: disabled 2023-09-23T22:22:41.968 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: inactive 2023-09-23T22:22:41.968 INFO:teuthology.orchestra.run.smithi052.stderr:firewalld.service is not enabled 2023-09-23T22:22:41.969 INFO:teuthology.orchestra.run.smithi052.stderr:Not possible to enable service . firewalld.service is not available 2023-09-23T22:22:41.976 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: disabled 2023-09-23T22:22:41.984 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: inactive 2023-09-23T22:22:41.984 INFO:teuthology.orchestra.run.smithi052.stderr:firewalld.service is not enabled 2023-09-23T22:22:41.985 INFO:teuthology.orchestra.run.smithi052.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T22:22:41.985 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for mgr to start... 2023-09-23T22:22:41.985 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for mgr... 2023-09-23T22:22:42.204 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:41 smithi052 bash[100630]: cf5f0eb823e68cc6547018e792249d21e9e25d7a3d57e4cb95c274ff56052b1b 2023-09-23T22:22:42.204 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:41 smithi052 systemd[1]: Started Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:22:42.204 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:42 smithi052 conmon[100670]: debug 2023-09-23T22:22:42.098+0000 7fa48700e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:22:42.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:42 smithi052 conmon[100670]: debug 2023-09-23T22:22:42.202+0000 7fa48700e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:22:42.708 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:42.708 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:22:42.708 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsid": "891acd3a-5a5f-11ee-8db4-212e2dc638e7", 2023-09-23T22:22:42.708 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "health": { 2023-09-23T22:22:42.708 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 0 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "a" 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-09-23T22:22:42.709 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T22:22:42.710 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T22:22:42.713 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T22:22:42.713 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T22:22:42.714 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "available": false, 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "iostat", 2023-09-23T22:22:42.715 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "nfs", 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "restful" 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modified": "2023-09-23T22:22:34.778302+0000", 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T22:22:42.716 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:22:42.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:42 smithi052 conmon[100670]: debug 2023-09-23T22:22:42.675+0000 7fa48700e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:22:42.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:42 smithi052 conmon[100224]: audit 2023-09-23T22:22:42.705261+0000 mon.a (mon.0) 2023-09-23T22:22:42.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:42 smithi052 conmon[100224]: 7 : audit [DBG] from='client.? 172.21.15.52:0/3432281072' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:22:43.340 INFO:teuthology.orchestra.run.smithi052.stderr:mgr not available, waiting (1/15)... 2023-09-23T22:22:43.965 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:43 smithi052 conmon[100670]: debug 2023-09-23T22:22:43.623+0000 7fa48700e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:22:43.966 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:43 smithi052 conmon[100670]: debug 2023-09-23T22:22:43.718+0000 7fa48700e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:22:45.357 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:43 smithi052 conmon[100670]: debug 2023-09-23T22:22:43.964+0000 7fa48700e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:22:45.357 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:45 smithi052 conmon[100670]: debug 2023-09-23T22:22:45.101+0000 7fa48700e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:22:45.358 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:45 smithi052 conmon[100670]: debug 2023-09-23T22:22:45.262+0000 7fa48700e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:22:45.642 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:45 smithi052 conmon[100670]: debug 2023-09-23T22:22:45.356+0000 7fa48700e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:22:45.642 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:45 smithi052 conmon[100670]: debug 2023-09-23T22:22:45.477+0000 7fa48700e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:22:45.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:45 smithi052 conmon[100670]: debug 2023-09-23T22:22:45.703+0000 7fa48700e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:22:46.141 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:46.141 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:22:46.141 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsid": "891acd3a-5a5f-11ee-8db4-212e2dc638e7", 2023-09-23T22:22:46.141 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "health": { 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 0 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "a" 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T22:22:46.143 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.144 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "available": false, 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "iostat", 2023-09-23T22:22:46.145 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "nfs", 2023-09-23T22:22:46.146 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "restful" 2023-09-23T22:22:46.146 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:46.146 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:46.146 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modified": "2023-09-23T22:22:34.778302+0000", 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T22:22:46.147 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:22:46.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:46 smithi052 conmon[100224]: audit 2023-09-23T22:22:46.136150+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.52:0/3372952060' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:22:46.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:46 smithi052 conmon[100670]: debug 2023-09-23T22:22:46.072+0000 7fa48700e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:22:46.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:46 smithi052 conmon[100670]: debug 2023-09-23T22:22:46.192+0000 7fa48700e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:22:46.705 INFO:teuthology.orchestra.run.smithi052.stderr:mgr not available, waiting (2/15)... 2023-09-23T22:22:47.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:47 smithi052 conmon[100670]: debug 2023-09-23T22:22:47.074+0000 7fa48700e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:22:47.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:47 smithi052 conmon[100670]: debug 2023-09-23T22:22:47.171+0000 7fa48700e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:22:47.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:47 smithi052 conmon[100670]: debug 2023-09-23T22:22:47.284+0000 7fa48700e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:22:47.780 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:47 smithi052 conmon[100670]: debug 2023-09-23T22:22:47.493+0000 7fa48700e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:22:48.108 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:47 smithi052 conmon[100670]: debug 2023-09-23T22:22:47.779+0000 7fa48700e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:22:48.108 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:47 smithi052 conmon[100670]: debug 2023-09-23T22:22:47.939+0000 7fa48700e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:22:48.419 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100670]: debug 2023-09-23T22:22:48.106+0000 7fa48700e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:22:48.419 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100670]: debug 2023-09-23T22:22:48.319+0000 7fa48700e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:22:48.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100670]: debug 2023-09-23T22:22:48.418+0000 7fa48700e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:22:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: cluster 2023-09-23T22:22:48.433196+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-09-23T22:22:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: cluster 2023-09-23T22:22:48.435692+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00258412s) 2023-09-23T22:22:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 2023-09-23T22:22: 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 48.436634+0000 mon.a (mon.0) 11 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: : audit [DBG] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 22:48.436838+0000 mon.a (mon.0 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: ) 12 : audit [DBG] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: -23T22:22:48.436981+0000 mon.a (mon.0 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: ) 13 : audit [DBG] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:22:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: -23T22:22:48. 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 437134+0000 mon.a (mon.0) 14 : 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit [DBG] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 09-23T22:22:48 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: .437338+0000 mon.a (mon.0) 15 : 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit [DBG] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 09-23T22:22:48 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: .450871+0000 mon.a (mon. 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 0) 16 : cluster [INF] Manager daemon y is now available 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:22:48.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 22:48.459610+0000 mon.a (mon.0 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: ) 17 : audit [INF] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 22:48.465400+0000 mon.a (mon 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: .0) 18 : audit [INF] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 2023-09-23T22:22 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: :48.468501+0000 mon.a (mon.0) 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 19 : audit [INF] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 09-23T22:22:48. 2023-09-23T22:22:48.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 471449+0000 mon.a (mon.0) 20 : 2023-09-23T22:22:48.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit [INF] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' 2023-09-23T22:22:48.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:22:48.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 09-23T22:22:48. 2023-09-23T22:22:48.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: 474044+0000 mon.a (mon.0) 21 : 2023-09-23T22:22:48.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:48 smithi052 conmon[100224]: audit [INF] from='mgr.14100 172.21.15.52:0/3667720024' entity='mgr.y' 2023-09-23T22:22:49.294 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsid": "891acd3a-5a5f-11ee-8db4-212e2dc638e7", 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "health": { 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T22:22:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 0 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "a" 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_age": 9, 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:49.296 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T22:22:49.297 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T22:22:49.297 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T22:22:49.299 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T22:22:49.300 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "available": false, 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "iostat", 2023-09-23T22:22:49.301 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "nfs", 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "restful" 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modified": "2023-09-23T22:22:34.778302+0000", 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T22:22:49.302 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:22:49.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:49 smithi052 conmon[100224]: audit 2023-09-23T22:22:49.291915+0000 mon.a ( 2023-09-23T22:22:49.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:49 smithi052 conmon[100224]: mon.0) 22 : audit [DBG] from='client.? 172.21.15.52:0/3566424109' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:22:49.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:49 smithi052 conmon[100224]: cluster 2023-09-23T22:22:49.438090+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e3: y(active, since 1.00498s) 2023-09-23T22:22:49.831 INFO:teuthology.orchestra.run.smithi052.stderr:mgr not available, waiting (3/15)... 2023-09-23T22:22:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:52 smithi052 conmon[100224]: cluster 2023-09-23T22:22:51 2023-09-23T22:22:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:52 smithi052 conmon[100224]: .067513+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsid": "891acd3a-5a5f-11ee-8db4-212e2dc638e7", 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "health": { 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "checks": {}, 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mutes": [] 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum": [ 2023-09-23T22:22:52.777 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 0 2023-09-23T22:22:52.779 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:52.779 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_names": [ 2023-09-23T22:22:52.779 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "a" 2023-09-23T22:22:52.779 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:52.779 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "quorum_age": 12, 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "monmap": { 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_mons": 1 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osdmap": { 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_osds": 0, 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-09-23T22:22:52.780 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-09-23T22:22:52.783 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-09-23T22:22:52.783 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgmap": { 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_pools": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_objects": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "bytes_total": 0 2023-09-23T22:22:52.784 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "fsmap": { 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "by_rank": [], 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "up:standby": 0 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mgrmap": { 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "available": true, 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modules": [ 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "iostat", 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "nfs", 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "restful" 2023-09-23T22:22:52.785 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ], 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "servicemap": { 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 1, 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "modified": "2023-09-23T22:22:34.778302+0000", 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "services": {} 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: }, 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "progress_events": {} 2023-09-23T22:22:52.786 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:22:53.347 INFO:teuthology.orchestra.run.smithi052.stderr:mgr is available 2023-09-23T22:22:53.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:53 smithi052 conmon[100224]: audit 2023-09-23T22:22:52.775476+0000 2023-09-23T22:22:53.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:53 smithi052 conmon[100224]: mon.a (mon.0) 25 : audit [DBG] from='client.? 172.21.15.52:0/3808431789' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T22:22:54.105 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: [global] 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: fsid = 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: log_to_journald = false 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: [mgr] 2023-09-23T22:22:54.106 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-09-23T22:22:54.107 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 2023-09-23T22:22:54.107 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: [osd] 2023-09-23T22:22:54.107 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-09-23T22:22:54.107 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-09-23T22:22:54.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:54 smithi052 conmon[100224]: audit 2023-09-23T22:22:54.103388+0000 2023-09-23T22:22:54.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:54 smithi052 conmon[100224]: mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.52:0/2766844989' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T22:22:54.622 INFO:teuthology.orchestra.run.smithi052.stderr:Enabling cephadm module... 2023-09-23T22:22:56.363 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:56 smithi052 conmon[100224]: audit 2023-09-23T22:22:55.464379+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.52:0/3472452802' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T22:22:56.364 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:56 smithi052 conmon[100670]: ignoring --setuser ceph since I am not root 2023-09-23T22:22:56.364 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:56 smithi052 conmon[100670]: ignoring --setgroup ceph since I am not root 2023-09-23T22:22:56.364 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:56 smithi052 conmon[100670]: debug 2023-09-23T22:22:56.263+0000 7fa33281e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:22:56.704 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:56 smithi052 conmon[100670]: debug 2023-09-23T22:22:56.362+0000 7fa33281e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:22:56.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:56 smithi052 conmon[100670]: debug 2023-09-23T22:22:56.831+0000 7fa33281e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:22:57.354 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:22:57.354 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 5, 2023-09-23T22:22:57.354 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "available": true, 2023-09-23T22:22:57.355 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "active_name": "y", 2023-09-23T22:22:57.355 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_standby": 0 2023-09-23T22:22:57.355 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:22:57.376 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:57 smithi052 conmon[100224]: audit 2023-09-23T22:22:56.072138+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.52:0/3472452802' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T22:22:57.376 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:57 smithi052 conmon[100224]: cluster 2023-09-23T22:22:56.072216+0000 mon.a (mon.0) 29 : cluster [DBG] mgrmap e5: y(active, since 7s) 2023-09-23T22:22:57.821 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for the mgr to restart... 2023-09-23T22:22:57.821 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for mgr epoch 5... 2023-09-23T22:22:58.037 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:57 smithi052 conmon[100670]: debug 2023-09-23T22:22:57.776+0000 7fa33281e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:22:58.037 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:57 smithi052 conmon[100670]: debug 2023-09-23T22:22:57.881+0000 7fa33281e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:22:58.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:58 smithi052 conmon[100224]: audit 2023-09-23T22:22:57.353314+0000 2023-09-23T22:22:58.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:22:58 smithi052 conmon[100224]: mon.a (mon.0) 30 : audit [DBG] from='client.? 172.21.15.52:0/3146675903' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T22:22:58.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:58 smithi052 conmon[100670]: debug 2023-09-23T22:22:58.141+0000 7fa33281e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:22:59.530 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:59 smithi052 conmon[100670]: debug 2023-09-23T22:22:59.274+0000 7fa33281e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:22:59.530 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:59 smithi052 conmon[100670]: debug 2023-09-23T22:22:59.435+0000 7fa33281e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:22:59.856 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:59 smithi052 conmon[100670]: debug 2023-09-23T22:22:59.529+0000 7fa33281e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:22:59.856 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:59 smithi052 conmon[100670]: debug 2023-09-23T22:22:59.638+0000 7fa33281e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:22:59.856 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:22:59 smithi052 conmon[100670]: debug 2023-09-23T22:22:59.855+0000 7fa33281e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:23:00.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:00 smithi052 conmon[100670]: debug 2023-09-23T22:23:00.217+0000 7fa33281e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:23:00.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:00 smithi052 conmon[100670]: debug 2023-09-23T22:23:00.334+0000 7fa33281e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:23:01.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:01 smithi052 conmon[100670]: debug 2023-09-23T22:23:01.196+0000 7fa33281e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:23:01.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:01 smithi052 conmon[100670]: debug 2023-09-23T22:23:01.294+0000 7fa33281e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:23:01.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:01 smithi052 conmon[100670]: debug 2023-09-23T22:23:01.404+0000 7fa33281e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:23:01.897 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:01 smithi052 conmon[100670]: debug 2023-09-23T22:23:01.614+0000 7fa33281e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:23:02.224 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:01 smithi052 conmon[100670]: debug 2023-09-23T22:23:01.897+0000 7fa33281e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:23:02.224 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100670]: debug 2023-09-23T22:23:02.056+0000 7fa33281e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:23:02.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100670]: debug 2023-09-23T22:23:02.223+0000 7fa33281e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:23:02.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100670]: debug 2023-09-23T22:23:02.436+0000 7fa33281e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:23:02.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100670]: debug 2023-09-23T22:23:02.535+0000 7fa33281e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:23:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: cluster 2023-09-23T22:23:02.549153+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:23:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: cluster 2023-09-23T22:23:02.549382+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2023-09-23T22:23:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: cluster 2023-09-23T22:23:02.552450 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: +0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: cluster 2023-09-23T22: 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: 23:02.552547+0000 mon.a (mon.0 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: ) 34 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00324646s) 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: T22:23:02.553654+0000 mon.a ( 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: 09-23T22:23:02.554374+0000 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.a (mon.0) 36 : audit [DBG] 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:23:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: audit 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: 2023-09-23T22:23:02.555091+0000 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.a (mon.0) 37 : audit [DBG] 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: audit 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: 2023-09-23T22:23:02.555348+0000 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.a (mon.0) 38 : audit [DBG] 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: audit 2023 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: -09-23T22:23:02.555597+0000 mon.a ( 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.0) 39 : audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:23:02.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: 09-23T22:23:02.569684+0000 2023-09-23T22:23:02.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.a (mon.0) 40 : cluster [INF] 2023-09-23T22:23:02.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: Manager daemon y is now available 2023-09-23T22:23:02.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: audit 2023-09-23T22:23:02.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: 2023-09-23T22:23:02.584222+0000 2023-09-23T22:23:02.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: mon.a (mon.0) 41 : audit [INF] 2023-09-23T22:23:02.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:02 smithi052 conmon[100224]: from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:03.566 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:23:03.566 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-09-23T22:23:03.567 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "initialized": true 2023-09-23T22:23:03.567 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:23:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:02.581690+0000 mgr.y ( 2023-09-23T22:23:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit 2023-09-23T22:23:02.702402+0000 mon.a (mon 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: .0) 42 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 09-23T22:23:02.703434 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: +0000 mon.a (mon.0) 43 : 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit 2023 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: -09-23T22:23:02. 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 704977+0000 mon.a (mon.0) 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 44 : audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit 2023 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: -09-23T22:23:02. 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 707192+0000 mon.a (mon.0) 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 45 : audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 09-23T22:23:02. 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 721478+0000 mon.a (mon.0) 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 46 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: audit 2023 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: -09-23T22:23:02 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: .724581+0000 mon.a (mon.0 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: ) 47 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:23:03.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:23:03.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: -09-23T22:23:03. 2023-09-23T22:23:03.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: 555167+0000 mon.a (mon.0 2023-09-23T22:23:03.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:03 smithi052 conmon[100224]: ) 48 : cluster [DBG] mgrmap e7: y(active, since 1.00586s) 2023-09-23T22:23:04.087 INFO:teuthology.orchestra.run.smithi052.stderr:mgr epoch 5 is available 2023-09-23T22:23:04.087 INFO:teuthology.orchestra.run.smithi052.stderr:Setting orchestrator backend to cephadm... 2023-09-23T22:23:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:04 smithi052 conmon[100224]: audit 2023-09-23T22:23:03.554201+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T22:23:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:04 smithi052 conmon[100224]: audit 2023-09-23T22:23:03.565528+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T22:23:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:05 smithi052 conmon[100224]: cluster 2023-09-23T22:23:04.704582+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-09-23T22:23:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:05 smithi052 conmon[100224]: audit 2023-09-23T22:23:04.889914+0000 mgr.y (mgr.14118) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:05 smithi052 conmon[100224]: audit 2023-09-23T22:23:04.894056+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:05 smithi052 conmon[100224]: audit 2023-09-23T22:23:04.928722+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:06.197 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: value unchanged 2023-09-23T22:23:06.720 INFO:teuthology.orchestra.run.smithi052.stderr:Generating ssh key... 2023-09-23T22:23:07.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100224]: audit 2023-09-23T22:23: 2023-09-23T22:23:07.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100224]: 06.197048+0000 mgr.y (mgr.14118) 5 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:07.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: Generating public/private rsa key pair. 2023-09-23T22:23:07.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: Your identification has been saved in /tmp/tmpf6ed68hr/key. 2023-09-23T22:23:07.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: Your public key has been saved in /tmp/tmpf6ed68hr/key.pub. 2023-09-23T22:23:07.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: The key fingerprint is: 2023-09-23T22:23:07.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: SHA256:xIvDAvPPECtKQ550xgh6YNhKuCuvXZAPHGZkdLrGKrg ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:23:07.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: The key's randomart image is: 2023-09-23T22:23:07.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: +---[RSA 3072]----+ 2023-09-23T22:23:07.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |=++ . | 2023-09-23T22:23:07.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |*=+o . | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |o*O+. o | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |**+B + o . | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: | *X = + S | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |=+.= = . | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |*. o o | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |.o.. | 2023-09-23T22:23:07.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: |E.. | 2023-09-23T22:23:07.996 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:07 smithi052 conmon[100670]: +----[SHA256]-----+ 2023-09-23T22:23:08.883 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGd4e3bZi52v1e/XDO58/g78HeAg2NHbfcMema/n4WudfGT98qsuRo7dXAC5i/ykkP+Z/Qeg6qzBcQvJmNRfKbKeRK7B3IzWTXtSD918b2cFf1OrETsySAto9RcQvMrZE1Y0JpwjYv39SdGMtpfjQgYbX4Zvd49k3n21x0irrZRR5pJWNew9bNNtQbrEG7S7B7JnEV9jr+9FXHjZAeZcQtwxI6DEgAEx0a3nt6M+NrzpU0rgR2/+KVS7I0cK/+JaLlpd45M8b5jr/D4/wPE+glVHHlWvnGb2dVWcm0us0ZCj5pOf1PJr8WH4n9/WqZi0ayD+EdfNkXeyloz1Y4yEL0n7xfCfih7TZJ4aoUbHQ4BIq+Pv3vY4TcpVl+12oZIJ4Z0zP8S10PXYN5SKdTlExWbKhWdcm/jPw1Y7OZM7B0fcgKCpLtOnULCRGuxvJc34ZyOo1I+GOLk+LOZ7kQMUU8DSuHdRIiXFU8jQZAxu9+unDJSr/0Jr1oz5JJQHHQd5c= ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: audit 2023-09-23T22:23:07. 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: 482195+0000 mgr.y (mgr.14118) 6 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:07.482430+0000 mgr.y (mgr.14118) 7 : cephadm [INF] Generating ssh key... 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: audit 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: 2023-09-23T22:23:07.592958 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: +0000 mon.a (mon.0) 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: 52 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: audit 2023 2023-09-23T22:23:08.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: -09-23T22:23:07 2023-09-23T22:23:08.908 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: .594685+0000 mon.a (mon. 2023-09-23T22:23:08.908 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: 0) 53 : audit [INF] 2023-09-23T22:23:08.908 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:08 smithi052 conmon[100224]: from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:09.378 INFO:teuthology.orchestra.run.smithi052.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T22:23:09.378 INFO:teuthology.orchestra.run.smithi052.stderr:Adding key to root@localhost authorized_keys... 2023-09-23T22:23:09.379 INFO:teuthology.orchestra.run.smithi052.stderr:Adding host smithi052... 2023-09-23T22:23:09.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:09 smithi052 conmon[100224]: audit 2023-09-23T22:23:08.883468+0000 2023-09-23T22:23:09.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:09 smithi052 conmon[100224]: mgr.y (mgr.14118) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:10.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:10 smithi052 conmon[100670]: [79B blob data] 2023-09-23T22:23:10.874 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: Added host 'smithi052' with addr '172.21.15.52' 2023-09-23T22:23:11.445 INFO:teuthology.orchestra.run.smithi052.stderr:Deploying unmanaged mon service... 2023-09-23T22:23:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:11 smithi052 conmon[100224]: audit 2023-09-23T22:23:10.179131+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi052", "addr": "172.21.15.52", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:11 smithi052 conmon[100224]: audit 2023-09-23T22:23:10.873084+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:11 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:10.873393+0000 mgr.y (mgr.14118) 10 : cephadm [INF] Added host smithi052 2023-09-23T22:23:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:11 smithi052 conmon[100224]: audit 2023-09-23T22:23:10.905790+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:12.291 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: Scheduled mon update... 2023-09-23T22:23:12.911 INFO:teuthology.orchestra.run.smithi052.stderr:Deploying unmanaged mgr service... 2023-09-23T22:23:13.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:13 smithi052 conmon[100224]: audit 2023-09-23T22:23:12.287184+0000 mgr.y (mgr.14118) 11 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:13.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:12.288577+0000 mgr.y (mgr.14118) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-09-23T22:23:13.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:13 smithi052 conmon[100224]: audit 2023-09-23T22:23:12.290841+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:14.132 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-09-23T22:23:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:14 smithi052 conmon[100224]: audit 2023-09-23T22:23:13.346276+0000 mon.a (mon.0) 2023-09-23T22:23:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:14 smithi052 conmon[100224]: 57 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:14 smithi052 conmon[100224]: audit 2023-09-23T22:23:13.623540+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:14 smithi052 conmon[100224]: audit 2023-09-23T22:23:14.132112+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:15 smithi052 conmon[100224]: audit 2023-09-23T22:23:14.128262+0000 mgr.y 2023-09-23T22:23:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:15 smithi052 conmon[100224]: (mgr.14118) 13 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:14.129761+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-09-23T22:23:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:15 smithi052 conmon[100224]: audit 2023-09-23T22:23:14.954737+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14118 172.21.15.52:0/1155701295' entity='mgr.y' 2023-09-23T22:23:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:15 smithi052 conmon[100224]: audit 2023-09-23T22:23:15.684413+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.52:0/1416684892' entity='client.admin' 2023-09-23T22:23:17.919 INFO:teuthology.orchestra.run.smithi052.stderr:Enabling the dashboard module... 2023-09-23T22:23:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:18 smithi052 conmon[100224]: audit 2023-09-23T22:23:17.353957+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.52:0/2693780144' entity='client.admin' 2023-09-23T22:23:19.656 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:19 smithi052 conmon[100224]: audit 2023-09-23T22:23:18.797339+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.52:0/624406573' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T22:23:19.656 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:19 smithi052 conmon[100670]: ignoring --setuser ceph since I am not root 2023-09-23T22:23:19.657 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:19 smithi052 conmon[100670]: ignoring --setgroup ceph since I am not root 2023-09-23T22:23:19.657 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:19 smithi052 conmon[100670]: debug 2023-09-23T22:23:19.556+0000 7f214ff38680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:23:19.657 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:19 smithi052 conmon[100670]: debug 2023-09-23T22:23:19.655+0000 7f214ff38680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:23:20.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:20 smithi052 conmon[100670]: debug 2023-09-23T22:23:20.119+0000 7f214ff38680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:23:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:20 smithi052 conmon[100224]: audit 2023-09-23T22:23:19.357792+0000 mon.a (mon. 2023-09-23T22:23:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:20 smithi052 conmon[100224]: 0) 64 : audit [INF] from='client.? 172.21.15.52:0/624406573' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T22:23:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:20 smithi052 conmon[100224]: cluster 2023-09-23T22:23:19.357886+0000 mon.a (mon.0) 65 : cluster [DBG] mgrmap e9: y(active, since 16s) 2023-09-23T22:23:20.771 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:23:20.772 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "epoch": 9, 2023-09-23T22:23:20.772 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "available": true, 2023-09-23T22:23:20.772 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "active_name": "y", 2023-09-23T22:23:20.772 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "num_standby": 0 2023-09-23T22:23:20.772 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:23:21.335 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for the mgr to restart... 2023-09-23T22:23:21.335 INFO:teuthology.orchestra.run.smithi052.stderr:Waiting for mgr epoch 9... 2023-09-23T22:23:21.340 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:21 smithi052 conmon[100670]: debug 2023-09-23T22:23:21.064+0000 7f214ff38680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:23:21.340 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:21 smithi052 conmon[100670]: debug 2023-09-23T22:23:21.160+0000 7f214ff38680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:23:21.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:21 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:23:21.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:21 smithi052 conmon[100224]: 22:23:20.771032+0000 mon.a (mon.0) 66 : audit [DBG] from='client.? 172.21.15.52:0/2141276132' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T22:23:21.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:21 smithi052 conmon[100670]: debug 2023-09-23T22:23:21.416+0000 7f214ff38680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:23:22.815 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:22 smithi052 conmon[100670]: debug 2023-09-23T22:23:22.557+0000 7f214ff38680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:23:22.815 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:22 smithi052 conmon[100670]: debug 2023-09-23T22:23:22.719+0000 7f214ff38680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:23:23.142 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:22 smithi052 conmon[100670]: debug 2023-09-23T22:23:22.814+0000 7f214ff38680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:23:23.142 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:22 smithi052 conmon[100670]: debug 2023-09-23T22:23:22.923+0000 7f214ff38680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:23:23.142 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:23 smithi052 conmon[100670]: debug 2023-09-23T22:23:23.141+0000 7f214ff38680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:23:23.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:23 smithi052 conmon[100670]: debug 2023-09-23T22:23:23.505+0000 7f214ff38680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:23:23.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:23 smithi052 conmon[100670]: debug 2023-09-23T22:23:23.622+0000 7f214ff38680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:23:24.700 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:24 smithi052 conmon[100670]: debug 2023-09-23T22:23:24.491+0000 7f214ff38680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:23:24.700 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:24 smithi052 conmon[100670]: debug 2023-09-23T22:23:24.589+0000 7f214ff38680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:23:24.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:24 smithi052 conmon[100670]: debug 2023-09-23T22:23:24.699+0000 7f214ff38680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:23:24.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:24 smithi052 conmon[100670]: debug 2023-09-23T22:23:24.909+0000 7f214ff38680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:23:25.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:25 smithi052 conmon[100670]: debug 2023-09-23T22:23:25.194+0000 7f214ff38680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:23:25.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:25 smithi052 conmon[100670]: debug 2023-09-23T22:23:25.355+0000 7f214ff38680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:23:25.837 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:25 smithi052 conmon[100670]: debug 2023-09-23T22:23:25.522+0000 7f214ff38680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:23:25.837 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:25 smithi052 conmon[100670]: debug 2023-09-23T22:23:25.735+0000 7f214ff38680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:23:26.109 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:25 smithi052 conmon[100670]: debug 2023-09-23T22:23:25.836+0000 7f214ff38680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:23:26.109 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100670]: [79B blob data] 2023-09-23T22:23:26.109 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: cluster 2023-09-23T22:23:25.849894+0000 mon.a 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: (mon.0) 67 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: cluster 2023-09-23T22: 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 23:25.850126+0000 mon.a (mon.0) 68 : cluster [INF] 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: Activating manager daemon y 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: cluster 2023-09-23T 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 22:23:25.853015+0000 mon.a (mon.0) 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 69 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: cluster 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 2023-09-23T22:23:25. 2023-09-23T22:23:26.110 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 853131+0000 mon.a (mon.0) 70 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: : cluster [DBG] mgrmap e10: y(active, starting, since 0.00310587s) 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 2023-09-23T22:23:25. 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 854241+0000 mon.a (mon.0) 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 71 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: -23T22:23:25.854975+0000 mon.a ( 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.0) 72 : audit [DBG] 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023-09-23T22:23:26.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 2023-09-23T22:23:25. 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 855674+0000 mon.a (mon.0) 73 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 23:25.855880+0000 mon.a ( 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.0) 74 : audit [DBG] 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 09-23T22:23:25.856049+0000 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.a (mon.0) 75 : audit 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:23:26.112 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 09-23T22:23:25.869781+0000 mon.a ( 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.0) 76 : cluster [INF] 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: Manager daemon y is now available 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 23:25.998410+0000 mon.a 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: (mon.0) 77 : audit 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 09-23T22:23:25.999957+0000 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.a (mon.0) 78 2023-09-23T22:23:26.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: 09-23T22:23:26.026770+0000 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.a (mon.0) 79 : audit 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: audit 2023 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: -09-23T22:23:26.029939+0000 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: mon.a (mon.0) 80 2023-09-23T22:23:26.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:26 smithi052 conmon[100224]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:23:26.870 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: { 2023-09-23T22:23:26.870 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-09-23T22:23:26.870 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: "initialized": true 2023-09-23T22:23:26.870 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: } 2023-09-23T22:23:27.526 INFO:teuthology.orchestra.run.smithi052.stderr:mgr epoch 9 is available 2023-09-23T22:23:27.526 INFO:teuthology.orchestra.run.smithi052.stderr:Generating a dashboard self-signed certificate... 2023-09-23T22:23:28.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:27 smithi052 conmon[100224]: audit 2023-09-23T22:23:26.854490+0000 mgr.y (mgr.14148) 1 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T22:23:28.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:27 smithi052 conmon[100224]: cluster 2023-09-23T22:23:26.855115+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e11: y(active, since 1.00509s) 2023-09-23T22:23:28.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:27 smithi052 conmon[100224]: audit 2023-09-23T22:23:26.869873+0000 mgr.y (mgr.14148) 2 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T22:23:28.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:27 smithi052 conmon[100224]: audit 2023-09-23T22:23:27.515585+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:28.409 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: Self-signed certificate created 2023-09-23T22:23:29.316 INFO:teuthology.orchestra.run.smithi052.stderr:Creating initial admin user... 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: audit 2023-09-23T22:23:28. 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: 336041+0000 mgr.y (mgr.14148) 3 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: audit 2023-09-23T22:23:28. 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: 407149+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: audit 2023-09-23T22:23: 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: 28.409026+0000 mon.a (mon.0) 84 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: cluster 2023-09-23T22:23 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: :28.517003+0000 mon.a (mon.0) 2023-09-23T22:23:29.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:29 smithi052 conmon[100224]: 85 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-09-23T22:23:30.381 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$Qdtke7/aoEduNQKoXhYmb.BBSzgpsmwn27xLi3gbjckNVmiLPvStq", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695507810, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T22:23:30.975 INFO:teuthology.orchestra.run.smithi052.stderr:Fetching dashboard port number... 2023-09-23T22:23:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:31 smithi052 conmon[100224]: audit 2023-09-23T22:23:30.136769+0000 mgr.y (mgr.14148) 4 : audit [DBG] from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:31 smithi052 conmon[100224]: audit 2023-09-23T22:23:30.381422+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:31.791 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: 8443 2023-09-23T22:23:32.326 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: disabled 2023-09-23T22:23:32.334 INFO:teuthology.orchestra.run.smithi052.stderr:systemctl: inactive 2023-09-23T22:23:32.334 INFO:teuthology.orchestra.run.smithi052.stderr:firewalld.service is not enabled 2023-09-23T22:23:32.334 INFO:teuthology.orchestra.run.smithi052.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T22:23:32.336 INFO:teuthology.orchestra.run.smithi052.stderr:Ceph Dashboard is now available at: 2023-09-23T22:23:32.337 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:32.337 INFO:teuthology.orchestra.run.smithi052.stderr: URL: https://smithi052:8443/ 2023-09-23T22:23:32.337 INFO:teuthology.orchestra.run.smithi052.stderr: User: admin 2023-09-23T22:23:32.337 INFO:teuthology.orchestra.run.smithi052.stderr: Password: 49r9j0ve9l 2023-09-23T22:23:32.337 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:32.337 INFO:teuthology.orchestra.run.smithi052.stderr:Enabling autotune for osd_memory_target 2023-09-23T22:23:32.578 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:32 smithi052 conmon[100224]: audit 2023-09-23T22:23:31.791431+0000 mon.a 2023-09-23T22:23:32.578 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:32 smithi052 conmon[100224]: (mon.0) 87 : audit [DBG] from='client.? 172.21.15.52:0/562040522' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T22:23:33.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:33 smithi052 conmon[100224]: cluster 2023-09-23T22:23:32.385461+0000 mon.a (mon.0) 88 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-09-23T22:23:34.754 INFO:teuthology.orchestra.run.smithi052.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-09-23T22:23:35.316 INFO:teuthology.orchestra.run.smithi052.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T22:23:35.317 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.317 INFO:teuthology.orchestra.run.smithi052.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:23:35.317 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.317 INFO:teuthology.orchestra.run.smithi052.stderr:Or, if you are only running a single cluster on this host: 2023-09-23T22:23:35.317 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.317 INFO:teuthology.orchestra.run.smithi052.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T22:23:35.319 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.319 INFO:teuthology.orchestra.run.smithi052.stderr:Please consider enabling telemetry to help improve Ceph: 2023-09-23T22:23:35.319 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.319 INFO:teuthology.orchestra.run.smithi052.stderr: ceph telemetry on 2023-09-23T22:23:35.320 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.320 INFO:teuthology.orchestra.run.smithi052.stderr:For more information see: 2023-09-23T22:23:35.320 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.320 INFO:teuthology.orchestra.run.smithi052.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-09-23T22:23:35.320 INFO:teuthology.orchestra.run.smithi052.stderr: 2023-09-23T22:23:35.320 INFO:teuthology.orchestra.run.smithi052.stderr:Bootstrap complete. 2023-09-23T22:23:35.355 INFO:tasks.cephadm:Fetching config... 2023-09-23T22:23:35.355 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:23:35.356 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T22:23:35.373 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T22:23:35.373 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:23:35.373 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T22:23:35.430 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T22:23:35.431 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:23:35.431 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd if=/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T22:23:35.499 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T22:23:35.499 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:23:35.499 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T22:23:35.557 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T22:23:35.557 DEBUG:teuthology.orchestra.run.smithi052:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGd4e3bZi52v1e/XDO58/g78HeAg2NHbfcMema/n4WudfGT98qsuRo7dXAC5i/ykkP+Z/Qeg6qzBcQvJmNRfKbKeRK7B3IzWTXtSD918b2cFf1OrETsySAto9RcQvMrZE1Y0JpwjYv39SdGMtpfjQgYbX4Zvd49k3n21x0irrZRR5pJWNew9bNNtQbrEG7S7B7JnEV9jr+9FXHjZAeZcQtwxI6DEgAEx0a3nt6M+NrzpU0rgR2/+KVS7I0cK/+JaLlpd45M8b5jr/D4/wPE+glVHHlWvnGb2dVWcm0us0ZCj5pOf1PJr8WH4n9/WqZi0ayD+EdfNkXeyloz1Y4yEL0n7xfCfih7TZJ4aoUbHQ4BIq+Pv3vY4TcpVl+12oZIJ4Z0zP8S10PXYN5SKdTlExWbKhWdcm/jPw1Y7OZM7B0fcgKCpLtOnULCRGuxvJc34ZyOo1I+GOLk+LOZ7kQMUU8DSuHdRIiXFU8jQZAxu9+unDJSr/0Jr1oz5JJQHHQd5c= ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T22:23:35.638 INFO:teuthology.orchestra.run.smithi052.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGd4e3bZi52v1e/XDO58/g78HeAg2NHbfcMema/n4WudfGT98qsuRo7dXAC5i/ykkP+Z/Qeg6qzBcQvJmNRfKbKeRK7B3IzWTXtSD918b2cFf1OrETsySAto9RcQvMrZE1Y0JpwjYv39SdGMtpfjQgYbX4Zvd49k3n21x0irrZRR5pJWNew9bNNtQbrEG7S7B7JnEV9jr+9FXHjZAeZcQtwxI6DEgAEx0a3nt6M+NrzpU0rgR2/+KVS7I0cK/+JaLlpd45M8b5jr/D4/wPE+glVHHlWvnGb2dVWcm0us0ZCj5pOf1PJr8WH4n9/WqZi0ayD+EdfNkXeyloz1Y4yEL0n7xfCfih7TZJ4aoUbHQ4BIq+Pv3vY4TcpVl+12oZIJ4Z0zP8S10PXYN5SKdTlExWbKhWdcm/jPw1Y7OZM7B0fcgKCpLtOnULCRGuxvJc34ZyOo1I+GOLk+LOZ7kQMUU8DSuHdRIiXFU8jQZAxu9+unDJSr/0Jr1oz5JJQHHQd5c= ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:23:35.659 DEBUG:teuthology.orchestra.run.smithi151:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGd4e3bZi52v1e/XDO58/g78HeAg2NHbfcMema/n4WudfGT98qsuRo7dXAC5i/ykkP+Z/Qeg6qzBcQvJmNRfKbKeRK7B3IzWTXtSD918b2cFf1OrETsySAto9RcQvMrZE1Y0JpwjYv39SdGMtpfjQgYbX4Zvd49k3n21x0irrZRR5pJWNew9bNNtQbrEG7S7B7JnEV9jr+9FXHjZAeZcQtwxI6DEgAEx0a3nt6M+NrzpU0rgR2/+KVS7I0cK/+JaLlpd45M8b5jr/D4/wPE+glVHHlWvnGb2dVWcm0us0ZCj5pOf1PJr8WH4n9/WqZi0ayD+EdfNkXeyloz1Y4yEL0n7xfCfih7TZJ4aoUbHQ4BIq+Pv3vY4TcpVl+12oZIJ4Z0zP8S10PXYN5SKdTlExWbKhWdcm/jPw1Y7OZM7B0fcgKCpLtOnULCRGuxvJc34ZyOo1I+GOLk+LOZ7kQMUU8DSuHdRIiXFU8jQZAxu9+unDJSr/0Jr1oz5JJQHHQd5c= ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T22:23:35.703 INFO:teuthology.orchestra.run.smithi151.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGd4e3bZi52v1e/XDO58/g78HeAg2NHbfcMema/n4WudfGT98qsuRo7dXAC5i/ykkP+Z/Qeg6qzBcQvJmNRfKbKeRK7B3IzWTXtSD918b2cFf1OrETsySAto9RcQvMrZE1Y0JpwjYv39SdGMtpfjQgYbX4Zvd49k3n21x0irrZRR5pJWNew9bNNtQbrEG7S7B7JnEV9jr+9FXHjZAeZcQtwxI6DEgAEx0a3nt6M+NrzpU0rgR2/+KVS7I0cK/+JaLlpd45M8b5jr/D4/wPE+glVHHlWvnGb2dVWcm0us0ZCj5pOf1PJr8WH4n9/WqZi0ayD+EdfNkXeyloz1Y4yEL0n7xfCfih7TZJ4aoUbHQ4BIq+Pv3vY4TcpVl+12oZIJ4Z0zP8S10PXYN5SKdTlExWbKhWdcm/jPw1Y7OZM7B0fcgKCpLtOnULCRGuxvJc34ZyOo1I+GOLk+LOZ7kQMUU8DSuHdRIiXFU8jQZAxu9+unDJSr/0Jr1oz5JJQHHQd5c= ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:23:35.717 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T22:23:35.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:35 smithi052 conmon[100224]: audit 2023-09-23T22:23:34.751878+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.52:0/802582760' entity='client.admin' 2023-09-23T22:23:38.375 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T22:23:38.376 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T22:23:39.079 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:38 smithi052 conmon[100224]: audit 2023-09-23T22:23:37.822521+0000 mon.a (mon.0) 90 : audit [INF] from='client.? 172.21.15.52:0/1297976458' entity='client.admin' 2023-09-23T22:23:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:40 smithi052 conmon[100224]: audit 2023-09-23T22:23:39.152738+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:40 smithi052 conmon[100224]: audit 2023-09-23T22:23:39.739179+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi052", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:23:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:40 smithi052 conmon[100224]: audit 2023-09-23T22:23:39.743176+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:40 smithi052 conmon[100224]: audit 2023-09-23T22:23:39.744908+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:40 smithi052 conmon[100224]: audit 2023-09-23T22:23:39.748733+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:40.974 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi151 2023-09-23T22:23:40.974 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:23:40.974 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/etc/ceph/ceph.conf 2023-09-23T22:23:40.994 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:23:40.995 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:23:41.054 INFO:tasks.cephadm:Adding host smithi151 to orchestrator... 2023-09-23T22:23:41.054 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi151 2023-09-23T22:23:41.722 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.367943+0000 mgr.y 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: (mgr.14148) 5 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.370443+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.414451+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.416164+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.416962+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.483581+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:41.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:41 smithi052 conmon[100224]: audit 2023-09-23T22:23:40.485802+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:42 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:40.418299+0000 mgr.y (mgr.14148) 6 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:23:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:42 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:40.437581+0000 mgr.y (mgr.14148) 7 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:23:43.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:42 smithi052 conmon[100670]: [90B blob data] 2023-09-23T22:23:43.524 INFO:teuthology.orchestra.run.smithi052.stdout:Added host 'smithi151' with addr '172.21.15.151' 2023-09-23T22:23:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:23:43 smithi052 conmon[100670]: [80B blob data] 2023-09-23T22:23:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:43 smithi052 conmon[100224]: audit 2023-09-23T22:23:42.822260+0000 mgr.y (mgr 2023-09-23T22:23:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:43 smithi052 conmon[100224]: .14148) 8 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi151", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:44.107 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T22:23:44.897 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: audit 2023-09-23T22:23:43.523245+0000 mon.a 2023-09-23T22:23:44.897 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: (mon.0) 102 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:44.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:43.523563+0000 mgr.y (mgr.14148) 9 : cephadm [INF] Added host smithi151 2023-09-23T22:23:44.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: audit 2023-09-23T22:23:43.558030+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:44.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: audit 2023-09-23T22:23:44.025791+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:44.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:23:44.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:44 smithi052 conmon[100224]: T22:23:44.378813+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:45.795 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:23:45.796 INFO:teuthology.orchestra.run.smithi052.stdout:[{"addr": "172.21.15.52", "hostname": "smithi052", "labels": [], "status": ""}, {"addr": "172.21.15.151", "hostname": "smithi151", "labels": [], "status": ""}] 2023-09-23T22:23:46.406 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T22:23:46.406 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T22:23:47.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:46 smithi052 conmon[100224]: audit 2023-09-23T22:23:45.795346+0000 mgr.y (mgr.14148) 10 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:47.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:46 smithi052 conmon[100224]: cluster 2023-09-23T22:23:45.856757+0000 mgr.y (mgr.14148) 11 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:23:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:47.259252+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:48.069989+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:48.070707+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi151", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:23:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:48.071621+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:48.072368+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:23:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:48.129719+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:23:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: 23T22:23:48.132018+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:48 smithi052 conmon[100224]: audit 2023-09-23T22:23:48.194887+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.52:0/4197738986' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T22:23:49.137 INFO:teuthology.orchestra.run.smithi052.stderr:adjusted tunables profile to default 2023-09-23T22:23:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:49 smithi052 conmon[100224]: cluster 2023-09-23T22:23:47.857012+0000 mgr.y (mgr.14148) 12 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:23:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:49 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:48.073627+0000 mgr.y (mgr.14148) 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:23:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:49 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:48.102580+0000 mgr.y (mgr.14148) 14 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:23:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:49 smithi052 conmon[100224]: audit 2023-09-23T22:23:49.133536+0000 mon.a (mon.0) 114 : audit [INF] from='client.? 172.21.15.52:0/4197738986' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T22:23:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:49 smithi052 conmon[100224]: cluster 2023-09-23T22:23:49.133603+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:23:49.700 INFO:tasks.cephadm:Adding mon.a on smithi052 2023-09-23T22:23:49.701 INFO:tasks.cephadm:Adding mon.c on smithi052 2023-09-23T22:23:49.701 INFO:tasks.cephadm:Adding mon.b on smithi151 2023-09-23T22:23:49.701 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi052:172.21.15.52=a;smithi052:[v2:172.21.15.52:3301,v1:172.21.15.52:6790]=c;smithi151:172.21.15.151=b' 2023-09-23T22:23:51.418 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled mon update... 2023-09-23T22:23:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:51 smithi052 conmon[100224]: cluster 2023-09-23T22:23:49.857314+0000 mgr.y (mgr.14148) 15 2023-09-23T22:23:51.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:51 smithi052 conmon[100224]: : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:23:51.978 DEBUG:teuthology.orchestra.run.smithi052:mon.c> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T22:23:51.981 DEBUG:teuthology.orchestra.run.smithi151:mon.b> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T22:23:51.983 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T22:23:51.985 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T22:23:52.014 INFO:journalctl@ceph.mon.b.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:23:52.018 INFO:journalctl@ceph.mon.c.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.412013+0000 mgr.y (mgr.14148) 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: 16 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi052:172.21.15.52=a;smithi052:[v2:172.21.15.52:3301,v1:172.21.15.52:6790]=c;smithi151:172.21.15.151=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.417176+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.454267+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.456249+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.457188+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:23:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.460930+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:23:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.462942+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:23:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:52 smithi052 conmon[100224]: audit 2023-09-23T22:23:51.463841+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:53.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:51.414779+0000 2023-09-23T22:23:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: mgr.y (mgr.14148) 17 : cephadm [INF] Saving service mon spec with placement smithi052:172.21.15.52=a;smithi052:[v2:172.21.15.52:3301,v1:172.21.15.52:6790]=c;smithi151:172.21.15.151=b;count:3 2023-09-23T22:23:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: cephadm 2023-09-23T22:23:51.464684 2023-09-23T22:23:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 18 : cephadm [INF] Deploying daemon mon.b on smithi151 2023-09-23T22:23:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: cluster 2023-09-23T22:23: 2023-09-23T22:23:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: 51.857669+0000 mgr.y (mgr.14148) 19 : 2023-09-23T22:23:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:53 smithi052 conmon[100224]: cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:23:54.497 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:23:54.498 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":1,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","modified":"2023-09-23T22:22:32.333200Z","created":"2023-09-23T22:22:32.333200Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:3300","nonce":0},{"type":"v1","addr":"172.21.15.52:6789","nonce":0}]},"addr":"172.21.15.52:6789/0","public_addr":"172.21.15.52:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T22:23:54.499 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 1 2023-09-23T22:23:55.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:55 smithi052 conmon[100224]: cluster 2023-09-23T22:23:53.857965+0000 mgr.y (mgr.14148) 20 2023-09-23T22:23:55.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:55 smithi052 conmon[100224]: : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:23:55.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:23:55 smithi052 conmon[100224]: audit 2023-09-23T22:23:54.497346+0000 mon.a (mon.0) 123 : audit [DBG] from='client.? 172.21.15.151:0/2336864733' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:23:55.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:55 smithi151 systemd[1]: Starting Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:23:56.229 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T22:23:56.229 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T22:23:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.066+0000 7fdd2ee78880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:23:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.066+0000 7fdd2ee78880 0 ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable), process ceph-mon, pid 7 2023-09-23T22:23:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.066+0000 7fdd2ee78880 0 pidfile_write: ignore empty --pid-file 2023-09-23T22:23:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 0 load: jerasure load: lrc load: isa 2023-09-23T22:23:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: DB SUMMARY 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: CURRENT file: CURRENT 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:23:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.create_if_missing: 0 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.env: 0x563b94777080 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.fs: Posix File System 2023-09-23T22:23:56.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.info_log: 0x563b95f6fa80 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.statistics: (nil) 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.use_fsync: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T22:23:56.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.db_log_dir: 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:23:56.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.write_buffer_manager: 0x563b96c6a330 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:23:56.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.unordered_write: 0 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.row_cache: None 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.wal_filter: None 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T22:23:56.455 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T22:23:56.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.max_open_files: -1 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:23:56.457 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.070+0000 7fdd2ee78880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Compression algorithms supported: 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kZSTD supported: 0 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T22:23:56.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.merge_operator: 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_filter: None 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:23:56.459 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:23:56.460 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:23:56.460 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563b95f19d18) 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: cache_index_and_filter_blocks: 1 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: pin_top_level_index_and_filter: 1 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: index_type: 0 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: data_block_index_type: 0 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: index_shortening: 1 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: hash_index_allow_collision: 1 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: checksum: 1 2023-09-23T22:23:56.461 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: no_block_cache: 0 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_cache: 0x563b95f52f10 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_cache_name: BinnedLRUCache 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_cache_options: 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: capacity : 536870912 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: num_shard_bits : 4 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: strict_capacity_limit : 0 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: high_pri_pool_ratio: 0.000 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_cache_compressed: (nil) 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: persistent_cache: (nil) 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_size: 4096 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_size_deviation: 10 2023-09-23T22:23:56.462 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_restart_interval: 16 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: index_block_restart_interval: 1 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: metadata_block_size: 4096 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: partition_filters: 0 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: use_delta_encoding: 1 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: whole_key_filtering: 1 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: verify_compression: 0 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: read_amp_bytes_per_bit: 0 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: format_version: 2 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: enable_index_compression: 1 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: block_align: 0 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.463 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression: NoCompression 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.num_levels: 7 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:23:56.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:23:56.465 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:23:56.466 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:23:56.467 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:23:56.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:23:56.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:23:56.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:23:56.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:23:56.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:23:56.468 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:23:56.469 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.ttl: 2592000 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-09-23T22:23:56.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507836072420, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.071+0000 7fdd2ee78880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.072+0000 7fdd2ee78880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507836072894, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695507836, "oldest_key_time": 3, "file_creation_time": 0}} 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.072+0000 7fdd2ee78880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.072+0000 7fdd2ee78880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507836073253, "job": 1, "event": "recovery_finished"} 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.073+0000 7fdd2ee78880 4 rocksdb: DB pointer 0x563b9601d800 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.073+0000 7fdd15c80700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.073+0000 7fdd15c80700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T22:23:56.471 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** DB Stats ** 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:23:56.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** Compaction Stats [default] ** 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** Compaction Stats [default] ** 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:23:56.473 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.474 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** Compaction Stats [default] ** 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** Compaction Stats [default] ** 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-09-23T22:23:56.475 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.073+0000 7fdd2ee78880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.073+0000 7fdd2ee78880 0 using public_addr v2:172.21.15.151:0/0 -> [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.074+0000 7fdd2ee78880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.075+0000 7fdd2ee78880 1 mon.b@-1(???) e0 preinit fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 bash[100413]: 4a3cec32371a1a87e3092bbbd4727088a62baa73c648ed17d686cc0c15ff2bf3 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 systemd[1]: Started Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 0 mon.b@-1(synchronizing).mds e1 new map 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: e1 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: legacy client fscid: -1 2023-09-23T22:23:56.477 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: No filesystems configured 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.093+0000 7fdd18485700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.096+0000 7fdd18485700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.096+0000 7fdd18485700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.096+0000 7fdd18485700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:23:56.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.096+0000 7fdd18485700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: cluster 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: 2023-09-23T22:23:53.857965 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: +0000 mgr.y (mgr.14148) 20 : cluster [DBG] 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: audit 2023-09-23T22:23:54 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: .497346+0000 mon.a (mon.0) 123 : audit 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: [DBG] from='client.? 172.21.15.151:0/2336864733' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:23:56.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:23:56 smithi151 conmon[100454]: debug 2023-09-23T22:23:56.096+0000 7fdd18485700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T22:24:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:23:55.858405+0000 mgr.y (mgr.14148 2023-09-23T22:24:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: ) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23T22:23:56.103083+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:23 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: :56.103290+0000 mon.a (mon.0) 126 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23T22:23:56.116109+0000 mon.a (mon.0) 127 : audit 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23T22:23:57 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: .100339+0000 mon.a (mon.0 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: ) 128 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 09-23T22:23:57.858671+0000 mgr.y (mgr 2023-09-23T22:24:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: .14148) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:23:58.100757 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: +0000 mon.a (mon.0) 129 : audit [DBG] 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:23:58. 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 101478+0000 mon.b (mon.1) 1 : cluster 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: [INF] mon.b calling monitor election 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: -09-23T22:23:59.100428 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: +0000 mon.a (mon.0) 130 : audit [DBG] 2023-09-23T22:24:01.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:23 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: :59.858988+0000 mgr.y (mgr.14148) 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 22:24:00.100998+0000 mon.a (mon.0 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: ) 131 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: T22:24:01.100929+0000 mon.a (mon 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: .0) 132 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 09-23T22:24:01.129421+0000 2023-09-23T22:24:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.a (mon.0) 133 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.138021+0000 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.a (mon.0) 134 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0]} removed_ranks: {} 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: -09-23T22:24:01.150506+0000 mon.a ( 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.0) 135 : cluster [DBG] fsmap 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 09-23T22:24:01.150548+0000 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.a (mon.0) 136 : cluster [DBG] 2023-09-23T22:24:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.150753+0000 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.a (mon.0) 137 : cluster [DBG] 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mgrmap e13: y(active, since 35s) 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: cluster 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: :01.153698+0000 mon.a (mon.0) 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 138 : cluster [INF] overall HEALTH_OK 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: -23T22:24:01.158962+0000 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.a (mon.0) 139 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.160013+0000 mon.a ( 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.0) 140 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 09-23T22:24:01.160927+0000 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: mon.a (mon.0) 141 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:01.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:01 smithi052 conmon[100224]: 2023-09-23T22:24:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:23:55.858405+0000 mgr.y (mgr.14148) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:23:56.103083+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:24:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:23:56.103290+0000 mon.a (mon.0) 126 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:23:56.116109+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:23:57.100339+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:23:57.858671+0000 mgr.y (mgr.14148) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:23:58.100757+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:23:58.101478+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:23:59.100428+0000 mon.a (mon.0) 130 : 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:23:59.858988+0000 mgr.y (mgr.14148) 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:24:00.100998+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:24:01.100929+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.129421+0000 mon.a (mon.0) 133 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.138021+0000 mon.a (mon.0) 134 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0]} removed_ranks: {} 2023-09-23T22:24:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.150506+0000 mon.a (mon.0) 135 : cluster [DBG] fsmap 2023-09-23T22:24:01.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.150548+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:01.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.150753+0000 mon.a (mon.0) 137 : cluster [DBG] mgrmap e13: y(active, since 35s) 2023-09-23T22:24:01.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.153698+0000 mon.a (mon.0) 138 : cluster [INF] overall HEALTH_OK 2023-09-23T22:24:01.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:24:01.158962+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:01.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:24:01.160013+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:01.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:01 smithi151 conmon[100454]: audit 2023-09-23T22:24:01.160927+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:01.759 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:24:01.759 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":2,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","modified":"2023-09-23T22:23:56.100557Z","created":"2023-09-23T22:22:32.333200Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:3300","nonce":0},{"type":"v1","addr":"172.21.15.52:6789","nonce":0}]},"addr":"172.21.15.52:6789/0","public_addr":"172.21.15.52:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T22:24:01.761 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 2 2023-09-23T22:24:02.470 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:02 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:01.161767+0000 mgr.y (mgr.14148) 24 : cephadm [INF] Deploying daemon mon.c on smithi052 2023-09-23T22:24:02.471 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:02 smithi052 conmon[100224]: audit 2023-09-23T22:24:01.758978+0000 mon.a (mon.0) 142 : audit [DBG] from='client.? 172.21.15.151:0/3088646072' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:24:02.471 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:02 smithi052 conmon[100224]: audit 2023-09-23T22:24:02.100913+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:02.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:02 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:01.161767+0000 mgr.y (mgr 2023-09-23T22:24:02.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:02 smithi151 conmon[100454]: .14148) 24 : cephadm [INF] Deploying daemon mon.c on smithi052 2023-09-23T22:24:02.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:02 smithi151 conmon[100454]: audit 2023-09-23T22:24:01.758978+0000 mon.a (mon.0) 142 : audit [DBG] from='client.? 172.21.15.151:0/3088646072' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:24:02.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:02 smithi151 conmon[100454]: audit 2023-09-23T22:24:02.100913+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:03.272 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T22:24:03.272 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T22:24:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:03 smithi052 conmon[100224]: cluster 2023-09-23T22:24:01 2023-09-23T22:24:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:03 smithi052 conmon[100224]: .859245+0000 mgr.y (mgr.14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:03.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:03 smithi151 conmon[100454]: cluster 2023-09-23T22:24:01.859245+0000 mgr.y (mgr.14148) 25 : cluster 2023-09-23T22:24:03.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:03 smithi151 conmon[100454]: [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:03 smithi052 systemd[1]: Starting Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:24:04.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.327+0000 7f7689555880 0 set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.327+0000 7f7689555880 0 ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable), process ceph-mon, pid 7 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.327+0000 7f7689555880 0 pidfile_write: ignore empty --pid-file 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 0 load: jerasure load: lrc load: isa 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: RocksDB version: 6.8.1 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T22:24:04.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Compile date Sep 21 2023 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: DB SUMMARY 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: CURRENT file: CURRENT 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 636 ; 2023-09-23T22:24:04.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.error_if_exists: 0 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.create_if_missing: 0 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.paranoid_checks: 1 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.env: 0x55598290f080 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.fs: Posix File System 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.info_log: 0x555984e2fa80 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.statistics: (nil) 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.use_fsync: 0 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.max_log_file_size: 0 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:24:04.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.allow_fallocate: 1 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.use_direct_reads: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.db_log_dir: 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:24:04.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.max_subcompactions: 1 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.max_background_flushes: -1 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.write_buffer_manager: 0x555985b2a300 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:24:04.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 bash[110041]: 58c863a9b1690898cce0a821af07f1b4f2349db2dc27f0c816900d59674b48b0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.unordered_write: 0 2023-09-23T22:24:04.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.row_cache: None 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.wal_filter: None 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.332+0000 7f7689555880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.preserve_deletes: 0 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.two_write_queues: 0 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:24:04.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.atomic_flush: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.log_readahead_size: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_background_jobs: 2 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_background_compactions: -1 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:24:04.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_open_files: -1 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Compression algorithms supported: 2023-09-23T22:24:04.754 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kZSTD supported: 0 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kXpressCompression supported: 0 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kLZ4Compression supported: 1 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kBZip2Compression supported: 0 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kZlibCompression supported: 1 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: kSnappyCompression supported: 1 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.755 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.merge_operator: 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_filter: None 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555984dd9d18) 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: cache_index_and_filter_blocks: 1 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:24:04.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: pin_top_level_index_and_filter: 1 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: index_type: 0 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: data_block_index_type: 0 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: index_shortening: 1 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: hash_index_allow_collision: 1 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: checksum: 1 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: no_block_cache: 0 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_cache: 0x555984e12f10 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_cache_name: BinnedLRUCache 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_cache_options: 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: capacity : 536870912 2023-09-23T22:24:04.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: num_shard_bits : 4 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: strict_capacity_limit : 0 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: high_pri_pool_ratio: 0.000 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_cache_compressed: (nil) 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: persistent_cache: (nil) 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_size: 4096 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_size_deviation: 10 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_restart_interval: 16 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: index_block_restart_interval: 1 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: metadata_block_size: 4096 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: partition_filters: 0 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: use_delta_encoding: 1 2023-09-23T22:24:04.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: whole_key_filtering: 1 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: verify_compression: 0 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: read_amp_bytes_per_bit: 0 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: format_version: 2 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: enable_index_compression: 1 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: block_align: 0 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression: NoCompression 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:24:04.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.num_levels: 7 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:24:04.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:24:04.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:24:04.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.arena_block_size: 4194304 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:24:04.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.table_properties_collectors: 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.inplace_update_support: 0 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:24:04.764 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.bloom_locality: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.max_successive_merges: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.force_consistency_checks: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.ttl: 2592000 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.333+0000 7f7689555880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-09-23T22:24:04.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507844335209, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507844335681, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695507844, "oldest_key_time": 3, "file_creation_time": 0}} 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.334+0000 7f7689555880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.335+0000 7f7689555880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1695507844336190, "job": 1, "event": "recovery_finished"} 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.335+0000 7f7689555880 4 rocksdb: DB pointer 0x555984edd800 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.335+0000 7f767035d700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-09-23T22:24:04.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.335+0000 7f767035d700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** DB Stats ** 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** Compaction Stats [default] ** 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:24:04.767 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** Compaction Stats [default] ** 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:24:04.768 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** Compaction Stats [default] ** 2023-09-23T22:24:04.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** Compaction Stats [default] ** 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2023-09-23T22:24:04.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.336+0000 7f7689555880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.336+0000 7f7689555880 0 using public_addrv [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.336+0000 7f7689555880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] at bind addrs [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.337+0000 7f7689555880 1 mon.c@-1(???) e0 preinit fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 systemd[1]: Started Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.353+0000 7f7672b62700 0 mon.c@-1(synchronizing).mds e1 new map 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.353+0000 7f7672b62700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: e1 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: legacy client fscid: -1 2023-09-23T22:24:04.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: No filesystems configured 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.353+0000 7f7672b62700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:24:04.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: cluster 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24: 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 01.859245+0000 mgr.y (mgr. 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: 2023-09-23T22:24:04.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:04 smithi052 conmon[110082]: debug 2023-09-23T22:24:04.354+0000 7f7672b62700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:03.859497+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:04.363657+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:04.363861+0000 mon.a (mon.0) 146 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:04.363963+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:04.377005+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:04.377159+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:05.359951+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:05.860165+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:06.359942+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:07.360208+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:07.860582+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:08.360119+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:09.360215+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.377149+0000 mon.a (mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.380453+0000 mon.a (mon.0) 155 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.392590+0000 mon.a (mon.0) 156 : cluster [DBG] fsmap 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.392652+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.392886+0000 mon.a (mon.0) 158 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-09-23T22:24:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.393110+0000 mon.a (mon.0) 159 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:24:09.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.397100+0000 mon.a (mon.0) 160 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:24:09.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.397130+0000 mon.a (mon.0) 161 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:24:09.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.397150+0000 mon.a (mon.0) 162 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] is down (out of quorum) 2023-09-23T22:24:09.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:09.399535+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:09.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:09 smithi151 conmon[100454]: audit 2023-09-23T22:24:09.400979+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:09.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22:24:03.859497+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:09.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09-23T22:24:04.363657+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22:24:04.363861+0000 mon.a (mon.0 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: ) 146 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: -23T22:24:04.363963+0000 mon.b (mon. 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 1) 2 : cluster [INF] mon.b calling monitor election 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 23T22:24:04.377005+0000 mon.a 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:24:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: -23T22:24:04.377159+0000 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 148 : audit [DBG] 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:05.359951+0000 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 149 : audit [DBG] 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:05.860165+0000 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mgr.y (mgr.14148) 27 : cluster [DBG] 2023-09-23T22:24:09.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:06.359942+0000 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: -09-23T22:24:07.360208+0000 mon.a ( 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:24:09.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 09-23T22:24:07.860582+0000 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mgr.y (mgr.14148) 28 : cluster [DBG] 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09-23T22:24: 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 08.360119+0000 mon.a (mon.0) 152 : audit 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 24:09.360215+0000 mon.a (mon.0 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: ) 153 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:09.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: T22:24:09.377149+0000 mon.a ( 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.0) 154 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: -23T22:24:09.380453+0000 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 155 : cluster [DBG] 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:09.392590+0000 mon.a 2023-09-23T22:24:09.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: (mon.0) 156 : cluster [DBG] fsmap 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 09-23T22:24:09.392652+0000 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 157 : cluster [DBG] 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22: 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 24:09.392886+0000 mon.a (mon.0 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: ) 158 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09- 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 23T22:24:09.393110+0000 mon.a 2023-09-23T22:24:09.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: (mon.0) 159 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 09-23T22:24:09.397100+0000 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 160 : cluster [WRN] 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:09.397130+0000 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 161 : cluster [WRN] 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: cluster 2023-09-23T22:24:09.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:09.397150+0000 mon.a ( 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.0) 162 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] is down (out of quorum) 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: -23T22:24:09.399535+0000 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 163 : audit [INF] 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: audit 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:09.400979+0000 2023-09-23T22:24:09.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: mon.a (mon.0) 164 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:09.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:09 smithi052 conmon[100224]: 2023-09-23T22:24:10.991 INFO:teuthology.orchestra.run.smithi151.stdout: 2023-09-23T22:24:10.991 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":3,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","modified":"2023-09-23T22:24:04.360408Z","created":"2023-09-23T22:22:32.333200Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:3300","nonce":0},{"type":"v1","addr":"172.21.15.52:6789","nonce":0}]},"addr":"172.21.15.52:6789/0","public_addr":"172.21.15.52:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:3301","nonce":0},{"type":"v1","addr":"172.21.15.52:6790","nonce":0}]},"addr":"172.21.15.52:6790/0","public_addr":"172.21.15.52:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T22:24:10.993 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 3 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: cluster 2023-09-23T22:24:09.860851+0000 mgr.y (mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24:10.229420+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24:10 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: .360288+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24:10 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: .423098+0000 mon.a (mon.0) 167 : audit 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24:10. 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 423675+0000 mon.a (mon.0) 168 : audit 2023-09-23T22:24:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 2023-09-23T22:24:10.424354+0000 mon.a (mon 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: .0) 169 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 22:24:10.440878+0000 mon.a (mon. 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 0) 170 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 22:24:10.455285+0000 mon.a (mon.0 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: ) 171 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 22:24:10.457756+0000 mon.a (mon.0 2023-09-23T22:24:11.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: ) 172 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 22:24:10.459025+0000 mon.a (mon.0 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: ) 173 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 22:24:10.459630+0000 mon.a (mon.0) 174 : audit 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: -09-23T22:24:10.460222+0000 mon.a (mon.0) 175 : audit [DBG] 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24:11.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: 2023-09-23T22:24:10.990714+0000 mon.a (mon 2023-09-23T22:24:11.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: .0) 176 : audit [DBG] from='client.? 172.21.15.151:0/4214171210' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:24:11.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: audit 2023-09-23T22:24 2023-09-23T22:24:11.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: :11.031911+0000 mon.a (mon.0) 177 2023-09-23T22:24:11.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:11 smithi151 conmon[100454]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: cluster 2023-09-23T22:24:09.860851+0000 mgr.y (mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.229420+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.360288+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.423098+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.423675+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: 2023-09-23T22:24:10.424354+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.440878+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.455285+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.457756+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.459025+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.459630+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.460222+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:10.990714+0000 mon.a (mon.0) 176 : audit [DBG] from='client.? 172.21.15.151:0/4214171210' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: audit 2023-09-23T22:24:11.031911+0000 mon.a (mon.0) 2023-09-23T22:24:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[100224]: 177 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: cluster 2023-09-23T22:24:09.860851+0000 mgr.y (mgr. 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023-09-23T22:24:10.229420+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023-09-23T22:24:10 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: .360288+0000 mon.a (mon.0) 166 : audit [DBG] 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023-09-23T22:24:10. 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 423098+0000 mon.a (mon.0) 167 : audit [INF] 2023-09-23T22:24:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023-09-23T22:24:10.423675 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: +0000 mon.a (mon.0) 168 : audit [DBG] 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: -09-23T22:24:10.424354+0000 mon.a 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: (mon.0) 169 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 2023-09-23T22:24:10.440878+0000 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023-09-23T22:24:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 2023-09-23T22:24:10.455285+0000 mon.a (mon 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: .0) 171 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 09-23T22:24:10.457756+0000 mon.a ( 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: mon.0) 172 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: -09-23T22:24:10.459025+0000 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: mon.a (mon.0) 173 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:11.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: -09-23T22:24:10.459630+0000 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: mon.a (mon.0) 174 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: -09-23T22:24:10.460222+0000 mon.a 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: (mon.0) 175 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: -09-23T22:24:10.990714+0000 mon.a ( 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: mon.0) 176 : audit [DBG] from='client.? 172.21.15.151:0/4214171210' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:24:11.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:24:11.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: 09-23T22:24:11.031911+0000 mon.a ( 2023-09-23T22:24:11.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:11 smithi052 conmon[110082]: mon.0) 177 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:11.603 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T22:24:11.603 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T22:24:12.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:06.361530+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:10.425348+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:10.425558+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:10.458813+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:10.460762+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi052 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: 2023-09-23T22:24:11.379008+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.379292+0000 mon.a (mon.0) 179 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.379310+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.392551+0000 mon.a (mon.0) 180 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.394857+0000 mon.a (mon.0) 181 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:24:12.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.395249+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.395276+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.395429+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.395568+0000 mon.a (mon.0) 185 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.395585+0000 mon.a (mon.0) 186 : cluster [INF] Cluster is now healthy 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.398076+0000 mon.a (mon.0) 187 : cluster 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: [INF] overall HEALTH_OK 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: audit 2023-09-23T22:24:11.695891+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: 09-23T22:24:11.696763+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: audit 2023-09-23T22:24:12.329 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: 2023-09-23T22:24:11.697405+0000 mon.a ( 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: 09-23T22:24:11.697995+0000 mon.a 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[100224]: (mon.0) 191 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:06.361530+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:10.425348+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:10.425558+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:10.458813+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:24:12.330 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: 24:10.460762+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi052 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.379008+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.379292+0000 mon.a (mon.0) 179 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.379310+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.392551+0000 mon.a (mon.0) 180 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.394857+0000 mon.a (mon.0) 181 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.395249+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.395276+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.395429+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: 24:11.395568+0000 mon.a (mon.0) 185 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.395585+0000 mon.a (mon.0) 186 : 2023-09-23T22:24:12.331 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster [INF] Cluster is now healthy 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: cluster 2023 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: -09-23T22:24:11.398076+0000 mon.a (mon.0 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: ) 187 : cluster [INF] overall HEALTH_OK 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: 23T22:24:11.695891+0000 mon.a (mon 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: .0) 188 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: 09-23T22:24:11.696763+0000 mon.a 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: (mon.0) 189 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: audit 2023-09-23T22:24:11 2023-09-23T22:24:12.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: .697405+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:12.333 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: audit 2023-09-23T22:24:11. 2023-09-23T22:24:12.333 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:12 smithi052 conmon[110082]: 697995+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:06.361530+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:10.425348+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:10.425558+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:10.458813+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:10.460762+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi052 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.379008+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.379292+0000 mon.a (mon.0) 179 : cluster [INF] mon.a calling monitor election 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.379310+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-09-23T22:24:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.392551+0000 mon.a (mon.0) 180 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.394857+0000 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: mon.a (mon.0) 181 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.395249+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 2023-09-23T22:24:11.395276+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: -23T22:24:11.395429+0000 mon.a (mon 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: .0) 184 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: T22:24:11.395568+0000 mon.a (mon.0 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: ) 185 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:24:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24:11.395585+0000 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: mon.a (mon.0) 186 : cluster [INF] Cluster is now healthy 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: cluster 2023-09-23T22:24: 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 11.398076+0000 mon.a (mon.0) 187 : cluster 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: [INF] overall HEALTH_OK 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: audit 2023-09-23T22:24:11.695891 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: +0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: audit 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 2023-09-23T22:24:11.696763+0000 mon.a (mon. 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 0) 189 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:24:12.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: T22:24:11.697405+0000 mon.a (mon. 2023-09-23T22:24:12.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 0) 190 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:12.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: audit 2023-09-23T22:24:11.697995 2023-09-23T22:24:12.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: +0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:12.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:12 smithi151 conmon[100454]: 2023-09-23T22:24:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:11.696521+0000 mgr.y (mgr 2023-09-23T22:24:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[100224]: .14148) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T22:24:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:11.698614+0000 mgr.y (mgr.14148) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi052 2023-09-23T22:24:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[100224]: cluster 2023-09-23T22:24:11.861108+0000 mgr.y (mgr.14148) 36 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[100224]: audit 2023-09-23T22:24:12.360414+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:11.696521+0000 mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:11.698614+0000 mgr.y (mgr.14148) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi052 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: cluster 2023-09-23T22:24:11.861108+0000 mgr.y (mgr.14148 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: ) 36 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: 23T22:24:12.360414+0000 mon.a (mon. 2023-09-23T22:24:13.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:13 smithi052 conmon[110082]: 0) 192 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:13.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:11.696521+0000 2023-09-23T22:24:13.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: mgr.y (mgr.14148) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:11.698614+0000 mgr.y (mgr. 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: 14148) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi052 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: cluster 2023-09- 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: 23T22:24:11.861108+0000 mgr.y (mgr.14148) 36 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: audit 2023 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: -09-23T22:24:12.360414+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:24:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:13 smithi151 conmon[100454]: 2023-09-23T22:24:13.865 INFO:teuthology.orchestra.run.smithi052.stdout:# minimal ceph.conf for 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:24:13.865 INFO:teuthology.orchestra.run.smithi052.stdout:[global] 2023-09-23T22:24:13.865 INFO:teuthology.orchestra.run.smithi052.stdout: fsid = 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:24:13.865 INFO:teuthology.orchestra.run.smithi052.stdout: mon_host = [v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] 2023-09-23T22:24:14.380 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T22:24:14.568 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:24:14.568 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T22:24:14.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: audit 2023-09-23T22:24:13.363296+0000 mon.a (mon.0 2023-09-23T22:24:14.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: ) 193 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:14.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:13.363787+0000 mgr.y (mgr.14148) 37 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T22:24:14.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: audit 2023-09-23T22:24:13.364021+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:14.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: audit 2023-09-23T22:24:13.364673+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:14.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: audit 2023-09-23T22:24:13.365331+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:14.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:13.365895+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring daemon mon.b on smithi151 2023-09-23T22:24:14.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[100224]: audit 2023-09-23T22:24:13.864778+0000 mon.a (mon.0) 197 : audit [DBG] from='client.? 172.21.15.52:0/1440629858' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:14.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:24:14.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: T22:24:13.363296+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:14.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:13.363787+0000 mgr.y (mgr.14148) 37 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T22:24:14.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: audit 2023-09-23T22:24:13.364021+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:14.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: audit 2023-09-23T22:24:13.364673+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:14.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: audit 2023-09-23T22:24:13.365331+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:14.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:13.365895+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring daemon mon.b on smithi151 2023-09-23T22:24:14.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:14 smithi052 conmon[110082]: audit 2023-09-23T22:24:13.864778+0000 mon.a (mon.0) 197 : audit [DBG] from='client.? 172.21.15.52:0/1440629858' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:14.615 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:24:14.615 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:24:14.685 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:24:14.685 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T22:24:14.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: audit 2023-09-23T22:24:13.363296+0000 mon.a (mon.0) 193 : audit [INF] 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:13.363787+0000 mgr.y (mgr.14148) 37 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: audit 2023-09-23T22:24:13.364021+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: audit 2023-09-23T22:24:13.364673+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: audit 2023-09-23T22:24:13.365331+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:13.365895+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring daemon mon.b on smithi151 2023-09-23T22:24:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:14 smithi151 conmon[100454]: audit 2023-09-23T22:24:13.864778+0000 mon.a (mon.0) 197 : audit [DBG] from='client.? 172.21.15.52:0/1440629858' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:14.727 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:24:14.727 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:24:14.795 INFO:tasks.cephadm:Adding mgr.y on smithi052 2023-09-23T22:24:14.795 INFO:tasks.cephadm:Adding mgr.x on smithi151 2023-09-23T22:24:14.795 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi052=y;smithi151=x' 2023-09-23T22:24:15.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:15 smithi151 conmon[100454]: cluster 2023-09-23T22:24:13.861354+0000 mgr.y (mgr.14148) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:15.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:15 smithi151 conmon[100454]: audit 2023-09-23T22:24:14.549994+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:15.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:15 smithi151 conmon[100454]: audit 2023-09-23T22:24:14.550965+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:15.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:15 smithi151 conmon[100454]: audit 2023-09-23T22:24:14.552417+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:15.621 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:15 smithi151 conmon[100454]: audit 2023-09-23T22:24:14.553166+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:15.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:15 smithi151 conmon[100454]: audit 2023-09-23T22:24:14.556768+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:15.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[100224]: cluster 2023-09-23T22:24:13.861354+0000 mgr.y (mgr.14148) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:15.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[100224]: audit 2023-09-23T22:24:14.549994+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:15.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[100224]: audit 2023-09-23T22:24:14.550965+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[100224]: audit 2023-09-23T22:24:14.552417+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[100224]: audit 2023-09-23T22:24:14.553166+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[100224]: audit 2023-09-23T22:24:14.556768+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[110082]: cluster 2023-09-23T22:24:13.861354+0000 mgr.y (mgr.14148) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[110082]: audit 2023-09-23T22:24:14.549994+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[110082]: audit 2023-09-23T22:24:14.550965+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[110082]: audit 2023-09-23T22:24:14.552417+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[110082]: audit 2023-09-23T22:24:14.553166+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:15.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:15 smithi052 conmon[110082]: audit 2023-09-23T22:24:14.556768+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:16.551 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled mgr update... 2023-09-23T22:24:17.008 DEBUG:teuthology.orchestra.run.smithi151:mgr.x> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T22:24:17.011 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T22:24:17.011 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:24:17.011 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T22:24:17.032 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T22:24:17.032 DEBUG:teuthology.orchestra.run.smithi052:> stat /dev/vg_nvme/lv_1 2023-09-23T22:24:17.041 INFO:journalctl@ceph.mgr.x.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:24:17.093 INFO:teuthology.orchestra.run.smithi052.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T22:24:17.093 INFO:teuthology.orchestra.run.smithi052.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout:Device: 6h/6d Inode: 389475 Links: 1 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout:Access: 2023-09-23 22:24:03.752394273 +0000 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout:Modify: 2023-09-23 22:15:50.891341040 +0000 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout:Change: 2023-09-23 22:15:50.891341040 +0000 2023-09-23T22:24:17.094 INFO:teuthology.orchestra.run.smithi052.stdout: Birth: - 2023-09-23T22:24:17.094 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T22:24:17.165 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records in 2023-09-23T22:24:17.165 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records out 2023-09-23T22:24:17.165 INFO:teuthology.orchestra.run.smithi052.stderr:512 bytes copied, 0.000140255 s, 3.7 MB/s 2023-09-23T22:24:17.166 DEBUG:teuthology.orchestra.run.smithi052:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T22:24:17.228 DEBUG:teuthology.orchestra.run.smithi052:> stat /dev/vg_nvme/lv_2 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout:Device: 6h/6d Inode: 387475 Links: 1 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout:Access: 2023-09-23 22:24:03.753394261 +0000 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout:Modify: 2023-09-23 22:15:51.388334584 +0000 2023-09-23T22:24:17.287 INFO:teuthology.orchestra.run.smithi052.stdout:Change: 2023-09-23 22:15:51.388334584 +0000 2023-09-23T22:24:17.288 INFO:teuthology.orchestra.run.smithi052.stdout: Birth: - 2023-09-23T22:24:17.288 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T22:24:17.356 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records in 2023-09-23T22:24:17.357 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records out 2023-09-23T22:24:17.357 INFO:teuthology.orchestra.run.smithi052.stderr:512 bytes copied, 0.000108976 s, 4.7 MB/s 2023-09-23T22:24:17.358 DEBUG:teuthology.orchestra.run.smithi052:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T22:24:17.419 DEBUG:teuthology.orchestra.run.smithi052:> stat /dev/vg_nvme/lv_3 2023-09-23T22:24:17.478 INFO:teuthology.orchestra.run.smithi052.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T22:24:17.478 INFO:teuthology.orchestra.run.smithi052.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:17.478 INFO:teuthology.orchestra.run.smithi052.stdout:Device: 6h/6d Inode: 393230 Links: 1 2023-09-23T22:24:17.478 INFO:teuthology.orchestra.run.smithi052.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:17.479 INFO:teuthology.orchestra.run.smithi052.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:17.479 INFO:teuthology.orchestra.run.smithi052.stdout:Access: 2023-09-23 22:24:03.753394261 +0000 2023-09-23T22:24:17.479 INFO:teuthology.orchestra.run.smithi052.stdout:Modify: 2023-09-23 22:15:51.897327972 +0000 2023-09-23T22:24:17.479 INFO:teuthology.orchestra.run.smithi052.stdout:Change: 2023-09-23 22:15:51.897327972 +0000 2023-09-23T22:24:17.479 INFO:teuthology.orchestra.run.smithi052.stdout: Birth: - 2023-09-23T22:24:17.480 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T22:24:17.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:24:17.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: :24:15.861795+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:17.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:17.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: -09-23T22:24:16.550237+0000 mon.a (mon.0) 203 : audit 2023-09-23T22:24:17.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 22:24:16.577614+0000 mon.a (mon.0 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: ) 204 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: -09-23T22:24:16. 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 579446+0000 mon.a (mon.0) 2023-09-23T22:24:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 205 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 22:24:16.580332+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 09-23T22:24:16.585035+0000 mon.a (mon 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: .0) 207 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 09-23T22:24:16.586689+0000 mon.a (mon 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: .0) 208 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023-09-23T22:24:17.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 2023-09-23 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: T22:24:16.589305+0000 mon.a (mon 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: .0) 209 : audit [INF] 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: -23T22:24:16.590247+0000 mon.a (mon. 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 0) 210 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:24:17.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: -23T22:24:16.591221+0000 mon.a (mon.0) 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[110082]: 211 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: cluster 2023-09-23T22:24:15.861795+0000 mgr.y ( 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: mgr.14148) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 24:16.550237+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:17.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 09-23T22:24:16 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: .577614+0000 mon.a (mon.0) 204 : audit 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: -23T22:24:16.579446+0000 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: T22:24:16.580332+0000 mon.a (mon.0) 206 2023-09-23T22:24:17.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 24:16.585035+0000 mon.a (mon.0 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: ) 207 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: T22:24:16.586689+0000 mon.a ( 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: mon.0) 208 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: -23T22:24:16.589305+0000 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: mon.a (mon.0) 209 : audit [INF] 2023-09-23T22:24:17.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 2023-09-23T22:24:16.590247 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: +0000 mon.a (mon.0) 210 : audit 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: -23T22:24:16. 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: 591221+0000 mon.a (mon.0) 211 2023-09-23T22:24:17.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:17 smithi052 conmon[100224]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:17.527 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records in 2023-09-23T22:24:17.527 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records out 2023-09-23T22:24:17.527 INFO:teuthology.orchestra.run.smithi052.stderr:512 bytes copied, 0.000132316 s, 3.9 MB/s 2023-09-23T22:24:17.529 DEBUG:teuthology.orchestra.run.smithi052:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T22:24:17.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: cluster 2023-09-23T22:24:15.861795+0000 mgr.y (mgr.14148) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:17.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.550237+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:17.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.577614+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:17.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.579446+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.580332+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: :16.585035+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.586689+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.589305+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.590247+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:24:17.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:17 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.591221+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:17.587 DEBUG:teuthology.orchestra.run.smithi052:> stat /dev/vg_nvme/lv_4 2023-09-23T22:24:17.645 INFO:teuthology.orchestra.run.smithi052.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T22:24:17.645 INFO:teuthology.orchestra.run.smithi052.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:17.645 INFO:teuthology.orchestra.run.smithi052.stdout:Device: 6h/6d Inode: 389780 Links: 1 2023-09-23T22:24:17.645 INFO:teuthology.orchestra.run.smithi052.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:17.646 INFO:teuthology.orchestra.run.smithi052.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:17.646 INFO:teuthology.orchestra.run.smithi052.stdout:Access: 2023-09-23 22:24:03.753394261 +0000 2023-09-23T22:24:17.646 INFO:teuthology.orchestra.run.smithi052.stdout:Modify: 2023-09-23 22:15:52.399321451 +0000 2023-09-23T22:24:17.646 INFO:teuthology.orchestra.run.smithi052.stdout:Change: 2023-09-23 22:15:52.399321451 +0000 2023-09-23T22:24:17.646 INFO:teuthology.orchestra.run.smithi052.stdout: Birth: - 2023-09-23T22:24:17.646 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T22:24:17.712 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records in 2023-09-23T22:24:17.712 INFO:teuthology.orchestra.run.smithi052.stderr:1+0 records out 2023-09-23T22:24:17.712 INFO:teuthology.orchestra.run.smithi052.stderr:512 bytes copied, 0.000110321 s, 4.6 MB/s 2023-09-23T22:24:17.713 DEBUG:teuthology.orchestra.run.smithi052:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T22:24:17.772 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:24:17.772 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T22:24:17.790 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T22:24:17.790 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_1 2023-09-23T22:24:17.847 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T22:24:17.847 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:17.847 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 389327 Links: 1 2023-09-23T22:24:17.847 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:17.848 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:17.848 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-09-23 22:23:55.515220447 +0000 2023-09-23T22:24:17.848 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-09-23 22:15:21.452938545 +0000 2023-09-23T22:24:17.848 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-09-23 22:15:21.452938545 +0000 2023-09-23T22:24:17.848 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-09-23T22:24:17.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T22:24:17.922 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-09-23T22:24:17.922 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-09-23T22:24:17.922 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000236074 s, 2.2 MB/s 2023-09-23T22:24:17.923 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T22:24:17.982 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_2 2023-09-23T22:24:18.039 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T22:24:18.039 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:18.039 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 389434 Links: 1 2023-09-23T22:24:18.039 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:18.039 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:18.039 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-09-23 22:24:17.959952390 +0000 2023-09-23T22:24:18.040 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-09-23 22:15:21.969932142 +0000 2023-09-23T22:24:18.040 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-09-23 22:15:21.969932142 +0000 2023-09-23T22:24:18.040 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-09-23T22:24:18.040 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T22:24:18.127 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-09-23T22:24:18.128 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-09-23T22:24:18.128 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000230812 s, 2.2 MB/s 2023-09-23T22:24:18.129 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T22:24:18.148 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_3 2023-09-23T22:24:18.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:18 smithi151 systemd[1]: Starting Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:24:18.209 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 389593 Links: 1 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-09-23 22:24:18.081950933 +0000 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-09-23 22:15:22.471925925 +0000 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-09-23 22:15:22.471925925 +0000 2023-09-23T22:24:18.210 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-09-23T22:24:18.210 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T22:24:18.278 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-09-23T22:24:18.278 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-09-23T22:24:18.278 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.0002274 s, 2.3 MB/s 2023-09-23T22:24:18.279 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T22:24:18.339 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_4 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 6h/6d Inode: 392273 Links: 1 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2023-09-23 22:24:18.081950933 +0000 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2023-09-23 22:15:22.959919881 +0000 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2023-09-23 22:15:22.959919881 +0000 2023-09-23T22:24:18.397 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: - 2023-09-23T22:24:18.398 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T22:24:18.462 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2023-09-23T22:24:18.462 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2023-09-23T22:24:18.463 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000238111 s, 2.2 MB/s 2023-09-23T22:24:18.463 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T22:24:18.522 INFO:tasks.cephadm:Deploying osd.0 on smithi052 with /dev/vg_nvme/lv_4... 2023-09-23T22:24:18.523 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T22:24:18.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[100454]: audit 2023-09-23T22:24:16.545283+0000 mgr.y (mgr.14148) 41 : audit 2023-09-23T22:24:18.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[100454]: [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi052=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:18.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:16.546979+0000 mgr.y (mgr.14148) 42 : cephadm [INF] Saving service mgr spec with placement smithi052=y;smithi151=x;count:2 2023-09-23T22:24:18.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:16.591978+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:24:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[100224]: audit 2023-09-23T22:24:16.545283+0000 mgr.y (mgr. 2023-09-23T22:24:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[100224]: 14148) 41 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi052=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:16.546979+0000 mgr.y (mgr.14148) 42 : cephadm [INF] Saving service mgr spec with placement smithi052=y;smithi151=x;count:2 2023-09-23T22:24:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[100224]: cephadm 2023-09- 2023-09-23T22:24:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[100224]: 23T22:24:16.591978+0000 mgr.y (mgr.14148) 43 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:24:18.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: audit 2023-09-23T22:24:16.545283+0000 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: mgr.y (mgr.14148) 41 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi052=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:16.546979+0000 mgr.y (mgr. 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: 14148) 42 : cephadm [INF] Saving service mgr spec with placement smithi052=y;smithi151=x;count:2 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: 2023-09-23T22:24:16.591978+0000 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: mgr.y (mgr.14148) 43 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:24:18.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:18 smithi052 conmon[110082]: 2023-09-23T22:24:18.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:18 smithi151 bash[102651]: 14a110bae0953f543bca13c4341d725fb2fe4a825cce52651685c12ca9b275d9 2023-09-23T22:24:18.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[102744]: debug 2023-09-23T22:24:18.668+0000 7f3edc7c5700 1 -- 172.21.15.151:0/1151984580 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56427c3804e0 con 0x56427c357400 2023-09-23T22:24:18.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:18 smithi151 systemd[1]: Started Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:24:18.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[102744]: debug 2023-09-23T22:24:18.798+0000 7f3eecdaa680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:24:18.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:18 smithi151 conmon[102744]: debug 2023-09-23T22:24:18.883+0000 7f3eecdaa680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:24:19.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:19 smithi151 conmon[102744]: debug 2023-09-23T22:24:19.318+0000 7f3eecdaa680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:24:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:19 smithi151 conmon[100454]: cluster 2023-09-23T22:24:17.862155+0000 mgr.y (mgr.14148 2023-09-23T22:24:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:19 smithi151 conmon[100454]: ) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:19 smithi151 conmon[100454]: audit 2023-09-23T22:24:18.733609+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:19 smithi151 conmon[100454]: audit 2023-09-23T22:24:18.735458+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[110082]: cluster 2023-09-23T22:24:17.862155+0000 2023-09-23T22:24:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[110082]: mgr.y (mgr.14148) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[110082]: audit 2023-09-23T22:24:18.733609+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:24:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[110082]: :24:18.735458+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:19.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[100224]: cluster 2023-09-23T22:24:17. 2023-09-23T22:24:19.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[100224]: 862155+0000 mgr.y (mgr.14148) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:19.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[100224]: audit 2023-09-23T22:24:18.733609+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:19.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:19 smithi052 conmon[100224]: audit 2023-09-23T22:24:18.735458+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:20.401 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[102744]: debug 2023-09-23T22:24:20.107+0000 7f3eecdaa680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:24:20.401 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[102744]: debug 2023-09-23T22:24:20.190+0000 7f3eecdaa680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:24:20.522 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T22:24:20.522 INFO:teuthology.orchestra.run.smithi052.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T22:24:20.522 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 10+0 records in 2023-09-23T22:24:20.522 INFO:teuthology.orchestra.run.smithi052.stderr:10+0 records out 2023-09-23T22:24:20.522 INFO:teuthology.orchestra.run.smithi052.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0081833 s, 1.3 GB/s 2023-09-23T22:24:20.522 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 2023-09-23T22:24:20.523 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping successful for: 2023-09-23T22:24:20.684 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[102744]: debug 2023-09-23T22:24:20.399+0000 7f3eecdaa680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:24:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23T22:24:19.684314+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23T22:24:19.684982+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23T22:24:19.685734+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23T22:24:19.689196+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: 2023-09-23T22:24:19.690787+0000 mgr.y (mgr.14148) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23T22:24:19.691049+0000 mon.a (mon.0) 218 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:24:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23T22:24 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: :19.691806+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: T22:24:19.692453+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: 2023-09-23T22:24:19.693074+0000 mgr.y (mgr.14148) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi052 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: cluster 2023-09-23T22:24:19.862446+0000 2023-09-23T22:24:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:20 smithi151 conmon[100454]: mgr.y (mgr.14148) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.684314+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.684982+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.685734+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.689196+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:19.690787+0000 mgr.y (mgr.14148) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:24:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.691049+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:24:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.691806+0000 2023-09-23T22:24:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: mon.a (mon.0) 219 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:24:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: audit 2023-09-23T22:24:19.692453+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:20.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:19.693074+0000 mgr.y (mgr.14148) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi052 2023-09-23T22:24:20.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[110082]: cluster 2023-09-23T22:24:19.862446+0000 mgr.y (mgr.14148) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22:24:19.684314+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22:24:19.684982+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22:24:19.685734+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22:24:19.689196+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:19.690787+0000 mgr.y (mgr.14148) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: 24:19.691049+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22:24:19.691806+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: audit 2023-09-23T22:24:19.692453+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:19.693074+0000 mgr.y (mgr.14148) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi052 2023-09-23T22:24:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:20 smithi052 conmon[100224]: cluster 2023-09-23T22:24:19.862446+0000 mgr.y (mgr.14148) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:21.451 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi052:vg_nvme/lv_4 2023-09-23T22:24:21.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:21 smithi151 conmon[102744]: debug 2023-09-23T22:24:21.343+0000 7f3eecdaa680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:24:21.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:21 smithi151 conmon[102744]: debug 2023-09-23T22:24:21.470+0000 7f3eecdaa680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:24:21.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:21 smithi151 conmon[102744]: debug 2023-09-23T22:24:21.547+0000 7f3eecdaa680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:24:21.637 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:21 smithi151 conmon[102744]: debug 2023-09-23T22:24:21.635+0000 7f3eecdaa680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:24:21.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:21 smithi151 conmon[102744]: debug 2023-09-23T22:24:21.829+0000 7f3eecdaa680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:24:22.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[100224]: audit 2023-09-23T22:24:21.036550+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[100224]: audit 2023-09-23T22:24:21.301405+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[100224]: audit 2023-09-23T22:24:21.302451+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:22.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[100224]: audit 2023-09-23T22:24:21.303833+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:22.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[100224]: audit 2023-09-23T22:24:21.304535+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[100224]: audit 2023-09-23T22:24:21.308129+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[110082]: audit 2023-09-23T22:24:21.036550+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[110082]: audit 2023-09-23T22:24:21.301405+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[110082]: audit 2023-09-23T22:24:21.302451+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[110082]: audit 2023-09-23T22:24:21.303833+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[110082]: audit 2023-09-23T22:24:21.304535+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:22.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:22 smithi052 conmon[110082]: audit 2023-09-23T22:24:21.308129+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[102744]: debug 2023-09-23T22:24:22.107+0000 7f3eecdaa680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[102744]: debug 2023-09-23T22:24:22.205+0000 7f3eecdaa680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit 2023-09-23T22:24:21.036550+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit 2023-09-23T22:24:21.301405+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit 2023-09-23T22:24:21.302451+0000 mon.a (mon.0) 223 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit 2023-09-23T22:24:21.303833+0000 mon.a (mon.0) 224 : 2023-09-23T22:24:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:22.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit 2023-09-23T22:24: 2023-09-23T22:24:22.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: 21.304535+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:22.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: audit 2023-09-23T22:24:22.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[100454]: 2023-09-23T22:24:21.308129+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:23.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:22 smithi151 conmon[102744]: debug 2023-09-23T22:24:22.930+0000 7f3eecdaa680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:24:23.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.011+0000 7f3eecdaa680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:24:23.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.101+0000 7f3eecdaa680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:24:23.531 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.272+0000 7f3eecdaa680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: cluster 2023-09-23T22:24:21.862661+0000 mgr.y (mgr.14148) 48 : cluster 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: audit 2023-09-23T22:24:23.199326+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: audit 2023-09-23T22:24:23.201706+0000 mon.a 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: (mon.0) 228 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:23.532 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[100454]: 22:24:23.202263+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:23.725 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[110082]: cluster 2023-09-23T22:24:21.862661+0000 2023-09-23T22:24:23.725 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[110082]: mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:23.725 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[110082]: audit 2023-09-23T22:24:23.199326+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:24:23.725 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[110082]: audit 2023-09-23T22:24:23.201706+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:24:23.725 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[110082]: audit 2023-09-23T22:24:23.202263+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:23.726 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[100224]: cluster 2023-09-23T22:24:21 2023-09-23T22:24:23.726 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[100224]: .862661+0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:23.726 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[100224]: audit 2023-09-23T22:24:23.199326+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:24:23.726 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[100224]: audit 2023-09-23T22:24:23.201706+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:24:23.726 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:23 smithi052 conmon[100224]: audit 2023-09-23T22:24:23.202263+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:23.793 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.530+0000 7f3eecdaa680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:24:23.793 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.660+0000 7f3eecdaa680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:24:24.047 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.791+0000 7f3eecdaa680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:24:24.047 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:23 smithi151 conmon[102744]: debug 2023-09-23T22:24:23.963+0000 7f3eecdaa680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:24:24.313 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[102744]: debug 2023-09-23T22:24:24.045+0000 7f3eecdaa680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:24:24.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: audit 2023-09-23T22:24:23.196464+0000 mgr.y 2023-09-23T22:24:24.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: (mgr.14148) 49 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: cluster 2023-09-23T22:24:24.060088+0000 mon.a (mon.0) 230 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: audit 2023-09-23T22:24:24.061146+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: audit 2023-09-23T22:24:24.061477+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: audit 2023-09-23T22:24:24.062363+0000 mon.b (mon.1) 6 : 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: audit 2023-09-23T22:24:24 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[100224]: .062661+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: audit 2023-09-23T22:24:23.196464+0000 mgr.y ( 2023-09-23T22:24:24.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: mgr.14148) 49 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: cluster 2023-09-23T22:24:24.060088+0000 mon.a (mon.0 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: ) 230 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: audit 2023-09-23T22:24:24. 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: 061146+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: audit 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: 2023-09-23T22:24:24.061477+0000 mon.b (mon.1) 5 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: audit 2023-09-23T22:24:24.062363+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: audit 2023-09-23T22:24:24.062661+0000 2023-09-23T22:24:24.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:24 smithi052 conmon[110082]: mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: audit 2023-09-23T22:24:23.196464+0000 mgr.y (mgr.14148) 49 : 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: cluster 2023-09-23T22:24:24.060088+0000 mon.a (mon.0) 230 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: audit 2023-09-23T22:24:24.061146+0000 mon.b (mon.1 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: ) 4 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: 24:24.061477+0000 mon.b (mon.1 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: ) 5 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: audit 2023-09-23T22:24:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: 2023-09-23T22:24:24.062363+0000 2023-09-23T22:24:24.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:24:24.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:24:24.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: :24:24.062661+0000 mon.b (mon.1) 7 : audit [DBG] 2023-09-23T22:24:24.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:24 smithi151 conmon[100454]: from='mgr.? 172.21.15.151:0/335298488' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:24:25.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:25 smithi151 conmon[100454]: cluster 2023-09-23T22:24:23.863058+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:25.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:25 smithi151 conmon[100454]: 2023-09-23T22:24:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:25 smithi151 conmon[100454]: cluster 2023-09-23T22:24:24.316949+0000 mon.a (mon.0) 231 : cluster [DBG] mgrmap e14: y(active, since 58s), standbys: x 2023-09-23T22:24:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:25 smithi151 conmon[100454]: audit 2023-09-23T22:24: 2023-09-23T22:24:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:25 smithi151 conmon[100454]: 24.317189+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:24:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:25 smithi052 conmon[100224]: cluster 2023-09-23T22:24:23.863058+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:25 smithi052 conmon[100224]: cluster 2023-09-23T22:24:24.316949+0000 mon.a (mon.0) 231 : cluster [DBG] mgrmap e14: y(active, since 58s), standbys: x 2023-09-23T22:24:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:25 smithi052 conmon[100224]: audit 2023-09-23T22:24:24.317189+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:24:25.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:25 smithi052 conmon[110082]: cluster 2023-09-23T22:24:23.863058+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:25.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:25 smithi052 conmon[110082]: cluster 2023-09-23T22:24:24.316949+0000 mon.a (mon.0) 231 : cluster [DBG] mgrmap e14: y(active, since 58s), standbys: x 2023-09-23T22:24:25.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:25 smithi052 conmon[110082]: audit 2023-09-23T22:24:24.317189+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:24:26.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: audit 2023-09-23T22:24:25.393497+0000 mon.a (mon.0) 233 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: : audit [INF] from='client.? 172.21.15.52:0/1236639615' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c382e099-ff66-4b35-af2f-978f52c83cfc"}]: dispatch 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: audit 2023-09-23T22:24:25.397014+0000 mon.a (mon.0) 234 : audit [INF] from='client.? 172.21.15.52:0/1236639615' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c382e099-ff66-4b35-af2f-978f52c83cfc"}]': finished 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: cluster 2023-09-23T22:24:25.397099+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: 23T22:24:25.397216+0000 mon.a (mon.0 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: ) 236 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: 24:25.996372+0000 mon.c (mon.2) 3 : audit 2023-09-23T22:24:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:26 smithi151 conmon[100454]: [DBG] from='client.? 172.21.15.52:0/282611178' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:24:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[100224]: audit 2023-09-23T22:24:25. 2023-09-23T22:24:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[100224]: 393497+0000 mon.a (mon.0) 233 : audit [INF] from='client.? 172.21.15.52:0/1236639615' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c382e099-ff66-4b35-af2f-978f52c83cfc"}]: dispatch 2023-09-23T22:24:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[100224]: audit 2023-09-23T22:24:25.397014+0000 mon.a (mon.0) 234 : audit [INF] from='client.? 172.21.15.52:0/1236639615' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c382e099-ff66-4b35-af2f-978f52c83cfc"}]': finished 2023-09-23T22:24:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[100224]: cluster 2023-09-23T22:24:25.397099+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:24:26.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[100224]: audit 2023-09-23T22:24:25.397216+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:26.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[100224]: audit 2023-09-23T22:24:25.996372+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.52:0/282611178' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: audit 2023-09-23T22:24:25.393497+0000 mon.a (mon. 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: 0) 233 : audit [INF] from='client.? 172.21.15.52:0/1236639615' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c382e099-ff66-4b35-af2f-978f52c83cfc"}]: dispatch 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: audit 2023-09-23T22:24:25.397014+0000 mon.a (mon 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: .0) 234 : audit [INF] from='client.? 172.21.15.52:0/1236639615' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c382e099-ff66-4b35-af2f-978f52c83cfc"}]': finished 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: cluster 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: 2023-09-23T22:24:25.397099+0000 mon.a (mon. 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: 0) 235 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:24:26.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: audit 2023-09-23T22:24 2023-09-23T22:24:26.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: :25.397216+0000 mon.a (mon.0) 236 : 2023-09-23T22:24:26.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:26.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:24:26.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: -23T22:24:25.996372+0000 2023-09-23T22:24:26.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.52:0/282611178' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:24:26.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:26 smithi052 conmon[110082]: 2023-09-23T22:24:27.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:27 smithi151 conmon[100454]: cluster 2023-09-23T22:24:25.863285+0000 mgr.y (mgr.14148 2023-09-23T22:24:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:27 smithi151 conmon[100454]: ) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:27 smithi052 conmon[100224]: cluster 2023-09-23T22:24:25.863285+0000 mgr.y (mgr 2023-09-23T22:24:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:27 smithi052 conmon[100224]: .14148) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:27 smithi052 conmon[110082]: cluster 2023-09-23T22:24:25.863285+0000 2023-09-23T22:24:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:27 smithi052 conmon[110082]: mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:29.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:29 smithi052 conmon[100224]: cluster 2023-09-23T22:24:27.863615+0000 mgr.y (mgr.14148) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:29.652 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:29 smithi052 conmon[110082]: cluster 2023-09-23T22:24:27.863615+0000 mgr.y (mgr.14148) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:29 smithi151 conmon[100454]: cluster 2023-09-23T22:24:27.863615+0000 mgr.y (mgr.14148) 52 : 2023-09-23T22:24:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:29 smithi151 conmon[100454]: cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:31.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:31 smithi151 conmon[100454]: cluster 2023-09-23T22:24:29.863889+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:31 smithi151 conmon[100454]: audit 2023-09-23T22:24:31.381873+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:24:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:31 smithi151 conmon[100454]: audit 2023-09-23T22:24:31.382638+0000 mon.a (mon.0) 238 : 2023-09-23T22:24:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:31 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[100224]: cluster 2023-09-23T22 2023-09-23T22:24:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[100224]: :24:29.863889+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[100224]: audit 2023-09-23T22:24:31.381873+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:24:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[100224]: audit 2023-09-23T22:24:31.382638+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[110082]: cluster 2023-09-23T22:24 2023-09-23T22:24:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[110082]: :29.863889+0000 mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[110082]: audit 2023-09-23T22:24:31.381873+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:24:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:31 smithi052 conmon[110082]: audit 2023-09-23T22:24:31.382638+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:32.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:32 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:31.383276+0000 mgr.y (mgr.14148) 54 : cephadm [INF] Deploying daemon osd.0 on smithi052 2023-09-23T22:24:32.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:32 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:31.383276+0000 mgr.y (mgr.14148) 54 : cephadm [INF] Deploying daemon osd.0 on smithi052 2023-09-23T22:24:32.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:32 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:31.383276+0000 mgr.y (mgr.14148) 54 : cephadm [INF] Deploying daemon osd.0 on smithi052 2023-09-23T22:24:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:33 smithi052 conmon[110082]: cluster 2023-09-23T22:24 2023-09-23T22:24:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:33 smithi052 conmon[110082]: :31.864059+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:33.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:33 smithi052 conmon[100224]: cluster 2023-09-23T22:24: 2023-09-23T22:24:33.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:33 smithi052 conmon[100224]: 31.864059+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:33.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:33 smithi151 conmon[100454]: cluster 2023-09-23T22:24:31.864059+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:35.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:35 smithi151 conmon[100454]: cluster 2023-09-23T22:24:33.864498+0000 mgr.y (mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:35 smithi052 conmon[100224]: cluster 2023-09-23T22:24:33.864498 2023-09-23T22:24:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:35 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:35 smithi052 conmon[110082]: cluster 2023-09-23T22:24:33.864498+0000 mgr.y ( 2023-09-23T22:24:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:35 smithi052 conmon[110082]: mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[110082]: cluster 2023-09-23T22:24:35.864943+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:37.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[110082]: audit 2023-09-23T22:24:36.243006+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:37.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:24:37.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[110082]: 24:36.251773+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:37.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[100224]: cluster 2023-09-23T22:24:35.864943+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:37.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[100224]: audit 2023-09-23T22:24:36.243006+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:37.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:37 smithi052 conmon[100224]: audit 2023-09-23T22:24:36.251773+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:37 smithi151 conmon[100454]: cluster 2023-09-23T22:24:35.864943+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:37 smithi151 conmon[100454]: audit 2023-09-23T22:24:36.243006+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:37 smithi151 conmon[100454]: audit 2023-09-23T22:24:36.251773+0000 mon.a (mon.0) 240 : 2023-09-23T22:24:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:37 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:39.184 INFO:teuthology.orchestra.run.smithi052.stdout:Created osd(s) 0 on host 'smithi052' 2023-09-23T22:24:39.399 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[100224]: cluster 2023-09-23T22:24:37.865360+0000 mgr.y (mgr.14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:39.399 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[100224]: audit 2023-09-23T22:24:38.128414+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:39.399 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[100224]: audit 2023-09-23T22:24:38.129135+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[100224]: audit 2023-09-23T22:24:38.129854+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[100224]: audit 2023-09-23T22:24:38.133694+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[110082]: cluster 2023-09-23T22:24:37.865360+0000 mgr.y (mgr.14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[110082]: audit 2023-09-23T22:24:38.128414+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[110082]: audit 2023-09-23T22:24:38.129135+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[110082]: audit 2023-09-23T22:24:38.129854+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:39.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:39 smithi052 conmon[110082]: audit 2023-09-23T22:24:38.133694+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: cluster 2023-09-23T22:24:37.865360+0000 mgr.y (mgr.14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: audit 2023-09-23T22:24:38.128414+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: audit 2023-09-23T22:24: 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: 38.129135+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: audit 2023-09-23T22:24:38.129854+0000 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: mon.a (mon.0) 243 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: audit 2023-09-23T22:24:38 2023-09-23T22:24:39.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: .133694+0000 mon.a (mon.0) 244 : audit [INF] 2023-09-23T22:24:39.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:39 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:39.763 DEBUG:teuthology.orchestra.run.smithi052:osd.0> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T22:24:39.765 INFO:tasks.cephadm:Deploying osd.1 on smithi052 with /dev/vg_nvme/lv_3... 2023-09-23T22:24:39.766 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T22:24:39.798 INFO:journalctl@ceph.osd.0.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:24:40.408 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[100224]: audit 2023-09-23T22:24:39.183745+0000 mon.a ( 2023-09-23T22:24:40.408 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[100224]: mon.0) 245 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:40.409 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[100224]: audit 2023-09-23T22:24:39.201205+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:40.409 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[100224]: audit 2023-09-23T22:24:39.427238+0000 mon.a (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:24:40.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[110082]: audit 2023-09-23T22:24:39.183745 2023-09-23T22:24:40.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[110082]: +0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:40.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[110082]: audit 2023-09-23T22:24:39.201205+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:40.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:40 smithi052 conmon[110082]: audit 2023-09-23T22:24:39.427238+0000 mon.a (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:24:40.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:40 smithi151 conmon[100454]: audit 2023-09-23T22:24:39.183745+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:40 smithi151 conmon[100454]: audit 2023-09-23T22:24:39.201205+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:40 smithi151 conmon[100454]: audit 2023-09-23T22:24:39.427238+0000 mon.a 2023-09-23T22:24:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:40 smithi151 conmon[100454]: (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:24:41.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: cluster 2023-09-23T22:24:39.865640+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:41.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: audit 2023-09-23T22:24:40.187392+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: cluster 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: 2023-09-23T22:24:40.187450+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: T22:24:40.187570+0000 mon.a (mon.0) 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: 250 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: 24:40.187985+0000 mon.a (mon.0) 251 : 2023-09-23T22:24:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:41 smithi151 conmon[100454]: audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:24:41.468 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[115173]: debug 2023-09-23T22:24:41.191+0000 7f1aeade5700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T22:24:41.468 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[115173]: debug 2023-09-23T22:24:41.194+0000 7f1ae5d5e700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[110082]: cluster 2023-09-23T22:24:39.865640+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[110082]: audit 2023-09-23T22:24:40.187392+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[110082]: cluster 2023-09-23T22:24:40.187450+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[110082]: audit 2023-09-23T22:24:40.187570+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[110082]: audit 2023-09-23T22:24:40.187985+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[100224]: cluster 2023-09-23T22:24:39.865640+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:41.469 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[100224]: audit 2023-09-23T22:24:40.187392+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:24:41.470 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[100224]: cluster 2023-09-23T22:24:40.187450+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:24:41.470 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[100224]: audit 2023-09-23T22:24:40.187570+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:41.470 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:41 smithi052 conmon[100224]: audit 2023-09-23T22:24:40.187985+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 10+0 records in 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr:10+0 records out 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00814365 s, 1.3 GB/s 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 2023-09-23T22:24:42.471 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping successful for: 2023-09-23T22:24:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[100224]: audit 2023-09-23T22:24:41.188654+0000 mon.a (mon.0) 252 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:24:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[100224]: cluster 2023-09-23T22:24:41.188732+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:24:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[100224]: audit 2023-09-23T22:24:41.189038+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[100224]: audit 2023-09-23T22:24:41.192890+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:42.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[110082]: audit 2023-09-23T22:24:41.188654+0000 mon.a (mon.0) 252 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:24:42.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[110082]: cluster 2023-09-23T22:24:41.188732+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:24:42.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[110082]: audit 2023-09-23T22:24:41.189038+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:42.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[110082]: audit 2023-09-23T22:24:41.192890+0000 mon.a (mon.0) 255 : audit 2023-09-23T22:24:42.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:42 smithi052 conmon[110082]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:42.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:42 smithi151 conmon[100454]: audit 2023-09-23T22:24:41.188654+0000 mon.a (mon.0) 252 : audit [INF] from='osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:24:42.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:42 smithi151 conmon[100454]: cluster 2023-09-23T22:24:41.188732+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:24:42.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:42 smithi151 conmon[100454]: audit 2023-09-23T22:24:41.189038 2023-09-23T22:24:42.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:42 smithi151 conmon[100454]: +0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:42.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:42 smithi151 conmon[100454]: audit 2023-09-23T22:24:41.192890+0000 2023-09-23T22:24:42.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:42 smithi151 conmon[100454]: mon.a (mon.0) 255 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:43.147 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi052:vg_nvme/lv_3 2023-09-23T22:24:43.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: cluster 2023-09-23T22:24: 2023-09-23T22:24:43.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: 40.471988+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:24:43.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: cluster 2023-09-23T22:24:40.472096+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: cluster 2023-09-23T22:24:41.865898+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: cluster 2023-09-23T22:24:42.193975+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560] boot 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: cluster 2023-09-23T22:24:42.194018+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[100224]: audit 2023-09-23T22:24:42.194432+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: cluster 2023-09-23T22:24 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: :40.471988+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: cluster 2023-09-23T22:24:40.472096+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:24:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: cluster 2023-09-23T22:24:41.865898+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: cluster 2023-09-23T22:24:42.193975+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560] boot 2023-09-23T22:24:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: cluster 2023-09-23T22:24:42.194018+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:24:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:43 smithi052 conmon[110082]: audit 2023-09-23T22:24:42.194432+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:43.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: cluster 2023-09-23T22:24:40.471988+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:24:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: cluster 2023-09-23T22:24:40.472096+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:24:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:24:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: -09-23T22:24:41.865898+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:24:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: cluster 2023-09-23T 2023-09-23T22:24:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: 22:24:42.193975+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.52:6802/858289560,v1:172.21.15.52:6803/858289560] boot 2023-09-23T22:24:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: cluster 2023- 2023-09-23T22:24:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: 09-23T22:24:42.194018+0000 mon.a 2023-09-23T22:24:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:24:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:24:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: -23T22:24:42.194432+0000 mon.a ( 2023-09-23T22:24:43.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:43 smithi151 conmon[100454]: mon.0) 258 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: cluster 2023-09-23T22:24:43.206849+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:43.235484+0000 mgr.y (mgr.14148) 61 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: audit 2023-09-23T22:24:43.239919+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: audit 2023-09-23T22:24:43.564199+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: audit 2023-09-23T22:24:43.568630+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: audit 2023-09-23T22:24:43.569588+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: audit 2023-09-23T22:24:43.570277+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[110082]: audit 2023-09-23T22:24:43.574571+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: cluster 2023-09-23T22:24:43.206849+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:43.235484+0000 mgr.y (mgr.14148) 61 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: audit 2023-09-23T22:24:43.239919+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: audit 2023-09-23T22:24:43.564199+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: audit 2023-09-23T22:24:43.568630+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: audit 2023-09-23T22:24:43.569588+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: audit 2023-09-23T22:24:43.570277+0000 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: mon.a (mon.0) 264 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:44 smithi052 conmon[100224]: audit 2023-09-23T22:24:43.574571+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: cluster 2023-09-23T22:24:43.206849+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:43.235484+0000 mgr.y (mgr.14148) 61 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: audit 2023-09-23T22:24:43.239919 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: +0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: audit 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: 2023-09-23T22:24:43.564199+0000 mon.a (mon.0 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: ) 261 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: audit 2023-09-23T22:24:43 2023-09-23T22:24:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: .568630+0000 mon.a (mon.0) 262 : audit [INF] 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: audit 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: 2023-09-23T22:24:43.569588+0000 mon.a (mon.0) 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: 263 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: audit 2023-09-23T22:24: 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: 43.570277+0000 mon.a (mon.0) 264 : audit 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: audit 2023-09-23T22:24:43. 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: 574571+0000 mon.a (mon.0) 265 : audit [INF] 2023-09-23T22:24:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:44 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:43.564819+0000 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[110082]: mgr.y (mgr.14148) 62 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 16149M 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[110082]: cluster 2023-09-23T22:24:43.866157+0000 mgr.y (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[110082]: audit 2023-09-23T22:24:44.919156+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[110082]: audit 2023-09-23T22:24:44.921480+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[110082]: audit 2023-09-23T22:24:44.922105+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:45.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[100224]: cephadm 2023-09-23T22:24: 2023-09-23T22:24:45.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[100224]: 43.564819+0000 mgr.y (mgr.14148) 62 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 16149M 2023-09-23T22:24:45.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[100224]: cluster 2023-09-23T22:24:43.866157+0000 mgr.y (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:45.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[100224]: audit 2023-09-23T22:24:44.919156+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:24:45.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[100224]: audit 2023-09-23T22:24:44.921480+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:24:45.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:45 smithi052 conmon[100224]: audit 2023-09-23T22:24:44.922105+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:45.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:43.564819+0000 mgr.y (mgr.14148) 62 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 16149M 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: cluster 2023-09-23T22:24:43.866157+0000 mgr.y (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: audit 2023-09-23T22:24:44. 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: 919156+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: audit 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: 2023-09-23T22:24:44.921480+0000 mon.a (mon.0) 267 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:24:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:45 smithi151 conmon[100454]: audit 2023-09-23T22:24:44.922105+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:46.483 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:46 smithi052 conmon[100224]: audit 2023-09-23T22:24:44.916531+0000 mgr.y (mgr.14148) 64 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:46.483 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:46 smithi052 conmon[110082]: audit 2023-09-23T22:24:44.916531+0000 mgr.y (mgr.14148) 64 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:46.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:46 smithi151 conmon[100454]: audit 2023-09-23T22:24:44.916531+0000 mgr.y (mgr.14148) 64 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:24:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[100224]: cluster 2023-09-23T22:24:45.866414+0000 mgr.y 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[100224]: (mgr.14148) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[100224]: audit 2023-09-23T22:24:47.141916+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.52:0/612425661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8704f27e-d1a5-49b1-9382-0ebc689e3f63"}]: dispatch 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[100224]: audit 2023-09-23T22:24:47.145546+0000 mon.a (mon.0) 270 : audit [INF] from='client.? 172.21.15.52:0/612425661' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8704f27e-d1a5-49b1-9382-0ebc689e3f63"}]': finished 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[100224]: cluster 2023-09-23T22:24:47.145600+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[100224]: audit 2023-09-23T22:24:47.145708+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[110082]: cluster 2023-09-23T22:24:45.866414+0000 2023-09-23T22:24:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[110082]: mgr.y (mgr.14148) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:47.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[110082]: audit 2023-09-23T22:24:47.141916+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.52:0/612425661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8704f27e-d1a5-49b1-9382-0ebc689e3f63"}]: dispatch 2023-09-23T22:24:47.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[110082]: audit 2023-09-23T22:24:47.145546+0000 mon.a (mon.0) 270 : audit [INF] from='client.? 172.21.15.52:0/612425661' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8704f27e-d1a5-49b1-9382-0ebc689e3f63"}]': finished 2023-09-23T22:24:47.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[110082]: cluster 2023-09-23T22:24:47.145600+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:24:47.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:47 smithi052 conmon[110082]: audit 2023-09-23T22:24:47.145708+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:24:47.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: cluster 2023-09-23T22:24:45.866414+0000 mgr.y (mgr.14148) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: audit 2023-09-23T22:24:47.141916+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.52:0/612425661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8704f27e-d1a5-49b1-9382-0ebc689e3f63"}]: dispatch 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: T22:24:47.145546+0000 mon.a (mon.0) 270 : audit [INF] from='client.? 172.21.15.52:0/612425661' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8704f27e-d1a5-49b1-9382-0ebc689e3f63"}]': finished 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: cluster 2023-09-23T22:24:47 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: .145600+0000 mon.a (mon.0) 271 : cluster [DBG] 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: audit 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: 2023-09-23T22:24:47.145708+0000 mon.a (mon.0) 272 : 2023-09-23T22:24:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:47 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:24:48.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:48 smithi052 conmon[100224]: audit 2023-09-23T22:24:47.722079+0000 mon.c (mon.2) 4 : 2023-09-23T22:24:48.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:48 smithi052 conmon[100224]: audit [DBG] from='client.? 172.21.15.52:0/119175256' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:24:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:48 smithi052 conmon[110082]: audit 2023-09-23T22:24:47.722079+0000 mon.c 2023-09-23T22:24:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:48 smithi052 conmon[110082]: (mon.2) 4 : audit [DBG] from='client.? 172.21.15.52:0/119175256' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:24:48.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:48 smithi151 conmon[100454]: audit 2023-09-23T22:24:47.722079+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.52:0/119175256' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:24:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:49 smithi052 conmon[100224]: cluster 2023-09-23T22:24:47.866708+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:49.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:49 smithi052 conmon[110082]: cluster 2023-09-23T22:24:47.866708+0000 2023-09-23T22:24:49.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:49 smithi052 conmon[110082]: mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:49.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:49 smithi151 conmon[100454]: cluster 2023-09-23T22:24:47.866708+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:51.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:51 smithi052 conmon[100224]: cluster 2023-09-23T22:24:49.867208+0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:51.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:51 smithi052 conmon[110082]: cluster 2023-09-23T22:24:49.867208+0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:51.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:51 smithi151 conmon[100454]: cluster 2023-09-23T22:24:49.867208+0000 mgr.y (mgr 2023-09-23T22:24:51.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:51 smithi151 conmon[100454]: .14148) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:53 smithi052 conmon[100224]: cluster 2023-09-23T22:24:51.867484+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:53 smithi052 conmon[100224]: audit 2023-09-23T22:24:53.079704+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:24:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:53 smithi052 conmon[100224]: audit 2023-09-23T22:24:53.080464+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:53.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:53 smithi052 conmon[110082]: cluster 2023-09-23T22:24:51.867484+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:53 smithi052 conmon[110082]: audit 2023-09-23T22:24:53.079704+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:24:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:53 smithi052 conmon[110082]: audit 2023-09-23T22:24:53.080464+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:53.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:53 smithi151 conmon[100454]: cluster 2023-09-23T22:24:51.867484+0000 mgr.y (mgr.14148) 68 : cluster [DBG] 2023-09-23T22:24:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:53 smithi151 conmon[100454]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:53 smithi151 conmon[100454]: audit 2023-09-23T22:24:53.079704+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:24:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:53 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:24:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:53 smithi151 conmon[100454]: 22:24:53.080464+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:54 smithi052 conmon[110082]: cephadm 2023-09-23T22:24:53.081087+0000 2023-09-23T22:24:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:54 smithi052 conmon[110082]: mgr.y (mgr.14148) 69 : cephadm [INF] Deploying daemon osd.1 on smithi052 2023-09-23T22:24:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:54 smithi052 conmon[100224]: cephadm 2023-09-23T22:24:53.081087+0000 2023-09-23T22:24:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:54 smithi052 conmon[100224]: mgr.y (mgr.14148) 69 : cephadm [INF] Deploying daemon osd.1 on smithi052 2023-09-23T22:24:54.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:54 smithi151 conmon[100454]: cephadm 2023-09-23T22:24:53.081087+0000 mgr.y ( 2023-09-23T22:24:54.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:54 smithi151 conmon[100454]: mgr.14148) 69 : cephadm [INF] Deploying daemon osd.1 on smithi052 2023-09-23T22:24:55.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:55 smithi052 conmon[100224]: cluster 2023-09-23T22:24:53.867869+0000 mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:55 smithi052 conmon[110082]: cluster 2023-09-23T22:24:53.867869+0000 mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:55.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:55 smithi151 conmon[100454]: cluster 2023-09-23T22:24:53.867869+0000 mgr.y (mgr.14148) 2023-09-23T22:24:55.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:55 smithi151 conmon[100454]: 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:57.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:57 smithi052 conmon[100224]: cluster 2023-09-23T22:24: 2023-09-23T22:24:57.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:57 smithi052 conmon[100224]: 55.868169+0000 mgr.y (mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:57.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:57 smithi052 conmon[110082]: cluster 2023-09-23T22:24:55 2023-09-23T22:24:57.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:57 smithi052 conmon[110082]: .868169+0000 mgr.y (mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:57.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:57 smithi151 conmon[100454]: cluster 2023-09-23T22:24:55.868169+0000 mgr.y (mgr.14148) 2023-09-23T22:24:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:57 smithi151 conmon[100454]: 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[100224]: audit 2023-09-23T22:24:57.794008 2023-09-23T22:24:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[100224]: +0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[100224]: audit 2023-09-23T22:24:57.819552+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[100224]: cluster 2023-09-23T22:24:57.868696+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[110082]: audit 2023-09-23T22:24: 2023-09-23T22:24:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[110082]: 57.794008+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[110082]: audit 2023-09-23T22:24:57.819552+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:24:58 smithi052 conmon[110082]: cluster 2023-09-23T22:24:57.868696+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:24:59.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:58 smithi151 conmon[100454]: audit 2023-09-23T22:24:57.794008+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:24:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:58 smithi151 conmon[100454]: audit 2023-09-23T22:24:57.819552+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:24:58 smithi151 conmon[100454]: cluster 2023-09-23T22:24:57.868696+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:00.793 INFO:teuthology.orchestra.run.smithi052.stdout:Created osd(s) 1 on host 'smithi052' 2023-09-23T22:25:00.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[110082]: audit 2023-09-23T22:24:59.701660+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:00.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[110082]: audit 2023-09-23T22:24:59.702394+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:00.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[110082]: audit 2023-09-23T22:24:59.703096+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:00.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[110082]: audit 2023-09-23T22:24:59.707674+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:00.962 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:25:00.962 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[110082]: 24:59.869002+0000 mgr.y (mgr.14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:00.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[100224]: audit 2023-09-23T22:24:59.701660+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:00.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[100224]: audit 2023-09-23T22:24:59.702394+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:00.963 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[100224]: audit 2023-09-23T22:24:59.703096+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:00.963 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[100224]: audit 2023-09-23T22:24:59.707674+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:00.963 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:00 smithi052 conmon[100224]: cluster 2023-09-23T22:24:59.869002+0000 mgr.y (mgr.14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:01.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: audit 2023-09-23T22:24:59.701660+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: audit 2023-09-23T22:24:59.702394+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: audit 2023-09-23T22:24:59.703096+0000 mon.a (mon 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: .0) 279 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: -23T22:24:59.707674+0000 mon.a (mon.0) 280 : audit [INF] 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: cluster 2023-09-23T22:25:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: 2023-09-23T22:24:59.869002+0000 mgr.y (mgr. 2023-09-23T22:25:01.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:00 smithi151 conmon[100454]: 14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:01.335 DEBUG:teuthology.orchestra.run.smithi052:osd.1> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T22:25:01.338 INFO:tasks.cephadm:Deploying osd.2 on smithi052 with /dev/vg_nvme/lv_2... 2023-09-23T22:25:01.338 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T22:25:01.369 INFO:journalctl@ceph.osd.1.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:25:01.969 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:01 smithi052 conmon[110082]: audit 2023-09-23T22:25:00.790386+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:01.969 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:01 smithi052 conmon[110082]: audit 2023-09-23T22:25:00.825859+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:01.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:01 smithi052 conmon[110082]: audit 2023-09-23T22:25:00.964501+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:25:01.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:01 smithi052 conmon[100224]: audit 2023-09-23T22:25:00.790386+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:01.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:01 smithi052 conmon[100224]: audit 2023-09-23T22:25:00.825859+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:01.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:01 smithi052 conmon[100224]: audit 2023-09-23T22:25:00.964501+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:25:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:01 smithi151 conmon[100454]: audit 2023-09-23T22:25:00.790386+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:01 smithi151 conmon[100454]: audit 2023-09-23T22:25:00.825859+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:01 smithi151 conmon[100454]: audit 2023 2023-09-23T22:25:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:01 smithi151 conmon[100454]: -09-23T22:25:00.964501+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:25:03.122 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[121096]: debug 2023-09-23T22:25:02.801+0000 7fbe4fac0700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T22:25:03.123 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[121096]: debug 2023-09-23T22:25:02.806+0000 7fbe4ba3b700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:25:03.123 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[110082]: audit 2023-09-23T22:25:01.796309+0000 2023-09-23T22:25:03.123 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[110082]: mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:25:03.123 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[110082]: cluster 2023-09-23T22:25:01.796386+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:25:03.124 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[110082]: audit 2023-09-23T22:25:01.796481+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:03.124 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[110082]: audit 2023-09-23T22:25:01.796859+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:03.124 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[110082]: cluster 2023-09-23T22:25:01.869224+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:03.124 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[100224]: audit 2023-09-23T22:25:01 2023-09-23T22:25:03.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[100224]: .796309+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:25:03.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[100224]: cluster 2023-09-23T22:25:01.796386+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:25:03.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[100224]: audit 2023-09-23T22:25:01.796481+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:03.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[100224]: audit 2023-09-23T22:25:01.796859+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:03.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:02 smithi052 conmon[100224]: cluster 2023-09-23T22:25:01.869224+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:03.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: audit 2023-09-23T22:25:01.796309+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:25:03.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: cluster 2023-09-23T22:25:01.796386+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: audit 2023-09-23T22:25:01. 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: 796481+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: audit 2023-09-23T22:25:01.796859+0000 mon.a ( 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: mon.0) 287 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: cluster 2023-09- 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: 23T22:25:01.869224+0000 mgr.y (mgr. 2023-09-23T22:25:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:02 smithi151 conmon[100454]: 14148) 74 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:03.632 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T22:25:03.632 INFO:teuthology.orchestra.run.smithi052.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T22:25:03.633 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 10+0 records in 2023-09-23T22:25:03.633 INFO:teuthology.orchestra.run.smithi052.stderr:10+0 records out 2023-09-23T22:25:03.633 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00805996 s, 1.3 GB/s 2023-09-23T22:25:03.633 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping successful for: 2023-09-23T22:25:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[100224]: audit 2023-09-23T22:25:02.797878+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[100224]: cluster 2023-09-23T22:25:02.797964+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:25:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[100224]: audit 2023-09-23T22:25:02.798378+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[100224]: audit 2023-09-23T22:25:02.802855+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[110082]: audit 2023-09-23T22:25:02.797878+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[110082]: cluster 2023-09-23T22:25:02.797964+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:25:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[110082]: audit 2023-09-23T22:25:02.798378+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:03 smithi052 conmon[110082]: audit 2023-09-23T22:25:02.802855+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: audit 2023-09-23T22:25:02.797878+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: cluster 2023-09-23T22:25:02.797964+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:25:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:25:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: T22:25:02.798378+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:25:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: :25:02.802855+0000 mon.a (mon.0) 291 2023-09-23T22:25:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:03 smithi151 conmon[100454]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:04.579 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi052:vg_nvme/lv_2 2023-09-23T22:25:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[100224]: cluster 2023-09-23T22:25:01.994014+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[100224]: cluster 2023-09-23T22:25:01.994127+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[100224]: cluster 2023-09-23T22:25:03.803568+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544] boot 2023-09-23T22:25:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[100224]: cluster 2023-09-23T22:25:03.803615+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:25:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[100224]: audit 2023-09-23T22:25:03.803954+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[100224]: cluster 2023-09-23T22:25:03.869487+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: cluster 2023-09-23T22:25:01.994014+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: cluster 2023-09-23T22:25:01.994127+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: cluster 2023-09-23T22:25:03.803568+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544] boot 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: cluster 2023-09-23T22:25:03.803615+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: audit 2023-09-23T22:25:03.803954+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: 2023-09-23T22:25:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:04 smithi052 conmon[110082]: cluster 2023-09-23T22:25:03.869487+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: cluster 2023-09-23T22:25:01.994014+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: cluster 2023-09-23T22:25:01.994127+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: cluster 2023-09-23T22:25:03.803568+0000 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.52:6810/3743472544,v1:172.21.15.52:6811/3743472544] boot 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: cluster 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: 2023-09-23T22:25:03.803615+0000 mon.a (mon.0 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: ) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:25:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: audit 2023-09-23T22:25:03 2023-09-23T22:25:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: .803954+0000 mon.a (mon.0) 294 : audit [DBG] 2023-09-23T22:25:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:25:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: cluster 2023-09-23T22:25 2023-09-23T22:25:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: :03.869487+0000 mgr.y (mgr.14148) 75 : cluster 2023-09-23T22:25:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:04 smithi151 conmon[100454]: [DBG] pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:25:06.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: cluster 2023-09-23T22:25:04.812418+0000 mon.a ( 2023-09-23T22:25:06.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:25:06.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:05.022245+0000 mgr.y (mgr.14148) 76 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:06.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.027613+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.028588+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:06.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.029302+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:05.029841+0000 mgr.y (mgr.14148) 77 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 8074M 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.033225+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.034631+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.035276+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[100224]: audit 2023-09-23T22:25:05.039153+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: cluster 2023-09-23T22:25:04.812418 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: +0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: cephadm 2023-09-23T22:25:05.022245+0000 mgr.y (mgr.14148) 76 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023-09-23T22:25:05.027613+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.040 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023-09-23T22:25:05.028588+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023-09-23T22:25:05.029302+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: cephadm 2023-09-23T22:25:05.029841+0000 mgr.y (mgr.14148) 77 : cephadm 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: [INF] Adjusting osd_memory_target on smithi052 to 8074M 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023-09-23T22:25:05.033225+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023-09-23T22:25:05.034631 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: +0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: 09-23T22:25:05.035276+0000 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: mon.a (mon.0) 301 : audit [INF] 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: audit 2023-09-23T22:25:05. 2023-09-23T22:25:06.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:05 smithi052 conmon[110082]: 039153+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: cluster 2023-09-23T22:25:04.812418+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:25:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:05.022245+0000 mgr.y (mgr.14148) 76 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09-23T22:25:05 2023-09-23T22:25:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: .027613+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: :25:05.028588+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: -23T22:25:05.029302+0000 mon.a (mon 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: .0) 298 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: cephadm 2023-09-23 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: T22:25:05.029841+0000 mgr.y (mgr.14148 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: ) 77 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 8074M 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: 22:25:05.033225+0000 mon.a (mon.0) 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: 299 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09-23T22:25 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: :05.034631+0000 mon.a (mon.0) 300 : audit 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09-23T22:25:05.035276 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: +0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: audit 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: 2023-09-23T22:25:05.039153+0000 mon.a (mon 2023-09-23T22:25:06.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:05 smithi151 conmon[100454]: .0) 302 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:07.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[100224]: cluster 2023-09-23T22:25:05.869760+0000 mgr.y ( 2023-09-23T22:25:07.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[100224]: mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:07.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[100224]: audit 2023-09-23T22:25:06.392621+0000 mgr.y (mgr.14148) 79 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:07.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[100224]: audit 2023-09-23T22:25:06.395250+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:07.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[100224]: audit 2023-09-23T22:25:06.397537+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:07.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[100224]: audit 2023-09-23T22:25:06.398182+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:07.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: cluster 2023-09-23T22:25:05. 2023-09-23T22:25:07.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: 869760+0000 mgr.y (mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:07.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: audit 2023-09-23T22:25:07.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: 2023-09-23T22:25:06.392621+0000 mgr.y 2023-09-23T22:25:07.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: (mgr.14148) 79 : audit [DBG] 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: audit 2023 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: -09-23T22:25: 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: 06.395250+0000 mon.a (mon 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: .0) 303 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: 2023-09-23T22:25:07.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: audit 2023 2023-09-23T22:25:07.105 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: -09-23T22:25:06 2023-09-23T22:25:07.105 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: .397537+0000 mon.a (mon.0) 2023-09-23T22:25:07.105 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: 304 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:07.105 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:06 smithi052 conmon[110082]: audit 2023-09-23T22:25:06.398182+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:07.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: cluster 2023-09-23T22:25:05.869760+0000 mgr.y (mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: audit 2023-09-23T22:25:06.392621+0000 mgr.y (mgr.14148) 79 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: audit 2023-09-23T22:25:06.395250+0000 mon.a (mon. 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: 0) 303 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: -23T22:25:06.397537+0000 mon.a (mon.0) 304 : audit [INF] 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: audit 2023 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: -09-23T22:25:06.398182+0000 mon.a (mon.0) 305 2023-09-23T22:25:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:06 smithi151 conmon[100454]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:09.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: cluster 2023-09-23T22:25:07.870176+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:09.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: 22:25:08.597972+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.52:0/2654174416' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]: dispatch 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: audit 2023-09-23T22:25:08.598323+0000 mon.a (mon.0) 306 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]: dispatch 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: audit 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: 2023-09-23T22:25:08.601941+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]': finished 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: cluster 2023-09-23T22:25 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: :08.602003+0000 mon.a (mon.0) 308 : cluster 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:25:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: audit 2023-09-23T22:25:08. 2023-09-23T22:25:09.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: 602114+0000 mon.a (mon.0) 309 : audit [DBG] 2023-09-23T22:25:09.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:08 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:09.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[100224]: cluster 2023-09-23T22:25:07.870176+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[100224]: audit 2023-09-23T22:25:08.597972+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.52:0/2654174416' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]: dispatch 2023-09-23T22:25:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[100224]: audit 2023-09-23T22:25:08.598323+0000 mon.a (mon.0) 306 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]: dispatch 2023-09-23T22:25:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[100224]: audit 2023-09-23T22:25:08.601941+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]': finished 2023-09-23T22:25:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[100224]: cluster 2023-09-23T22:25:08.602003+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:25:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[100224]: audit 2023-09-23T22:25:08.602114+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: cluster 2023-09-23T22:25:07.870176+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: audit 2023-09-23T22:25:08.597972+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.52:0/2654174416' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]: dispatch 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: audit 2023-09-23T22:25:08.598323+0000 mon.a (mon.0) 306 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]: dispatch 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: audit 2023-09-23T22:25:08.601941 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: +0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "80877779-5cd3-4a3c-bb3e-ffd41e160fb5"}]': finished 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: cluster 2023-09-23T22:25:08.602003+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:25:09.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:08 smithi052 conmon[110082]: audit 2023-09-23T22:25:08.602114+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:10.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:09 smithi151 conmon[100454]: audit 2023-09-23T22:25:09.207991+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.52:0/1412823450' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:10.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:09 smithi052 conmon[100224]: audit 2023-09-23T22:25:09.207991+0000 mon.c (mon. 2023-09-23T22:25:10.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:09 smithi052 conmon[100224]: 2) 6 : audit [DBG] from='client.? 172.21.15.52:0/1412823450' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:10.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:09 smithi052 conmon[110082]: audit 2023-09-23T22:25:09.207991+0000 mon.c (mon.2) 6 2023-09-23T22:25:10.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:09 smithi052 conmon[110082]: : audit [DBG] from='client.? 172.21.15.52:0/1412823450' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:11.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:11 smithi151 conmon[100454]: cluster 2023-09-23T22:25:09.870432+0000 mgr.y ( 2023-09-23T22:25:11.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:11 smithi151 conmon[100454]: mgr.14148) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:11.470 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:11 smithi052 conmon[110082]: cluster 2023-09-23T22:25:09.870432+0000 2023-09-23T22:25:11.470 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:11 smithi052 conmon[110082]: mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:11.470 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:11 smithi052 conmon[100224]: cluster 2023-09-23T22:25:09 2023-09-23T22:25:11.471 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:11 smithi052 conmon[100224]: .870432+0000 mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:13.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:13 smithi052 conmon[100224]: cluster 2023-09-23T22:25:11.870863+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:13 smithi052 conmon[110082]: cluster 2023-09-23T22:25:11.870863+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:13.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:13 smithi151 conmon[100454]: cluster 2023-09-23T22:25:11.870863+0000 mgr.y (mgr.14148) 2023-09-23T22:25:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:13 smithi151 conmon[100454]: 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:15.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:15 smithi052 conmon[110082]: cluster 2023-09-23T22:25:13.871198+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:15.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:15 smithi052 conmon[110082]: audit 2023-09-23T22:25:14.484339+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:25:15.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:15 smithi052 conmon[110082]: audit 2023-09-23T22:25:14.484977+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:15.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:15 smithi052 conmon[100224]: cluster 2023-09-23T22:25:13.871198+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:15.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:15 smithi052 conmon[100224]: audit 2023-09-23T22:25:14.484339+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:25:15.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:15 smithi052 conmon[100224]: audit 2023-09-23T22:25:14.484977+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:15.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:15 smithi151 conmon[100454]: cluster 2023-09-23T22:25:13.871198+0000 mgr.y (mgr.14148) 83 2023-09-23T22:25:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:15 smithi151 conmon[100454]: : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:15 smithi151 conmon[100454]: audit 2023-09-23T22:25:14.484339+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:25:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:15 smithi151 conmon[100454]: audit 2023-09-23T22:25:14 2023-09-23T22:25:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:15 smithi151 conmon[100454]: .484977+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:15 smithi151 conmon[100454]: 2023-09-23T22:25:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:16 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:14.485569+0000 mgr.y 2023-09-23T22:25:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:16 smithi052 conmon[100224]: (mgr.14148) 84 : cephadm [INF] Deploying daemon osd.2 on smithi052 2023-09-23T22:25:16.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:16 smithi052 conmon[110082]: cephadm 2023-09-23T22:25:14.485569+0000 mgr.y 2023-09-23T22:25:16.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:16 smithi052 conmon[110082]: (mgr.14148) 84 : cephadm [INF] Deploying daemon osd.2 on smithi052 2023-09-23T22:25:16.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:16 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:14.485569+0000 mgr.y (mgr.14148) 84 : cephadm [INF] 2023-09-23T22:25:16.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:16 smithi151 conmon[100454]: Deploying daemon osd.2 on smithi052 2023-09-23T22:25:17.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:17 smithi151 conmon[100454]: cluster 2023-09-23T22:25:15.871585 2023-09-23T22:25:17.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:17 smithi151 conmon[100454]: +0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:17.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:17 smithi052 conmon[100224]: cluster 2023-09-23T22:25:15.871585+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:17 smithi052 conmon[110082]: cluster 2023-09-23T22:25:15.871585+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:19 smithi151 conmon[100454]: cluster 2023-09-23T22:25:17.871851+0000 mgr.y ( 2023-09-23T22:25:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:19 smithi151 conmon[100454]: mgr.14148) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:19 smithi052 conmon[100224]: cluster 2023-09-23T22:25:17.871851 2023-09-23T22:25:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:19 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:19.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:19 smithi052 conmon[110082]: cluster 2023-09-23T22:25:17.871851 2023-09-23T22:25:19.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:19 smithi052 conmon[110082]: +0000 mgr.y (mgr.14148) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:20.394 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:20 smithi052 conmon[110082]: audit 2023-09-23T22:25:19.262140+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:20.394 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:20 smithi052 conmon[110082]: audit 2023-09-23T22:25:19.281840+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:20.395 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:20 smithi052 conmon[100224]: audit 2023-09-23T22:25:19.262140+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:20.395 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:20 smithi052 conmon[100224]: audit 2023-09-23T22:25:19.281840+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:20.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:20 smithi151 conmon[100454]: audit 2023-09-23T22:25:19.262140+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:20.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:20 smithi151 conmon[100454]: audit 2023-09-23T22:25:19.281840+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:21.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:21 smithi151 conmon[100454]: cluster 2023-09-23T22:25:19.872147+0000 2023-09-23T22:25:21.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:21 smithi151 conmon[100454]: mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:21.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:21 smithi052 conmon[100224]: cluster 2023-09-23T22:25:19.872147+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:21.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:21 smithi052 conmon[110082]: cluster 2023-09-23T22:25:19.872147+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:22.371 INFO:teuthology.orchestra.run.smithi052.stdout:Created osd(s) 2 on host 'smithi052' 2023-09-23T22:25:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:25:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 25:21.492982+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 2023-09-23T22:25:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023-09-23T22:25:21. 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 493758+0000 mon.a (mon.0) 315 : audit [DBG] 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023-09-23T22:25: 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 21.494514+0000 mon.a (mon.0) 316 : audit 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 2023-09-23T22:25:21.500118+0000 mon.a (mon.0) 317 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: cluster 2023-09-23T22:25:21. 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 872408+0000 mgr.y (mgr.14148) 88 : cluster [DBG] 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:22.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: :25:22.367915+0000 mon.a (mon. 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 0) 318 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 23T22:25:22.370776+0000 mon.a ( 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 09-23T22:25:22.456238+0000 mon.c 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: (mon.2) 7 : audit [INF] from='osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: 09-23T22:25:22.456655+0000 mon.a 2023-09-23T22:25:22.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[110082]: (mon.0) 320 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023-09-23T22:25:21.492982+0000 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: mon.a (mon.0) 314 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 2023-09-23T22:25:21.493758 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: +0000 mon.a (mon.0) 315 : audit 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 23T22:25:21.494514+0000 mon.a 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: (mon.0) 316 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 2023-09-23T22:25:22.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: T22:25:21.500118+0000 mon.a ( 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: mon.0) 317 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: -23T22:25:21.872408+0000 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: mgr.y (mgr.14148) 88 : cluster [DBG] 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 2023-09-23T22:25:22.367915+0000 mon.a 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: (mon.0) 318 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:22.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: -09-23T22:25:22.370776+0000 mon.a ( 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: mon.0) 319 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: -09-23T22:25:22.456238 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: +0000 mon.c (mon.2) 7 : audit 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: [INF] from='osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: audit 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 2023-09-23T 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: 22:25:22.456655+0000 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: mon.a (mon.0) 320 2023-09-23T22:25:22.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:22 smithi052 conmon[100224]: : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:25:22.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09-23T22:25:21.492982+0000 mon.a (mon.0 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: ) 314 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09-23T22:25:21.493758+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09-23T22:25:21.494514+0000 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: mon.a (mon.0) 316 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09-23T22:25: 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: 21.500118+0000 mon.a (mon.0) 317 : audit [INF] 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: cluster 2023-09-23T22:25:21.872408 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: +0000 mgr.y (mgr.14148) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: 09-23T22:25:22.367915+0000 mon.a ( 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: mon.0) 318 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: T22:25:22.370776+0000 mon.a (mon.0 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: ) 319 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: 23T22:25:22.456238+0000 mon.c (mon. 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: 2) 7 : audit [INF] from='osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: 22:25:22.456655+0000 mon.a (mon.0) 2023-09-23T22:25:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:22 smithi151 conmon[100454]: 320 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:25:22.975 DEBUG:teuthology.orchestra.run.smithi052:osd.2> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T22:25:22.977 INFO:tasks.cephadm:Deploying osd.3 on smithi052 with /dev/vg_nvme/lv_1... 2023-09-23T22:25:22.977 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T22:25:23.010 INFO:journalctl@ceph.osd.2.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:25:24.596 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[100224]: audit 2023-09-23T22:25:23.372003+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:25:24.596 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[100224]: cluster 2023-09-23T22:25:23.372063+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[100224]: audit 2023-09-23T22:25:23.372159+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[100224]: audit 2023-09-23T22:25:23.372602+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[100224]: audit 2023-09-23T22:25:23.372908+0000 mon.a (mon.0) 324 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[110082]: audit 2023-09-23T22:25:23.372003+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[110082]: cluster 2023-09-23T22:25:23.372063+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[110082]: audit 2023-09-23T22:25:23.372159+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[110082]: audit 2023-09-23T22:25:23.372602+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:24.597 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[110082]: audit 2023-09-23T22:25:23.372908+0000 mon.a (mon.0) 324 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:24.598 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[126863]: debug 2023-09-23T22:25:24.376+0000 7f698a14b700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T22:25:24.598 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:25:24 smithi052 conmon[126863]: debug 2023-09-23T22:25:24.379+0000 7f69868c7700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: audit 2023-09-23T22:25:23.372003+0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: cluster 2023-09-23T22:25:23.372063+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: audit 2023-09-23T22:25:23.372159+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: audit 2023-09-23T22:25:23.372602+0000 mon.c (mon 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: .2) 8 : audit [INF] from='osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: 22:25:23.372908+0000 mon.a (mon.0) 324 2023-09-23T22:25:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:24 smithi151 conmon[100454]: : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:25.591 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T22:25:25.591 INFO:teuthology.orchestra.run.smithi052.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T22:25:25.591 INFO:teuthology.orchestra.run.smithi052.stderr: stderr: 10+0 records in 2023-09-23T22:25:25.591 INFO:teuthology.orchestra.run.smithi052.stderr:10+0 records out 2023-09-23T22:25:25.591 INFO:teuthology.orchestra.run.smithi052.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00836512 s, 1.3 GB/s 2023-09-23T22:25:25.591 INFO:teuthology.orchestra.run.smithi052.stderr:--> Zapping successful for: 2023-09-23T22:25:26.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[100224]: cluster 2023-09-23T22:25:23.872719+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:26.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[100224]: audit 2023-09-23T22:25:24.373241+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[100224]: cluster 2023-09-23T22:25:24.373327+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[100224]: 2023-09-23T22:25:24.373631+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[100224]: audit 2023-09-23T22:25:24.377572+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[110082]: cluster 2023-09-23T22:25:23.872719+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[110082]: audit 2023-09-23T22:25:24.373241+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[110082]: cluster 2023-09-23T22:25:24.373327+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[110082]: audit 2023-09-23T22:25:24.373631+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[110082]: audit 2023-09-23T22:25:24.377572+0000 mon.a (mon. 2023-09-23T22:25:26.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:25 smithi052 conmon[110082]: 0) 328 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: cluster 2023-09-23T22:25:23.872719+0000 mgr.y (mgr.14148) 89 : cluster 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: audit 2023-09-23T22:25:24.373241+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: cluster 2023-09-23T22:25:24.373327 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: +0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: 09-23T22:25:24.373631+0000 mon.a (mon.0) 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: 327 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: T22:25:24.377572+0000 mon.a (mon 2023-09-23T22:25:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:25 smithi151 conmon[100454]: .0) 328 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.608 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi052:vg_nvme/lv_1 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: cluster 2023-09-23T22:25:23.486064+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: cluster 2023-09-23T22:25:23.486112+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: cluster 2023-09-23T22:25:25.378349+0000 mon.a (mon.0) 329 : cluster [INF] osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972] boot 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: cluster 2023-09-23T22:25:25.378397+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:25.379748+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:25.380280+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: cluster 2023-09-23T22:25:25.873065+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:26.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.379670+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: cluster 2023-09-23T22:25:26.379740+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.380566+0000 mon.a (mon.0) 335 : audit [INF] 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.701553+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.702576+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.703262+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.703888+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.707914+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.709058+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.709802+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:26.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[100224]: audit 2023-09-23T22:25:26.714126+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: cluster 2023-09-23T22:25:23.486064+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: cluster 2023-09-23T22:25:23.486112+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: cluster 2023-09-23T22:25:25.378349+0000 mon.a (mon.0) 329 : cluster [INF] osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972] boot 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: cluster 2023-09-23T22:25:25.378397+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:25.379748+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:25.380280+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:25:27.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: cluster 2023-09-23T22:25:25.873065+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.379670+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: 23T22:25:26.379740+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.380566+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.701553+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.702576+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.703262+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:27.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.703888+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:27.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.707914+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:27.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.709058+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:27.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.709802+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:27.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:26 smithi052 conmon[110082]: audit 2023-09-23T22:25:26.714126+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: cluster 2023-09-23T22:25:23.486064+0000 osd.2 (osd.2) 2023-09-23T22:25:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: cluster 2023-09-23T22:25:23.486112+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: cluster 2023-09-23 2023-09-23T22:25:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: T22:25:25.378349+0000 mon.a ( 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: mon.0) 329 : cluster [INF] osd.2 [v2:172.21.15.52:6818/1781181972,v1:172.21.15.52:6819/1781181972] boot 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: -09-23T22:25:25. 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 378397+0000 mon.a (mon.0) 330 : cluster [DBG] 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T22:25 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: :25.379748+0000 mon.a (mon.0) 331 : audit 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 22:25:25.380280+0000 mon.a (mon.0 2023-09-23T22:25:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: ) 332 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: cluster 2023-09 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: -23T22:25:25.873065+0000 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 2023-09-23T22:25: 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 26.379670+0000 mon.a (mon.0) 333 : audit 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: cluster 2023-09- 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 23T22:25:26.379740+0000 mon.a 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: (mon.0) 334 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:25:27.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 2023-09-23T22:25:26 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: .380566+0000 mon.a (mon.0) 335 : audit 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: :25:26.701553+0000 mon.a (mon.0 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: ) 336 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 09-23T22:25:26.702576+0000 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: mon.a (mon.0) 337 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: -09-23T22:25:26. 2023-09-23T22:25:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 703262+0000 mon.a (mon.0) 338 : audit [INF] 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T22:25:26. 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 703888+0000 mon.a (mon.0) 339 : audit [INF] 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 25:26.707914+0000 mon.a (mon.0) 340 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: -23T22:25:26.709058 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: +0000 mon.a (mon.0) 341 : audit [DBG] 2023-09-23T22:25:27.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 23T22:25:26.709802+0000 mon.a ( 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: mon.0) 342 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: audit 2023 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: -09-23T22:25:26. 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: 714126+0000 mon.a (mon.0) 343 : audit [INF] 2023-09-23T22:25:27.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:26 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:28.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:26.696878+0000 mgr.y (mgr.14148 2023-09-23T22:25:28.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: ) 91 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:26.704373+0000 mgr.y (mgr.14148) 92 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 5383M 2023-09-23T22:25:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: audit 2023-09-23T22:25:27.380558+0000 mon.a (mon.0) 344 : audit 2023-09-23T22:25:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T22:25:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: cluster 2023-09-23T22:25:27 2023-09-23T22:25:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: .380677+0000 mon.a (mon.0) 345 : cluster [DBG] 2023-09-23T22:25:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:28 smithi151 conmon[100454]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:25:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[100224]: cephadm 2023-09-23T22:25: 2023-09-23T22:25:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[100224]: 26.696878+0000 mgr.y (mgr.14148) 91 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:26.704373+0000 mgr.y (mgr.14148) 92 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 5383M 2023-09-23T22:25:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[100224]: audit 2023-09-23T22:25:27.380558+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T22:25:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[100224]: cluster 2023-09-23T22:25:27.380677+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:25:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:25:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[110082]: 25:26.696878+0000 mgr.y (mgr.14148) 91 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[110082]: cephadm 2023-09-23T22:25:26.704373+0000 mgr.y (mgr.14148) 92 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 5383M 2023-09-23T22:25:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[110082]: audit 2023-09-23T22:25:27.380558+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T22:25:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:28 smithi052 conmon[110082]: cluster 2023-09-23T22:25:27.380677+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:25:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: cluster 2023-09-23T22:25:27.873352+0000 mgr.y (mgr. 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: 14148) 93 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: cluster 2023-09-23T22:25:28.384914+0000 mon.a (mon.0) 346 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: audit 2023 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: -09-23T22:25:28.466157+0000 mon.a (mon.0) 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: 347 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: 23T22:25:28.468439+0000 mon.a 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:25:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: 22:25:28.469030+0000 mon.a (mon.0) 2023-09-23T22:25:29.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:29 smithi151 conmon[100454]: 349 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:29.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[110082]: cluster 2023-09-23T22:25:27.873352+0000 mgr.y (mgr.14148) 93 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:29.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[110082]: cluster 2023-09-23T22:25:28.384914+0000 mon.a (mon.0) 346 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:25:29.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[110082]: audit 2023-09-23T22:25:28.466157+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:29.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[110082]: audit 2023-09-23T22:25:28.468439+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[110082]: audit 2023-09-23T22:25:28.469030+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[100224]: cluster 2023-09-23T22:25:27.873352+0000 mgr.y (mgr.14148) 93 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[100224]: cluster 2023-09-23T22:25:28.384914+0000 mon.a (mon.0) 346 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:25:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[100224]: audit 2023-09-23T22:25:28.466157+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[100224]: audit 2023-09-23T22:25:28.468439+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:29 smithi052 conmon[100224]: audit 2023-09-23T22:25:28.469030+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:30.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:30 smithi151 conmon[100454]: audit 2023-09-23T22:25:28.463531+0000 mgr.y (mgr.14148) 94 : audit 2023-09-23T22:25:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:30 smithi151 conmon[100454]: [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:30 smithi052 conmon[100224]: audit 2023-09-23T22:25:28. 2023-09-23T22:25:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:30 smithi052 conmon[100224]: 463531+0000 mgr.y (mgr.14148) 94 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:30 smithi052 conmon[110082]: audit 2023-09-23T22:25:28.463531+0000 mgr.y 2023-09-23T22:25:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:30 smithi052 conmon[110082]: (mgr.14148) 94 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi052:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:31.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: cluster 2023-09-23T22:25:29.873684+0000 mgr.y (mgr.14148) 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: 95 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: audit 2023-09-23T22:25:30.752514+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.52:0/1487138982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]: dispatch 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: audit 2023-09-23T22:25:30.752867+0000 mon.a (mon.0) 350 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]: dispatch 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: audit 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: 2023-09-23T22:25:30.756739+0000 mon.a (mon.0) 351 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]': finished 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: cluster 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: 2023-09-23T22:25:30.756800+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: audit 2023-09-23T22:25:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: 2023-09-23 2023-09-23T22:25:31.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: T22:25:30.756898+0000 mon.a (mon. 2023-09-23T22:25:31.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: 0) 353 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:31.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:25:31.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: T22:25:31.367123+0000 mon.c (mon.2 2023-09-23T22:25:31.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:31 smithi151 conmon[100454]: ) 10 : audit [DBG] from='client.? 172.21.15.52:0/1056298828' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: cluster 2023-09-23T22:25:29.873684+0000 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: audit 2023-09-23T22:25:30.752514+0000 mon.c (mon.2) 9 : audit [INF] from='client.? 172.21.15.52:0/1487138982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]: dispatch 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: audit 2023-09-23T22:25:30.752867+0000 mon.a (mon.0) 350 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]: dispatch 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: audit 2023-09-23T22:25:30.756739+0000 mon.a (mon.0) 351 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]': finished 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: -23T22:25:30.756800+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: audit 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: 2023-09-23T22:25:30.756898+0000 2023-09-23T22:25:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: mon.a (mon.0) 353 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: -09-23T22:25:31.367123+0000 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[100224]: mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.52:0/1056298828' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 25:29.873684+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 23T22:25:30.752514+0000 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: mon.c (mon.2) 9 : audit 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: [INF] from='client.? 172.21.15.52:0/1487138982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]: dispatch 2023-09-23T22:25:31.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 09-23T22:25:30. 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 752867+0000 mon.a (mon.0) 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 350 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]: dispatch 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: audit 2023 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: -09-23T22:25:30.756739+0000 mon.a 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: (mon.0) 351 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c94bebc9-a105-4d74-9a03-c88e10cc2876"}]': finished 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: cluster 2023-09-23T22:25 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: :30.756800+0000 mon.a (mon.0) 352 : cluster [DBG] 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: audit 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:25:31.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 25:30.756898+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:31.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: audit 2023-09-23T22:25:31.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 2023-09-23T22:25:31.367123+0000 mon.c 2023-09-23T22:25:31.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: (mon.2) 10 : audit [DBG] from='client.? 172.21.15.52:0/1056298828' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:31.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:31 smithi052 conmon[110082]: 2023-09-23T22:25:33.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:33 smithi052 conmon[110082]: cluster 2023-09-23T22:25:31.874209+0000 mgr.y ( 2023-09-23T22:25:33.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:33 smithi052 conmon[110082]: mgr.14148) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:33.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:33 smithi052 conmon[100224]: cluster 2023-09-23T22:25:31.874209+0000 mgr.y ( 2023-09-23T22:25:33.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:33 smithi052 conmon[100224]: mgr.14148) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:33.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:33 smithi151 conmon[100454]: cluster 2023-09-23T22:25:31.874209+0000 mgr.y (mgr.14148) 96 : 2023-09-23T22:25:33.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:33 smithi151 conmon[100454]: cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:35.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:35 smithi052 conmon[110082]: cluster 2023-09-23T22:25:33.874511+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:35 smithi052 conmon[100224]: cluster 2023-09-23T22:25:33.874511+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:35.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:35 smithi151 conmon[100454]: cluster 2023-09-23T22:25:33.874511+0000 mgr.y (mgr.14148 2023-09-23T22:25:35.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:35 smithi151 conmon[100454]: ) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: cluster 2023-09-23T22:25:35.874794+0000 mgr.y 2023-09-23T22:25:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: audit 2023-09-23T22:25:36.887733+0000 mon.a (mon.0) 354 : audit 2023-09-23T22:25:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:25:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:25:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: T22:25:36.888575+0000 mon.a (mon.0) 355 : audit 2023-09-23T22:25:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:37 smithi151 conmon[100454]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:37 smithi052 conmon[100224]: cluster 2023-09-23T22:25:35.874794+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:37 smithi052 conmon[100224]: audit 2023-09-23T22:25:36.887733+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:25:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:37 smithi052 conmon[100224]: audit 2023-09-23T22:25:36.888575+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:37 smithi052 conmon[110082]: cluster 2023-09-23T22:25:35.874794+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:37 smithi052 conmon[110082]: audit 2023-09-23T22:25:36.887733+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:25:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:37 smithi052 conmon[110082]: audit 2023-09-23T22:25:36.888575+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:38.429 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:38 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:36.889284+0000 mgr.y (mgr. 2023-09-23T22:25:38.429 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:38 smithi052 conmon[100224]: 14148) 99 : cephadm [INF] Deploying daemon osd.3 on smithi052 2023-09-23T22:25:38.430 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:38 smithi052 conmon[110082]: cephadm 2023-09-23T22:25 2023-09-23T22:25:38.430 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:38 smithi052 conmon[110082]: :36.889284+0000 mgr.y (mgr.14148) 99 : cephadm [INF] Deploying daemon osd.3 on smithi052 2023-09-23T22:25:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:38 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:36.889284+0000 2023-09-23T22:25:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:38 smithi151 conmon[100454]: mgr.y (mgr.14148) 99 : cephadm [INF] Deploying daemon osd.3 on smithi052 2023-09-23T22:25:39.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:39 smithi052 conmon[110082]: cluster 2023-09-23T22:25:37.875095+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:39.616 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:39 smithi052 conmon[100224]: cluster 2023-09-23T22:25:37.875095+0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:39.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:39 smithi151 conmon[100454]: cluster 2023-09-23T22:25:37.875095 2023-09-23T22:25:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:39 smithi151 conmon[100454]: +0000 mgr.y (mgr.14148) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:41.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:41 smithi151 conmon[100454]: cluster 2023-09-23T22:25:39.875382+0000 mgr.y (mgr.14148) 2023-09-23T22:25:41.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:41 smithi151 conmon[100454]: 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:41.702 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:41 smithi052 conmon[100224]: cluster 2023-09-23T22:25:39.875382+0000 mgr.y 2023-09-23T22:25:41.702 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:41 smithi052 conmon[100224]: (mgr.14148) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:41.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:41 smithi052 conmon[110082]: cluster 2023-09-23T22:25:39.875382+0000 mgr.y 2023-09-23T22:25:41.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:41 smithi052 conmon[110082]: (mgr.14148) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:42 smithi052 conmon[110082]: audit 2023-09-23T22:25:41.826528+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:42 smithi052 conmon[110082]: audit 2023-09-23T22:25:41.860763+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:42 smithi052 conmon[110082]: cluster 2023-09-23T22:25:41.875683+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:42.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:42 smithi052 conmon[100224]: audit 2023-09-23T22:25:41.826528+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:42.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:42 smithi052 conmon[100224]: audit 2023-09-23T22:25:41.860763+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:42.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:42 smithi052 conmon[100224]: cluster 2023-09-23T22:25:41.875683+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:43.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:42 smithi151 conmon[100454]: audit 2023-09-23T22:25:41.826528+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:42 smithi151 conmon[100454]: audit 2023-09-23T22:25:41.860763+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:42 smithi151 conmon[100454]: cluster 2023-09-23T22:25:41.875683 2023-09-23T22:25:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:42 smithi151 conmon[100454]: +0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:44.910 INFO:teuthology.orchestra.run.smithi052.stdout:Created osd(s) 3 on host 'smithi052' 2023-09-23T22:25:45.034 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[110082]: audit 2023-09-23T22:25:43.777966+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:45.034 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[110082]: cluster 2023-09-23T22:25:43.876025+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:45.034 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[110082]: audit 2023-09-23T22:25:44.111617+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:45.034 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[110082]: audit 2023-09-23T22:25:44.112517+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:45.034 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[110082]: audit 2023 2023-09-23T22:25:45.034 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[110082]: -09-23T22:25:44.118063+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:45.035 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[100224]: audit 2023-09-23T22:25:43.777966+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:45.035 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[100224]: cluster 2023-09-23T22:25:43.876025+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:45.035 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[100224]: audit 2023-09-23T22:25:44.111617+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:45.035 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[100224]: audit 2023-09-23T22:25:44.112517+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:45.035 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:44 smithi052 conmon[100224]: audit 2023-09-23T22:25:44.118063+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: audit 2023-09-23T22:25:43.777966+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: cluster 2023-09-23T22:25:43.876025+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: 09-23T22:25:44.111617+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: 09-23T22:25:44.112517 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: +0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:25:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:44 smithi151 conmon[100454]: :25:44.118063+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:45.488 DEBUG:teuthology.orchestra.run.smithi052:osd.3> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T22:25:45.490 INFO:tasks.cephadm:Deploying osd.4 on smithi151 with /dev/vg_nvme/lv_4... 2023-09-23T22:25:45.490 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T22:25:45.520 INFO:journalctl@ceph.osd.3.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:25:46.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[100224]: audit 2023-09-23T22:25:44.908968+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:46.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[100224]: audit 2023-09-23T22:25:44.936287+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:46.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[100224]: audit 2023-09-23T22:25:45.037997+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:25:46.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[100224]: audit 2023-09-23T22:25:45.038322+0000 mon.a ( 2023-09-23T22:25:46.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[100224]: mon.0) 364 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:25:46.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[110082]: audit 2023-09-23T22:25:44.908968+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:46.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[110082]: audit 2023-09-23T22:25:44.936287+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:46.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[110082]: audit 2023-09-23T22:25:45.037997+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:25:46.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:45 smithi052 conmon[110082]: audit 2023-09-23T22:25:45.038322+0000 mon.a (mon.0) 364 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:25:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:45 smithi151 conmon[100454]: audit 2023-09-23T22:25:44.908968+0000 mon.a (mon. 2023-09-23T22:25:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:45 smithi151 conmon[100454]: 0) 362 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:45 smithi151 conmon[100454]: audit 2023-09-23T22:25:44.936287+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:45 smithi151 conmon[100454]: audit 2023-09-23T22:25:45.037997+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:25:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:45 smithi151 conmon[100454]: audit 2023-09-23T22:25:45.038322+0000 mon.a (mon.0) 364 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:25:46.927 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: cluster 2023-09-23T22:25:45.876507+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:47.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:25:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: -23T22:25:45.913616+0000 mon.a (mon.0) 365 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:25:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: cluster 2023-09-23T22:25:45.913678+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:25:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: audit 2023-09-23T22:25:45.913804+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: audit 2023-09-23T22:25:45.914219+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:46 smithi151 conmon[100454]: audit 2023-09-23T22:25:45.914520+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:47.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: cluster 2023-09-23T22:25:45.876507 2023-09-23T22:25:47.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:47.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: audit 2023-09-23T22:25:45.913616+0000 mon.a (mon.0) 365 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:25:47.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: cluster 2023-09-23T22:25:45.913678+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:25:47.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: audit 2023-09-23T22:25:45.913804+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:47.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: audit 2023-09-23T22:25:45.914219+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:47.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[100224]: audit 2023-09-23T22:25:45.914520+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:47.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[110082]: cluster 2023-09-23T22:25:45.876507+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:47.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[110082]: audit 2023-09-23T22:25:45.913616+0000 mon.a (mon.0) 365 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:25:47.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[110082]: cluster 2023-09-23T22:25:45.913678+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:25:47.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[110082]: audit 2023-09-23T22:25:45.913804+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:47.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[110082]: audit 2023-09-23T22:25:45.914219+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:47.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[110082]: audit 2023-09-23T22:25:45.914520+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:25:47.245 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[132705]: debug 2023-09-23T22:25:46.919+0000 7f72551b2700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T22:25:47.245 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:25:46 smithi052 conmon[132705]: debug 2023-09-23T22:25:46.923+0000 7f725192e700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:25:47.378 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T22:25:47.379 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T22:25:47.379 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-09-23T22:25:47.379 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-09-23T22:25:47.379 INFO:teuthology.orchestra.run.smithi151.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110834 s, 946 MB/s 2023-09-23T22:25:47.379 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-09-23T22:25:47.897 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_4 2023-09-23T22:25:48.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:47 smithi151 conmon[100454]: audit 2023-09-23T22:25:46.916037+0000 mon.a (mon.0) 369 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:48.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:47 smithi151 conmon[100454]: cluster 2023-09-23T22:25:46.916229+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:25:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:47 smithi151 conmon[100454]: audit 2023-09-23T22:25:46.916689+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:47 smithi151 conmon[100454]: audit 2023-09-23T22:25:46.920585+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[110082]: audit 2023-09-23T22:25:46.916037+0000 mon.a (mon.0) 369 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[110082]: cluster 2023-09-23T22:25:46.916229+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:25:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[110082]: audit 2023-09-23T22:25:46.916689+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[110082]: audit 2023-09-23T22:25:46.920585+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[100224]: audit 2023-09-23T22:25:46.916037+0000 mon.a (mon.0) 369 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]': finished 2023-09-23T22:25:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[100224]: cluster 2023-09-23T22:25:46.916229+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:25:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[100224]: audit 2023-09-23T22:25:46.916689+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:47 smithi052 conmon[100224]: audit 2023-09-23T22:25:46.920585+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cluster 2023-09-23T22:25:46.033377+0000 osd.3 (osd.3 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: ) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cluster 2023-09-23T22:25:46.033423+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cluster 2023-09-23T22:25:47.876848+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cluster 2023-09-23T22:25:47.921017+0000 mon.a (mon.0) 373 : cluster [INF] osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400] boot 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cluster 2023-09-23T22:25:47.921086+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:25:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:47.921561+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:48.120307+0000 mgr.y (mgr.14148) 106 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.125022+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.126009+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.126739+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.127447+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.128155+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:48.128722+0000 mgr.y (mgr.14148) 107 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 4037M 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.131987+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.133448+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.134139+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:48 smithi151 conmon[100454]: audit 2023-09-23T22:25:48.139087+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cluster 2023-09-23T22:25:46.033377 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: +0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cluster 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 2023-09-23T22:25:46.033423+0000 osd.3 ( 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: osd.3) 2 : cluster [DBG] 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: purged_snaps scrub ok 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:47.876848+0000 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: mgr.y (mgr.14148) 105 : 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:47.921017+0000 mon.a 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: (mon.0) 373 : cluster [INF] osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400] boot 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: T22:25:47.921086+0000 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: mon.a (mon.0) 374 : cluster 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:47.921561 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: +0000 mon.a (mon.0) 375 : 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 2023-09-23T22:25: 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 48.120307+0000 mgr.y (mgr 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: .14148) 106 : cephadm [INF] 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: Detected new or changed devices on smithi052 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 2023-09-23T22:25 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: :48.125022+0000 mon.a (mon.0 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: ) 376 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:48.126009+0000 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: mon.a (mon.0) 377 : audit 2023-09-23T22:25:49.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:48.126739+0000 mon.a (mon 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: .0) 378 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -23T22:25:48.127447+0000 mon.a (mon.0) 379 : 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023-09-23T22:25 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: :48.128155+0000 mon.a (mon.0) 380 : audit 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: cephadm 2023-09-23T 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 22:25:48.128722+0000 mgr.y (mgr.14148) 107 2023-09-23T22:25:49.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: : cephadm [INF] Adjusting osd_memory_target on smithi052 to 4037M 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cluster 2023-09-23T22:25:46.033377 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: +0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cluster 2023-09-23T22:25 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: :46.033423+0000 osd.3 (osd.3) 2 : cluster 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: [DBG] purged_snaps scrub ok 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:25:49.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: T22:25:47.876848+0000 mgr.y (mgr.14148 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: ) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cluster 2023-09 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: -23T22:25:47.921017+0000 mon.a (mon 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: .0) 373 : cluster [INF] osd.3 [v2:172.21.15.52:6826/1128061400,v1:172.21.15.52:6827/1128061400] boot 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 09-23T22:25:47.921086+0000 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: mon.a (mon.0) 374 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:25:49.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:25:49.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 23T22:25:47.921561+0000 mon.a (mon.0) 375 : 2023-09-23T22:25:49.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:25:49.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cephadm 2023-09- 2023-09-23T22:25:49.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 23T22:25:48.120307+0000 mgr.y (mgr.14148 2023-09-23T22:25:49.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: ) 106 : cephadm [INF] Detected new or changed devices on smithi052 2023-09-23T22:25:49.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:25:49.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: -23T22:25:48.125022+0000 mon.a (mon. 2023-09-23T22:25:49.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 0) 376 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023 2023-09-23T22:25:49.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: -09-23T22:25:48.126009 2023-09-23T22:25:49.252 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: +0000 mon.a (mon.0) 377 2023-09-23T22:25:49.252 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.252 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09-23T22:25:49.252 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:25:49.252 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 25:48.126739+0000 mon.a (mon 2023-09-23T22:25:49.252 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: .0) 378 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: -23T22:25:48.127447+0000 mon.a 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: (mon.0) 379 : audit 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09-23T22:25:48.128155+0000 mon.a 2023-09-23T22:25:49.253 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: (mon.0) 380 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:25:49.254 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: cephadm 2023 2023-09-23T22:25:49.254 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: -09-23T22:25:48.128722+0000 mgr.y (mgr. 2023-09-23T22:25:49.254 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 14148) 107 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 4037M 2023-09-23T22:25:49.254 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:25:49.254 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 09-23T22:25:48.131987+0000 mon.a (mon.0) 2023-09-23T22:25:49.254 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 381 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: 23T22:25:48.131987+0000 mon.a (mon.0 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: ) 381 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:48.133448+0000 mon.a ( 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:49.255 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:48.134139+0000 mon.a 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: (mon.0) 383 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: -09-23T22:25:48.139087+0000 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[100224]: mon.a (mon.0) 384 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 22:25:48.133448+0000 mon.a (mon.0) 382 : audit 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09-23T22:25:48 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: .134139+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 2023-09-23T22:25:49.256 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: audit 2023-09-23T22:25:49.257 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 2023-09-23T22:25:48. 2023-09-23T22:25:49.257 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 139087+0000 mon.a (mon.0) 2023-09-23T22:25:49.257 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:48 smithi052 conmon[110082]: 384 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:25:50.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:49 smithi151 conmon[100454]: cluster 2023-09-23T22:25:48.932635+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:25:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:49 smithi151 conmon[100454]: audit 2023-09-23T22:25:49.594537+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:49 smithi151 conmon[100454]: audit 2023-09-23T22:25:49.597425+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:49 smithi151 conmon[100454]: audit 2023-09-23T22:25:49.598211+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: cluster 2023-09-23T22:25:48.932635+0000 mon.a (mon. 2023-09-23T22:25:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: 0) 385 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:25:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: audit 2023-09-23T22:25:49.594537+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: audit 2023-09-23T22:25:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: 2023-09-23T22 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: :25:49.597425+0000 mon.a (mon 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: .0) 387 : audit [INF] 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[100224]: 22:25:49.598211+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: cluster 2023-09-23T22:25: 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: 48.932635+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: :25:49.594537+0000 mon.a (mon.0) 386 : audit 2023-09-23T22:25:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:25:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: audit 2023-09-23T22:25:49. 2023-09-23T22:25:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: 597425+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:25:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: 2023-09-23T22:25:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: audit 2023 2023-09-23T22:25:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: -09-23T22:25:49.598211+0000 mon.a (mon.0) 388 2023-09-23T22:25:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:49 smithi052 conmon[110082]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:51 smithi151 conmon[100454]: audit 2023-09-23T22:25:49.591263+0000 mgr.y (mgr.14148) 108 : audit [DBG] from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:51 smithi151 conmon[100454]: cluster 2023-09-23T22:25:49.877217+0000 mgr.y (mgr.14148) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:51.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[100224]: audit 2023-09-23T22:25:49.591263 2023-09-23T22:25:51.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 108 : audit [DBG] from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:51.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[100224]: cluster 2023-09-23T22:25:49.877217 2023-09-23T22:25:51.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:51.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[110082]: audit 2023-09-23T22:25: 2023-09-23T22:25:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[110082]: 49.591263+0000 mgr.y (mgr.14148) 108 : audit [DBG] from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:25:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:25:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[110082]: 09-23T22:25:49.877217 2023-09-23T22:25:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[110082]: +0000 mgr.y (mgr.14148) 109 2023-09-23T22:25:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:51 smithi052 conmon[110082]: : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:52 smithi151 conmon[100454]: audit 2023-09-23T22:25:51.671375+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.151:0/1039819275' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]: dispatch 2023-09-23T22:25:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:52 smithi151 conmon[100454]: audit 2023-09-23T22:25:51.672001+0000 mon.a (mon.0) 389 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]: dispatch 2023-09-23T22:25:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:52 smithi151 conmon[100454]: audit 2023-09-23T22:25:51.677554+0000 mon.a (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]': finished 2023-09-23T22:25:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:52 smithi151 conmon[100454]: cluster 2023-09-23T22:25:51.677663+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:25:52.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:52 smithi151 conmon[100454]: audit 2023-09-23T22:25:51.677807+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:25:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: audit 2023-09-23T22:25:51.671375+0000 2023-09-23T22:25:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.151:0/1039819275' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]: dispatch 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: audit 2023-09-23T22:25:51. 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: 672001+0000 mon.a (mon.0) 389 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]: dispatch 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: audit 2023-09-23T22:25:51.677554+0000 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: mon.a (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]': finished 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: -09-23T22:25:51.677663+0000 mon.a (mon.0 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: ) 391 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: audit 2023-09-23T22:25: 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: 51.677807+0000 mon.a (mon.0) 392 : 2023-09-23T22:25:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[100224]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: audit 2023-09-23T22:25:51.671375+0000 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.151:0/1039819275' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]: dispatch 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: audit 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: 2023-09-23T22:25:51.672001+0000 mon.a (mon.0) 389 : audit [INF] 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]: dispatch 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: audit 2023-09-23T22:25:51.677554+0000 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: mon.a (mon.0) 390 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a8b1470a-6d28-44d9-9921-6f8fbcb52b59"}]': finished 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: cluster 2023-09 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: -23T22:25:51.677663+0000 mon.a (mon.0) 391 : cluster [DBG] 2023-09-23T22:25:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:25:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:25:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:52 smithi052 conmon[110082]: 22:25:51.677807+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:25:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:53 smithi151 conmon[100454]: cluster 2023-09-23T22:25:51.877593+0000 mgr.y (mgr. 2023-09-23T22:25:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:53 smithi151 conmon[100454]: 14148) 110 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:53 smithi151 conmon[100454]: audit 2023-09-23T22:25:52.256631+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.151:0/3780081460' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[100224]: cluster 2023-09-23T22:25:51. 2023-09-23T22:25:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[100224]: 877593+0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[100224]: audit 2023-09-23T22:25:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[100224]: 2023-09-23T22:25:52.256631+0000 mon.b (mon.1) 9 : audit [DBG] 2023-09-23T22:25:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[100224]: from='client.? 172.21.15.151:0/3780081460' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:53.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[110082]: cluster 2023-09-23T22:25:51.877593 2023-09-23T22:25:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[110082]: +0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[110082]: audit 2023-09-23T22:25:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:53 smithi052 conmon[110082]: 2023-09-23T22:25:52.256631+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.151:0/3780081460' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:25:55.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:55 smithi151 conmon[100454]: cluster 2023-09-23T22:25:53.878020+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:55.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:55 smithi052 conmon[100224]: cluster 2023-09-23T22:25:53.878020+0000 mgr.y (mgr 2023-09-23T22:25:55.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:55 smithi052 conmon[100224]: .14148) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:55 smithi052 conmon[110082]: cluster 2023-09-23T22:25:53 2023-09-23T22:25:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:55 smithi052 conmon[110082]: .878020+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:57.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:57 smithi151 conmon[100454]: cluster 2023-09-23T22:25:55.878598+0000 mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:57.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:57 smithi052 conmon[100224]: cluster 2023-09-23T22:25:55.878598+0000 2023-09-23T22:25:57.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:57 smithi052 conmon[100224]: mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:57 smithi052 conmon[110082]: cluster 2023-09-23T22:25: 2023-09-23T22:25:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:57 smithi052 conmon[110082]: 55.878598+0000 mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:58.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:58 smithi151 conmon[100454]: audit 2023-09-23T22:25:57.495709+0000 mon.a (mon.0) 393 : audit 2023-09-23T22:25:58.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:58 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:25:58.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:58 smithi151 conmon[100454]: audit 2023-09-23T22:25:57.496626+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:58.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[100224]: audit 2023-09-23T22:25:57.495709+0000 mon.a (mon.0 2023-09-23T22:25:58.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[100224]: ) 393 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:25:58.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[100224]: audit 2023-09-23T22:25:57.496626+0000 mon.a (mon.0) 394 : audit [DBG] 2023-09-23T22:25:58.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[110082]: audit 2023-09-23T22:25:57. 2023-09-23T22:25:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[110082]: 495709+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:25:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:25:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[110082]: 23T22:25:57.496626+0000 2023-09-23T22:25:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[110082]: mon.a (mon.0) 394 : audit 2023-09-23T22:25:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:58 smithi052 conmon[110082]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:59.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:59 smithi151 conmon[100454]: cephadm 2023-09-23T22:25:57.497439+0000 mgr.y (mgr.14148) 113 : cephadm [INF] Deploying daemon osd.4 on smithi151 2023-09-23T22:25:59.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:25:59 smithi151 conmon[100454]: cluster 2023-09-23T22:25:57.879166+0000 mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[100224]: cephadm 2023-09-23T22:25:57.497439+0000 mgr.y (mgr. 2023-09-23T22:25:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[100224]: 14148) 113 : cephadm [INF] Deploying daemon osd.4 on smithi151 2023-09-23T22:25:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[100224]: cluster 2023-09-23T22:25:57.879166+0000 2023-09-23T22:25:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[100224]: mgr.y (mgr.14148) 114 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:25:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[110082]: cephadm 2023-09-23T22:25:57. 2023-09-23T22:25:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[110082]: 497439+0000 mgr.y (mgr.14148) 113 : cephadm [INF] Deploying daemon osd.4 on smithi151 2023-09-23T22:25:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:25:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[110082]: 23T22:25:57.879166 2023-09-23T22:25:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[110082]: +0000 mgr.y (mgr.14148) 114 2023-09-23T22:25:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:25:59 smithi052 conmon[110082]: : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:01 smithi151 conmon[100454]: cluster 2023-09-23T22:25:59.879555+0000 mgr.y (mgr 2023-09-23T22:26:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:01 smithi151 conmon[100454]: .14148) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:01.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:01 smithi052 conmon[110082]: cluster 2023-09-23T22:25:59.879555+0000 mgr.y (mgr.14148) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:01 smithi052 conmon[100224]: cluster 2023-09-23T22:25:59.879555+0000 mgr.y 2023-09-23T22:26:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:01 smithi052 conmon[100224]: (mgr.14148) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:03 smithi151 conmon[100454]: cluster 2023-09-23T22:26:01.879973+0000 mgr.y (mgr.14148) 2023-09-23T22:26:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:03 smithi151 conmon[100454]: 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:03 smithi151 conmon[100454]: audit 2023-09-23T22:26:02.180611+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:03 smithi151 conmon[100454]: audit 2023-09-23T22:26:02.193247+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: cluster 2023-09-23T22:26:01.879973 2023-09-23T22:26:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:26:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: 22:26:02.180611+0000 mon.a (mon.0) 395 : audit [INF] 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: 23T22:26:02.193247+0000 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: mon.a (mon.0) 396 : audit [DBG] 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: cluster 2023-09-23T22:26:01. 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: 879973+0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: audit 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: 2023-09-23T 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: 22:26:02.180611+0000 mon.a 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: (mon.0) 395 : audit 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: audit 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: 2023-09-23T22:26:02. 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: 193247+0000 mon.a (mon.0) 2023-09-23T22:26:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:03 smithi052 conmon[110082]: 396 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:05.007 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 4 on host 'smithi151' 2023-09-23T22:26:05.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:04 smithi151 conmon[100454]: cluster 2023-09-23T22:26:03.880407+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:04 smithi151 conmon[100454]: audit 2023-09-23T22:26:03.929763+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:04 smithi151 conmon[100454]: audit 2023-09-23T22:26:04.243001+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:04 smithi151 conmon[100454]: audit 2023-09-23T22:26:04.243989+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:04 smithi151 conmon[100454]: audit 2023-09-23T22:26:04.249870+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: cluster 2023-09-23T22:26:03. 2023-09-23T22:26:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: 880407+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: audit 2023-09-23T22:26:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: 2023-09-23T22:26:03.929763+0000 mon.a (mon. 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: 0) 397 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: 09-23T22:26:04.243001+0000 mon.a 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: (mon.0) 398 : audit [DBG] 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: :26:04.243989+0000 mon.a (mon.0) 399 : 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: audit 2023-09-23T22:26 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: :04.249870+0000 mon.a (mon.0) 400 : audit 2023-09-23T22:26:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: cluster 2023-09-23T22:26:03.880407+0000 mgr.y (mgr.14148) 117 : cluster 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: audit 2023-09-23T22:26:03.929763+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: audit 2023 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: -09-23T22:26 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: :04.243001+0000 mon.a (mon. 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: 0) 398 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: 09-23T22:26:04.243989+0000 2023-09-23T22:26:05.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: mon.a (mon.0) 399 : audit 2023-09-23T22:26:05.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:05.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: audit 2023 2023-09-23T22:26:05.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: -09-23T22:26:04.249870+0000 2023-09-23T22:26:05.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: mon.a (mon.0) 400 : audit 2023-09-23T22:26:05.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:04 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:05.589 DEBUG:teuthology.orchestra.run.smithi151:osd.4> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T22:26:05.591 INFO:tasks.cephadm:Deploying osd.5 on smithi151 with /dev/vg_nvme/lv_3... 2023-09-23T22:26:05.591 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T22:26:05.621 INFO:journalctl@ceph.osd.4.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:26:06.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:06 smithi151 conmon[100454]: audit 2023-09-23T22:26:05.006719+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:06 smithi151 conmon[100454]: audit 2023-09-23T22:26:05.018520+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:06 smithi151 conmon[100454]: audit 2023-09-23T22:26:05.417015+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:26:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:06 smithi151 conmon[100454]: audit 2023-09-23T22:26:05.417421+0000 mon.a (mon.0) 403 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:26:06.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: audit 2023-09-23T22:26:05.006719+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:06.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: audit 2023-09-23T22:26:05.018520+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:06.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: audit 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: 2023-09-23T22:26:05.417015+0000 mon.b (mon.1) 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: 10 : audit [INF] from='osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: 09-23T22:26:05.417421+0000 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: mon.a (mon.0) 403 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[100224]: : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: audit 2023-09-23T22:26:05.006719+0000 mon.a ( 2023-09-23T22:26:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: mon.0) 401 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: audit 2023-09-23T22:26: 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: 05.018520+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: audit 2023-09-23T22:26:05 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: .417015+0000 mon.b (mon.1) 10 : audit [INF] 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: from='osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: 26:05.417421+0000 mon.a (mon.0) 403 : audit 2023-09-23T22:26:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:06 smithi052 conmon[110082]: [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:26:07.388 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[106754]: debug 2023-09-23T22:26:07.114+0000 7f24d77bf700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T22:26:07.388 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[106754]: debug 2023-09-23T22:26:07.119+0000 7f24d1f37700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: cluster 2023-09-23T22:26:05.880833+0000 mgr.y (mgr.14148) 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: audit 2023-09-23T22:26:06.111031+0000 mon.a (mon.0) 404 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: cluster 2023-09-23T22:26:06.111158+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: audit 2023-09-23T22:26:06.111323+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: audit 2023-09-23T22:26:06.111801+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:07.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:07 smithi151 conmon[100454]: audit 2023-09-23T22:26:06.112203+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: cluster 2023-09-23T22:26:05.880833+0000 mgr.y 2023-09-23T22:26:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: (mgr.14148) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: audit 2023-09-23T22:26:06.111031+0000 mon.a (mon. 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: 0) 404 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: cluster 2023-09-23T22:26:06.111158+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: audit 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: 2023-09-23T22:26:06.111323+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: audit 2023-09-23T22:26:06.111801+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: audit 2023-09-23T22:26:06.112203+0000 mon.a (mon 2023-09-23T22:26:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[110082]: .0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: cluster 2023-09-23T22:26:05. 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: 880833+0000 mgr.y (mgr.14148) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: audit 2023-09-23T22:26:06.111031+0000 mon.a (mon.0) 404 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: cluster 2023-09-23T22:26:06.111158+0000 mon.a (mon.0) 405 : cluster [DBG] 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: audit 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: 2023-09-23T22:26:06.111323+0000 mon.a (mon.0) 406 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:07.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: audit 2023-09-23T22:26:07.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: 2023-09-23T22:26:06.111801+0000 mon.b (mon. 2023-09-23T22:26:07.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: 1) 11 : audit [INF] from='osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:07.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: audit 2023-09-23T22:26: 2023-09-23T22:26:07.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: 06.112203+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:07.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:07 smithi052 conmon[100224]: 2023-09-23T22:26:08.059 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T22:26:08.059 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T22:26:08.059 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-09-23T22:26:08.059 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-09-23T22:26:08.059 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114477 s, 916 MB/s 2023-09-23T22:26:08.060 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-09-23T22:26:08.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:08 smithi151 conmon[100454]: audit 2023-09-23T22:26:07.112933+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:08 smithi151 conmon[100454]: cluster 2023-09-23T22:26:07.113113+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:26:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:08 smithi151 conmon[100454]: audit 2023-09-23T22:26:07.113785+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:08 smithi151 conmon[100454]: audit 2023-09-23T22:26:07.117887+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:08.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[110082]: audit 2023-09-23T22:26:07.112933+0000 mon.a (mon.0) 408 : audit 2023-09-23T22:26:08.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[110082]: [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:08.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[110082]: cluster 2023-09-23T22:26:07.113113+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[110082]: audit 2023-09-23T22:26:07.113785+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[110082]: audit 2023-09-23T22:26:07.117887+0000 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[110082]: mon.a (mon.0) 411 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: audit 2023-09-23T22:26:07.112933 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: +0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: cluster 2023-09-23T22:26:07.113113+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: audit 2023-09-23T22:26:07.113785+0000 mon.a (mon.0) 410 : audit 2023-09-23T22:26:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: audit 2023-09-23T22:26:07. 2023-09-23T22:26:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: 117887+0000 mon.a (mon.0) 411 : audit [DBG] 2023-09-23T22:26:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:08 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:08.934 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_3 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: cluster 2023-09-23T22:26:06.424597+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: cluster 2023-09-23T22:26:06.424675+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: cluster 2023-09-23T22:26:07.881311+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: cluster 2023-09-23T22:26:08.114625+0000 mon.a (mon.0) 412 : cluster [INF] osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857] boot 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: cluster 2023-09-23T22:26:08.114770+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: audit 2023-09-23T22:26:08.114971+0000 mon.a (mon.0) 414 : audit [DBG] 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:09.392 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:09 smithi151 conmon[100454]: cluster 2023-09-23T22:26:09.116452+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:26:09.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: cluster 2023-09-23T22:26:06.424597+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: cluster 2023-09-23T22:26:06.424675+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: cluster 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: 2023-09-23T22:26:07.881311+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: -09-23T22:26:08.114625+0000 mon.a (mon.0) 412 : cluster [INF] osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857] boot 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: cluster 2023-09-23T22:26: 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: 08.114770+0000 mon.a (mon.0) 413 : cluster 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:26:09.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: -23T22:26:08.114971+0000 mon.a (mon.0) 414 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: cluster 2023-09-23T22:26:09.116452+0000 mon.a (mon.0 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[100224]: ) 415 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: cluster 2023-09-23T22:26:06.424597+0000 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: cluster 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: 2023-09-23T22:26:06. 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: 424675+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: cluster 2023-09 2023-09-23T22:26:09.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: -23T22:26:07.881311+0000 mgr.y (mgr.14148) 119 : cluster [DBG] 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: cluster 2023-09-23T22:26:08.114625+0000 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: mon.a (mon.0) 412 : cluster [INF] osd.4 [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857] boot 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: cluster 2023-09-23T22:26:08.114770+0000 mon.a (mon.0) 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: 413 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: audit 2023 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: -09-23T22:26:08.114971+0000 mon.a (mon.0) 414 : 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: cluster 2023-09-23T22:26:09.116452+0000 mon.a (mon.0) 415 : cluster 2023-09-23T22:26:09.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:09 smithi052 conmon[110082]: [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:26:10.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:09.113589+0000 mgr.y (mgr.14148) 2023-09-23T22:26:10.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: 120 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:10.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: audit 2023-09-23T22:26:09.120457+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: audit 2023-09-23T22:26:09.121664+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:10.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:09.122505+0000 mgr.y (mgr.14148) 121 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 17168M 2023-09-23T22:26:10.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: audit 2023-09-23T22:26:09.129621+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: audit 2023-09-23T22:26:09.130752+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:10.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: audit 2023-09-23T22:26:09.131671+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:10.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: audit 2023-09-23T22:26:09.136482+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:10 smithi151 conmon[100454]: cluster 2023-09-23T22:26:10.116674+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:26:10.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: cephadm 2023-09-23T22:26:09.113589+0000 mgr.y (mgr.14148) 120 : 2023-09-23T22:26:10.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: audit 2023-09-23T22:26:09.120457+0000 mon.a (mon.0) 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 416 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: audit 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 2023-09-23T22:26:09.121664+0000 mon.a (mon. 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 0) 417 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: cephadm 2023-09-23T22:26:09.122505+0000 mgr.y 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: (mgr.14148) 121 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 17168M 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: audit 2023-09-23T22:26:09.129621+0000 mon.a (mon.0) 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 418 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: -23T22:26:09.130752+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: audit 2023-09-23T22:26:09.131671+0000 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 09-23T22:26:09.136482+0000 mon.a (mon 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: .0) 421 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: cluster 2023-09- 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 23T22:26:10.116674+0000 mon.a (mon.0) 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[100224]: 422 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:26:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: cephadm 2023-09-23T22:26:09.113589+0000 mgr.y (mgr. 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: 14148) 120 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit 2023-09-23T22:26:09.120457+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit 2023-09-23T22:26:09.121664+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: cephadm 2023-09-23T22:26:09.122505+0000 mgr.y (mgr.14148 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: ) 121 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 17168M 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: 09-23T22:26:09.129621+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit 2023-09-23T22:26:09.130752+0000 mon.a (mon.0) 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: 419 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit 2023-09-23T22:26 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: :09.131671+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: -23T22:26:09.136482 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: +0000 mon.a (mon.0) 421 : 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: cluster 2023 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: -09-23T22:26:10.116674 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: +0000 mon.a (mon.0) 422 2023-09-23T22:26:10.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:10 smithi052 conmon[110082]: : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:26:11.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:11 smithi151 conmon[100454]: cluster 2023-09-23T22:26:09.881832+0000 mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v103: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:11.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:11 smithi151 conmon[100454]: audit 2023-09-23T22:26:10.761059+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:11.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:11 smithi151 conmon[100454]: audit 2023-09-23T22:26:10.763991+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:11.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:11 smithi151 conmon[100454]: audit 2023-09-23T22:26:10.764825+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: cluster 2023-09-23T22:26:09. 2023-09-23T22:26:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: 881832+0000 mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v103: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: audit 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: 2023-09-23T22:26:10.761059+0000 mon.a (mon.0) 423 : audit [DBG] 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: 23T22:26:10.763991+0000 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: mon.a (mon.0) 424 : audit 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: -09-23T22:26:10.764825 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: +0000 mon.a (mon.0) 425 : 2023-09-23T22:26:11.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[100224]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: cluster 2023-09-23T22:26:09.881832+0000 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: mgr.y (mgr.14148) 122 : cluster [DBG] pgmap v103: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: audit 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: 2023-09-23T22:26:10.761059+0000 mon.a (mon 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: .0) 423 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: 23T22:26:10.763991+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: 2023-09-23T22:26:11.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:11.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:11 smithi052 conmon[110082]: 09-23T22:26:10.764825+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:12.302 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:12 smithi151 conmon[100454]: audit 2023-09-23T22:26:10.758067+0000 mgr.y (mgr.14148) 123 : audit [DBG] from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:12.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:12 smithi052 conmon[100224]: audit 2023-09-23T22:26:10.758067+0000 mgr.y ( 2023-09-23T22:26:12.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:12 smithi052 conmon[100224]: mgr.14148) 123 : audit [DBG] from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:12 smithi052 conmon[110082]: audit 2023-09-23T22:26:10.758067+0000 mgr.y 2023-09-23T22:26:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:12 smithi052 conmon[110082]: (mgr.14148) 123 : audit [DBG] from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: cluster 2023-09-23T22:26:11.882220+0000 mgr.y (mgr.14148 2023-09-23T22:26:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: ) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: audit 2023-09-23T22:26:12.997458+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.151:0/1562013872' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]: dispatch 2023-09-23T22:26:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: audit 2023-09-23T22:26:12.997964+0000 mon.a (mon.0) 426 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]: dispatch 2023-09-23T22:26:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: audit 2023-09-23T22:26:13.002850+0000 mon.a (mon.0) 427 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]': finished 2023-09-23T22:26:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: cluster 2023-09-23T22:26:13.002953+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T22:26:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:13 smithi151 conmon[100454]: audit 2023-09-23T22:26:13.003131+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: cluster 2023-09-23T22:26:11.882220+0000 mgr.y (mgr. 2023-09-23T22:26:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: 14148) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: audit 2023-09-23T22:26:12.997458+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.151:0/1562013872' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]: dispatch 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: audit 2023-09-23T22:26:12.997964+0000 mon.a ( 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: mon.0) 426 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]: dispatch 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: audit 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: 2023-09-23T22:26: 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: 13.002850+0000 mon.a (mon.0) 427 : 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]': finished 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: -23T22:26:13. 2023-09-23T22:26:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: 002953+0000 mon.a (mon.0) 428 : cluster 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: 23T22:26:13.003131+0000 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: mon.a (mon.0) 429 : audit [DBG] 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: cluster 2023-09-23T22:26:11 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: .882220+0000 mgr.y (mgr.14148) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: audit 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: 2023-09-23T22:26:12.997458+0000 mon.b (mon 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: .1) 12 : audit [INF] from='client.? 172.21.15.151:0/1562013872' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]: dispatch 2023-09-23T22:26:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: audit 2023-09-23T22:26:12 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: .997964+0000 mon.a (mon.0) 426 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]: dispatch 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: audit 2023-09-23T22:26:13.002850+0000 mon.a (mon.0) 427 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "768d49a4-59ee-43e4-802b-a58f2b77580b"}]': finished 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: cluster 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: 2023-09-23T22:26:13.002953 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: +0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: audit 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: 2023-09-23T22 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: :26:13.003131+0000 mon.a (mon.0) 429 : 2023-09-23T22:26:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:13 smithi052 conmon[110082]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:14.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:14 smithi151 conmon[100454]: audit 2023-09-23T22:26:13.598671+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.151:0/2448383387' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:14 smithi052 conmon[100224]: audit 2023-09-23T22:26:13.598671+0000 mon.b (mon 2023-09-23T22:26:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:14 smithi052 conmon[100224]: .1) 13 : audit [DBG] from='client.? 172.21.15.151:0/2448383387' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:14.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:14 smithi052 conmon[110082]: audit 2023-09-23T22:26:13.598671+0000 mon.b 2023-09-23T22:26:14.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:14 smithi052 conmon[110082]: (mon.1) 13 : audit [DBG] from='client.? 172.21.15.151:0/2448383387' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:15 smithi151 conmon[100454]: cluster 2023-09-23T22:26:13.882577+0000 mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:15.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:15 smithi052 conmon[100224]: cluster 2023-09-23T22:26:13.882577+0000 mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:15.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:15 smithi052 conmon[100224]: 2023-09-23T22:26:15.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:15 smithi052 conmon[110082]: cluster 2023-09-23T22:26:13.882577+0000 2023-09-23T22:26:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:15 smithi052 conmon[110082]: mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:17.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:17 smithi151 conmon[100454]: cluster 2023-09-23T22:26:15.882965+0000 mgr.y ( 2023-09-23T22:26:17.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:17 smithi151 conmon[100454]: mgr.14148) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:17.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:17 smithi052 conmon[100224]: cluster 2023-09-23T22:26:15.882965+0000 2023-09-23T22:26:17.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:17 smithi052 conmon[100224]: mgr.y (mgr.14148) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:17.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:17 smithi052 conmon[110082]: cluster 2023-09-23T22:26:15.882965+0000 mgr.y 2023-09-23T22:26:17.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:17 smithi052 conmon[110082]: (mgr.14148) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:19.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:19 smithi151 conmon[100454]: cluster 2023-09-23T22:26:17.883377+0000 mgr.y (mgr.14148) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:19.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:19 smithi151 conmon[100454]: audit 2023-09-23T22:26:18.934386+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:26:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:19 smithi151 conmon[100454]: audit 2023-09-23T22:26:18.935376+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[100224]: cluster 2023-09-23T22:26:17.883377+0000 mgr.y (mgr.14148) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[100224]: audit 2023-09-23T22:26:18.934386+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:26:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[100224]: audit 2023-09-23T22:26:18.935376+0000 mon.a (mon.0) 431 : audit 2023-09-23T22:26:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:19.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[110082]: cluster 2023-09-23T22:26:17.883377+0000 mgr.y (mgr.14148) 127 : cluster 2023-09-23T22:26:19.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[110082]: [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:19.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[110082]: audit 2023-09-23T22:26:18.934386+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:26:19.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[110082]: audit 2023-09-23T22:26:18.935376+0000 mon.a (mon.0) 2023-09-23T22:26:19.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:19 smithi052 conmon[110082]: 431 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:20.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:18.936197+0000 mgr.y (mgr.14148 2023-09-23T22:26:20.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:20 smithi151 conmon[100454]: ) 128 : cephadm [INF] Deploying daemon osd.5 on smithi151 2023-09-23T22:26:20.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:20 smithi052 conmon[100224]: cephadm 2023-09-23T22:26:18.936197+0000 mgr.y (mgr.14148 2023-09-23T22:26:20.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:20 smithi052 conmon[100224]: ) 128 : cephadm [INF] Deploying daemon osd.5 on smithi151 2023-09-23T22:26:20.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:20 smithi052 conmon[110082]: cephadm 2023-09-23T22:26:18.936197+0000 mgr.y ( 2023-09-23T22:26:20.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:20 smithi052 conmon[110082]: mgr.14148) 128 : cephadm [INF] Deploying daemon osd.5 on smithi151 2023-09-23T22:26:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:21 smithi151 conmon[100454]: cluster 2023-09-23T22:26:19.883747+0000 mgr.y (mgr.14148) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:21.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:21 smithi052 conmon[100224]: cluster 2023-09-23T22:26:19.883747+0000 mgr.y (mgr.14148) 2023-09-23T22:26:21.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:21 smithi052 conmon[100224]: 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:21.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:21 smithi052 conmon[110082]: cluster 2023-09-23T22:26:19 2023-09-23T22:26:21.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:21 smithi052 conmon[110082]: .883747+0000 mgr.y (mgr.14148) 129 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:23.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:23 smithi151 conmon[100454]: cluster 2023-09-23T22:26:21. 2023-09-23T22:26:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:23 smithi151 conmon[100454]: 884125+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:23.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:23 smithi052 conmon[100224]: cluster 2023-09-23T22 2023-09-23T22:26:23.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:23 smithi052 conmon[100224]: :26:21.884125+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:23.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:23 smithi052 conmon[110082]: cluster 2023-09-23T22:26:21.884125+0000 mgr.y (mgr 2023-09-23T22:26:23.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:23 smithi052 conmon[110082]: .14148) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:24.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:24 smithi151 conmon[100454]: audit 2023-09-23T22:26:23.654062+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:24 smithi151 conmon[100454]: audit 2023-09-23T22:26:23.692548+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:24 smithi151 conmon[100454]: cluster 2023-09-23T22:26:23.884667+0000 mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[100224]: audit 2023-09-23T22:26:23.654062+0000 mon.a ( 2023-09-23T22:26:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[100224]: mon.0) 432 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[100224]: audit 2023-09-23T22:26:23.692548+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[100224]: cluster 2023-09-23T22:26:23.884667+0000 mgr.y ( 2023-09-23T22:26:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[100224]: mgr.14148) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[110082]: audit 2023-09-23T22:26:23.654062+0000 2023-09-23T22:26:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[110082]: mon.a (mon.0) 432 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[110082]: audit 2023-09-23T22:26:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[110082]: 2023-09-23T22:26:23.692548+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[110082]: cluster 2023-09-23T 2023-09-23T22:26:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:24 smithi052 conmon[110082]: 22:26:23.884667+0000 mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:26.480 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 5 on host 'smithi151' 2023-09-23T22:26:26.634 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:26 smithi151 conmon[100454]: audit 2023-09-23T22:26:25.365080+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:26.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:26 smithi151 conmon[100454]: audit 2023-09-23T22:26:25.366116+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:26.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:26 smithi151 conmon[100454]: audit 2023-09-23T22:26:25.367250+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:26.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:26 smithi151 conmon[100454]: audit 2023-09-23T22:26:25.372273+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[100224]: audit 2023-09-23T22:26:25.365080+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[100224]: audit 2023-09-23T22:26:25.366116+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[100224]: audit 2023-09-23T22:26:25.367250+0000 mon.a (mon.0) 436 : 2023-09-23T22:26:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[100224]: audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[100224]: audit 2023-09-23T22:26:25 2023-09-23T22:26:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[100224]: .372273+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: audit 2023-09-23T22:26:25.365080+0000 mon.a (mon.0) 2023-09-23T22:26:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: 434 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: audit 2023-09-23T22:26:25.366116+0000 2023-09-23T22:26:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: mon.a (mon.0) 435 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: audit 2023-09-23T22:26:25.367250 2023-09-23T22:26:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: +0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: 2023-09-23T22:26:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: audit 2023-09-23T22:26:25.372273+0000 2023-09-23T22:26:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:26 smithi052 conmon[110082]: mon.a (mon.0) 437 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:27.007 DEBUG:teuthology.orchestra.run.smithi151:osd.5> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T22:26:27.009 INFO:tasks.cephadm:Deploying osd.6 on smithi151 with /dev/vg_nvme/lv_2... 2023-09-23T22:26:27.009 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T22:26:27.038 INFO:journalctl@ceph.osd.5.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:26:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:27 smithi151 conmon[100454]: cluster 2023-09-23T22:26:25.885091 2023-09-23T22:26:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:27 smithi151 conmon[100454]: +0000 mgr.y (mgr.14148) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:27 smithi151 conmon[100454]: audit 2023-09-23T22:26:26.479510+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:27 smithi151 conmon[100454]: audit 2023-09-23T22:26:26.491848+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:27 smithi151 conmon[100454]: audit 2023-09-23T22:26:26.896341+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:26:27.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:27 smithi151 conmon[100454]: audit 2023-09-23T22:26:26.896675+0000 mon.a (mon.0) 440 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:26:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: cluster 2023-09-23T22:26:25.885091+0000 mgr.y (mgr 2023-09-23T22:26:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: .14148) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: audit 2023-09-23T22:26:26.479510 2023-09-23T22:26:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: +0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: 23T22:26:26.491848+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: audit 2023-09-23T22:26:26.896341+0000 mon.b (mon.1) 14 : audit [INF] 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: from='osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: audit 2023-09-23T22:26:26.896675 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[100224]: +0000 mon.a (mon.0) 440 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: cluster 2023-09-23T22:26:25.885091+0000 mgr.y (mgr 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: .14148) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: 22:26:26.479510+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: 23T22:26:26.491848+0000 mon.a (mon.0) 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: 439 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: audit 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: 2023-09-23T22:26:26 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: .896341+0000 mon.b (mon.1) 14 : audit [INF] 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: from='osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: audit 2023-09-23T22:26:26 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: .896675+0000 mon.a (mon.0) 440 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:26:27.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:27 smithi052 conmon[110082]: 2023-09-23T22:26:28.597 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[100454]: audit 2023-09-23T22:26:27.480551+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:26:28.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[100454]: cluster 2023-09-23T22:26:27.480636+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T22:26:28.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[100454]: audit 2023-09-23T22:26:27.480803+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:28.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[100454]: audit 2023-09-23T22:26:27.481252+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:28.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[100454]: audit 2023-09-23T22:26:27.481537+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:28.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[100454]: cluster 2023-09-23T22:26:27.885481+0000 mgr.y (mgr.14148) 133 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:28.598 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[112515]: debug 2023-09-23T22:26:28.486+0000 7ff06ddc8700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T22:26:28.598 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 22:26:28 smithi151 conmon[112515]: debug 2023-09-23T22:26:28.491+0000 7ff064d39700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:26:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: audit 2023-09-23T22:26:27.480551+0000 mon.a (mon.0) 441 2023-09-23T22:26:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:26:28.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: cluster 2023-09-23T22:26:27.480636+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: 26:27.480803+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: audit 2023-09-23T22:26:27.481252+0000 mon.b 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: audit 2023-09-23T22:26:27 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: .481537+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: cluster 2023-09-23T22:26:27.885481+0000 mgr.y (mgr.14148 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[100224]: ) 133 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: audit 2023-09-23T22:26:27.480551+0000 2023-09-23T22:26:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: cluster 2023-09-23T22:26: 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: 27.480636+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: 23T22:26:27.480803+0000 mon.a (mon. 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: 0) 443 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: 22:26:27.481252+0000 mon.b (mon.1) 15 : audit [INF] 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: from='osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: audit 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: 2023-09-23T22:26:27.481537+0000 mon.a 2023-09-23T22:26:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:28.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: cluster 2023-09-23T22:26:28.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: 2023-09-23T22:26:27.885481+0000 2023-09-23T22:26:28.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:28 smithi052 conmon[110082]: mgr.y (mgr.14148) 133 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-09-23T22:26:29.446 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T22:26:29.446 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T22:26:29.446 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-09-23T22:26:29.446 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-09-23T22:26:29.446 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112289 s, 934 MB/s 2023-09-23T22:26:29.446 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-09-23T22:26:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: audit 2023-09-23T22:26:28.482109+0000 mon.a (mon.0 2023-09-23T22:26:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: ) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: cluster 2023-09-23T22:26:28.482222+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:26:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: :26:28.482780+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: audit 2023-09-23T22:26:28.487136+0000 mon.a (mon 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[100224]: .0) 448 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: audit 2023-09-23T22:26:28.482109+0000 mon.a (mon.0) 445 : audit 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: cluster 2023-09-23T22:26:28.482222+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: audit 2023-09-23T22:26:28.482780+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: 2023-09-23T22:26:29.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:29.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: 23T22:26:28.487136+0000 mon.a (mon.0 2023-09-23T22:26:29.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:29 smithi052 conmon[110082]: ) 448 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:29.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:29 smithi151 conmon[100454]: audit 2023-09-23T22:26:28.482109+0000 2023-09-23T22:26:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:29 smithi151 conmon[100454]: mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:29 smithi151 conmon[100454]: cluster 2023-09-23T22:26:28.482222+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:26:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:29 smithi151 conmon[100454]: audit 2023-09-23T22:26:28.482780+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:29 smithi151 conmon[100454]: audit 2023-09-23T22:26:28.487136+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:30.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_2 2023-09-23T22:26:30.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: cluster 2023-09-23T22:26:27.894616+0000 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: cluster 2023-09-23T22:26:27.894694+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: cluster 2023-09-23T22:26:29.484550+0000 mon.a (mon.0) 449 : cluster [INF] osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865] boot 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: cluster 2023-09-23T22:26:29.484624+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: audit 2023-09-23T22:26:29.484785+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: cluster 2023-09-23T22:26:29.885938+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:30 smithi151 conmon[100454]: cluster 2023-09-23T22:26:30.110398+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T22:26:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: cluster 2023-09-23T22:26:27.894616+0000 osd.5 ( 2023-09-23T22:26:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: cluster 2023-09-23T22:26:27.894694+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: cluster 2023-09-23T22:26:29.484550+0000 mon.a (mon.0) 449 : cluster [INF] osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865] boot 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: cluster 2023-09-23T22:26:29.484624+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: audit 2023-09-23T22:26: 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: 29.484785+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: cluster 2023-09-23T22:26:29.885938 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: cluster 2023-09-23T22:26:30.110398+0000 mon.a (mon. 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[100224]: 0) 452 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: cluster 2023-09-23T22:26:27.894616+0000 osd.5 (osd.5) 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: T22:26:27.894694+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: cluster 2023-09-23T22:26:29.484550+0000 mon.a ( 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: mon.0) 449 : cluster [INF] osd.5 [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865] boot 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: cluster 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: 2023-09-23T22:26:29.484624+0000 mon.a ( 2023-09-23T22:26:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: mon.0) 450 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: audit 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: 2023-09-23T22:26:29.484785+0000 mon.a ( 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: cluster 2023-09-23T22:26 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: :29.885938+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: :26:30.110398+0000 mon.a (mon.0) 452 : cluster 2023-09-23T22:26:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:30 smithi052 conmon[110082]: [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-09-23T22:26:31.998 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:30.903207+0000 mgr.y (mgr.14148) 135 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:31.998 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.908120+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:31.998 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.909281+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.910315+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:30.911028+0000 mgr.y (mgr.14148) 136 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 8584M 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.915039+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.916135+0000 mon.a (mon 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: .0) 457 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.917107+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: audit 2023-09-23T22:26:30.921700+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:31.999 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:31 smithi151 conmon[100454]: cluster 2023-09-23T22:26:31.115866+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:26:32.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: cephadm 2023-09-23T22:26:30.903207+0000 2023-09-23T22:26:32.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: mgr.y (mgr.14148) 135 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09-23T22:26:30.908120+0000 mon.a 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: (mon.0) 453 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09-23T22:26:30.909281+0000 mon.a 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: (mon.0) 454 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: 23T22:26:30.910315+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: cephadm 2023-09-23T22:26:30.911028+0000 mgr.y (mgr.14148) 136 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: : cephadm [INF] Adjusting osd_memory_target on smithi151 to 8584M 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: 23T22:26:30.915039+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09-23T22:26:30.916135+0000 mon.a (mon. 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: 0) 457 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: 2023-09-23T22:26:30.917107+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: -23T22:26:30.921700+0000 mon.a (mon.0 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: ) 459 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: cluster 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: 26:31.115866+0000 mon.a (mon.0) 460 2023-09-23T22:26:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[110082]: : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: cephadm 2023-09-23T22:26:30.903207+0000 mgr.y (mgr. 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 14148) 135 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023-09-23T22:26:30.908120+0000 mon.a (mon.0) 453 : audit [INF] 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 09-23T22:26:30.909281+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023-09-23T22:26:30.910315+0000 mon.a 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: (mon.0) 455 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: cephadm 2023 2023-09-23T22:26:32.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: -09-23T22:26:30.911028+0000 mgr.y (mgr.14148 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: ) 136 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 8584M 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: -23T22:26:30.915039+0000 mon.a (mon.0) 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 456 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: -23T22:26:30.916135+0000 mon.a (mon.0) 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 457 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 09-23T22:26:30.917107+0000 mon.a (mon. 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 0) 458 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: audit 2023-09-23T22:26:32.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 2023-09-23 2023-09-23T22:26:32.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: T22:26:30.921700+0000 mon.a (mon.0) 2023-09-23T22:26:32.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 459 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:32.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:26:32.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: -09-23T22:26:31.115866+0000 mon.a (mon. 2023-09-23T22:26:32.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:31 smithi052 conmon[100224]: 0) 460 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:26:33.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:33 smithi151 conmon[100454]: cluster 2023-09-23T22:26:31.886327+0000 mgr.y (mgr 2023-09-23T22:26:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:33 smithi151 conmon[100454]: .14148) 137 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:33 smithi151 conmon[100454]: audit 2023-09-23T22:26:32.319265+0000 mgr.y (mgr.14148) 138 : audit [DBG] from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:33 smithi151 conmon[100454]: audit 2023-09-23T22:26:32.322382+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:33 smithi151 conmon[100454]: audit 2023-09-23T22:26:32.325401+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:33 smithi151 conmon[100454]: audit 2023-09-23T22:26:32.326343+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: cluster 2023-09-23T22:26:31.886327+0000 2023-09-23T22:26:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: audit 2023-09-23T22:26:32 2023-09-23T22:26:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: .319265+0000 mgr.y (mgr.14148) 138 : audit [DBG] from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: audit 2023-09-23T22:26:32.322382 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: +0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: audit 2023-09-23T22:26:32.325401+0000 mon.a (mon 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: .0) 462 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: audit 2023-09-23T22:26:32.326343+0000 mon.a (mon.0) 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[110082]: 463 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: cluster 2023-09-23T22:26:31.886327+0000 mgr.y (mgr 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: .14148) 137 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:33.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: 09-23T22:26:32.319265+0000 mgr.y (mgr.14148) 138 : audit [DBG] from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: 09-23T22:26:32.322382+0000 mon.a (mon.0) 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: 461 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: T22:26:32.325401+0000 mon.a (mon.0) 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: 462 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: audit 2023 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: -09-23T22:26:32.326343+0000 mon.a ( 2023-09-23T22:26:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:33 smithi052 conmon[100224]: mon.0) 463 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: cluster 2023-09-23T22:26:33.886678+0000 mgr.y (mgr.14148) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: audit 2023-09-23T22:26:34.488911+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.151:0/1039522930' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]: dispatch 2023-09-23T22:26:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: audit 2023-09-23T22:26:34.489344+0000 mon.a (mon.0) 464 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]: dispatch 2023-09-23T22:26:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: audit 2023-09-23T22:26:34.494204+0000 mon.a (mon.0) 465 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]': finished 2023-09-23T22:26:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: cluster 2023-09-23T22:26:34.494282+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T22:26:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: audit 2023-09-23T22:26:34.494414+0000 mon.a (mon.0) 467 : 2023-09-23T22:26:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:35 smithi151 conmon[100454]: audit 2023-09-23T22:26:35.085793+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.151:0/330310310' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: cluster 2023-09-23T22:26:33.886678+0000 mgr.y (mgr. 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: 14148) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: audit 2023-09-23T22:26:34.488911+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.151:0/1039522930' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]: dispatch 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: audit 2023-09-23T22:26:34.489344 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: +0000 mon.a (mon.0) 464 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]: dispatch 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: audit 2023-09-23T22:26:34.494204+0000 mon.a (mon.0) 465 : audit 2023-09-23T22:26:35.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]': finished 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: cluster 2023-09-23T22:26:34.494282+0000 mon.a ( 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: mon.0) 466 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: audit 2023-09-23T22:26 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: :34.494414+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: audit 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: 2023-09-23T22:26:35.085793+0000 mon.b (mon.1 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[100224]: ) 17 : audit [DBG] from='client.? 172.21.15.151:0/330310310' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: cluster 2023-09-23T22:26:33 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: .886678+0000 mgr.y (mgr.14148) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:35.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: 26:34.488911+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.151:0/1039522930' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]: dispatch 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: 09-23T22:26:34.489344+0000 mon.a (mon.0 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: ) 464 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]: dispatch 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: -23T22:26:34.494204+0000 mon.a (mon.0) 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: 465 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c295677-55e8-41ff-8733-982844192fbc"}]': finished 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: 23T22:26:34.494282+0000 mon.a (mon.0) 466 : cluster [DBG] 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: osdmap e39: 7 total, 6 up, 7 in 2023-09-23T22:26:35.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:26:35.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: -23T22:26:34.494414+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:35.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: audit 2023-09-23T22:26:35.085793+0000 mon.b (mon. 2023-09-23T22:26:35.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:35 smithi052 conmon[110082]: 1) 17 : audit [DBG] from='client.? 172.21.15.151:0/330310310' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:37.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:37 smithi151 conmon[100454]: cluster 2023-09-23T22:26:35.887037+0000 mgr.y (mgr.14148) 140 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:37 smithi052 conmon[100224]: cluster 2023-09-23T22:26:35.887037+0000 mgr.y (mgr.14148) 140 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:37 smithi052 conmon[110082]: cluster 2023-09-23T22:26:35.887037+0000 mgr.y 2023-09-23T22:26:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:37 smithi052 conmon[110082]: (mgr.14148) 140 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:39.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:39 smithi151 conmon[100454]: cluster 2023-09-23T22:26:37.887484+0000 mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:39.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:39 smithi052 conmon[100224]: cluster 2023-09-23T22:26:37. 2023-09-23T22:26:39.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:39 smithi052 conmon[100224]: 887484+0000 mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:39 smithi052 conmon[110082]: cluster 2023-09-23T22:26:37.887484+0000 2023-09-23T22:26:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:39 smithi052 conmon[110082]: mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:41 smithi151 conmon[100454]: cluster 2023-09-23T22:26:39.888024+0000 mgr.y (mgr.14148) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:41 smithi151 conmon[100454]: audit 2023-09-23T22:26:40.372579+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:26:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:41 smithi151 conmon[100454]: audit 2023-09-23T22:26:40.373479+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:41 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:40.374270+0000 mgr.y (mgr.14148) 143 : cephadm [INF] Deploying daemon osd.6 on smithi151 2023-09-23T22:26:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: cluster 2023-09-23T22:26:39.888024+0000 mgr.y (mgr.14148) 142 : 2023-09-23T22:26:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: audit 2023-09-23T22:26:40. 2023-09-23T22:26:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: 372579+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:26:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: audit 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: 2023-09-23T22:26: 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: 40.373479+0000 mon.a (mon.0) 469 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: cephadm 2023-09-23T22 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[100224]: :26:40.374270+0000 mgr.y (mgr.14148) 143 : cephadm [INF] Deploying daemon osd.6 on smithi151 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: cluster 2023-09-23T22:26:39.888024+0000 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: mgr.y (mgr.14148) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: audit 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: 2023-09-23T22:26 2023-09-23T22:26:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: :40.372579+0000 mon.a (mon.0) 468 : audit 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: 09-23T22:26:40.373479+0000 mon.a (mon.0) 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: 469 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: cephadm 2023-09- 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: 23T22:26:40.374270+0000 mgr.y (mgr.14148) 2023-09-23T22:26:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:41 smithi052 conmon[110082]: 143 : cephadm [INF] Deploying daemon osd.6 on smithi151 2023-09-23T22:26:43.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:43 smithi151 conmon[100454]: cluster 2023-09-23T22:26:41 2023-09-23T22:26:43.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:43 smithi151 conmon[100454]: .888574+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:43.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:43 smithi052 conmon[110082]: cluster 2023-09-23T22:26:41.888574+0000 2023-09-23T22:26:43.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:43 smithi052 conmon[110082]: mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:43.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:43 smithi052 conmon[100224]: cluster 2023-09-23T22:26:41.888574+0000 2023-09-23T22:26:43.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:43 smithi052 conmon[100224]: mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:45.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:45 smithi151 conmon[100454]: cluster 2023-09-23T22:26:43.889049+0000 mgr.y ( 2023-09-23T22:26:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:45 smithi151 conmon[100454]: mgr.14148) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:45 smithi151 conmon[100454]: audit 2023-09-23T22:26:45.120565+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:45 smithi151 conmon[100454]: audit 2023-09-23T22:26:45.126167+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:45.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[100224]: cluster 2023-09-23T22:26:43.889049+0000 mgr.y (mgr 2023-09-23T22:26:45.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[100224]: .14148) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:45.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[100224]: audit 2023-09-23T22:26:45.120565+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:45.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[100224]: audit 2023-09-23T22:26:45.126167+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: cluster 2023-09-23T22:26:43.889049 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: +0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: 23T22:26:45.120565+0000 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: mon.a (mon.0) 470 : audit 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: audit 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: 2023-09-23T22:26:45.126167+0000 mon.a ( 2023-09-23T22:26:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: mon.0) 471 : audit [DBG] 2023-09-23T22:26:45.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:45 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:47.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:47 smithi151 conmon[100454]: cluster 2023-09-23T22:26:45.889409+0000 mgr.y (mgr.14148) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:47.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:47 smithi151 conmon[100454]: audit 2023-09-23T22:26:46.775689+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:47.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:47 smithi151 conmon[100454]: audit 2023-09-23T22:26:46.776589+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:47.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:47 smithi151 conmon[100454]: audit 2023-09-23T22:26:46.777613+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:47.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:47 smithi151 conmon[100454]: audit 2023-09-23T22:26:46.782601+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: cluster 2023-09-23T22:26:45.889409+0000 mgr.y (mgr.14148) 2023-09-23T22:26:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: audit 2023-09-23T22:26:46.775689+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: audit 2023-09-23T22:26:46.776589+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: audit 2023-09-23T22:26:46.777613+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: audit 2023-09-23T22:26:46.782601+0000 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[100224]: mon.a (mon.0) 475 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: cluster 2023-09-23T22:26:45.889409+0000 mgr.y (mgr.14148 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: ) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: 23T22:26:46.775689+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: 26:46.776589+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:26:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: T22:26:46.777613+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:47.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:26:47.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:47 smithi052 conmon[110082]: 26:46.782601+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:47.733 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 6 on host 'smithi151' 2023-09-23T22:26:48.278 DEBUG:teuthology.orchestra.run.smithi151:osd.6> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T22:26:48.281 INFO:tasks.cephadm:Deploying osd.7 on smithi151 with /dev/vg_nvme/lv_1... 2023-09-23T22:26:48.281 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T22:26:48.311 INFO:journalctl@ceph.osd.6.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:26:48.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:48 smithi151 conmon[100454]: audit 2023-09-23T22:26:47 2023-09-23T22:26:48.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:48 smithi151 conmon[100454]: .732830+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:48.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:48 smithi151 conmon[100454]: audit 2023-09-23T22:26:47.751542+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:48.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:48 smithi151 conmon[100454]: cluster 2023-09-23T22:26:47.889756+0000 mgr.y (mgr.14148) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:48.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:48 smithi151 conmon[100454]: audit 2023-09-23T22:26:48.298978+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:26:48.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:48 smithi151 conmon[100454]: audit 2023-09-23T22:26:48.299366+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:26:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: audit 2023-09-23T22:26:47.732830+0000 mon.a (mon.0) 476 : audit [INF] 2023-09-23T22:26:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: audit 2023-09-23T22:26:47.751542+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: cluster 2023-09-23T22:26:47.889756+0000 mgr.y (mgr.14148) 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: audit 2023-09-23T22:26:48.298978+0000 mon.b 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: audit 2023-09-23T22:26:48.299366+0000 mon.a (mon.0) 478 : audit 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[100224]: [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: audit 2023-09-23T22:26:47.732830+0000 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: mon.a (mon.0) 476 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: audit 2023-09-23T22:26:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: 2023-09-23T22:26:47.751542+0000 mon.a (mon.0) 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: 477 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: 09-23T22:26:47.889756+0000 mgr.y (mgr.14148) 147 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: T22:26:48.298978+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: 23T22:26:48.299366+0000 mon.a (mon.0) 478 : audit [INF] 2023-09-23T22:26:48.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:48 smithi052 conmon[110082]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:26:49.949 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[118391]: debug 2023-09-23T22:26:49.744+0000 7fcc13c72700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T22:26:49.949 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[118391]: debug 2023-09-23T22:26:49.750+0000 7fcc0e3ea700 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:26:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[100454]: audit 2023-09-23T22:26:48.738579+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:26:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[100454]: cluster 2023-09-23T22:26:48.738701+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T22:26:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[100454]: audit 2023-09-23T22:26:48.738867+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[100454]: audit 2023-09-23T22:26:48.740450+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:49 smithi151 conmon[100454]: audit 2023-09-23T22:26:48.740803+0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:49.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: audit 2023-09-23T22:26:48.738579+0000 mon.a (mon.0 2023-09-23T22:26:49.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: ) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:26:49.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: cluster 2023-09-23T22:26:48.738701+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: audit 2023-09-23T22:26:48.738867+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: audit 2023-09-23T22:26:48.740450+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: audit 2023-09-23T22:26:48.740803 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[100224]: +0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: audit 2023-09-23T22:26:48.738579+0000 mon.a ( 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: cluster 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: 2023-09-23T22:26:48.738701+0000 mon.a (mon.0) 480 2023-09-23T22:26:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: 09-23T22:26:48.738867+0000 mon.a (mon. 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: 0) 481 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: audit 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: 2023-09-23T22:26:48.740450 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: +0000 mon.b (mon.1 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: ) 19 : audit [INF] from='osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: audit 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: 2023-09-23T22:26:48.740803+0000 2023-09-23T22:26:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:49 smithi052 conmon[110082]: mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:26:50.546 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T22:26:50.546 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T22:26:50.546 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2023-09-23T22:26:50.546 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2023-09-23T22:26:50.546 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129696 s, 808 MB/s 2023-09-23T22:26:50.547 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: audit 2023-09-23T22:26:49.740023+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: cluster 2023-09-23T22:26:49.740170+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: audit 2023-09-23T22:26:49.740671+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: audit 2023-09-23T22:26:49.744958+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: cluster 2023-09-23T22:26:49.890219+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: cluster 2023-09-23T22:26:50.115041+0000 mon.a (mon.0) 487 : cluster [INF] osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922] boot 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: cluster 2023-09-23T22:26:50.115124+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T22:26:50.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:50 smithi151 conmon[100454]: audit 2023-09-23T22:26:50.115653+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: audit 2023-09-23T22:26:49.740023+0000 mon.a (mon. 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: cluster 2023-09-23T22:26:49.740170+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 09-23T22:26:49.740671+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: audit 2023-09-23T22:26: 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 49.744958+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: cluster 2023-09- 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 23T22:26:49.890219+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: cluster 2023-09-23T22:26: 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 50.115041+0000 mon.a (mon.0) 487 : cluster [INF] osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922] boot 2023-09-23T22:26:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: cluster 2023-09-23T22:26: 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 50.115124+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[100224]: 09-23T22:26:50.115653+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: audit 2023-09-23T22:26:49.740023+0000 mon.a (mon.0) 483 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: cluster 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: 2023-09-23T22:26:49.740170+0000 mon.a (mon.0) 484 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: 23T22:26:49.740671+0000 mon.a (mon.0) 485 2023-09-23T22:26:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: T22:26:49.744958+0000 mon.a (mon.0) 486 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: 09-23T22:26:49.890219+0000 mgr.y (mgr.14148) 148 : 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: cluster 2023-09-23T22:26:50.115041+0000 mon.a (mon.0) 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: 487 : cluster [INF] osd.6 [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922] boot 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: cluster 2023-09-23T 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: 22:26:50.115124+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: audit 2023-09-23T22:26:50.115653+0000 mon.a (mon.0) 489 2023-09-23T22:26:51.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:50 smithi052 conmon[110082]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:26:51.612 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi151:vg_nvme/lv_1 2023-09-23T22:26:52.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: cluster 2023-09-23T22:26:49.343698+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:52.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: cluster 2023-09-23T22:26:49.343742+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:52.380 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: cluster 2023-09-23T22:26:51.115869+0000 mon.a (mon.0) 490 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.036768+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.037918+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.038852+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.039785+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.045138+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.046305+0000 mon.a (mon.0) 496 : audit 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.047168+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:52.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:52 smithi151 conmon[100454]: audit 2023-09-23T22:26:52.051785+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: cluster 2023-09-23T22:26:49.343698+0000 osd.6 ( 2023-09-23T22:26:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: cluster 2023-09-23T22:26:49.343742+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: cluster 2023-09-23T22:26:51.115869+0000 2023-09-23T22:26:52.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: mon.a (mon.0) 490 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22:26:52.036768+0000 mon.a (mon. 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 0) 491 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22:26:52.037918+0000 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: mon.a (mon.0) 492 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: :26:52.038852+0000 mon.a (mon.0) 493 : audit [INF] 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22:26:52.039785+0000 mon.a (mon.0) 494 : audit [INF] 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: -09-23T22:26:52.045138+0000 mon.a (mon.0) 495 : audit 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22:26:52.046305+0000 mon.a (mon.0) 496 : audit 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: -09-23T22:26:52.047168+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: audit 2023-09-23T22:26:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: 2023-09-23T22:26:52.051785+0000 mon.a (mon.0) 498 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[100224]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: cluster 2023-09-23T22:26:49.343698+0000 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: cluster 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 2023-09-23T22:26:49.343742+0000 osd.6 ( 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: cluster 2023-09-23T 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 22:26:51.115869+0000 mon.a (mon.0) 490 : cluster 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:26:52.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: T22:26:52.036768+0000 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: mon.a (mon.0) 491 : audit 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 2023-09-23T22:26:52.037918+0000 mon.a (mon.0) 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 492 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 2023-09-23T22:26:52.038852+0000 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: mon.a (mon.0) 493 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 09-23T22:26:52.039785+0000 mon.a (mon.0) 494 : audit 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:26:52.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 26:52.045138+0000 mon.a (mon.0) 495 : audit [INF] 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 2023-09-23T22:26:52.046305+0000 mon.a (mon 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: .0) 496 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: 09-23T22:26:52.047168+0000 mon.a (mon.0) 497 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: -23T22:26:52.051785+0000 mon.a (mon.0) 498 2023-09-23T22:26:52.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:52 smithi052 conmon[110082]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:26:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:53 smithi151 conmon[100454]: cluster 2023-09-23T22:26:51.890649+0000 mgr.y (mgr.14148) 149 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:53 smithi151 conmon[100454]: cephadm 2023-09-23T22 2023-09-23T22:26:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:53 smithi151 conmon[100454]: :26:52.031016+0000 mgr.y (mgr.14148) 150 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:53 smithi151 conmon[100454]: cephadm 2023-09-23T22:26:52.040513+0000 mgr.y (mgr.14148) 151 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 5722M 2023-09-23T22:26:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:53 smithi151 conmon[100454]: cluster 2023-09-23T22:26:52.121069+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T22:26:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: cluster 2023-09-23T22:26:51.890649+0000 mgr.y (mgr. 2023-09-23T22:26:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: 14148) 149 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: cephadm 2023-09-23T22:26:52.031016+0000 mgr.y (mgr.14148) 150 : cephadm [INF] 2023-09-23T22:26:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: Detected new or changed devices on smithi151 2023-09-23T22:26:53.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: cephadm 2023-09-23T22: 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: 26:52.040513+0000 mgr.y (mgr.14148) 151 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 5722M 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: 09-23T22:26:52.121069+0000 mon.a (mon.0) 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[100224]: 499 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: cluster 2023-09-23T22:26:51.890649+0000 mgr.y (mgr.14148) 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: 149 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: cephadm 2023-09-23T22:26: 2023-09-23T22:26:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: 52.031016+0000 mgr.y (mgr.14148) 150 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: cephadm 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: 2023-09-23T22 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: :26:52.040513+0000 mgr.y (mgr. 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: 14148) 151 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 5722M 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: 23T22:26:52.121069+0000 mon.a (mon.0) 499 2023-09-23T22:26:53.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:53 smithi052 conmon[110082]: : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-09-23T22:26:54.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:54 smithi151 conmon[100454]: audit 2023-09-23T22:26:53.420909+0000 mgr.y (mgr.14148) 152 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:54.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:54 smithi151 conmon[100454]: audit 2023-09-23T22:26:53.423995+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:54.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:54 smithi151 conmon[100454]: audit 2023-09-23T22:26:53.426910+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:54.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:54 smithi151 conmon[100454]: audit 2023-09-23T22:26:53.427795+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: audit 2023-09-23T22:26:53.420909+0000 mgr.y (mgr. 2023-09-23T22:26:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: 14148) 152 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: audit 2023-09-23T22:26:53.423995+0000 mon.a 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: (mon.0) 500 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: audit 2023-09-23T22:26:53.426910+0000 mon.a (mon.0 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: ) 501 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: 23T22:26:53.427795+0000 mon.a (mon.0) 502 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[110082]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[100224]: audit 2023-09-23T22:26:53.420909+0000 mgr.y (mgr.14148) 152 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[100224]: 2023-09-23T22:26:54.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[100224]: audit 2023-09-23T22:26:53.423995+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:26:54.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[100224]: audit 2023-09-23T22:26:53.426910+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:26:54.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:54 smithi052 conmon[100224]: audit 2023-09-23T22:26:53.427795+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:26:55.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:55 smithi151 conmon[100454]: cluster 2023-09-23T22:26:53.891200+0000 mgr.y (mgr 2023-09-23T22:26:55.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:55 smithi151 conmon[100454]: .14148) 153 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:55.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:55 smithi052 conmon[100224]: cluster 2023-09-23T22:26:53.891200+0000 mgr.y (mgr. 2023-09-23T22:26:55.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:55 smithi052 conmon[100224]: 14148) 153 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:55 smithi052 conmon[110082]: cluster 2023-09-23T22:26:53.891200+0000 mgr.y (mgr 2023-09-23T22:26:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:55 smithi052 conmon[110082]: .14148) 153 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:56.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:56 smithi151 conmon[100454]: audit 2023-09-23T22:26:55.708139+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.151:0/39140573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]: dispatch 2023-09-23T22:26:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:56 smithi151 conmon[100454]: audit 2023-09-23T22:26:55.708504+0000 mon.a (mon.0) 503 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]: dispatch 2023-09-23T22:26:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:56 smithi151 conmon[100454]: audit 2023-09-23T22:26:55.713363+0000 mon.a (mon.0) 504 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]': finished 2023-09-23T22:26:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:56 smithi151 conmon[100454]: cluster 2023-09-23T22:26:55.713440+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T22:26:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:56 smithi151 conmon[100454]: audit 2023-09-23T22:26:55.713600+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:26:56.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: audit 2023-09-23T22:26:55. 2023-09-23T22:26:56.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: 708139+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.151:0/39140573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]: dispatch 2023-09-23T22:26:56.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: audit 2023-09-23T22:26:55.708504+0000 mon.a ( 2023-09-23T22:26:56.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: mon.0) 503 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]: dispatch 2023-09-23T22:26:56.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: audit 2023-09-23T22:26 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: :55.713363+0000 mon.a (mon.0) 504 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]': finished 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: cluster 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: 2023-09-23T22:26:55.713440+0000 mon.a (mon.0) 505 : 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: 26:55.713600+0000 mon.a (mon.0) 506 : audit 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: audit 2023-09-23T22:26:55.708139+0000 2023-09-23T22:26:56.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.151:0/39140573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]: dispatch 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: audit 2023-09-23T22:26:55.708504+0000 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: mon.a (mon.0) 503 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]: dispatch 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: T22:26:55.713363+0000 mon.a (mon.0) 504 : 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c1b48eff-20d5-4d6d-bb17-0c44ffdf866e"}]': finished 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: cluster 2023-09-23T22:26:55. 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: 713440+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: audit 2023 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: -09-23T22:26:55.713600+0000 mon.a ( 2023-09-23T22:26:56.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:56 smithi052 conmon[110082]: mon.0) 506 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:26:57.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:57 smithi151 conmon[100454]: cluster 2023-09-23T22:26:55.891653+0000 mgr.y (mgr.14148) 154 : cluster 2023-09-23T22:26:57.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:57 smithi151 conmon[100454]: [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:57.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:57 smithi151 conmon[100454]: audit 2023-09-23T22:26:56.281900+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.151:0/2823522410' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[110082]: cluster 2023-09-23T22:26:55.891653+0000 2023-09-23T22:26:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[110082]: mgr.y (mgr.14148) 154 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[110082]: audit 2023-09-23T22:26 2023-09-23T22:26:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[110082]: :56.281900+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.151:0/2823522410' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:57.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[100224]: cluster 2023-09-23T22:26:55.891653+0000 2023-09-23T22:26:57.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[100224]: mgr.y (mgr.14148) 154 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:57.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:26:57.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:57 smithi052 conmon[100224]: T22:26:56.281900+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.151:0/2823522410' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:26:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:26:59 smithi151 conmon[100454]: cluster 2023-09-23T22:26:57.892085+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:59 smithi052 conmon[100224]: cluster 2023-09-23T22:26:57.892085+0000 mgr.y (mgr 2023-09-23T22:26:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:26:59 smithi052 conmon[100224]: .14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:26:59.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:59 smithi052 conmon[110082]: cluster 2023-09-23T22:26:57.892085+0000 2023-09-23T22:26:59.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:26:59 smithi052 conmon[110082]: mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:01 smithi151 conmon[100454]: cluster 2023-09-23T22:26:59.892618+0000 mgr.y (mgr.14148) 156 : cluster 2023-09-23T22:27:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:01 smithi151 conmon[100454]: [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:01 smithi052 conmon[100224]: cluster 2023-09-23T22:26:59.892618+0000 mgr.y (mgr.14148) 2023-09-23T22:27:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:01 smithi052 conmon[100224]: 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:01.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:01 smithi052 conmon[110082]: cluster 2023-09-23T22:26:59.892618+0000 mgr.y 2023-09-23T22:27:01.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:01 smithi052 conmon[110082]: (mgr.14148) 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:02.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:02 smithi151 conmon[100454]: audit 2023-09-23T22:27:01.670954+0000 mon.a (mon.0) 507 2023-09-23T22:27:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:02 smithi151 conmon[100454]: : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:27:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:02 smithi151 conmon[100454]: audit 2023-09-23T22:27:01.671899+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:02.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[100224]: audit 2023-09-23T22:27:01.670954+0000 mon.a (mon.0) 507 : 2023-09-23T22:27:02.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[100224]: audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:27:02.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[100224]: audit 2023-09-23T22:27:01.671899+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:02.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[110082]: audit 2023-09-23T22:27:01.670954+0000 mon.a (mon.0) 507 : 2023-09-23T22:27:02.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[110082]: audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:27:02.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:27:02.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:02 smithi052 conmon[110082]: :27:01.671899+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:03 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:01.672718+0000 mgr.y (mgr.14148) 157 : 2023-09-23T22:27:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:03 smithi151 conmon[100454]: cephadm [INF] Deploying daemon osd.7 on smithi151 2023-09-23T22:27:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:03 smithi151 conmon[100454]: cluster 2023-09-23T22:27:01.893034+0000 mgr.y (mgr.14148) 158 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:01.672718+0000 mgr.y (mgr. 2023-09-23T22:27:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[100224]: 14148) 157 : cephadm [INF] Deploying daemon osd.7 on smithi151 2023-09-23T22:27:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[100224]: cluster 2023-09-23T22:27:01.893034+0000 mgr.y (mgr.14148) 158 : cluster 2023-09-23T22:27:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[100224]: [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:01.672718+0000 mgr.y (mgr.14148) 2023-09-23T22:27:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[110082]: 157 : cephadm [INF] Deploying daemon osd.7 on smithi151 2023-09-23T22:27:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[110082]: cluster 2023-09-23T22:27:01.893034+0000 mgr.y (mgr.14148) 158 : cluster [DBG] 2023-09-23T22:27:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:03 smithi052 conmon[110082]: pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:05 smithi151 conmon[100454]: cluster 2023-09-23T22:27:03.893347+0000 mgr.y (mgr 2023-09-23T22:27:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:05 smithi151 conmon[100454]: .14148) 159 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:05.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:05 smithi052 conmon[100224]: cluster 2023-09-23T22:27:03.893347 2023-09-23T22:27:05.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:05 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 159 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:05 smithi052 conmon[110082]: cluster 2023-09-23T22:27:03.893347+0000 mgr.y (mgr.14148) 2023-09-23T22:27:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:05 smithi052 conmon[110082]: 159 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:07 smithi151 conmon[100454]: cluster 2023-09-23T22:27:05.893875+0000 mgr.y (mgr.14148) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:07 smithi151 conmon[100454]: audit 2023-09-23T22:27:06.370762+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:07 smithi151 conmon[100454]: audit 2023-09-23T22:27:06.405118+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[100224]: cluster 2023-09-23T22:27:05.893875 2023-09-23T22:27:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[100224]: audit 2023-09-23T22:27:06.370762+0000 mon.a 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[100224]: (mon.0) 509 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[100224]: audit 2023-09-23T22:27:06.405118+0000 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[100224]: mon.a (mon.0) 510 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: cluster 2023-09-23T22:27:05.893875+0000 mgr.y (mgr 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: .14148) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: audit 2023-09-23T22:27:06.370762+0000 mon.a (mon 2023-09-23T22:27:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: .0) 509 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: audit 2023-09-23T22:27:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: 2023-09-23T22:27:06 2023-09-23T22:27:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: .405118+0000 mon.a (mon.0) 510 : audit 2023-09-23T22:27:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:07 smithi052 conmon[110082]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:09.273 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 7 on host 'smithi151' 2023-09-23T22:27:09.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:09 smithi151 conmon[100454]: cluster 2023-09-23T22:27:07.894397+0000 mgr.y (mgr.14148) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:09.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:09 smithi151 conmon[100454]: audit 2023-09-23T22:27:08.249964+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:09.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:09 smithi151 conmon[100454]: audit 2023-09-23T22:27:08.567213+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:09.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:09 smithi151 conmon[100454]: audit 2023-09-23T22:27:08.568276+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:09.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:09 smithi151 conmon[100454]: audit 2023-09-23T22:27:08.573605+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:09.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: cluster 2023-09-23T22:27:07.894397+0000 mgr.y (mgr.14148) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:09.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: audit 2023-09-23T22:27:08.249964+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:09.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: audit 2023-09-23T22:27:08.567213+0000 mon.a (mon.0) 512 : audit 2023-09-23T22:27:09.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: :27:08.568276+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: audit 2023-09-23T22:27:08.573605+0000 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[100224]: mon.a (mon.0) 514 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: cluster 2023-09-23T22:27:07 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: .894397+0000 mgr.y (mgr.14148) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:27:09.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: 22:27:08.249964+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: audit 2023-09-23T22:27:08.567213+0000 mon.a (mon. 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: 0) 512 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: audit 2023-09-23T22:27:08 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: .568276+0000 mon.a (mon.0) 513 : audit 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: audit 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: 2023-09-23T22:27:08.573605+0000 mon.a 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: (mon.0) 514 : audit [INF] 2023-09-23T22:27:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:09 smithi052 conmon[110082]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:09.799 DEBUG:teuthology.orchestra.run.smithi151:osd.7> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T22:27:09.801 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T22:27:09.802 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T22:27:09.831 INFO:journalctl@ceph.osd.7.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:27:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:10 smithi151 conmon[100454]: audit 2023-09-23T22:27:09.272521+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:10 smithi151 conmon[100454]: audit 2023-09-23T22:27:09.292525+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:10 smithi151 conmon[100454]: audit 2023-09-23T22:27:09.567781+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:27:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:10 smithi151 conmon[100454]: audit 2023-09-23T22:27:09.568360+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:27:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[110082]: audit 2023-09-23T22:27:09.272521+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[110082]: audit 2023-09-23T22:27:09.292525+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[110082]: audit 2023-09-23T22:27:09.567781+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:27:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[110082]: audit 2023-09-23T22:27:09.568360+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:27:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[100224]: audit 2023-09-23T22:27:09.272521+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[100224]: audit 2023-09-23T22:27:09.292525+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[100224]: audit 2023-09-23T22:27:09.567781+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:27:10.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:10 smithi052 conmon[100224]: audit 2023-09-23T22:27:09.568360+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:27:11.596 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:27:11.622 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[110082]: cluster 2023-09-23T22:27:09.894812+0000 mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:11.622 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[110082]: audit 2023-09-23T22:27:10.277938+0000 mon.a (mon.0) 518 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[110082]: cluster 2023-09-23T22:27:10.278011+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[110082]: audit 2023-09-23T22:27:10.278342+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[110082]: audit 2023-09-23T22:27:10.278615+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[110082]: audit 2023-09-23T22:27:10.279082+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: cluster 2023-09-23T22:27:09.894812+0000 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: audit 2023-09-23T22:27:10.277938+0000 mon.a (mon.0) 518 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: cluster 2023-09-23T22:27:10.278011+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T22:27:11.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: audit 2023-09-23T22:27:10.278342+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:11.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: audit 2023-09-23T22:27:10.278615+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:27:11.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:11 smithi052 conmon[100224]: audit 2023-09-23T22:27:10.279082+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:27:11.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: cluster 2023-09-23T22:27:09.894812+0000 mgr.y (mgr. 2023-09-23T22:27:11.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: 14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:11.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: audit 2023-09-23T22:27:10.277938+0000 mon.a (mon.0) 518 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:27:11.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: cluster 2023-09-23T22:27:10.278011+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-09-23T22:27:11.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: audit 2023-09-23T22:27:10.278342+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:11.645 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: audit 2023-09-23T22:27:10.278615+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:27:11.645 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[100454]: audit 2023-09-23T22:27:10.279082+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2023-09-23T22:27:11.645 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[124295]: debug 2023-09-23T22:27:11.281+0000 7fd40020a700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T22:27:11.645 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 22:27:11 smithi151 conmon[124295]: debug 2023-09-23T22:27:11.287+0000 7fd3fb183700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:27:12.195 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":47,"num_osds":8,"num_up_osds":7,"osd_up_since":1695508010,"num_in_osds":8,"osd_in_since":1695508015,"num_remapped_pgs":0} 2023-09-23T22:27:12.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:12 smithi151 conmon[100454]: audit 2023-09-23T22:27:11.278569+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:27:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:12 smithi151 conmon[100454]: cluster 2023-09-23T22:27:11.278675+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T22:27:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:12 smithi151 conmon[100454]: audit 2023-09-23T22:27:11.278976+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:12 smithi151 conmon[100454]: audit 2023-09-23T22:27:11.282639+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:12 smithi151 conmon[100454]: audit 2023-09-23T22:27:11.595902+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.52:0/1957566454' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: audit 2023-09-23T22:27:11.278569+0000 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: cluster 2023-09-23T22:27:11.278675 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: +0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: T22:27:11.278976+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:12.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: :27:11.282639+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: audit 2023-09-23T22:27:11.595902+0000 mon.c (mon 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[110082]: .2) 13 : audit [DBG] from='client.? 172.21.15.52:0/1957566454' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: audit 2023-09-23T22:27:11.278569+0000 mon.a (mon. 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: 0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: cluster 2023-09-23T22:27:11.278675+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: audit 2023-09-23T22:27:11.278976+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: audit 2023-09-23T22:27:11.282639+0000 mon.a (mon.0) 525 : audit [DBG] 2023-09-23T22:27:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:12.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: audit 2023-09-23T22:27:11.595902+0000 2023-09-23T22:27:12.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:12 smithi052 conmon[100224]: mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.52:0/1957566454' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:27:13.196 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T22:27:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cluster 2023-09-23T22:27:10.551164+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:27:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cluster 2023-09-23T22:27:10.551211+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:27:13.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cluster 2023-09-23T22:27:11.895171+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cluster 2023-09-23T22:27:12.279923+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088] boot 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cluster 2023-09-23T22:27:12.280006+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.280173+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:12.367810+0000 mgr.y (mgr.14148) 164 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.372571+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.373580+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.374307+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.374968+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: -23T22:27:12.375646+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:12.376175+0000 mgr.y (mgr.14148) 165 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 4292M 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.379599+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.380491+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.381155+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: audit 2023-09-23T22:27:12.385774+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[100224]: cluster 2023-09-23T22:27:13.281977+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cluster 2023-09-23T22:27:10.551164+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cluster 2023-09-23T22:27:10.551211+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cluster 2023-09-23T22:27:11.895171+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:13.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cluster 2023-09-23T22:27:12.279923+0000 mon.a (mon.0) 526 : cluster [INF] osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088] boot 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cluster 2023-09-23T22:27:12.280006+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.280173+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:12.367810+0000 mgr.y (mgr.14148) 164 : cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.372571+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.373580+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.374307+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.374968+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.375646+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:12.376175+0000 mgr.y (mgr.14148) 165 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 4292M 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.379599+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.380491+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.381155+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: audit 2023-09-23T22:27:12.385774+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:13 smithi052 conmon[110082]: cluster 2023-09-23T22:27:13.281977+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cluster 2023-09-23T22:27:10.551164+0000 osd.7 (osd.7) 1 : cluster 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: [DBG] purged_snaps scrub starts 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cluster 2023-09-23T22:27:10.551211+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cluster 2023-09-23T22 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: :27:11.895171+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cluster 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 2023-09-23T22:27:12.279923+0000 mon.a (mon.0) 526 : cluster [INF] 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: osd.7 [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088] boot 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cluster 2023-09-23T22:27:12.280006 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: +0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 2023-09-23T22:27:12.280173+0000 mon.a (mon.0) 528 : 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cephadm 2023-09-23T 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 22:27:12.367810+0000 mgr.y (mgr.14148) 164 : 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cephadm [INF] Detected new or changed devices on smithi151 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 27:12.372571+0000 mon.a (mon.0) 529 : audit [INF] 2023-09-23T22:27:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27:12. 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 373580+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27: 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 12.374307+0000 mon.a (mon.0) 531 : audit 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27:12.374968+0000 mon.a (mon 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: .0) 532 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: -23T22:27:12.375646+0000 mon.a (mon.0 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: ) 533 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cephadm 2023-09-23T 2023-09-23T22:27:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 22:27:12.376175+0000 mgr.y (mgr.14148) 165 : 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cephadm [INF] Adjusting osd_memory_target on smithi151 to 4292M 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27:12 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: .379599+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27:12.380491+0000 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: mon.a (mon.0) 535 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 2023-09-23T22:27:12.381155+0000 mon.a 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: (mon.0) 536 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 09-23T22:27:12.385774+0000 mon.a (mon. 2023-09-23T22:27:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 0) 537 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: cluster 2023-09- 2023-09-23T22:27:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 23T22:27:13.281977+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-09-23T22:27:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:13 smithi151 conmon[100454]: 2023-09-23T22:27:14.852 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:27:15.369 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1695508032,"num_in_osds":8,"osd_in_since":1695508015,"num_remapped_pgs":0} 2023-09-23T22:27:15.370 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:27:15.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:15 smithi151 conmon[100454]: cluster 2023-09-23T22:27:13.895490+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v153: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:15 smithi151 conmon[100454]: cluster 2023-09-23T22:27:14.283007+0000 2023-09-23T22:27:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:15 smithi151 conmon[100454]: mon.a (mon.0) 539 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T22:27:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:15 smithi151 conmon[100454]: cluster 2023-09-23T22:27:14.285407+0000 mon.a (mon.0) 540 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:15 smithi151 conmon[100454]: audit 2023-09-23T22:27:14.851940+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.52:0/2986319805' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:27:15.726 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[110082]: cluster 2023-09-23T22:27:13 2023-09-23T22:27:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[110082]: .895490+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v153: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[110082]: cluster 2023-09-23T22:27:14.283007+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T22:27:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[110082]: cluster 2023-09-23T22:27:14.285407+0000 mon.a (mon.0) 540 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[110082]: audit 2023-09-23T22:27:14.851940+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.52:0/2986319805' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:27:15.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[100224]: cluster 2023-09-23T22:27 2023-09-23T22:27:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[100224]: :13.895490+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v153: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[100224]: cluster 2023-09-23T22:27:14.283007+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-09-23T22:27:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[100224]: cluster 2023-09-23T22:27:14.285407+0000 mon.a (mon.0) 540 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:15 smithi052 conmon[100224]: audit 2023-09-23T22:27:14.851940+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.52:0/2986319805' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:27:17.169 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:27:17.169 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":50,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:27:14.278676+0000","last_up_change":"2023-09-23T22:27:12.276151+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6803","nonce":858289560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6805","nonce":858289560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6809","nonce":858289560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6807","nonce":858289560}]},"public_addr":"172.21.15.52:6803/858289560","cluster_addr":"172.21.15.52:6805/858289560","heartbeat_back_addr":"172.21.15.52:6809/858289560","heartbeat_front_addr":"172.21.15.52:6807/858289560","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6811","nonce":3743472544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6813","nonce":3743472544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6817","nonce":3743472544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6815","nonce":3743472544}]},"public_addr":"172.21.15.52:6811/3743472544","cluster_addr":"172.21.15.52:6813/3743472544","heartbeat_back_addr":"172.21.15.52:6817/3743472544","heartbeat_front_addr":"172.21.15.52:6815/3743472544","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6819","nonce":1781181972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6821","nonce":1781181972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6825","nonce":1781181972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6823","nonce":1781181972}]},"public_addr":"172.21.15.52:6819/1781181972","cluster_addr":"172.21.15.52:6821/1781181972","heartbeat_back_addr":"172.21.15.52:6825/1781181972","heartbeat_front_addr":"172.21.15.52:6823/1781181972","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6827","nonce":1128061400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6829","nonce":1128061400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6833","nonce":1128061400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6831","nonce":1128061400}]},"public_addr":"172.21.15.52:6827/1128061400","cluster_addr":"172.21.15.52:6829/1128061400","heartbeat_back_addr":"172.21.15.52:6833/1128061400","heartbeat_front_addr":"172.21.15.52:6831/1128061400","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:27:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:17 smithi052 conmon[100224]: cluster 2023-09-23T22:27:15.896021 2023-09-23T22:27:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:17 smithi052 conmon[100224]: +0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v155: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:17 smithi052 conmon[100224]: audit 2023-09-23T22:27:17.168042+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.52:0/908280544' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:17 smithi052 conmon[110082]: cluster 2023-09-23T22:27:15.896021+0000 mgr.y (mgr.14148) 167 : cluster 2023-09-23T22:27:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:17 smithi052 conmon[110082]: [DBG] pgmap v155: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:17 smithi052 conmon[110082]: audit 2023-09-23T22:27:17.168042+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.52:0/908280544' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:17.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:17 smithi151 conmon[100454]: cluster 2023-09-23T22:27:15.896021+0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v155: 1 pgs: 1 remapped+peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:17.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:17 smithi151 conmon[100454]: audit 2023-09-23T22:27:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:17 smithi151 conmon[100454]: 2023-09-23T22:27:17.168042+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.52:0/908280544' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:17.700 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-09-23T22:25:25.839760+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-09-23T22:27:17.700 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-09-23T22:27:18.055 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:18.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:18 smithi052 conmon[110082]: cluster 2023-09-23T22:27:18.284377+0000 mon.a (mon.0) 541 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:27:18.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:18 smithi052 conmon[110082]: cluster 2023-09-23T22:27:18.284409+0000 mon.a (mon.0) 542 : cluster [INF] Cluster is now healthy 2023-09-23T22:27:18.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:18 smithi052 conmon[100224]: cluster 2023-09-23T22:27:18.284377+0000 mon.a (mon.0) 541 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:27:18.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:18 smithi052 conmon[100224]: cluster 2023-09-23T22:27:18.284409+0000 mon.a (mon.0) 542 : cluster [INF] Cluster is now healthy 2023-09-23T22:27:18.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:18 smithi151 conmon[100454]: cluster 2023-09-23T22:27:18.284377+0000 mon.a (mon.0) 541 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:27:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:18 smithi151 conmon[100454]: cluster 2023-09-23T22:27: 2023-09-23T22:27:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:18 smithi151 conmon[100454]: 18.284409+0000 mon.a (mon.0) 542 : cluster [INF] Cluster is now healthy 2023-09-23T22:27:19.537 INFO:teuthology.orchestra.run.smithi052.stdout:pg_num: 1 2023-09-23T22:27:19.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:19 smithi151 conmon[100454]: cluster 2023-09-23T22:27:17.896344+0000 mgr.y (mgr.14148) 168 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:19.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:19 smithi052 conmon[110082]: cluster 2023-09-23T22:27: 2023-09-23T22:27:19.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:19 smithi052 conmon[110082]: 17.896344+0000 mgr.y (mgr.14148) 168 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:19.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:19 smithi052 conmon[100224]: cluster 2023-09-23T22:27:17.896344+0000 mgr.y 2023-09-23T22:27:19.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:19 smithi052 conmon[100224]: (mgr.14148) 168 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:20.160 INFO:tasks.cephadm:Adding prometheus.a on smithi151 2023-09-23T22:27:20.161 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi151=a' 2023-09-23T22:27:20.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:20 smithi151 conmon[100454]: audit 2023-09-23T22:27:19.536453+0000 mon.c ( 2023-09-23T22:27:20.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:20 smithi151 conmon[100454]: mon.2) 16 : audit [DBG] from='client.? 172.21.15.52:0/3413423906' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T22:27:20.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:20 smithi052 conmon[100224]: audit 2023-09-23T22:27:19.536453 2023-09-23T22:27:20.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:20 smithi052 conmon[100224]: +0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.52:0/3413423906' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T22:27:20.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:20 smithi052 conmon[110082]: audit 2023-09-23T22:27 2023-09-23T22:27:20.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:20 smithi052 conmon[110082]: :19.536453+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.52:0/3413423906' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-09-23T22:27:21.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:21 smithi151 conmon[100454]: cluster 2023-09-23T22:27:19.896751+0000 mgr.y (mgr 2023-09-23T22:27:21.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:21 smithi151 conmon[100454]: .14148) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:21.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:21 smithi052 conmon[110082]: cluster 2023-09-23T22:27:19 2023-09-23T22:27:21.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:21 smithi052 conmon[110082]: .896751+0000 mgr.y (mgr.14148) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:21.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:21 smithi052 conmon[100224]: cluster 2023-09-23T22:27:19.896751+0000 2023-09-23T22:27:21.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:21 smithi052 conmon[100224]: mgr.y (mgr.14148) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:21.880 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled prometheus update... 2023-09-23T22:27:22.559 DEBUG:teuthology.orchestra.run.smithi151:prometheus.a> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T22:27:22.561 INFO:tasks.cephadm:Adding node-exporter.a on smithi052 2023-09-23T22:27:22.561 INFO:tasks.cephadm:Adding node-exporter.b on smithi151 2023-09-23T22:27:22.562 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi052=a;smithi151=b' 2023-09-23T22:27:22.590 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:27:23.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.873666+0000 mgr.y (mgr 2023-09-23T22:27:23.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: .14148) 170 : audit [DBG] from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:23.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:21.875371+0000 mgr.y (mgr.14148) 171 : cephadm [INF] Saving service prometheus spec with placement smithi151=a;count:1 2023-09-23T22:27:23.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.880244+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:23.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: cluster 2023-09-23T22:27:21.897188+0000 mgr.y (mgr.14148) 172 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:23.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.921977+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.923844+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.924754+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.930846+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[100454]: audit 2023-09-23T22:27:21.937283+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[102744]: ignoring --setuser ceph since I am not root 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[102744]: ignoring --setgroup ceph since I am not root 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:22 smithi151 conmon[102744]: debug 2023-09-23T22:27:22.969+0000 7f6fc1b45700 1 -- 172.21.15.151:0/4025297527 <== mon.0 v2:172.21.15.52:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564b9a6184e0 con 0x564b9a5f0c00 2023-09-23T22:27:23.190 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:23 smithi151 conmon[102744]: debug 2023-09-23T22:27:23.101+0000 7f6fd1d0c680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:27:23.191 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:23 smithi151 conmon[102744]: debug 2023-09-23T22:27:23.187+0000 7f6fd1d0c680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:27:23.226 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100670]: ignoring --setuser ceph since I am not root 2023-09-23T22:27:23.226 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100670]: ignoring --setgroup ceph since I am not root 2023-09-23T22:27:23.226 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100670]: debug 2023-09-23T22:27:22.980+0000 7fb1448c7700 1 -- 172.21.15.52:0/4125756074 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5559df5784e0 con 0x5559df550c00 2023-09-23T22:27:23.227 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[100670]: debug 2023-09-23T22:27:23.127+0000 7fb154a8e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:27:23.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023-09-23T22:27:21.873666+0000 mgr.y (mgr.14148 2023-09-23T22:27:23.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: ) 170 : audit [DBG] from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:23.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:21.875371+0000 mgr.y (mgr.14148) 171 : cephadm [INF] Saving service prometheus spec with placement smithi151=a;count:1 2023-09-23T22:27:23.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: 2023-09-23T22:27:21.880244+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: cluster 2023-09-23T22:27:21.897188+0000 mgr.y (mgr 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: .14148) 172 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023-09-23T22:27:21.921977+0000 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: mon.a (mon.0) 544 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023-09-23T22:27 2023-09-23T22:27:23.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: :21.923844+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: 09-23T22:27:21.924754+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023-09-23T22:27:21.930846+0000 mon.a (mon.0) 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: 547 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[100224]: T22:27:21.937283+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:27:23.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023-09-23T22:27 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: :21.873666+0000 mgr.y (mgr.14148) 170 : audit [DBG] from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: cephadm 2023-09-23T22:27 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: :21.875371+0000 mgr.y (mgr.14148) 171 : cephadm 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: [INF] Saving service prometheus spec with placement smithi151=a;count:1 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 22:27:21.880244+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 09-23T22:27:21.897188+0000 mgr.y (mgr.14148) 172 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 2023-09-23T22:27:21.921977+0000 mon.a (mon. 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 0) 544 : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:23.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 2023-09-23T22:27: 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 21.923844+0000 mon.a (mon.0) 545 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: : audit [DBG] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: -23T22:27:21.924754+0000 mon.a (mon 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: .0) 546 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 2023-09-23T22:27:21.930846+0000 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: mon.a (mon.0) 547 : audit 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' 2023-09-23T22:27:23.231 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:27:23.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 09-23T22:27:21.937283+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:27:23.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:22 smithi052 conmon[110082]: 2023-09-23T22:27:23.931 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[100670]: debug 2023-09-23T22:27:23.225+0000 7fb154a8e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:27:23.938 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[100670]: debug 2023-09-23T22:27:23.672+0000 7fb154a8e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:27:23.939 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:23 smithi151 conmon[102744]: debug 2023-09-23T22:27:23.572+0000 7f6fd1d0c680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:27:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:23 smithi151 conmon[100454]: audit 2023-09-23T22:27:22.935954 2023-09-23T22:27:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:23 smithi151 conmon[100454]: +0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:27:24.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:23 smithi151 conmon[100454]: cluster 2023-09-23T22:27:22.936063+0000 mon.a (mon.0) 550 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-09-23T22:27:24.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[100224]: audit 2023-09-23T22:27:22.935954+0000 mon.a (mon 2023-09-23T22:27:24.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[100224]: .0) 549 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:27:24.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[100224]: cluster 2023-09-23T22:27:22.936063+0000 mon.a (mon.0) 550 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-09-23T22:27:24.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[110082]: audit 2023-09-23T22:27:22.935954+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14148 172.21.15.52:0/3222191476' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:27:24.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[110082]: cluster 2023-09-23T22:27:22.936063+0000 mon.a (mon.0) 550 : cluster [DBG] 2023-09-23T22:27:24.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:23 smithi052 conmon[110082]: mgrmap e15: y(active, since 3m), standbys: x 2023-09-23T22:27:24.668 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:24 smithi151 conmon[102744]: debug 2023-09-23T22:27:24.367+0000 7f6fd1d0c680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:27:24.668 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:24 smithi151 conmon[102744]: debug 2023-09-23T22:27:24.454+0000 7f6fd1d0c680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:27:24.668 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:24 smithi151 conmon[102744]: debug 2023-09-23T22:27:24.666+0000 7f6fd1d0c680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:27:24.947 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:24 smithi052 conmon[100670]: debug 2023-09-23T22:27:24.602+0000 7fb154a8e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:27:24.947 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:24 smithi052 conmon[100670]: debug 2023-09-23T22:27:24.698+0000 7fb154a8e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:27:25.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:24 smithi052 conmon[100670]: debug 2023-09-23T22:27:24.946+0000 7fb154a8e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:27:25.910 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:25 smithi151 conmon[102744]: debug 2023-09-23T22:27:25.613+0000 7f6fd1d0c680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:27:25.910 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:25 smithi151 conmon[102744]: debug 2023-09-23T22:27:25.740+0000 7f6fd1d0c680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:27:25.910 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:25 smithi151 conmon[102744]: debug 2023-09-23T22:27:25.818+0000 7f6fd1d0c680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:27:26.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:25 smithi151 conmon[102744]: debug 2023-09-23T22:27:25.909+0000 7f6fd1d0c680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:27:26.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:26 smithi151 conmon[102744]: debug 2023-09-23T22:27:26.105+0000 7f6fd1d0c680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:27:26.337 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:26 smithi052 conmon[100670]: debug 2023-09-23T22:27:26.079+0000 7fb154a8e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:27:26.337 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:26 smithi052 conmon[100670]: debug 2023-09-23T22:27:26.241+0000 7fb154a8e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:27:26.667 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:26 smithi052 conmon[100670]: debug 2023-09-23T22:27:26.336+0000 7fb154a8e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:27:26.667 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:26 smithi052 conmon[100670]: debug 2023-09-23T22:27:26.447+0000 7fb154a8e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:27:26.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:26 smithi151 conmon[102744]: debug 2023-09-23T22:27:26.385+0000 7f6fd1d0c680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:27:26.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:26 smithi151 conmon[102744]: debug 2023-09-23T22:27:26.484+0000 7f6fd1d0c680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:27:26.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:26 smithi052 conmon[100670]: debug 2023-09-23T22:27:26.666+0000 7fb154a8e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:27:27.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:27 smithi052 conmon[100670]: debug 2023-09-23T22:27:27.030+0000 7fb154a8e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:27:27.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:27 smithi052 conmon[100670]: debug 2023-09-23T22:27:27.149+0000 7fb154a8e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:27:27.565 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:27 smithi151 conmon[102744]: debug 2023-09-23T22:27:27.212+0000 7f6fd1d0c680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:27:27.565 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:27 smithi151 conmon[102744]: debug 2023-09-23T22:27:27.295+0000 7f6fd1d0c680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:27:27.565 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:27 smithi151 conmon[102744]: debug 2023-09-23T22:27:27.387+0000 7f6fd1d0c680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:27:28.088 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:27 smithi151 conmon[102744]: debug 2023-09-23T22:27:27.563+0000 7f6fd1d0c680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:27:28.089 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:27 smithi151 conmon[102744]: debug 2023-09-23T22:27:27.823+0000 7f6fd1d0c680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:27:28.089 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:27 smithi151 conmon[102744]: debug 2023-09-23T22:27:27.954+0000 7f6fd1d0c680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:27:28.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: debug 2023-09-23T22:27:28.087+0000 7f6fd1d0c680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:27:28.346 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: debug 2023-09-23T22:27:28.260+0000 7f6fd1d0c680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:27:28.417 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100670]: debug 2023-09-23T22:27:28.019+0000 7fb154a8e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:27:28.417 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100670]: debug 2023-09-23T22:27:28.119+0000 7fb154a8e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:27:28.417 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100670]: debug 2023-09-23T22:27:28.231+0000 7fb154a8e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:27:28.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: debug 2023-09-23T22:27:28.344+0000 7f6fd1d0c680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:27:28.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: [23/Sep/2023:22:27:28] ENGINE Bus STARTING 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: CherryPy Checker: 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: The Application mounted at '' has an empty config. 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: [23/Sep/2023:22:27:28] ENGINE Serving on http://:::9283 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[102744]: [23/Sep/2023:22:27:28] ENGINE Bus STARTED 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: cluster 2023-09-23T22:27:28.360096+0000 mon.a (mon.0) 551 : 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: cluster 2023-09-23T22:27:28.360201+0000 mon.a (mon.0) 552 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: audit 2023-09-23T22:27:28. 2023-09-23T22:27:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: 361683+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: audit 2023 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: -09-23T22:27:28.362140+0000 mon.b (mon.1) 25 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: audit 2023-09-23T22:27:28.364049 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: +0000 mon.b (mon.1) 26 : audit [DBG] 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: audit 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: 2023-09-23T22:27:28.364927+0000 mon.b (mon.1 2023-09-23T22:27:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:28 smithi151 conmon[100454]: ) 27 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:27:28.732 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100670]: debug 2023-09-23T22:27:28.446+0000 7fb154a8e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:27:28.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: cluster 2023-09-23T22:27:28.360096+0000 mon.a (mon.0) 551 : cluster 2023-09-23T22:27:28.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: [DBG] Standby manager daemon x restarted 2023-09-23T22:27:28.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: cluster 2023-09-23T22:27:28.360201+0000 mon.a (mon.0) 552 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: audit 2023-09-23T22:27:28.361683+0000 mon.b 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: audit 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: 2023-09-23T22:27:28.362140+0000 mon.b ( 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: audit 2023 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: -09-23T22:27:28.364049+0000 mon.b 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:27:28.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100224]: :27:28.364927+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: cluster 2023-09-23T22:27:28.360096 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: +0000 mon.a (mon.0) 551 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: cluster 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 2023-09-23T22:27:28.360201+0000 mon.a (mon. 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 0) 552 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 23T22:27:28.361683+0000 mon.b 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: -23T22:27:28.362140+0000 mon.b 2023-09-23T22:27:28.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 09-23T22:27:28.364049+0000 mon.b (mon 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: .1) 26 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 23T22:27:28.364927+0000 mon.b 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.151:0/194900676' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:27:28.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[110082]: 2023-09-23T22:27:28.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100670]: debug 2023-09-23T22:27:28.731+0000 7fb154a8e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:27:28.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:28 smithi052 conmon[100670]: debug 2023-09-23T22:27:28.894+0000 7fb154a8e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:27:29.382 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: debug 2023-09-23T22:27:29.064+0000 7fb154a8e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:27:29.382 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: debug 2023-09-23T22:27:29.279+0000 7fb154a8e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:27:29.668 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: debug 2023-09-23T22:27:29.381+0000 7fb154a8e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:27:29.668 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: [23/Sep/2023:22:27:29] ENGINE Bus STARTING 2023-09-23T22:27:29.668 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: [80B blob data] 2023-09-23T22:27:29.668 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: [79B blob data] 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster 2023-09-23T22:27:28.415791+0000 mon.a ( 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: mon.0) 553 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster 2023-09-23T22:27:29.395547+0000 mon.a (mon. 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 0) 554 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 2023-09-23T22:27:29.395940+0000 mon.a (mon.0) 555 : 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster [INF] Activating manager daemon y 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 09-23T22:27:29.401984+0000 mon.a (mon.0) 556 : cluster [DBG] 2023-09-23T22:27:29.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: osdmap e51: 8 total, 8 up, 8 in 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 2023-09-23T22:27:29.402454 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: +0000 mon.a (mon.0) 557 : cluster [DBG] mgrmap e17: y(active, starting, since 0.00665329s), standbys: x 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 23T22:27:29.404097+0000 mon.a (mon.0) 558 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 09-23T22:27:29.404229+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: :29.404354+0000 mon.a (mon.0) 560 : audit [DBG] 2023-09-23T22:27:29.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: -23T22:27:29.405467+0000 mon.a ( 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: mon.0) 561 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: -23T22:27:29.405637+0000 mon.a ( 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: mon.0) 562 : audit [DBG] 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: -09-23T22:27:29.405834 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: +0000 mon.a (mon.0) 563 : audit 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:27:29.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27:29 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: .406145+0000 mon.a (mon.0) 564 : audit 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 2023-09-23T22:27:29. 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 406339+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 2023-09-23T22:27:29 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: .406565+0000 mon.a (mon.0) 566 : audit 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:27:29.672 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster 2023-09-23T22:27:28.415791+0000 mon.a ( 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: mon.0) 553 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster 2023-09-23T22:27:29.395547+0000 mon.a 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: (mon.0) 554 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: T22:27:29.395940+0000 mon.a (mon.0) 555 : 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster [INF] Activating manager daemon y 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 27:29.401984+0000 mon.a (mon.0) 556 : cluster [DBG] 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: osdmap e51: 8 total, 8 up, 8 in 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster 2023-09-23T22:27:29 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: .402454+0000 mon.a (mon.0) 557 : cluster 2023-09-23T22:27:29.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: [DBG] mgrmap e17: y(active, starting, since 0.00665329s), standbys: x 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: :29.404097+0000 mon.a (mon.0) 558 : audit 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 2023-09-23T22:27:29.404229+0000 mon.a (mon. 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 0) 559 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: :27:29.404354+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 2023-09-23T22:27:29.405467+0000 mon.a (mon.0 2023-09-23T22:27:29.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: ) 561 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 22:27:29.405637+0000 mon.a (mon.0) 562 : audit [DBG] 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 2023-09-23T22:27:29.405834+0000 mon.a (mon.0) 563 : audit [DBG] 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: T22:27:29.406145+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 22:27:29.406339+0000 mon.a (mon.0) 565 : audit [DBG] 2023-09-23T22:27:29.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: -23T22:27:29.406565+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: T22:27:29.406814+0000 mon.a (mon.0) 567 : 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.406814+0000 mon.a 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: (mon.0) 567 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 09-23T22:27:29.407112 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: +0000 mon.a (mon.0) 568 : audit 2023-09-23T22:27:29.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: -23T22:27:29.407306+0000 mon.a (mon.0 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: ) 569 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.407543 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: +0000 mon.a (mon.0) 570 : 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 23T22:27:29.407761+0000 mon.a (mon.0) 571 : audit 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09-23T22:27:29 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: .408058+0000 mon.a (mon.0) 572 : audit [DBG] 2023-09-23T22:27:29.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: -23T22:27:29.408545+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: cluster 2023-09-23T22 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: :27:29.411466+0000 mon.a (mon.0) 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100224]: 574 : cluster [INF] Manager daemon y is now available 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:27:29.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 27:29.407112+0000 mon.a (mon.0) 568 : audit [DBG] 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: -09-23T22:27:29.407306+0000 mon.a 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: (mon.0) 569 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: -23T22:27:29.407543+0000 mon.a (mon.0) 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 570 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 2023-09-23T22:27:29.407761+0000 mon.a 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: (mon.0) 571 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:27:29.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27: 2023-09-23T22:27:29.680 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: 29.408058+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:27:29.680 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.408545 2023-09-23T22:27:29.680 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: +0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:27:29.680 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: cluster 2023-09-23T22:27:29.411466+0000 mon.a ( 2023-09-23T22:27:29.680 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[110082]: mon.0) 574 : cluster [INF] Manager daemon y is now available 2023-09-23T22:27:29.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: cluster 2023-09-23T22:27:28.415791+0000 mon.a (mon.0) 553 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: cluster 2023-09-23T22:27:29.395547+0000 mon.a (mon.0) 554 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: cluster 2023-09-23T22:27:29.395940+0000 mon.a (mon.0) 555 : cluster [INF] Activating manager daemon y 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: cluster 2023-09-23T22: 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 27:29.401984+0000 mon.a (mon.0) 556 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: cluster 2023-09-23T22:27:29.402454+0000 mon.a (mon.0) 557 : cluster [DBG] 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: mgrmap e17: y(active, starting, since 0.00665329s), standbys: x 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.404097+0000 mon.a (mon 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: .0) 558 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:27:29.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: T22:27:29.404229+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.404354+0000 mon.a (mon.0) 560 : 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: .405467+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 22:27:29.405637+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.405834+0000 mon.a (mon.0) 563 : audit [DBG] 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:27:29.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: :27:29.406145+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.406339+0000 mon.a (mon.0) 565 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 2023-09-23T22:27:29.406565+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.406814+0000 mon.a (mon.0 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: ) 567 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 2023-09-23T22:27:29.407112+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 2023-09-23T22:27:29.407306+0000 mon.a (mon.0) 569 : 2023-09-23T22:27:29.683 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.407543+0000 mon.a (mon.0 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: ) 570 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.407761+0000 mon.a (mon.0) 571 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: -09-23T22:27:29.408058+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: audit 2023-09-23T22:27:29 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: .408545+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: cluster 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: 2023-09-23T22:27:29.411466+0000 2023-09-23T22:27:29.684 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:29 smithi151 conmon[100454]: mon.a (mon.0) 574 : cluster [INF] Manager daemon y is now available 2023-09-23T22:27:29.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: CherryPy Checker: 2023-09-23T22:27:29.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: The Application mounted at '' has an empty config. 2023-09-23T22:27:29.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: 2023-09-23T22:27:29.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: [23/Sep/2023:22:27:29] ENGINE Serving on http://:::9283 2023-09-23T22:27:29.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:29 smithi052 conmon[100670]: [23/Sep/2023:22:27:29] ENGINE Bus STARTED 2023-09-23T22:27:30.416 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled node-exporter update... 2023-09-23T22:27:30.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.543858+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:30.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.545298+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.581213+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.594010+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:27:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: audit 2023-09-23T22:27:29.603265+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:27:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: cluster 2023-09-23T22:27:30.406708+0000 mon.a (mon.0) 580 : cluster [DBG] mgrmap e18: y(active, since 1.01091s), standbys: x 2023-09-23T22:27:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:30 smithi151 conmon[100454]: audit 2023-09-23T22:27:30.415833+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:30.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.543858+0000 mon.a ( 2023-09-23T22:27:30.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: mon.0) 575 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:30.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.545298+0000 mon.a (mon. 2023-09-23T22:27:30.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: 0) 576 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: T22:27:29.581213+0000 mon.a (mon.0) 577 : 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: 09-23T22:27:29.594010+0000 mon.a (mon.0 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: ) 578 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit 2023-09-23T22:27:29.603265+0000 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: mon.a (mon.0) 579 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: T22:27:30.406708+0000 mon.a ( 2023-09-23T22:27:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: mon.0) 580 : cluster [DBG] mgrmap e18: y(active, since 1.01091s), standbys: x 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: audit 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: 2023-09-23T22:27:30.415833 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[110082]: +0000 mon.a (mon.0) 581 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.543858+0000 mon.a (mon.0 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: ) 575 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.545298+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.581213+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.594010+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:27:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit 2023-09-23T22:27:29.603265+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:27:30.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: cluster 2023-09-23T22:27:30.406708+0000 mon.a (mon.0) 580 : cluster [DBG] mgrmap e18: y(active, since 1.01091s), standbys: x 2023-09-23T22:27:30.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit 2023-09-23T22:27 2023-09-23T22:27:30.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: :30.415833+0000 mon.a (mon.0) 581 : 2023-09-23T22:27:30.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:30 smithi052 conmon[100224]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:31.303 DEBUG:teuthology.orchestra.run.smithi052:node-exporter.a> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T22:27:31.305 DEBUG:teuthology.orchestra.run.smithi151:node-exporter.b> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T22:27:31.308 INFO:tasks.cephadm:Adding alertmanager.a on smithi052 2023-09-23T22:27:31.308 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi052=a' 2023-09-23T22:27:31.339 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:27:31.340 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:27:31.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:31 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:30.412713+0000 mgr.y (mgr.24314 2023-09-23T22:27:31.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:31 smithi151 conmon[100454]: ) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi052=a;smithi151=b;count:2 2023-09-23T22:27:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:31 smithi151 conmon[100454]: audit 2023-09-23T22:27:31.121602+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:31 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:30.412713+0000 mgr.y (mgr 2023-09-23T22:27:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:31 smithi052 conmon[100224]: .24314) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi052=a;smithi151=b;count:2 2023-09-23T22:27:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:31 smithi052 conmon[100224]: audit 2023-09-23T22:27:31.121602+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:31 smithi052 conmon[110082]: cephadm 2023-09-23T22:27 2023-09-23T22:27:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:31 smithi052 conmon[110082]: :30.412713+0000 mgr.y (mgr.24314) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi052=a;smithi151=b;count:2 2023-09-23T22:27:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:31 smithi052 conmon[110082]: audit 2023-09-23T22:27:31.121602+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: cluster 2023-09-23T22:27:31.406975+0000 mgr.y (mgr.24314) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:31.549898+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:31.923301+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:31.924460+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:31.925986+0000 mgr.y (mgr.24314) 5 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:31.926333+0000 mgr.y (mgr.24314) 6 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:31.946706+0000 mgr.y (mgr.24314) 7 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:31.951566+0000 mgr.y (mgr.24314 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: ) 8 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:31.995247+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:31.999289+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:32.003456+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:32.009090+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:27:32.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: audit 2023-09-23T22:27:32.412855+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:27:32.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:32 smithi151 conmon[100454]: cluster 2023-09-23T22:27:32.412942+0000 mon.a (mon.0) 591 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T22:27:32.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: cluster 2023-09-23T22:27:31.406975+0000 mgr.y (mgr.24314) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23T22:27:31.549898+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: T22:27:31.923301+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23T22:27 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: :31.924460+0000 mon.a (mon.0) 585 : audit 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:31.925986 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 5 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:31.926333+0000 2023-09-23T22:27:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: mgr.y (mgr.24314) 6 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: cephadm 2023-09- 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: 23T22:27:31.946706+0000 mgr.y (mgr.24314 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: ) 7 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: cephadm 2023 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: -09-23T22:27:31.951566+0000 mgr.y (mgr.24314) 8 : cephadm [INF] 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23T22:27:31 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: .995247+0000 mon.a (mon.0) 586 : audit [INF] 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23T22:27:31 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: .999289+0000 mon.a (mon.0) 587 : audit [INF] 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23T22:27:32.003456 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: +0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: T22:27:32.009090+0000 mon.a (mon.0) 589 : audit [INF] 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: audit 2023-09-23T22:27:32.412855 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: +0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:27:32.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: -09-23T22:27:32.412942+0000 mon.a (mon 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[100224]: .0) 591 : cluster [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: cluster 2023-09-23T22:27:31.406975+0000 mgr.y (mgr.24314 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: ) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023-09-23T22:27:31.549898+0000 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: mon.a (mon.0) 583 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023-09-23T22:27:31. 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 923301+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:27:32.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 27:31.924460+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: cephadm 2023 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: -09-23T22:27:31.925986+0000 mgr.y (mgr.24314) 5 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 2023-09-23T22:27:31.926333+0000 mgr.y (mgr.24314) 6 : cephadm 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: cephadm 2023-09 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: -23T22:27:31.946706+0000 mgr.y (mgr.24314) 7 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: cephadm 2023-09 2023-09-23T22:27:32.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: -23T22:27:31.951566+0000 mgr.y (mgr.24314) 8 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: -09-23T22:27:31.995247+0000 mon.a (mon.0) 586 : audit [INF] 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 09-23T22:27:31.999289+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:27:32.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 22:27:32.003456+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: T22:27:32.009090+0000 mon.a (mon.0) 589 : 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: audit 2023 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: -09-23T22:27:32.412855+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: 27:32.412942+0000 mon.a (mon.0) 591 : cluster 2023-09-23T22:27:33.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:32 smithi052 conmon[110082]: [DBG] mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T22:27:33.201 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled alertmanager update... 2023-09-23T22:27:33.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:33 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:32.427462+0000 mgr.y (mgr.24314) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi151 2023-09-23T22:27:33.843 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:33 smithi151 conmon[100454]: audit 2023-09-23T22:27:33.200118+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:33.867 DEBUG:teuthology.orchestra.run.smithi052:alertmanager.a> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T22:27:33.869 INFO:tasks.cephadm:Adding grafana.a on smithi151 2023-09-23T22:27:33.870 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi151=a' 2023-09-23T22:27:33.897 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:32.427462+0000 mgr.y ( 2023-09-23T22:27:33.898 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[110082]: mgr.24314) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi151 2023-09-23T22:27:33.898 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:27:33.898 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[110082]: -23T22:27:33.200118+0000 mon.a (mon.0) 592 : audit [INF] 2023-09-23T22:27:33.898 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:33.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:32.427462+0000 mgr.y (mgr.24314) 2023-09-23T22:27:33.898 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[100224]: 9 : cephadm [INF] Deploying daemon prometheus.a on smithi151 2023-09-23T22:27:33.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:33 smithi052 conmon[100224]: audit 2023-09-23T22:27:33.200118+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:33.904 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:27:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:34 smithi151 conmon[100454]: audit 2023-09-23T22:27:33.194376+0000 mgr.y (mgr.24314) 10 : audit [DBG] from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi052=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:34 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:33.196095+0000 mgr.y (mgr.24314) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi052=a;count:1 2023-09-23T22:27:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:34 smithi151 conmon[100454]: cluster 2023-09-23T22:27:33.407332+0000 mgr.y (mgr.24314) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[100224]: audit 2023-09-23T22:27:33.194376+0000 mgr.y (mgr.24314) 10 : audit [DBG] from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi052=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:33.196095+0000 mgr.y (mgr.24314) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi052=a;count:1 2023-09-23T22:27:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:27:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[100224]: T22:27:33.407332+0000 mgr.y (mgr.24314) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: :27:33.194376+0000 mgr.y (mgr.24314) 10 : audit [DBG] from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi052=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: 2023-09-23T22:27:33.196095+0000 mgr.y (mgr.24314) 11 : cephadm 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: [INF] Saving service alertmanager spec with placement smithi052=a;count:1 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: cluster 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: 2023-09-23T22:27:33.407332+0000 mgr.y (mgr.24314) 12 : cluster [DBG] 2023-09-23T22:27:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:34 smithi052 conmon[110082]: pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:36.228 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled grafana update... 2023-09-23T22:27:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:36 smithi151 conmon[100454]: cluster 2023-09-23T22:27:35.407751+0000 mgr.y (mgr.24314) 13 2023-09-23T22:27:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:36 smithi151 conmon[100454]: : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:36.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:36 smithi052 conmon[110082]: cluster 2023-09-23T22:27:35.407751+0000 mgr.y 2023-09-23T22:27:36.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:36 smithi052 conmon[110082]: (mgr.24314) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:36.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:36 smithi052 conmon[100224]: cluster 2023-09-23T22:27:35.407751+0000 mgr.y (mgr.24314 2023-09-23T22:27:36.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:36 smithi052 conmon[100224]: ) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: audit 2023-09-23T22:27:36.221961+0000 mgr.y 2023-09-23T22:27:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: (mgr.24314) 14 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: cephadm 2023-09 2023-09-23T22:27:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: -23T22:27:36.223584+0000 mgr.y (mgr.24314) 15 : cephadm [INF] Saving service grafana spec with placement smithi151=a;count:1 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: -23T22:27:36.227982+0000 mon.a (mon 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[110082]: .0) 593 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: audit 2023-09-23T22:27:36. 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: 221961+0000 mgr.y (mgr.24314) 14 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: cephadm 2023-09-23T22 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: :27:36.223584+0000 mgr.y (mgr.24314) 15 : cephadm [INF] Saving service grafana spec with placement smithi151=a;count:1 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: 2023-09-23T22:27:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: audit 2023-09-23T22:27:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: 2023-09-23T22:27:36.227982 2023-09-23T22:27:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: +0000 mon.a (mon.0) 593 : 2023-09-23T22:27:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:37 smithi052 conmon[100224]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:37.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:37 smithi151 conmon[100454]: audit 2023-09-23T22:27:36.221961 2023-09-23T22:27:37.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:37 smithi151 conmon[100454]: +0000 mgr.y (mgr.24314) 14 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:27:37.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:37 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:36.223584+0000 mgr.y (mgr.24314) 15 : cephadm [INF] Saving service grafana spec with placement smithi151=a;count:1 2023-09-23T22:27:37.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:37 smithi151 conmon[100454]: audit 2023-09-23T22:27:37.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:37 smithi151 conmon[100454]: 2023-09-23T22:27:36.227982+0000 mon.a (mon.0) 593 2023-09-23T22:27:37.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:37 smithi151 conmon[100454]: : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:38 smithi052 conmon[110082]: cluster 2023-09-23T22:27:37. 2023-09-23T22:27:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:38 smithi052 conmon[110082]: 408326+0000 mgr.y (mgr.24314) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:38 smithi052 conmon[100224]: cluster 2023-09-23T22:27:37.408326+0000 mgr.y ( 2023-09-23T22:27:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:38 smithi052 conmon[100224]: mgr.24314) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:38 smithi151 conmon[100454]: cluster 2023-09-23T22:27:37.408326+0000 mgr.y (mgr.24314) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:39.930 DEBUG:teuthology.orchestra.run.smithi151:grafana.a> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T22:27:39.934 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T22:27:39.934 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T22:27:39.966 INFO:journalctl@ceph.grafana.a.smithi151.stdout:-- Logs begin at Sat 2023-09-23 22:06:15 UTC. -- 2023-09-23T22:27:40.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:40 smithi052 conmon[110082]: cluster 2023-09-23T22:27:39.408878+0000 mgr.y (mgr.24314) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:40 smithi052 conmon[100224]: cluster 2023-09-23T22:27:39.408878+0000 mgr.y (mgr.24314) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:40.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:40 smithi151 conmon[100454]: cluster 2023-09-23T22:27:39.408878+0000 mgr.y (mgr.24314) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi052.stdout:[client.0] 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi052.stdout: key = AQBdZg9lKJK2NBAA+gd+yQwculKkIb2Lser71g== 2023-09-23T22:27:42.433 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:27:42.433 DEBUG:teuthology.orchestra.run.smithi052:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T22:27:42.433 DEBUG:teuthology.orchestra.run.smithi052:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T22:27:42.478 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T22:27:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[100224]: cluster 2023-09-23T22:27:41. 2023-09-23T22:27:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[100224]: 409248+0000 mgr.y (mgr.24314) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[100224]: audit 2023-09-23T22:27:41.883925+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.52:0/2287571953' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[100224]: audit 2023-09-23T22:27:41.884224+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[100224]: audit 2023-09-23T22:27:41.887644+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: cluster 2023-09-23T22:27:41.409248+0000 mgr.y (mgr.24314 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: ) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: audit 2023-09-23T22:27:41.883925+0000 mon.c (mon.2) 17 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: : audit [INF] from='client.? 172.21.15.52:0/2287571953' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: audit 2023-09-23T22:27:41. 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: 884224+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:27:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:42 smithi052 conmon[110082]: 09-23T22:27:41.887644+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:27:42.920 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:42 smithi151 conmon[100454]: cluster 2023-09-23T22:27:41.409248+0000 mgr.y (mgr.24314) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:42.920 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:42 smithi151 conmon[100454]: audit 2023-09-23T22:27:41.883925+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.52:0/2287571953' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:42.920 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:42 smithi151 conmon[100454]: audit 2023-09-23T22:27:41.884224+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:42.920 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:42 smithi151 conmon[100454]: audit 2023-09-23T22:27:41.887644+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:27:44.330 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:44 smithi151 systemd[1]: Starting Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:27:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:44 smithi151 conmon[100454]: cluster 2023-09-23T22:27:43. 2023-09-23T22:27:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:44 smithi151 conmon[100454]: 409671+0000 mgr.y (mgr.24314) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:44.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:44 smithi052 conmon[100224]: cluster 2023-09-23T22:27:43.409671+0000 mgr.y (mgr.24314) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:44.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:44 smithi052 conmon[110082]: cluster 2023-09-23T22:27:43.409671+0000 mgr.y (mgr.24314) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:45.164 INFO:teuthology.orchestra.run.smithi151.stdout:[client.1] 2023-09-23T22:27:45.164 INFO:teuthology.orchestra.run.smithi151.stdout: key = AQBhZg9lUPaHCRAAwqegWMAVegUPSpWnwAYUKQ== 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.177Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.177Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.177Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi151 (none))" 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.177Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.177Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.179Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 bash[129758]: 47cd70b0fe355dd3423cf08c01c600e3ffeb4c1467c16faab66d499af271a50f 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.179Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T22:27:45.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.180Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 systemd[1]: Started Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.182Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.182Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.839µs 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.182Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.183Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.183Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.558µs wal_replay_duration=122.307µs total_replay_duration=160.287µs 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.183Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.183Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.183Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.199Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.188701ms db_storage=731ns remote_storage=1.393µs web_handler=356ns query_engine=711ns scrape=453.819µs scrape_sd=19.07µs notify=983ns notify_sd=1.405µs rules=15.452269ms 2023-09-23T22:27:45.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:27:45 smithi151 conmon[129826]: ts=2023-09-23T22:27:45.200Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T22:27:45.806 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2023-09-23T22:27:45.806 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T22:27:45.806 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T22:27:45.846 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T22:27:45.847 DEBUG:teuthology.orchestra.run.smithi052:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T22:27:45.867 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:45 smithi052 systemd[1]: Starting Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:27:46.224 INFO:teuthology.misc.health.smithi052.stdout: 2023-09-23T22:27:46.224 INFO:teuthology.misc.health.smithi052.stdout:{"epoch":51,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:27:29.395624+0000","last_up_change":"2023-09-23T22:27:12.276151+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6803","nonce":858289560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6805","nonce":858289560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6809","nonce":858289560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6807","nonce":858289560}]},"public_addr":"172.21.15.52:6803/858289560","cluster_addr":"172.21.15.52:6805/858289560","heartbeat_back_addr":"172.21.15.52:6809/858289560","heartbeat_front_addr":"172.21.15.52:6807/858289560","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6811","nonce":3743472544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6813","nonce":3743472544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6817","nonce":3743472544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6815","nonce":3743472544}]},"public_addr":"172.21.15.52:6811/3743472544","cluster_addr":"172.21.15.52:6813/3743472544","heartbeat_back_addr":"172.21.15.52:6817/3743472544","heartbeat_front_addr":"172.21.15.52:6815/3743472544","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6819","nonce":1781181972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6821","nonce":1781181972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6825","nonce":1781181972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6823","nonce":1781181972}]},"public_addr":"172.21.15.52:6819/1781181972","cluster_addr":"172.21.15.52:6821/1781181972","heartbeat_back_addr":"172.21.15.52:6825/1781181972","heartbeat_front_addr":"172.21.15.52:6823/1781181972","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6827","nonce":1128061400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6829","nonce":1128061400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6833","nonce":1128061400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6831","nonce":1128061400}]},"public_addr":"172.21.15.52:6827/1128061400","cluster_addr":"172.21.15.52:6829/1128061400","heartbeat_back_addr":"172.21.15.52:6833/1128061400","heartbeat_front_addr":"172.21.15.52:6831/1128061400","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:27:46.237 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T22:27:46.237 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T22:27:46.237 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T22:27:46.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:46 smithi151 conmon[100454]: audit 2023-09-23T22:27:45.159140+0000 mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.151:0/1268877446' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:46.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:46 smithi151 conmon[100454]: audit 2023-09-23T22:27:45.159712+0000 mon.a (mon.0) 596 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:46 smithi151 conmon[100454]: audit 2023-09-23T22:27:45.164164+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:27:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:46 smithi151 conmon[100454]: audit 2023-09-23T22:27:45.251483+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:46 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:45.253664+0000 mgr.y (mgr.24314) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi052 2023-09-23T22:27:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:46 smithi151 conmon[100454]: cluster 2023-09-23T22:27:45.410000+0000 mgr.y (mgr.24314) 21 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:46.493 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:46 smithi052 bash[138359]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-09-23T22:27:46.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: audit 2023-09-23T22:27:45.159140+0000 2023-09-23T22:27:46.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.151:0/1268877446' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:46.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: audit 2023-09-23T22:27:45.159712+0000 mon.a (mon.0) 596 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: audit 2023-09-23T22:27:45.164164+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: audit 2023-09-23T22:27:45.251483+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:45.253664+0000 mgr.y (mgr.24314) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi052 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[100224]: cluster 2023-09-23T22:27:45.410000+0000 mgr.y (mgr.24314) 21 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: audit 2023-09-23T22:27:45.159140+0000 mon.b (mon 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: .1) 28 : audit [INF] from='client.? 172.21.15.151:0/1268877446' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: audit 2023-09-23T22:27:45.159712+0000 mon.a (mon.0) 596 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: audit 2023-09-23T22:27:45.164164+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: audit 2023-09-23T22:27:45.251483+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:45.253664+0000 mgr.y (mgr.24314) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi052 2023-09-23T22:27:46.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:46 smithi052 conmon[110082]: cluster 2023-09-23T22:27:45.410000+0000 mgr.y (mgr.24314) 21 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:46.993 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:46 smithi052 bash[138359]: Getting image source signatures 2023-09-23T22:27:47.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:47 smithi151 conmon[100454]: audit 2023-09-23T22:27:46.223456+0000 mon.c (mon.2) 18 : audit 2023-09-23T22:27:47.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:47 smithi151 conmon[100454]: [DBG] from='client.? 172.21.15.52:0/1194438765' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:47 smithi052 conmon[110082]: audit 2023-09-23T22:27:46 2023-09-23T22:27:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:47 smithi052 conmon[110082]: .223456+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.52:0/1194438765' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:47.493 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:47 smithi052 bash[138359]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-09-23T22:27:47.493 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:47 smithi052 bash[138359]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-09-23T22:27:47.494 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:47 smithi052 bash[138359]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-09-23T22:27:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:47 smithi052 conmon[100224]: audit 2023-09-23T22:27:46 2023-09-23T22:27:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:47 smithi052 conmon[100224]: .223456+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.52:0/1194438765' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:48.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:48 smithi052 conmon[100224]: cluster 2023-09-23T22:27:47 2023-09-23T22:27:48.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:48 smithi052 conmon[100224]: .410299+0000 mgr.y (mgr.24314) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:48.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:48 smithi151 conmon[100454]: cluster 2023-09-23T22:27:47.410299+0000 mgr.y (mgr.24314) 22 : cluster 2023-09-23T22:27:48.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:48 smithi151 conmon[100454]: [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:48 smithi052 conmon[110082]: cluster 2023-09-23T22:27:47.410299+0000 mgr.y (mgr.24314 2023-09-23T22:27:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:48 smithi052 conmon[110082]: ) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:48.493 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:48 smithi052 bash[138359]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-09-23T22:27:48.493 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:48 smithi052 bash[138359]: Writing manifest to image destination 2023-09-23T22:27:48.494 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:48 smithi052 bash[138359]: Storing signatures 2023-09-23T22:27:49.144 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'rbd' created 2023-09-23T22:27:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:49 smithi151 conmon[100454]: audit 2023-09-23T22:27:48.428554+0000 mon.c (mon.2) 19 : audit 2023-09-23T22:27:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:49 smithi151 conmon[100454]: [INF] from='client.? 172.21.15.52:0/1749940809' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:27:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:49 smithi151 conmon[100454]: audit 2023-09-23T22:27:48.428839+0000 mon.a (mon.0) 599 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:27:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:27:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[110082]: :27:48.428554+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.52:0/1749940809' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:27:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:27:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[110082]: 22:27:48.428839+0000 mon.a (mon 2023-09-23T22:27:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[110082]: .0) 599 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:27:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[110082]: 2023-09-23T22:27:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:27:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[100224]: T22:27:48.428554+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.52:0/1749940809' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:27:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:27:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[100224]: 09-23T22:27:48.428839+0000 2023-09-23T22:27:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[100224]: mon.a (mon.0) 599 2023-09-23T22:27:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:49 smithi052 conmon[100224]: : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:27:50.243 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T22:27:50.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: audit 2023-09-23T22:27:49.141183+0000 mon.a (mon.0) 600 : audit [INF] 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: cluster 2023-09-23T22:27:49.141288+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: audit 2023-09-23T22:27:49.141616+0000 mon.a (mon.0) 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: 602 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: audit 2023-09-23T22:27:49.142386+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: cluster 2023-09-23T22:27:49.410608+0000 mgr.y (mgr 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: .24314) 23 : cluster [DBG] pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: audit 2023-09-23T22:27:49.587777 2023-09-23T22:27:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:50 smithi151 conmon[100454]: +0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: audit 2023-09-23T22:27:49.141183+0000 mon.a (mon.0 2023-09-23T22:27:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: ) 600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:27:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: cluster 2023-09-23T22:27:49.141288+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T22:27:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: audit 2023-09-23T22:27:49.141616+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:27:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: audit 2023-09-23T22:27:49.142386+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T22:27:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: cluster 2023-09-23T22:27:49.410608+0000 mgr.y (mgr.24314) 23 : cluster [DBG] pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[100224]: audit 2023-09-23T22:27:49.587777+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[110082]: audit 2023-09-23T22:27:49.141183+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:27:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[110082]: cluster 2023-09-23T22:27:49.141288+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-09-23T22:27:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[110082]: audit 2023-09-23T22:27:49.141616+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:27:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[110082]: audit 2023-09-23T22:27:49.142386+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-09-23T22:27:50.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[110082]: cluster 2023-09-23T22:27:49.410608+0000 mgr.y (mgr.24314) 23 : cluster [DBG] pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:50.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[110082]: audit 2023-09-23T22:27:49.587777+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:50.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.326Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T22:27:50.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.326Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T22:27:50.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.326Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T22:27:50.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.326Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:27:50.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T22:27:50.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T22:27:50.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T22:27:50.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 conmon[138767]: ts=2023-09-23T22:27:50.327Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 bash[138359]: 319ab89d682474b3091df72c468c6f1b87efb5353a58b18b03587f07a9587f3c 2023-09-23T22:27:50.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:27:50 smithi052 systemd[1]: Started Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:27:51.145 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:50 smithi151 systemd[1]: Starting Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:27:51.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[110082]: audit 2023-09-23T22:27:50.145876+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[110082]: cluster 2023-09-23T22:27:50.146106+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[110082]: audit 2023-09-23T22:27:50.407365+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:50.408388+0000 mgr.y (mgr.24314) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi151 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[100224]: audit 2023-09-23T22:27:50.145876+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[100224]: cluster 2023-09-23T22:27:50.146106+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[100224]: audit 2023-09-23T22:27:50.407365+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:51.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:51 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:50.408388+0000 mgr.y (mgr.24314) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi151 2023-09-23T22:27:51.448 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:51 smithi151 bash[130267]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-09-23T22:27:51.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: audit 2023-09-23T22:27: 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: 50.145876+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: cluster 2023-09-23T22:27:50.146106+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: audit 2023-09-23T22:27:50.407365+0000 mon.a (mon.0 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: ) 607 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: cephadm 2023- 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: 09-23T22:27:50.408388+0000 mgr.y (mgr. 2023-09-23T22:27:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:51 smithi151 conmon[100454]: 24314) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi151 2023-09-23T22:27:52.154 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:51 smithi151 bash[130267]: Getting image source signatures 2023-09-23T22:27:52.155 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:52 smithi151 bash[130267]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-09-23T22:27:52.155 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:52 smithi151 bash[130267]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-09-23T22:27:52.155 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:52 smithi151 bash[130267]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-09-23T22:27:52.422 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:52 smithi151 bash[130267]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-09-23T22:27:52.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:52 smithi151 conmon[100454]: cluster 2023-09-23T22:27:51.150302+0000 mon.a (mon.0) 608 : cluster 2023-09-23T22:27:52.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:52 smithi151 conmon[100454]: [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:27:52.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:52 smithi151 conmon[100454]: cluster 2023-09-23T22:27:51.411061+0000 mgr.y (mgr.24314) 25 : cluster [DBG] pgmap v17: 9 pgs: 5 creating+activating, 4 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:52 smithi052 conmon[100224]: cluster 2023-09-23T22:27: 2023-09-23T22:27:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:52 smithi052 conmon[100224]: 51.150302+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:27:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:52 smithi052 conmon[100224]: cluster 2023-09-23T22:27:51.411061+0000 mgr.y (mgr.24314) 25 : cluster [DBG] pgmap v17: 9 pgs: 5 creating+activating, 4 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:52 smithi052 conmon[110082]: cluster 2023-09-23T22:27:51.150302+0000 2023-09-23T22:27:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:52 smithi052 conmon[110082]: mon.a (mon.0) 608 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:27:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:52 smithi052 conmon[110082]: cluster 2023-09-23T22:27:51.411061+0000 mgr.y (mgr.24314) 25 : cluster [DBG] pgmap v17: 9 pgs: 5 creating+activating, 4 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:52.698 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:52 smithi151 bash[130267]: Writing manifest to image destination 2023-09-23T22:27:52.699 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:52 smithi151 bash[130267]: Storing signatures 2023-09-23T22:27:53.163 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T22:27:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[100454]: audit 2023-09-23T22:27:52.160902+0000 mon.a (mon.0) 609 : audit [INF] 2023-09-23T22:27:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/3905315450' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:27:53.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:53 smithi052 conmon[110082]: audit 2023-09-23T22:27:52.160902+0000 mon.a ( 2023-09-23T22:27:53.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:53 smithi052 conmon[110082]: mon.0) 609 : audit [INF] from='client.? 172.21.15.52:0/3905315450' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:27:53.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:53 smithi052 conmon[100224]: audit 2023-09-23T22:27:52.160902+0000 mon.a (mon.0) 609 2023-09-23T22:27:53.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:53 smithi052 conmon[100224]: : audit [INF] from='client.? 172.21.15.52:0/3905315450' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:27:53.745 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:27:53.745 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:27:53.745 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:27:53.949 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.683Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T22:27:53.949 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.683Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T22:27:53.949 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.683Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T22:27:53.949 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.683Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:27:53.949 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T22:27:53.950 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T22:27:53.951 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T22:27:53.952 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.684Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.685Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 conmon[130342]: ts=2023-09-23T22:27:53.685Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 bash[130267]: ab706ae51a5db211cac3634070ce70e71ce4e24c51d300df631d8b381d24a317 2023-09-23T22:27:53.953 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:27:53 smithi151 systemd[1]: Started Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:27:54.222 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:54.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[110082]: audit 2023-09-23T22:27:53.159845+0000 mon.a (mon.0) 610 : audit [INF] from='client.? 172.21.15.52:0/3905315450' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:27:54.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[110082]: cluster 2023-09-23T22:27:53.159936+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:27:54.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[110082]: cluster 2023-09-23T22:27:53.411427+0000 mgr.y (mgr.24314) 26 : cluster [DBG] pgmap v19: 9 pgs: 5 creating+activating, 4 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:54.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[110082]: audit 2023-09-23T22:27:53.756260+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:54.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[110082]: audit 2023-09-23T22:27:53.757857+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:54.433 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[100224]: audit 2023-09-23T22:27:53.159845+0000 mon.a (mon.0) 610 : audit [INF] from='client.? 172.21.15.52:0/3905315450' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:27:54.433 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[100224]: cluster 2023-09-23T22:27:53.159936+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:27:54.433 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[100224]: cluster 2023-09-23T22:27:53.411427+0000 mgr.y (mgr.24314) 26 : cluster [DBG] pgmap v19: 9 pgs: 5 creating+activating, 4 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:54.433 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[100224]: audit 2023-09-23T22:27:53.756260+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:54.433 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:54 smithi052 conmon[100224]: audit 2023-09-23T22:27:53.757857+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:54.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:54 smithi151 conmon[100454]: audit 2023-09-23T22:27:53.159845+0000 mon.a (mon.0) 610 : audit 2023-09-23T22:27:54.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:54 smithi151 conmon[100454]: [INF] from='client.? 172.21.15.52:0/3905315450' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:27:54.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:54 smithi151 conmon[100454]: cluster 2023-09-23T22:27:53.159936+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:27:54.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:54 smithi151 conmon[100454]: cluster 2023-09-23T22:27:53.411427+0000 mgr.y (mgr.24314) 26 : cluster [DBG] pgmap v19: 9 pgs: 5 creating+activating, 4 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:54.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:54 smithi151 conmon[100454]: audit 2023-09-23T22:27:53.756260+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:54.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:54 smithi151 conmon[100454]: audit 2023-09-23T22:27:53.757857+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:27:55.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:27:55 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:27:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:27:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:55 smithi052 conmon[110082]: audit 2023-09-23T22:27:54.594031+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:55.785 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:55 smithi052 conmon[100224]: audit 2023-09-23T22:27:54.594031+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:55.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:55 smithi151 conmon[100454]: audit 2023-09-23T22:27:54.594031+0000 2023-09-23T22:27:55.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:55 smithi151 conmon[100454]: mon.a (mon.0) 614 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:56.162 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:27:56.723 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":19,"active_gid":24314,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6800","nonce":2740388443},{"type":"v1","addr":"172.21.15.52:6801","nonce":2740388443}]},"active_addr":"172.21.15.52:6801/2740388443","active_change":"2023-09-23T22:27:29.395785+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":3070502744}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":1078027830}]}]} 2023-09-23T22:27:56.727 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:27:56.728 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:27:56.728 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:27:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: cluster 2023-09-23T22:27:55.411764+0000 mgr.y (mgr.24314) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: audit 2023-09-23T22:27:55.703131+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: audit 2023-09-23T22:27:56.158338+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.52:0/334606547' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: audit 2023-09-23T22:27:56.215956+0000 mon.a (mon.0 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: ) 616 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: audit 2023-09-23T22:27:56.216749+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: audit 2023-09-23T22:27:56.217432+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[100224]: audit 2023-09-23T22:27:56.221743+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: cluster 2023-09-23T22:27:55.411764+0000 mgr.y (mgr.24314) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:56.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: audit 2023-09-23T22:27:55.703131+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: audit 2023-09-23T22:27:56.158338+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.52:0/334606547' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:27:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: audit 2023-09-23T22:27:56.215956+0000 mon.a ( 2023-09-23T22:27:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: mon.0) 616 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: audit 2023-09-23T22:27:56.216749+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: audit 2023-09-23T22:27:56.217432+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:56 smithi052 conmon[110082]: audit 2023-09-23T22:27:56.221743+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:57.089 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: cluster 2023-09-23T22:27:55.411764+0000 mgr.y (mgr.24314) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit 2023-09-23T22:27:55.703131+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit 2023-09-23T22:27:56.158338+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.52:0/334606547' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:27:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit 2023-09-23T22:27:56.215956+0000 mon.a (mon 2023-09-23T22:27:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: .0) 616 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: :27:56.216749+0000 mon.a (mon.0) 617 : 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit 2023-09-23T22:27:56 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: .217432+0000 mon.a (mon.0) 618 : audit [INF] 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: audit 2023-09-23T22:27:56.221743 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: +0000 mon.a (mon.0) 619 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:27:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:56 smithi151 conmon[100454]: 2023-09-23T22:27:57.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:57 smithi052 conmon[100224]: cephadm 2023-09-23T22:27:56.234507 2023-09-23T22:27:57.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:57 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 28 : cephadm [INF] Deploying daemon alertmanager.a on smithi052 2023-09-23T22:27:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:57 smithi052 conmon[110082]: cephadm 2023-09-23T22:27:56.234507+0000 mgr.y (mgr.24314) 28 : cephadm [INF] Deploying daemon alertmanager.a on smithi052 2023-09-23T22:27:58.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:57 smithi151 conmon[100454]: cephadm 2023-09-23T22:27:56.234507+0000 mgr.y (mgr.24314) 28 : 2023-09-23T22:27:58.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:57 smithi151 conmon[100454]: cephadm [INF] Deploying daemon alertmanager.a on smithi052 2023-09-23T22:27:58.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:27:58 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:27:58] "GET /metrics HTTP/1.1" 200 153246 "" "Prometheus/2.33.4" 2023-09-23T22:27:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:58 smithi052 conmon[110082]: cluster 2023-09-23T22:27:57.412079+0000 mgr.y (mgr.24314) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:58 smithi052 conmon[100224]: cluster 2023-09-23T22:27:57.412079+0000 mgr.y (mgr.24314) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:59.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:58 smithi151 conmon[100454]: cluster 2023-09-23T22:27:57.412079+0000 mgr.y (mgr.24314) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:27:59.658 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:27:59.658 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":55,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:27:53.152659+0000","last_up_change":"2023-09-23T22:27:12.276151+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6803","nonce":858289560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6805","nonce":858289560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6809","nonce":858289560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6807","nonce":858289560}]},"public_addr":"172.21.15.52:6803/858289560","cluster_addr":"172.21.15.52:6805/858289560","heartbeat_back_addr":"172.21.15.52:6809/858289560","heartbeat_front_addr":"172.21.15.52:6807/858289560","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6811","nonce":3743472544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6813","nonce":3743472544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6817","nonce":3743472544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6815","nonce":3743472544}]},"public_addr":"172.21.15.52:6811/3743472544","cluster_addr":"172.21.15.52:6813/3743472544","heartbeat_back_addr":"172.21.15.52:6817/3743472544","heartbeat_front_addr":"172.21.15.52:6815/3743472544","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6819","nonce":1781181972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6821","nonce":1781181972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6825","nonce":1781181972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6823","nonce":1781181972}]},"public_addr":"172.21.15.52:6819/1781181972","cluster_addr":"172.21.15.52:6821/1781181972","heartbeat_back_addr":"172.21.15.52:6825/1781181972","heartbeat_front_addr":"172.21.15.52:6823/1781181972","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6827","nonce":1128061400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6829","nonce":1128061400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6833","nonce":1128061400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6831","nonce":1128061400}]},"public_addr":"172.21.15.52:6827/1128061400","cluster_addr":"172.21.15.52:6829/1128061400","heartbeat_back_addr":"172.21.15.52:6833/1128061400","heartbeat_front_addr":"172.21.15.52:6831/1128061400","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:27:59.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:59 smithi052 conmon[100224]: audit 2023-09-23T22:27:59.657422+0000 mon.a 2023-09-23T22:27:59.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:27:59 smithi052 conmon[100224]: (mon.0) 620 : audit [DBG] from='client.? 172.21.15.52:0/1419008307' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:27:59.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:59 smithi052 conmon[110082]: audit 2023-09-23T22:27:59 2023-09-23T22:27:59.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:27:59 smithi052 conmon[110082]: .657422+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.52:0/1419008307' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:00.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:27:59 smithi151 conmon[100454]: audit 2023-09-23T22:27:59.657422+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.52:0/1419008307' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:00.915 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:28:00.915 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:28:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:00 smithi052 conmon[110082]: cluster 2023-09-23T22:27:59.412399+0000 mgr.y (mgr.24314) 30 : cluster [DBG] 2023-09-23T22:28:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:00 smithi052 conmon[110082]: pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:00.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:00 smithi052 conmon[100224]: cluster 2023-09-23T22:27:59.412399+0000 mgr.y ( 2023-09-23T22:28:00.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:00 smithi052 conmon[100224]: mgr.24314) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:01.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:00 smithi151 conmon[100454]: cluster 2023-09-23T22:27:59.412399+0000 mgr.y (mgr.24314) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:01.881 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:02 smithi052 conmon[110082]: cluster 2023-09-23T22:28:01.412771+0000 mgr.y (mgr.24314) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:02 smithi052 conmon[100224]: cluster 2023-09-23T22:28:01.412771+0000 mgr.y (mgr.24314) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:03.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:02 smithi151 conmon[100454]: cluster 2023-09-23T22:28:01.412771+0000 mgr.y (mgr.24314) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:04.018 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:04.018 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":55,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:27:53.152659+0000","last_up_change":"2023-09-23T22:27:12.276151+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6803","nonce":858289560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6805","nonce":858289560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6809","nonce":858289560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6807","nonce":858289560}]},"public_addr":"172.21.15.52:6803/858289560","cluster_addr":"172.21.15.52:6805/858289560","heartbeat_back_addr":"172.21.15.52:6809/858289560","heartbeat_front_addr":"172.21.15.52:6807/858289560","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6811","nonce":3743472544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6813","nonce":3743472544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6817","nonce":3743472544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6815","nonce":3743472544}]},"public_addr":"172.21.15.52:6811/3743472544","cluster_addr":"172.21.15.52:6813/3743472544","heartbeat_back_addr":"172.21.15.52:6817/3743472544","heartbeat_front_addr":"172.21.15.52:6815/3743472544","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6819","nonce":1781181972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6821","nonce":1781181972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6825","nonce":1781181972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6823","nonce":1781181972}]},"public_addr":"172.21.15.52:6819/1781181972","cluster_addr":"172.21.15.52:6821/1781181972","heartbeat_back_addr":"172.21.15.52:6825/1781181972","heartbeat_front_addr":"172.21.15.52:6823/1781181972","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6827","nonce":1128061400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6829","nonce":1128061400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6833","nonce":1128061400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6831","nonce":1128061400}]},"public_addr":"172.21.15.52:6827/1128061400","cluster_addr":"172.21.15.52:6829/1128061400","heartbeat_back_addr":"172.21.15.52:6833/1128061400","heartbeat_front_addr":"172.21.15.52:6831/1128061400","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:28:04.632 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100670]: [23/Sep/2023:22:28:04] ENGINE Bus STOPPING 2023-09-23T22:28:04.632 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100670]: [23/Sep/2023:22:28:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:28:04.632 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100670]: [23/Sep/2023:22:28:04] ENGINE Bus STOPPED 2023-09-23T22:28:04.632 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100670]: [23/Sep/2023:22:28:04] ENGINE Bus STARTING 2023-09-23T22:28:04.632 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[141643]: level=info ts=2023-09-23T22:28:04.360Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T22:28:04.953 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100670]: [23/Sep/2023:22:28:04] ENGINE Serving on http://:::9283 2023-09-23T22:28:04.953 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100670]: [23/Sep/2023:22:28:04] ENGINE Bus STARTED 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: cluster 2023-09- 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 23T22:28:03.413140+0000 mgr.y (mgr.24314) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: -23T22:28:04.016819 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: +0000 mon.a (mon.0) 2023-09-23T22:28:04.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 621 : audit [DBG] from='client.? 172.21.15.52:0/4014078283' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: -09-23T22:28: 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 04.401494+0000 mon.a (mon 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: .0) 622 : audit [INF] 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: -23T22:28:04. 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 496137+0000 mon.a (mon.0) 623 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 2023-09-23T22:28 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: :04.498931+0000 mon.a (mon.0) 624 : 2023-09-23T22:28:04.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 28:04.501709+0000 mon.a (mon. 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 0) 625 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 2023-09-23T22:28:04.505454+0000 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: mon.a (mon.0) 626 : audit 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: audit 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 2023-09-23T 2023-09-23T22:28:04.956 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 22:28:04.602190+0000 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: mon.a (mon.0) 627 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[100224]: 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: cluster 2023-09-23T22:28:03.413140+0000 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: mgr.y (mgr.24314) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023-09-23T22:28:04 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: .016819+0000 mon.a (mon.0) 621 : audit 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/4014078283' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 23T22:28:04.401494+0000 2023-09-23T22:28:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: mon.a (mon.0) 622 : audit [INF] 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023-09-23T22:28 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: :04.496137+0000 mon.a ( 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: mon.0) 623 : audit [INF] 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 28:04.498931+0000 mon.a (mon.0 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: ) 624 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 09-23T22:28: 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 04.501709+0000 mon.a (mon. 2023-09-23T22:28:04.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 0) 625 : audit [INF] 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: -09-23T22:28 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: :04.505454+0000 mon.a ( 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: mon.0) 626 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 23T22:28:04.602190 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: +0000 mon.a (mon. 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 0) 627 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:04.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:04 smithi052 conmon[110082]: 2023-09-23T22:28:04.977 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:28:04.977 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:28:04.978 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:28:04.978 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:28:04.979 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:28:04.979 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:28:04.979 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:28:04.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:28:05.173 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: cluster 2023-09-23T22:28:03.413140+0000 mgr.y (mgr.24314) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:05.173 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.016819+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.52:0/4014078283' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:05.173 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.401494+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:05.173 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.496137+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:05.173 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.498931+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:05.173 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.501709+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:28:05.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.505454+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:05.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.602190+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:05.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:04 smithi151 conmon[100454]: 2023-09-23T22:28:05.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:28:05 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:28:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[100224]: audit 2023-09-23T22:28:04.502088+0000 mgr.y (mgr.24314) 33 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:28:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:04.517223+0000 mgr.y (mgr.24314) 34 : cephadm [INF] Deploying daemon grafana.a on smithi151 2023-09-23T22:28:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:28:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[110082]: 22:28:04.502088+0000 mgr.y (mgr.24314) 33 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:28:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[110082]: cephadm 2023- 2023-09-23T22:28:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[110082]: 09-23T22:28:04.517223 2023-09-23T22:28:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 34 2023-09-23T22:28:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:05 smithi052 conmon[110082]: : cephadm [INF] Deploying daemon grafana.a on smithi151 2023-09-23T22:28:06.137 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:05 smithi151 conmon[100454]: audit 2023-09-23T22:28:04.502088+0000 mgr.y (mgr.24314) 33 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:28:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:05 smithi151 conmon[100454]: cephadm 2023-09-23T22:28:04.517223+0000 mgr.y (mgr.24314) 34 : cephadm [INF] Deploying daemon grafana.a on smithi151 2023-09-23T22:28:06.255 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.256 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.341 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.431 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.484 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.485 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.496 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:06.724 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:06 smithi052 conmon[141643]: level=info ts=2023-09-23T22:28:06.325Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000384357s 2023-09-23T22:28:06.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:06 smithi052 conmon[110082]: cluster 2023-09-23T22:28:05.413640+0000 mgr.y (mgr.24314) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:06.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:06 smithi052 conmon[100224]: cluster 2023-09-23T22:28:05.413640+0000 mgr.y (mgr.24314) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:07.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:06 smithi151 conmon[100454]: cluster 2023-09-23T22:28:05.413640+0000 mgr.y (mgr.24314) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:08.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:08 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:08] "GET /metrics HTTP/1.1" 200 155645 "" "Prometheus/2.33.4" 2023-09-23T22:28:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:08 smithi052 conmon[110082]: cluster 2023-09-23T22:28:07. 2023-09-23T22:28:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:08 smithi052 conmon[110082]: 414231+0000 mgr.y (mgr.24314) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:08.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:08 smithi052 conmon[100224]: cluster 2023-09-23T22:28:07.414231+0000 2023-09-23T22:28:08.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:08 smithi052 conmon[100224]: mgr.y (mgr.24314) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:09.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:08 smithi151 conmon[100454]: cluster 2023-09-23T22:28:07.414231+0000 mgr.y (mgr.24314) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:10 smithi052 conmon[110082]: cluster 2023-09-23T22:28:09.414638+0000 mgr.y (mgr.24314) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:10 smithi052 conmon[100224]: cluster 2023-09-23T22:28:09.414638+0000 mgr.y (mgr.24314) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:11.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:10 smithi151 conmon[100454]: cluster 2023-09-23T22:28:09.414638+0000 mgr.y (mgr.24314) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:11.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:10 smithi151 conmon[100454]: 2023-09-23T22:28:12.432 INFO:teuthology.orchestra.run.smithi052.stdout:34359738410 2023-09-23T22:28:12.432 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:28:12.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:12 smithi052 conmon[100224]: cluster 2023-09-23T22:28:11.415040+0000 mgr.y (mgr.24314) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:12 smithi052 conmon[110082]: cluster 2023-09-23T22:28:11.415040+0000 mgr.y (mgr.24314) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:13.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:12 smithi151 conmon[100454]: cluster 2023-09-23T22:28:11. 2023-09-23T22:28:13.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:12 smithi151 conmon[100454]: 415040+0000 mgr.y (mgr.24314) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:14.674 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:14 smithi052 conmon[141643]: level=info ts=2023-09-23T22:28:14.327Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002555833s 2023-09-23T22:28:14.802 INFO:teuthology.orchestra.run.smithi052.stdout:55834574886 2023-09-23T22:28:14.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:28:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:14 smithi052 conmon[110082]: cluster 2023-09-23T22:28:13 2023-09-23T22:28:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:14 smithi052 conmon[110082]: .415543+0000 mgr.y (mgr.24314) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:14.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:14 smithi052 conmon[100224]: cluster 2023-09-23T22:28:13.415543+0000 mgr.y ( 2023-09-23T22:28:14.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:14 smithi052 conmon[100224]: mgr.24314) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:15.172 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:14 smithi151 conmon[100454]: cluster 2023-09-23T22:28:13.415543+0000 mgr.y (mgr.24314) 39 2023-09-23T22:28:15.172 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:14 smithi151 conmon[100454]: : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:15.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:28:15 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:28:15.678 INFO:teuthology.orchestra.run.smithi052.stdout:154618822678 2023-09-23T22:28:15.678 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:28:15.949 INFO:teuthology.orchestra.run.smithi052.stdout:206158430222 2023-09-23T22:28:15.950 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:28:16.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:16 smithi052 conmon[110082]: cluster 2023-09-23T22:28:15.415864+0000 mgr.y (mgr.24314) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:16 smithi052 conmon[100224]: cluster 2023-09-23T22:28:15.415864+0000 mgr.y (mgr.24314) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:16.262 INFO:teuthology.orchestra.run.smithi052.stdout:128849018906 2023-09-23T22:28:16.262 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:28:16.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:16 smithi151 conmon[100454]: cluster 2023-09-23T22:28:15.415864+0000 mgr.y (mgr.24314) 2023-09-23T22:28:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:16 smithi151 conmon[100454]: 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:16.664 INFO:teuthology.orchestra.run.smithi052.stdout:180388626450 2023-09-23T22:28:16.664 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:28:17.241 INFO:teuthology.orchestra.run.smithi052.stdout:107374182430 2023-09-23T22:28:17.241 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:28:17.451 INFO:teuthology.orchestra.run.smithi052.stdout:77309411365 2023-09-23T22:28:17.452 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:28:17.478 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:17.752 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:17.906 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:17.907 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:18.261 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:18.262 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:18.469 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:18 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:18] "GET /metrics HTTP/1.1" 200 155643 "" "Prometheus/2.33.4" 2023-09-23T22:28:18.584 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:18.608 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:18 smithi052 conmon[100224]: cluster 2023-09-23T22:28:17.416185+0000 mgr.y (mgr.24314) 41 2023-09-23T22:28:18.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:18 smithi052 conmon[100224]: : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:18.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:18 smithi052 conmon[110082]: cluster 2023-09-23T22:28:17.416185+0000 mgr.y (mgr 2023-09-23T22:28:18.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:18 smithi052 conmon[110082]: .24314) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:18.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:18 smithi151 conmon[100454]: cluster 2023-09-23T22:28:17.416185+0000 mgr.y (mgr.24314) 41 2023-09-23T22:28:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:18 smithi151 conmon[100454]: : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:20 smithi052 conmon[110082]: cluster 2023-09-23T22:28:19.416466+0000 mgr.y (mgr.24314) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:20 smithi052 conmon[100224]: cluster 2023-09-23T22:28:19.416466+0000 mgr.y (mgr.24314) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:20 smithi151 conmon[100454]: cluster 2023-09-23T22:28:19.416466+0000 mgr.y (mgr.24314) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:21.500 INFO:teuthology.orchestra.run.smithi052.stdout:34359738413 2023-09-23T22:28:22.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:22 smithi151 conmon[100454]: cluster 2023-09-23T22:28:21.416817+0000 mgr.y (mgr.24314) 43 : cluster [DBG] 2023-09-23T22:28:22.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:22 smithi151 conmon[100454]: pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:22.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:22 smithi151 conmon[100454]: audit 2023-09-23T22:28:21.500422+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.52:0/3156208476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:28:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:22 smithi052 conmon[110082]: cluster 2023-09-23T22:28:21.416817+0000 mgr.y (mgr.24314) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:22 smithi052 conmon[110082]: audit 2023-09-23T22:28:21.500422+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.52:0/3156208476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:28:22.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:22 smithi052 conmon[100224]: cluster 2023-09-23T22:28:21.416817+0000 mgr.y (mgr.24314) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:22.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:22 smithi052 conmon[100224]: audit 2023-09-23T22:28:21.500422+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.52:0/3156208476' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:28:23.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:23 smithi151 systemd[1]: Starting Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:28:23.875 INFO:teuthology.orchestra.run.smithi052.stdout:180388626452 2023-09-23T22:28:24.043 INFO:teuthology.orchestra.run.smithi052.stdout:128849018908 2023-09-23T22:28:24.498 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: cluster 2023-09-23T22:28:23.417141+0000 mgr.y (mgr.24314) 44 : cluster 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: audit 2023-09-23T22:28:23.781456+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.52:0/1685459937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: audit 2023-09-23T22:28:23. 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: 874967+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.52:0/4175644726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: audit 2023-09-23T22:28:24.334125+0000 mon.a (mon.0) 630 : audit 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: audit 2023-09-23T22:28:24.335482+0000 mon.a (mon. 2023-09-23T22:28:24.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[100454]: 0) 631 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T22:28:24.500 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="App mode production" logger=settings 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 bash[131748]: 6cfce387f58fdecf283a8af4f7ca3025a0b3b38189c94e7f1d8bdd7377c092f8 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 systemd[1]: Started Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:28:24.501 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-09-23T22:28:24.502 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-09-23T22:28:24.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[110082]: cluster 2023-09-23T22:28:23.417141+0000 mgr.y (mgr.24314) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:24.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[110082]: audit 2023-09-23T22:28:23.781456+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.52:0/1685459937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:28:24.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[110082]: audit 2023-09-23T22:28:23.874967+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.52:0/4175644726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:28:24.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[110082]: audit 2023-09-23T22:28:24.334125+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:24.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[110082]: audit 2023-09-23T22:28:24.335482+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:24.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[100224]: cluster 2023-09-23T22:28:23.417141+0000 mgr.y (mgr.24314) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:24.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[100224]: audit 2023-09-23T22:28:23.781456+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.52:0/1685459937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:28:24.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[100224]: audit 2023-09-23T22:28:23.874967+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.52:0/4175644726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:28:24.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[100224]: audit 2023-09-23T22:28:24.334125+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:24.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:24 smithi052 conmon[100224]: audit 2023-09-23T22:28:24.335482+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:24.751 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-09-23T22:28:24.751 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-09-23T22:28:24.751 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-09-23T22:28:24.751 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-09-23T22:28:24.752 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-09-23T22:28:24.752 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-09-23T22:28:24.752 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-09-23T22:28:24.752 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-09-23T22:28:24.752 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-09-23T22:28:25.048 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-09-23T22:28:25.049 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-09-23T22:28:25.049 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T22:28:25.049 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:24 smithi151 conmon[131789]: t=2023-09-23T22:28:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-09-23T22:28:25.085 INFO:teuthology.orchestra.run.smithi052.stdout:107374182431 2023-09-23T22:28:25.330 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-09-23T22:28:25.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-09-23T22:28:25.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-09-23T22:28:25.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-09-23T22:28:25.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-09-23T22:28:25.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-09-23T22:28:25.331 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:28:25.498 INFO:teuthology.orchestra.run.smithi052.stdout:77309411366 2023-09-23T22:28:25.591 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T22:28:25.591 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T22:28:25.592 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-09-23T22:28:25.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[100224]: audit 2023-09-23T22:28 2023-09-23T22:28:25.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[100224]: :24.616621+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:25.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[100224]: audit 2023-09-23T22:28:24.672917+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.52:0/955181533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:28:25.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[100224]: audit 2023-09-23T22:28:25.498203+0000 mon.a (mon.0) 633 : audit [DBG] from='client.? 172.21.15.52:0/99305786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:28:25.791 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[110082]: audit 2023-09-23T22:28:24.616621 2023-09-23T22:28:25.791 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[110082]: +0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:25.791 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[110082]: audit 2023-09-23T22:28:24.672917+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.52:0/955181533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:28:25.791 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:25 smithi052 conmon[110082]: audit 2023-09-23T22:28:25.498203+0000 mon.a (mon.0) 633 : audit [DBG] from='client.? 172.21.15.52:0/99305786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:28:25.792 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626450 got 180388626452 for osd.6 2023-09-23T22:28:25.792 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:25.835 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738410 got 34359738413 for osd.0 2023-09-23T22:28:25.835 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:25.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[100454]: audit 2023-09-23T22:28:24.616621+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:25.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[100454]: 2023-09-23T22:28:25.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[100454]: audit 2023-09-23T22:28:24.672917+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.52:0/955181533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:28:25.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[100454]: audit 2023-09-23T22:28:25.498203+0000 mon.a (mon.0) 633 : audit [DBG] 2023-09-23T22:28:25.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/99305786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:28:25.869 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-09-23T22:28:25.869 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-09-23T22:28:25.869 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-09-23T22:28:25.870 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-09-23T22:28:25.870 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-09-23T22:28:25.870 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T22:28:26.162 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-09-23T22:28:26.162 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-09-23T22:28:26.162 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:25 smithi151 conmon[131789]: t=2023-09-23T22:28:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-09-23T22:28:26.163 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-09-23T22:28:26.163 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-09-23T22:28:26.163 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-09-23T22:28:26.246 INFO:teuthology.orchestra.run.smithi052.stdout:206158430225 2023-09-23T22:28:26.430 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-09-23T22:28:26.431 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-09-23T22:28:26.431 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T22:28:26.431 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T22:28:26.431 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T22:28:26.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[100454]: cluster 2023-09-23T22:28:25.417456+0000 mgr.y (mgr.24314) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:28:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[100454]: 23T22:28:26.172951+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.52:0/411367818' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:28:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-09-23T22:28:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-09-23T22:28:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T22:28:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-09-23T22:28:26.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:26 smithi052 conmon[110082]: cluster 2023-09-23T22:28:25.417456+0000 mgr.y (mgr.24314) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:26.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:26 smithi052 conmon[110082]: audit 2023-09-23T22:28:26.172951+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.52:0/411367818' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:28:26.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:26 smithi052 conmon[100224]: cluster 2023-09-23T22:28:25.417456+0000 mgr.y (mgr.24314) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:26.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:26 smithi052 conmon[100224]: audit 2023-09-23T22:28:26.172951+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.52:0/411367818' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:28:27.042 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-09-23T22:28:27.042 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-09-23T22:28:27.043 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-09-23T22:28:27.043 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-09-23T22:28:27.043 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-09-23T22:28:27.043 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:26 smithi151 conmon[131789]: t=2023-09-23T22:28:26+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-09-23T22:28:27.043 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-09-23T22:28:27.148 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182430 got 107374182431 for osd.3 2023-09-23T22:28:27.148 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:27.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-09-23T22:28:27.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-09-23T22:28:27.331 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-09-23T22:28:27.606 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-09-23T22:28:27.626 INFO:teuthology.orchestra.run.smithi052.stdout:55834574889 2023-09-23T22:28:27.709 INFO:teuthology.orchestra.run.smithi052.stdout:154618822681 2023-09-23T22:28:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: audit 2023-09-23T22:28:27.105592 2023-09-23T22:28:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: +0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.52:0/107895955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:28:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: audit 2023-09-23T22:28:27.121688 2023-09-23T22:28:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: +0000 mon.a (mon.0) 635 : audit 2023-09-23T22:28:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/269084787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: 23T22:28:27.412689+0000 mon.a (mon 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[110082]: .0) 636 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[100224]: audit 2023-09-23T22:28:27.105592+0000 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[100224]: mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.52:0/107895955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[100224]: audit 2023-09-23T22:28:27.121688+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.52:0/269084787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[100224]: audit 2023-09-23T22:28:27.412689+0000 mon.a (mon.0) 636 : audit 2023-09-23T22:28:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:27 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:27.793 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411366 for osd.2 2023-09-23T22:28:27.793 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[100454]: audit 2023-09-23T22:28:27.105592+0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.52:0/107895955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:28:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[100454]: audit 2023-09-23T22:28:27.121688+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.52:0/269084787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:28:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[100454]: audit 2023-09-23T22:28:27.412689+0000 mon.a (mon.0) 636 : audit 2023-09-23T22:28:27.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T22:28:27.859 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-09-23T22:28:28.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-09-23T22:28:28.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-09-23T22:28:28.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-09-23T22:28:28.119 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:27 smithi151 conmon[131789]: t=2023-09-23T22:28:27+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-09-23T22:28:28.120 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-09-23T22:28:28.277 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018906 got 128849018908 for osd.4 2023-09-23T22:28:28.278 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:28.396 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-09-23T22:28:28.396 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-09-23T22:28:28.396 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-09-23T22:28:28.397 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-09-23T22:28:28.397 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-09-23T22:28:28.397 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-09-23T22:28:28.397 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-09-23T22:28:28.397 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-09-23T22:28:28.397 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-09-23T22:28:28.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:28 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:28] "GET /metrics HTTP/1.1" 200 155643 "" "Prometheus/2.33.4" 2023-09-23T22:28:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[100454]: cluster 2023-09-23T22:28:27.417912+0000 mgr.y (mgr.24314) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:28.674 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-09-23T22:28:28.674 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-09-23T22:28:28.674 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-09-23T22:28:28.674 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-09-23T22:28:28.674 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-09-23T22:28:28.674 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-09-23T22:28:28.675 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-09-23T22:28:28.787 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822678 got 154618822681 for osd.5 2023-09-23T22:28:28.787 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:28.821 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430225 for osd.7 2023-09-23T22:28:28.821 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:28.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:28 smithi052 conmon[110082]: cluster 2023-09-23T22:28 2023-09-23T22:28:28.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:28 smithi052 conmon[110082]: :27.417912+0000 mgr.y (mgr.24314) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:28.885 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:28 smithi052 conmon[100224]: cluster 2023-09-23T22:28:27.417912+0000 mgr.y (mgr.24314) 46 : cluster 2023-09-23T22:28:28.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:28 smithi052 conmon[100224]: [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:28.887 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574886 got 55834574889 for osd.1 2023-09-23T22:28:28.887 DEBUG:teuthology.parallel:result is None 2023-09-23T22:28:28.887 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:28:28.887 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:28:28.925 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-09-23T22:28:28.925 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-09-23T22:28:28.925 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-09-23T22:28:28.925 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-09-23T22:28:28.925 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T22:28:28.926 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-09-23T22:28:28.926 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-09-23T22:28:28.926 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-09-23T22:28:28.926 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-09-23T22:28:29.198 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:28 smithi151 conmon[131789]: t=2023-09-23T22:28:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-09-23T22:28:29.199 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-09-23T22:28:29.199 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-09-23T22:28:29.199 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-09-23T22:28:29.199 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-09-23T22:28:29.199 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-09-23T22:28:29.475 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:29.478 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-09-23T22:28:29.479 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T22:28:29.747 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-09-23T22:28:29.747 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T22:28:29.747 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-09-23T22:28:29.747 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T22:28:29.747 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-09-23T22:28:29.747 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-09-23T22:28:30.015 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T22:28:30.015 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-09-23T22:28:30.015 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-09-23T22:28:30.015 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-09-23T22:28:30.015 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-09-23T22:28:30.015 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:29 smithi151 conmon[131789]: t=2023-09-23T22:28:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-09-23T22:28:30.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-09-23T22:28:30.284 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-09-23T22:28:30.552 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T22:28:30.553 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T22:28:30.710 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:30 smithi052 conmon[110082]: cluster 2023-09-23T22:28:29.418408+0000 mgr.y (mgr.24314) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:30.710 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:30 smithi052 conmon[100224]: cluster 2023-09-23T22:28:29.418408+0000 mgr.y (mgr.24314) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:30.820 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[100454]: cluster 2023-09-23T22:28:29.418408+0000 mgr.y (mgr.24314) 47 : cluster 2023-09-23T22:28:30.820 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[100454]: [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-09-23T22:28:30.821 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-09-23T22:28:31.072 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-09-23T22:28:31.072 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-09-23T22:28:31.072 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-09-23T22:28:31.073 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:30 smithi151 conmon[131789]: t=2023-09-23T22:28:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-09-23T22:28:31.073 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-09-23T22:28:31.073 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-09-23T22:28:31.073 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-09-23T22:28:31.073 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T22:28:31.073 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-09-23T22:28:31.076 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:31.078 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-09-23T22:28:31.341 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-09-23T22:28:31.618 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-09-23T22:28:31.887 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-09-23T22:28:31.887 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-09-23T22:28:31.887 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-09-23T22:28:31.887 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-09-23T22:28:31.887 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-09-23T22:28:31.888 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-09-23T22:28:31.888 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-09-23T22:28:31.888 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T22:28:31.888 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T22:28:32.014 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2023-09-23T22:28:29.418180+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.973518+0000","last_change":"2023-09-23T22:27:50.149464+0000","last_active":"2023-09-23T22:27:51.973518+0000","last_peered":"2023-09-23T22:27:51.973518+0000","last_clean":"2023-09-23T22:27:51.973518+0000","last_became_active":"2023-09-23T22:27:50.149120+0000","last_became_peered":"2023-09-23T22:27:50.149120+0000","last_unstale":"2023-09-23T22:27:51.973518+0000","last_undegraded":"2023-09-23T22:27:51.973518+0000","last_fullsized":"2023-09-23T22:27:51.973518+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.973586+0000","last_change":"2023-09-23T22:27:50.607023+0000","last_active":"2023-09-23T22:27:51.973586+0000","last_peered":"2023-09-23T22:27:51.973586+0000","last_clean":"2023-09-23T22:27:51.973586+0000","last_became_active":"2023-09-23T22:27:50.606684+0000","last_became_peered":"2023-09-23T22:27:50.606684+0000","last_unstale":"2023-09-23T22:27:51.973586+0000","last_undegraded":"2023-09-23T22:27:51.973586+0000","last_fullsized":"2023-09-23T22:27:51.973586+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.472940+0000","last_change":"2023-09-23T22:27:50.607540+0000","last_active":"2023-09-23T22:27:51.472940+0000","last_peered":"2023-09-23T22:27:51.472940+0000","last_clean":"2023-09-23T22:27:51.472940+0000","last_became_active":"2023-09-23T22:27:50.607232+0000","last_became_peered":"2023-09-23T22:27:50.607232+0000","last_unstale":"2023-09-23T22:27:51.472940+0000","last_undegraded":"2023-09-23T22:27:51.472940+0000","last_fullsized":"2023-09-23T22:27:51.472940+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.472929+0000","last_change":"2023-09-23T22:27:50.607516+0000","last_active":"2023-09-23T22:27:51.472929+0000","last_peered":"2023-09-23T22:27:51.472929+0000","last_clean":"2023-09-23T22:27:51.472929+0000","last_became_active":"2023-09-23T22:27:50.607193+0000","last_became_peered":"2023-09-23T22:27:50.607193+0000","last_unstale":"2023-09-23T22:27:51.472929+0000","last_undegraded":"2023-09-23T22:27:51.472929+0000","last_fullsized":"2023-09-23T22:27:51.472929+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.472924+0000","last_change":"2023-09-23T22:27:14.671001+0000","last_active":"2023-09-23T22:27:51.472924+0000","last_peered":"2023-09-23T22:27:51.472924+0000","last_clean":"2023-09-23T22:27:51.472924+0000","last_became_active":"2023-09-23T22:27:14.670462+0000","last_became_peered":"2023-09-23T22:27:14.670462+0000","last_unstale":"2023-09-23T22:27:51.472924+0000","last_undegraded":"2023-09-23T22:27:51.472924+0000","last_fullsized":"2023-09-23T22:27:51.472924+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.405998+0000","last_change":"2023-09-23T22:27:50.149615+0000","last_active":"2023-09-23T22:27:51.405998+0000","last_peered":"2023-09-23T22:27:51.405998+0000","last_clean":"2023-09-23T22:27:51.405998+0000","last_became_active":"2023-09-23T22:27:50.149191+0000","last_became_peered":"2023-09-23T22:27:50.149191+0000","last_unstale":"2023-09-23T22:27:51.405998+0000","last_undegraded":"2023-09-23T22:27:51.405998+0000","last_fullsized":"2023-09-23T22:27:51.405998+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.149316+0000","last_change":"2023-09-23T22:27:50.607355+0000","last_active":"2023-09-23T22:27:51.149316+0000","last_peered":"2023-09-23T22:27:51.149316+0000","last_clean":"2023-09-23T22:27:51.149316+0000","last_became_active":"2023-09-23T22:27:50.607075+0000","last_became_peered":"2023-09-23T22:27:50.607075+0000","last_unstale":"2023-09-23T22:27:51.149316+0000","last_undegraded":"2023-09-23T22:27:51.149316+0000","last_fullsized":"2023-09-23T22:27:51.149316+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.406017+0000","last_change":"2023-09-23T22:27:50.150091+0000","last_active":"2023-09-23T22:27:51.406017+0000","last_peered":"2023-09-23T22:27:51.406017+0000","last_clean":"2023-09-23T22:27:51.406017+0000","last_became_active":"2023-09-23T22:27:50.149716+0000","last_became_peered":"2023-09-23T22:27:50.149716+0000","last_unstale":"2023-09-23T22:27:51.406017+0000","last_undegraded":"2023-09-23T22:27:51.406017+0000","last_fullsized":"2023-09-23T22:27:51.406017+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.973557+0000","last_change":"2023-09-23T22:27:50.607380+0000","last_active":"2023-09-23T22:27:51.973557+0000","last_peered":"2023-09-23T22:27:51.973557+0000","last_clean":"2023-09-23T22:27:51.973557+0000","last_became_active":"2023-09-23T22:27:50.607045+0000","last_became_peered":"2023-09-23T22:27:50.607045+0000","last_unstale":"2023-09-23T22:27:51.973557+0000","last_undegraded":"2023-09-23T22:27:51.973557+0000","last_fullsized":"2023-09-23T22:27:51.973557+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430225,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.752},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.625}]},{"osd":3,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":1.3169999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56000000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.53000000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.72499999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.68100000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.69599999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 22:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":0.71199999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.372,"15min":0.35599999999999998},"min":{"1min":0.253,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.372,"15min":0.35199999999999998},"min":{"1min":0.25900000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:27:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.39000000000000001,"15min":0.38400000000000001},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40100000000000002,"15min":0.39600000000000002},"min":{"1min":0.29099999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:27:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.498,"15min":0.503},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.48299999999999998,"15min":0.49099999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.67500000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.377}]},{"osd":4,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.621,"15min":0.628},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.70199999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:27:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:27:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":2.524,"5min":2.524,"15min":2.524},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.56100000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.34699999999999998},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.59399999999999997,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.36799999999999999,"15min":0.34599999999999997},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.254},"max":{"1min":0.60499999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:27:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.439,"5min":0.41099999999999998,"15min":0.40699999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47299999999999998,"15min":0.47099999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35999999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73899999999999999,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.625,"15min":0.627},"min":{"1min":0.42799999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":1.103},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.627,"15min":0.621},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:27:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":1.02}]},{"osd":6,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":1.0680000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.66200000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.34999999999999998,"15min":0.33900000000000002},"min":{"1min":0.32900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.35699999999999998,"15min":0.34799999999999998},"min":{"1min":0.29099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.52800000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:27:28 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.316,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.372,"15min":0.35999999999999999},"min":{"1min":0.33900000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55600000000000005,"5min":0.625,"15min":0.625},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.45600000000000002,"15min":0.46100000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.49299999999999999,"15min":0.49099999999999999},"min":{"1min":0.33100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59699999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:28:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.63400000000000001,"15min":0.64300000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.753,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68100000000000005,"15min":0.68899999999999995},"min":{"1min":0.52400000000000002,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.81599999999999995,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.77100000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.755},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.72899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:28:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.68300000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182432,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:48 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.433,"15min":0.442},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59699999999999998,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.45800000000000002,"15min":0.47199999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.61499999999999999,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.434}]},{"osd":1,"last update":"Sat Sep 23 22:27:48 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.54700000000000004,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.373},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.47899999999999998,"15min":0.48999999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.61799999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:27:48 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.52700000000000002,"15min":0.53700000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.51500000000000001,"15min":0.52500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.65700000000000003,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.41099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.82699999999999996,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:28:14 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.64100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018909,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.55200000000000005,"15min":0.56000000000000005},"min":{"1min":0.39300000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68200000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.377},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.60799999999999998,"15min":0.61899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77800000000000002,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.41399999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.76100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.62,"15min":0.63200000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.71999999999999997,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.59699999999999998,"15min":0.60899999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.749,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.59799999999999998,"15min":0.60699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79300000000000004,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.60799999999999998,"15min":0.621},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.755,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.434},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.61399999999999999,"15min":0.626},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.71899999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:27:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.52000000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 22:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.46200000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822681,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.505},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.64700000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.69899999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.72499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.751},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.70899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:28:32.016 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:31 smithi151 conmon[131789]: t=2023-09-23T22:28:31+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-09-23T22:28:32.156 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-09-23T22:28:32.397 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:32.432 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-09-23T22:28:32.432 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-09-23T22:28:32.432 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-09-23T22:28:32.432 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-09-23T22:28:32.432 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-09-23T22:28:32.433 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-09-23T22:28:32.433 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: audit 2023-09-23T22:28:31.072489+0000 mgr.y (mgr.24314) 48 : audit [DBG] from='client.14580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: cluster 2023-09-23T22:28:31.418792+0000 mgr.y (mgr.24314) 49 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: audit 2023-09-23T22:28 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: :32.136097+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: audit 2023-09-23T22:28:32.492224+0000 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: mon.a (mon.0) 638 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: 28:32.492928+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: audit 2023-09-23T22:28:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[100454]: 2023-09-23T22:28:32.498226+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-09-23T22:28:32.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-09-23T22:28:32.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: audit 2023-09-23T22:28:31.072489+0000 mgr.y (mgr.24314) 48 : audit [DBG] from='client.14580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: cluster 2023-09-23T22:28:31.418792+0000 mgr.y (mgr.24314) 49 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: audit 2023-09-23T22:28:32.136097+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: audit 2023-09-23T22:28 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: :32.492224+0000 mon.a (mon.0) 638 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: audit 2023-09-23T22:28:32.492928+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[110082]: audit 2023-09-23T22:28:32.498226+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: audit 2023-09-23T22:28:31.072489+0000 mgr.y (mgr.24314) 48 : audit [DBG] from='client.14580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: cluster 2023-09-23T22:28:31.418792+0000 mgr.y (mgr.24314) 49 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:32.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: audit 2023-09-23T22:28:32.136097+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:32.887 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: audit 2023-09-23T22:28:32.492224+0000 mon.a ( 2023-09-23T22:28:32.887 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: mon.0) 638 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:32.887 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: audit 2023-09-23T22:28:32.492928+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:32.887 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:32 smithi052 conmon[100224]: audit 2023-09-23T22:28:32.498226+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:32.962 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-09-23T22:28:32.962 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T22:28:32.962 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-09-23T22:28:32.962 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-09-23T22:28:32.962 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-09-23T22:28:32.963 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:32 smithi151 conmon[131789]: t=2023-09-23T22:28:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-09-23T22:28:33.240 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-09-23T22:28:33.241 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-09-23T22:28:33.241 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-09-23T22:28:33.508 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-09-23T22:28:33.508 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-09-23T22:28:33.508 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-09-23T22:28:33.508 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-09-23T22:28:33.508 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-09-23T22:28:33.508 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T22:28:33.776 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[100454]: cephadm 2023-09-23T22:28:32.501429+0000 mgr.y (mgr.24314) 50 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:28:33.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[100454]: cephadm 2023-09-23T22:28: 2023-09-23T22:28:33.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[100454]: 32.504698+0000 mgr.y (mgr.24314) 51 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi052 2023-09-23T22:28:33.777 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-09-23T22:28:33.777 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-09-23T22:28:33.777 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-09-23T22:28:33.777 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-09-23T22:28:33.777 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-09-23T22:28:33.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T22:28:33.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-09-23T22:28:33.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-09-23T22:28:33.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-09-23T22:28:33.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:33 smithi052 conmon[110082]: cephadm 2023-09-23T22:28:32.501429+0000 2023-09-23T22:28:33.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:33 smithi052 conmon[110082]: mgr.y (mgr.24314) 50 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:28:33.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:33 smithi052 conmon[110082]: cephadm 2023-09-23T22:28:32.504698+0000 mgr.y (mgr.24314) 51 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi052 2023-09-23T22:28:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:33 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:32.501429+0000 2023-09-23T22:28:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:33 smithi052 conmon[100224]: mgr.y (mgr.24314) 50 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:28:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:33 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:32.504698+0000 mgr.y (mgr.24314) 51 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi052 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:33 smithi151 conmon[131789]: t=2023-09-23T22:28:33+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-09-23T22:28:34.037 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-09-23T22:28:34.038 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-09-23T22:28:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:34.114 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:28:34.305 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-09-23T22:28:34.305 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-09-23T22:28:34.305 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-09-23T22:28:34.305 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-09-23T22:28:34.305 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-09-23T22:28:34.306 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-09-23T22:28:34.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-09-23T22:28:34.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-09-23T22:28:34.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-09-23T22:28:34.573 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-09-23T22:28:34.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T22:28:34.574 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-09-23T22:28:34.841 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[100454]: cluster 2023-09-23T22:28:33.419146+0000 mgr.y (mgr.24314) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:34.841 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-09-23T22:28:34.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-09-23T22:28:34.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T22:28:34.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-09-23T22:28:34.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-09-23T22:28:34.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-09-23T22:28:34.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-09-23T22:28:34.890 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:34 smithi052 conmon[110082]: cluster 2023-09-23T22:28 2023-09-23T22:28:34.890 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:34 smithi052 conmon[110082]: :33.419146+0000 mgr.y (mgr.24314) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:34.890 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:34 smithi052 conmon[100224]: cluster 2023-09-23T22:28:33.419146+0000 mgr.y (mgr.24314 2023-09-23T22:28:34.890 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:34 smithi052 conmon[100224]: ) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:34.909 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2023-09-23T22:28:33.419002+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.973518+0000","last_change":"2023-09-23T22:27:50.149464+0000","last_active":"2023-09-23T22:27:51.973518+0000","last_peered":"2023-09-23T22:27:51.973518+0000","last_clean":"2023-09-23T22:27:51.973518+0000","last_became_active":"2023-09-23T22:27:50.149120+0000","last_became_peered":"2023-09-23T22:27:50.149120+0000","last_unstale":"2023-09-23T22:27:51.973518+0000","last_undegraded":"2023-09-23T22:27:51.973518+0000","last_fullsized":"2023-09-23T22:27:51.973518+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.973586+0000","last_change":"2023-09-23T22:27:50.607023+0000","last_active":"2023-09-23T22:27:51.973586+0000","last_peered":"2023-09-23T22:27:51.973586+0000","last_clean":"2023-09-23T22:27:51.973586+0000","last_became_active":"2023-09-23T22:27:50.606684+0000","last_became_peered":"2023-09-23T22:27:50.606684+0000","last_unstale":"2023-09-23T22:27:51.973586+0000","last_undegraded":"2023-09-23T22:27:51.973586+0000","last_fullsized":"2023-09-23T22:27:51.973586+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.472940+0000","last_change":"2023-09-23T22:27:50.607540+0000","last_active":"2023-09-23T22:27:51.472940+0000","last_peered":"2023-09-23T22:27:51.472940+0000","last_clean":"2023-09-23T22:27:51.472940+0000","last_became_active":"2023-09-23T22:27:50.607232+0000","last_became_peered":"2023-09-23T22:27:50.607232+0000","last_unstale":"2023-09-23T22:27:51.472940+0000","last_undegraded":"2023-09-23T22:27:51.472940+0000","last_fullsized":"2023-09-23T22:27:51.472940+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.472929+0000","last_change":"2023-09-23T22:27:50.607516+0000","last_active":"2023-09-23T22:27:51.472929+0000","last_peered":"2023-09-23T22:27:51.472929+0000","last_clean":"2023-09-23T22:27:51.472929+0000","last_became_active":"2023-09-23T22:27:50.607193+0000","last_became_peered":"2023-09-23T22:27:50.607193+0000","last_unstale":"2023-09-23T22:27:51.472929+0000","last_undegraded":"2023-09-23T22:27:51.472929+0000","last_fullsized":"2023-09-23T22:27:51.472929+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.472924+0000","last_change":"2023-09-23T22:27:14.671001+0000","last_active":"2023-09-23T22:27:51.472924+0000","last_peered":"2023-09-23T22:27:51.472924+0000","last_clean":"2023-09-23T22:27:51.472924+0000","last_became_active":"2023-09-23T22:27:14.670462+0000","last_became_peered":"2023-09-23T22:27:14.670462+0000","last_unstale":"2023-09-23T22:27:51.472924+0000","last_undegraded":"2023-09-23T22:27:51.472924+0000","last_fullsized":"2023-09-23T22:27:51.472924+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.405998+0000","last_change":"2023-09-23T22:27:50.149615+0000","last_active":"2023-09-23T22:27:51.405998+0000","last_peered":"2023-09-23T22:27:51.405998+0000","last_clean":"2023-09-23T22:27:51.405998+0000","last_became_active":"2023-09-23T22:27:50.149191+0000","last_became_peered":"2023-09-23T22:27:50.149191+0000","last_unstale":"2023-09-23T22:27:51.405998+0000","last_undegraded":"2023-09-23T22:27:51.405998+0000","last_fullsized":"2023-09-23T22:27:51.405998+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.149316+0000","last_change":"2023-09-23T22:27:50.607355+0000","last_active":"2023-09-23T22:27:51.149316+0000","last_peered":"2023-09-23T22:27:51.149316+0000","last_clean":"2023-09-23T22:27:51.149316+0000","last_became_active":"2023-09-23T22:27:50.607075+0000","last_became_peered":"2023-09-23T22:27:50.607075+0000","last_unstale":"2023-09-23T22:27:51.149316+0000","last_undegraded":"2023-09-23T22:27:51.149316+0000","last_fullsized":"2023-09-23T22:27:51.149316+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.406017+0000","last_change":"2023-09-23T22:27:50.150091+0000","last_active":"2023-09-23T22:27:51.406017+0000","last_peered":"2023-09-23T22:27:51.406017+0000","last_clean":"2023-09-23T22:27:51.406017+0000","last_became_active":"2023-09-23T22:27:50.149716+0000","last_became_peered":"2023-09-23T22:27:50.149716+0000","last_unstale":"2023-09-23T22:27:51.406017+0000","last_undegraded":"2023-09-23T22:27:51.406017+0000","last_fullsized":"2023-09-23T22:27:51.406017+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-09-23T22:27:51.973557+0000","last_change":"2023-09-23T22:27:50.607380+0000","last_active":"2023-09-23T22:27:51.973557+0000","last_peered":"2023-09-23T22:27:51.973557+0000","last_clean":"2023-09-23T22:27:51.973557+0000","last_became_active":"2023-09-23T22:27:50.607045+0000","last_became_peered":"2023-09-23T22:27:50.607045+0000","last_unstale":"2023-09-23T22:27:51.973557+0000","last_undegraded":"2023-09-23T22:27:51.973557+0000","last_fullsized":"2023-09-23T22:27:51.973557+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430226,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.376},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.42899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.65300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.61699999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.68700000000000006}]},{"osd":6,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.71999999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626454,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.315},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.47199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":0.442}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.372,"15min":0.35599999999999998},"min":{"1min":0.253,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.372,"15min":0.35199999999999998},"min":{"1min":0.25900000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.375}]},{"osd":2,"last update":"Sat Sep 23 22:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.28299999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.40200000000000002,"15min":0.39600000000000002},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:27:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.498,"15min":0.503},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.48299999999999998,"15min":0.49099999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.67500000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.621,"15min":0.628},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:27:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.629}]},{"osd":6,"last update":"Sat Sep 23 22:27:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":2.524,"5min":2.524,"15min":2.524},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:28:12 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.57799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.34699999999999998},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.59399999999999997,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.36799999999999999,"15min":0.34599999999999997},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.254},"max":{"1min":0.60499999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:27:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.439,"5min":0.41099999999999998,"15min":0.40699999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47299999999999998,"15min":0.47099999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.373},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35999999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73899999999999999,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.40899999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.625,"15min":0.627},"min":{"1min":0.42799999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.627,"15min":0.621},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:27:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.625}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.46300000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36599999999999999,"15min":0.35099999999999998},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53900000000000003,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 22:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.374,"15min":0.34899999999999998},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.54400000000000004,"5min":0.629,"15min":0.629},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.376,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27000000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48299999999999998,"5min":0.625,"15min":0.625},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.45600000000000002,"15min":0.46100000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.49299999999999999,"15min":0.49099999999999999},"min":{"1min":0.33100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:28:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.63400000000000001,"15min":0.64300000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.753,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68100000000000005,"15min":0.68899999999999995},"min":{"1min":0.52400000000000002,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.81599999999999995,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.499}]},{"osd":7,"last update":"Sat Sep 23 22:28:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.57799999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182433,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:48 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.433,"15min":0.442},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59699999999999998,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.45800000000000002,"15min":0.47199999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.61499999999999999,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:27:48 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.54700000000000004,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.47899999999999998,"15min":0.48999999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.61799999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:27:48 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.52700000000000002,"15min":0.53700000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.51500000000000001,"15min":0.52500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.65700000000000003,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.41099999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.497},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.82699999999999996,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.60099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:28:14 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.57999999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.55200000000000005,"15min":0.56000000000000005},"min":{"1min":0.39300000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68200000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.60799999999999998,"15min":0.61899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77800000000000002,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.76100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.375},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.62,"15min":0.63200000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.71999999999999997,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.59699999999999998,"15min":0.60899999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.749,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.434},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.59799999999999998,"15min":0.60699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79300000000000004,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.60799999999999998,"15min":0.621},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.755,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.61399999999999999,"15min":0.626},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.71899999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:27:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.31},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.34499999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.503}]}]},{"osd":5,"up_from":36,"seq":154618822682,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.498}]},{"osd":2,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":0.442},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.65300000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:27:50 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.76300000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.73899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:28:34.911 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:28:34.911 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:28:34.911 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:28:34.912 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:28:35.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T22:28:35.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T22:28:35.102 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:34 smithi151 conmon[131789]: t=2023-09-23T22:28:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T22:28:35.103 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-09-23T22:28:35.354 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:28:35.354 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T22:28:35.354 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-09-23T22:28:35.355 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-09-23T22:28:35.355 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-09-23T22:28:35.355 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-09-23T22:28:35.355 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T22:28:35.355 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T22:28:35.355 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-09-23T22:28:35.622 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T22:28:35.623 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-09-23T22:28:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:35 smithi052 conmon[110082]: audit 2023-09-23T22:28:34.109694+0000 mgr.y (mgr.24314) 53 : audit [DBG] from='client.14586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:28:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:35 smithi052 conmon[100224]: audit 2023-09-23T22:28:34.109694+0000 mgr.y (mgr.24314) 53 : audit [DBG] from='client.14586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:28:35.862 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:35.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[100454]: audit 2023-09-23T22:28:34.109694+0000 mgr.y (mgr.24314) 53 : audit [DBG] from='client.14586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:28:35.874 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T22:28:35.874 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T22:28:35.874 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T22:28:35.874 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-09-23T22:28:35.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-09-23T22:28:35.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-09-23T22:28:35.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-09-23T22:28:35.875 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-09-23T22:28:36.134 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T22:28:36.134 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-09-23T22:28:36.135 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-09-23T22:28:36.135 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:35 smithi151 conmon[131789]: t=2023-09-23T22:28:35+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-09-23T22:28:36.135 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-09-23T22:28:36.135 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-09-23T22:28:36.135 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-09-23T22:28:36.136 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-09-23T22:28:36.136 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-09-23T22:28:36.136 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-09-23T22:28:36.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:35 smithi052 systemd[1]: Stopping Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:28:36.244 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:35 smithi052 bash[150363]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-alertmanager.a found: no such container 2023-09-23T22:28:36.403 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T22:28:36.403 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-09-23T22:28:36.403 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-09-23T22:28:36.403 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-09-23T22:28:36.404 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-09-23T22:28:36.404 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-09-23T22:28:36.404 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T22:28:36.635 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:36 smithi052 conmon[141643]: level=info ts=2023-09-23T22:28:36.364Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:28:36.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[100454]: cluster 2023-09-23T22:28:35.419501+0000 mgr.y (mgr.24314) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-09-23T22:28:36.656 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-09-23T22:28:36.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-09-23T22:28:36.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-09-23T22:28:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:36 smithi052 conmon[110082]: cluster 2023-09-23T22:28:35.419501 2023-09-23T22:28:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:36 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:36.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:36 smithi052 conmon[100224]: cluster 2023-09-23T22:28:35.419501+0000 mgr.y (mgr. 2023-09-23T22:28:36.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:36 smithi052 conmon[100224]: 24314) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:37.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-09-23T22:28:37.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:36 smithi151 conmon[131789]: t=2023-09-23T22:28:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-09-23T22:28:37.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-09-23T22:28:37.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-09-23T22:28:37.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T22:28:37.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-09-23T22:28:37.210 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-09-23T22:28:37.210 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-09-23T22:28:37.454 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:37 smithi052 bash[150528]: 3b798c4d1141ecaaa546cc620368bd883083c27913a69c2cae18ff932012edf5 2023-09-23T22:28:37.460 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=13.012665149s 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T22:28:37.461 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T22:28:37.462 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-09-23T22:28:37.462 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-09-23T22:28:37.743 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:37 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:28:37.743 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:37 smithi052 systemd[1]: Stopped Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:28:37.743 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:37 smithi052 systemd[1]: Starting Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:28:37.888 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:37.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:28:37 smithi151 conmon[131789]: t=2023-09-23T22:28:37+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T22:28:38.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:38] "GET /metrics HTTP/1.1" 200 155645 "" "Prometheus/2.33.4" 2023-09-23T22:28:38.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.243Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T22:28:38.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.243Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.245Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.52 port=9094 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.246Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 bash[151006]: f6ec125d25a1efdb37354e69173ef9460c7537a44b41332a79c4c44916b082ea 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 systemd[1]: Started Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.278Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.279Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.281Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T22:28:38.494 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:38.281Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T22:28:38.847 INFO:teuthology.orchestra.run.smithi052.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:28:38.847 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:28:38.848 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T22:28:38.848 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T22:28:38.858 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi052.front.sepia.ceph.com 2023-09-23T22:28:38.859 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T22:28:38.888 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[100224]: cluster 2023-09-23T22:28 2023-09-23T22:28:38.888 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[100224]: :37.419770+0000 mgr.y (mgr.24314) 55 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:38.888 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[100224]: audit 2023-09-23T22:28:37.888238+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.52:0/581614469' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:28:38.888 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[100224]: audit 2023-09-23T22:28:38.289374+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:38.888 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: cluster 2023-09-23T22:28:37 2023-09-23T22:28:38.888 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: .419770+0000 mgr.y (mgr.24314) 55 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:38.889 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: audit 2023-09-23T22:28:38.889 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: 2023-09-23T22:28:37.888238+0000 2023-09-23T22:28:38.889 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.52:0/581614469' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:28:38.889 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:28:38.889 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: :28:38.289374+0000 mon.a (mon.0) 641 : audit [INF] 2023-09-23T22:28:38.889 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:38 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:38.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:38 smithi151 conmon[100454]: cluster 2023-09-23T22:28:37.419770+0000 mgr.y (mgr.24314) 55 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:38.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:38 smithi151 conmon[100454]: audit 2023-09-23T22:28:37.888238+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.52:0/581614469' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:28:38.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:38 smithi151 conmon[100454]: audit 2023-09-23T22:28:38.289374+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:39.941 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:39 smithi151 conmon[100454]: cephadm 2023-09-23T22:28:38.290525+0000 mgr.y (mgr.24314) 56 2023-09-23T22:28:39.942 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:39 smithi151 conmon[100454]: : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:28:39.942 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:39 smithi151 conmon[100454]: cephadm 2023-09-23T22:28:38.294388+0000 mgr.y (mgr.24314) 57 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:28:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:39 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:38.290525+0000 mgr.y (mgr.24314) 56 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:28:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:39 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:38.294388+0000 mgr.y (mgr.24314) 57 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:28:39.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:39 smithi052 conmon[110082]: cephadm 2023-09-23T22:28:38.290525+0000 mgr.y (mgr.24314) 56 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:28:39.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:39 smithi052 conmon[110082]: cephadm 2023-09-23T22:28:38.294388+0000 mgr.y (mgr.24314) 57 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:28:40.404 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:40 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:40.246Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00042894s 2023-09-23T22:28:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:40 smithi052 conmon[100224]: cluster 2023-09-23T22:28:39.420176+0000 mgr.y (mgr.24314) 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:40.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:40 smithi052 conmon[110082]: cluster 2023-09-23T22:28 2023-09-23T22:28:40.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:40 smithi052 conmon[110082]: :39.420176+0000 mgr.y (mgr.24314) 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:40.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:40 smithi151 systemd[1]: Stopping Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:28:40.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:40 smithi151 bash[133077]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-prometheus.a found: no such container 2023-09-23T22:28:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:40 smithi151 conmon[100454]: cluster 2023-09-23T22:28:39.420176+0000 mgr.y (mgr.24314) 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:41.448 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T22:28:41.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[129826]: ts=2023-09-23T22:28:41.002Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T22:28:41.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[100454]: audit 2023-09-23T22:28:40.837439+0000 mgr.y (mgr.24314) 59 : 2023-09-23T22:28:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[100454]: audit [DBG] from='client.14598 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:28:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:41 smithi151 conmon[100454]: audit 2023-09-23T22:28:40.837975+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:28:41.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 bash[133194]: 47cd70b0fe355dd3423cf08c01c600e3ffeb4c1467c16faab66d499af271a50f 2023-09-23T22:28:41.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:28:41.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 systemd[1]: Stopped Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:28:41.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:41 smithi151 systemd[1]: Starting Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:28:41.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:41 smithi052 conmon[110082]: audit 2023-09-23T22:28:40.837439+0000 mgr.y (mgr.24314) 59 : audit [DBG] from='client.14598 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:28:41.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:41 smithi052 conmon[110082]: audit 2023-09-23T22:28:40.837975+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:28:41.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:41 smithi052 conmon[100224]: audit 2023-09-23T22:28:40.837439+0000 mgr.y (mgr.24314) 59 : audit [DBG] from='client.14598 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:28:41.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:41 smithi052 conmon[100224]: audit 2023-09-23T22:28:40.837975+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:28:42.647 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STOPPING 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STOPPED 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STARTING 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Serving on http://:::9283 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STARTED 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STOPPING 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STOPPED 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STARTING 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Serving on http://:::9283 2023-09-23T22:28:42.648 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STARTED 2023-09-23T22:28:42.649 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STOPPING 2023-09-23T22:28:42.649 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:28:42.649 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STOPPED 2023-09-23T22:28:42.649 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STARTING 2023-09-23T22:28:42.649 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Serving on http://:::9283 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 bash[133520]: 73d582b307e40d22ae8960aa76a02a5082f6fbbd2bb503a3e9d30629e6f5ac7b 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.254Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.254Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.254Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi151 (none))" 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.254Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.254Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.256Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.257Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 systemd[1]: Started Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:28:42.652 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.260Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.261Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.261Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.467µs 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.261Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.429Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.429Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.429Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.553µs wal_replay_duration=167.651126ms total_replay_duration=167.698751ms 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.431Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.431Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.431Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.453Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.242302ms db_storage=839ns remote_storage=1.491µs web_handler=509ns query_engine=997ns scrape=632.7µs scrape_sd=34.984µs notify=29.192µs notify_sd=15.769µs rules=19.960876ms 2023-09-23T22:28:42.653 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[133561]: ts=2023-09-23T22:28:42.453Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: cluster 2023-09-23T22:28:41.420645+0000 mgr.y (mgr.24314) 60 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:41.652025+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: cluster 2023-09-23T22:28:41.652124+0000 mon.a (mon.0) 644 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:41.656269+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.285287+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.288615+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: 28:42.289929+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052:9093"}]: dispatch 2023-09-23T22:28:42.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.294182+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.298596+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: :28:42.304117+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: .308573+0000 mon.a (mon.0) 652 : audit 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: .313048+0000 mon.a (mon.0) 653 : audit 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.316600 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: +0000 mon.a (mon.0) 654 : audit [INF] 2023-09-23T22:28:42.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:28:42.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: 2023-09-23T22:28:42.322056+0000 mon.a (mon.0 2023-09-23T22:28:42.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: ) 655 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:28:42.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: :28:42.326034+0000 mon.a (mon.0 2023-09-23T22:28:42.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:42 smithi151 conmon[100454]: ) 656 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:42.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100670]: [23/Sep/2023:22:28:42] ENGINE Bus STARTED 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: cluster 2023-09-23T22:28:41.420645+0000 mgr.y (mgr.24314) 60 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:41.652025+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: cluster 2023-09-23T22:28:41.652124+0000 mon.a (mon.0) 644 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:41.656269+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.285287+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: 09-23T22:28:42.288615+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:28:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.289929+0000 mon.a (mon 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: .0) 648 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052:9093"}]: dispatch 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: 2023-09-23T22:28:42.294182+0000 mon.a 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: (mon.0) 649 : audit 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: :28:42.298596+0000 mon.a (mon 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: .0) 650 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:28:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:42 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: .304117+0000 mon.a (mon.0) 651 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: -09-23T22:28:42.308573+0000 mon.a (mon.0) 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: 652 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: 2023-09-23T22:28:42.313048+0000 mon.a (mon.0) 653 : audit 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:28:42.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: T22:28:42.316600+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: 22:28:42.322056+0000 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: mon.a (mon.0) 655 : audit [INF] 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: audit 2023-09-23T22:28:42. 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[110082]: 326034+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:42.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: cluster 2023-09-23T22:28:41.420645+0000 mgr.y (mgr 2023-09-23T22:28:42.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: .24314) 60 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:42.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:41.652025 2023-09-23T22:28:42.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: +0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:28:42.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:28:42.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: -23T22:28 2023-09-23T22:28:42.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: :41.652124+0000 mon.a (mon.0) 644 : 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 2023-09-23T22:28:41.656269 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: +0000 mon.a (mon.0) 645 : audit [INF] 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: :28:42.285287+0000 mon.a (mon.0 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: ) 646 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 2023-09-23T22:28:42.288615+0000 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: mon.a (mon.0) 647 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: -09-23T22:28:42.289929+0000 mon.a (mon. 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 0) 648 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052:9093"}]: dispatch 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: T22:28:42.294182+0000 mon.a (mon.0 2023-09-23T22:28:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: ) 649 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: T22:28:42.298596+0000 mon.a (mon 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: .0) 650 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:42 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: .304117+0000 mon.a (mon.0) 651 : audit 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:42. 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 308573+0000 mon.a (mon.0) 652 : audit [INF] 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:43.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:42 2023-09-23T22:28:43.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: .313048+0000 mon.a (mon.0) 653 : audit [DBG] 2023-09-23T22:28:43.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:28:43.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:43.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 2023-09-23T22:28:42.316600+0000 mon.a 2023-09-23T22:28:43.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: (mon.0) 654 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:28:43.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 09-23T22:28:42.322056+0000 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: mon.a (mon.0) 655 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: audit 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: 2023-09-23T22:28:42.326034 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: +0000 mon.a (mon.0) 656 : audit [DBG] 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:43.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:42 smithi052 conmon[100224]: debug 2023-09-23T22:28:42.662+0000 7f1a56b62700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:28:43.462 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T22:28:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.288998+0000 mgr.y (mgr.24314) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:28:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.290231+0000 mgr.y (mgr.24314) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052:9093"}]: dispatch 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.299062+0000 mgr.y (mgr.24314) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.304469+0000 mgr.y (mgr.24314) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.313474+0000 mgr.y (mgr.24314) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.316973+0000 mgr.y (mgr.24314) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.658589+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: cluster 2023-09-23T22:28:42.658667+0000 mon.a (mon.0 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: ) 658 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.662991+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:28:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: cluster 2023-09-23T22:28:42.663643+0000 mon.a (mon.0) 660 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: cluster 2023-09-23T22:28:42.663676+0000 mon.a (mon.0) 661 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: cluster 2023-09-23T22:28:42.674237+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42.674351+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: cluster 2023-09-23T22:28:42.674422+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:0 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:42.687143+0000 mgr.y (mgr.24314) 67 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: audit 2023-09-23T22:28:42 2023-09-23T22:28:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[100224]: .691150+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.288998+0000 mgr.y (mgr.24314) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.290231+0000 mgr.y (mgr.24314) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052:9093"}]: dispatch 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.299062+0000 mgr.y (mgr.24314) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.304469+0000 mgr.y (mgr.24314) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.313474+0000 mgr.y (mgr.24314) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: 28:42.316973+0000 mgr.y (mgr.24314) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:28:43.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.658589+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: cluster 2023-09-23T22:28:42.658667+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.662991+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: cluster 2023-09-23T22:28:42.663643+0000 mon.a (mon.0) 660 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: cluster 2023-09-23T22:28:42.663676+0000 mon.a (mon.0) 661 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: cluster 2023-09-23T22:28:42.674237+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09-23T22:28:42.674351+0000 mon.a ( 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: mon.0) 663 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: cluster 2023-09-23T22:28:42.674422+0000 mon.a (mon.0) 664 : cluster 2023-09-23T22:28:43.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: [DBG] fsmap foo:0 2023-09-23T22:28:43.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: cephadm 2023 2023-09-23T22:28:43.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: -09-23T22:28:42.687143+0000 mgr.y (mgr 2023-09-23T22:28:43.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: .24314) 67 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T22:28:43.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:28:43.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: 23T22:28:42.691150+0000 mon.a ( 2023-09-23T22:28:43.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:43 smithi052 conmon[110082]: mon.0) 665 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.288998+0000 mgr.y (mgr.24314 2023-09-23T22:28:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: ) 61 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:28:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.290231+0000 mgr.y (mgr.24314) 62 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052:9093"}]: dispatch 2023-09-23T22:28:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.299062+0000 mgr.y (mgr.24314) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:28:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.304469+0000 mgr.y (mgr.24314) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:28:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.313474+0000 mgr.y (mgr.24314) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.316973+0000 mgr.y (mgr.24314) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22:28:42.658589+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cluster 2023-09 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: -23T22:28:42.658667+0000 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: mon.a (mon.0) 658 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: :28:42.662991+0000 mon.a (mon. 2023-09-23T22:28:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 0) 659 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cluster 2023- 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 09-23T22:28:42. 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 663643+0000 mon.a (mon.0) 660 : cluster 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cluster 2023-09-23 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: T22:28:42.663676+0000 mon.a ( 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: mon.0) 661 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cluster 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 2023-09-23T22:28: 2023-09-23T22:28:43.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 42.674237+0000 mon.a (mon.0) 662 : 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 23T22:28:42.674351+0000 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: mon.a (mon.0) 663 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cluster 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 2023-09-23T22:28:42.674422 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: +0000 mon.a (mon.0) 664 : cluster [DBG] 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: fsmap foo:0 2023-09-23T22:28:43.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: cephadm 2023-09-23T22 2023-09-23T22:28:43.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: :28:42.687143+0000 mgr.y (mgr.24314 2023-09-23T22:28:43.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: ) 67 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-09-23T22:28:43.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:28:43.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: 09-23T22:28:42.691150 2023-09-23T22:28:43.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:43 smithi151 conmon[100454]: +0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:44.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[100224]: cluster 2023-09-23T22:28:43.421038+0000 mgr.y (mgr.24314 2023-09-23T22:28:44.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[100224]: ) 68 : cluster [DBG] pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:44.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[100224]: cluster 2023-09-23T22:28:43.673621+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:28:44.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[100224]: audit 2023-09-23T22:28:44.589601+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:44 smithi151 conmon[100454]: cluster 2023-09-23T22:28:43.421038+0000 mgr.y ( 2023-09-23T22:28:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:44 smithi151 conmon[100454]: mgr.24314) 68 : cluster [DBG] pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:44 smithi151 conmon[100454]: cluster 2023-09-23T22:28:43.673621+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:28:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:44 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:28:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:44 smithi151 conmon[100454]: 28:44.589601+0000 mon.a (mon.0) 667 : 2023-09-23T22:28:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:44 smithi151 conmon[100454]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[110082]: cluster 2023-09-23T22:28:43. 2023-09-23T22:28:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[110082]: 421038+0000 mgr.y (mgr.24314) 68 : cluster [DBG] pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[110082]: cluster 2023-09-23T22:28:43.673621+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:28:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:44 smithi052 conmon[110082]: audit 2023-09-23T22:28:44.589601+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:45.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:45 smithi151 conmon[100454]: cluster 2023-09-23T22:28:44.674682+0000 mon.a (mon. 2023-09-23T22:28:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:45 smithi151 conmon[100454]: 0) 668 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:28:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:45 smithi151 conmon[100454]: audit 2023-09-23T22:28:44.814811+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:45.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:45 smithi052 conmon[110082]: cluster 2023-09-23T22:28:44.674682+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:28:45.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:45 smithi052 conmon[110082]: audit 2023-09-23T22:28:44.814811+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:45.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:45 smithi052 conmon[100224]: cluster 2023-09-23T22:28:44.674682+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:28:45.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:45 smithi052 conmon[100224]: audit 2023-09-23T22:28:44.814811+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.247 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T22:28:46.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: cluster 2023-09-23T22:28:45.421543+0000 mgr.y (mgr.24314) 69 : 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023-09-23T22:28:46.205936+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023-09-23T22:28:46.206804+0000 mon.a (mon.0) 671 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: 09-23T22:28:46.207564+0000 mon.a (mon.0) 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: 672 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023-09-23T22:28: 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: 46.211691+0000 mon.a (mon.0) 673 : audit [INF] 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023-09-23T22:28:46. 2023-09-23T22:28:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: 219614+0000 mon.a (mon.0) 674 : audit [INF] 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023-09-23T22:28:46.220632 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: +0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: -09-23T22:28:46.224227+0000 mon.a (mon 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: .0) 676 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: -23T22:28:46.225176+0000 mon.a (mon.0) 2023-09-23T22:28:46.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:46 smithi151 conmon[100454]: 677 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: cluster 2023-09-23T22:28:45.421543+0000 mgr.y (mgr 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: .24314) 69 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46.205936+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46.206804+0000 mon.a (mon.0) 671 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 2023-09-23T22:28:46.207564+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:46.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46.211691+0000 mon.a (mon. 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 0) 673 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 2023-09-23T22:28:46.219614+0000 mon.a (mon. 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 0) 674 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46.220632+0000 mon.a 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: (mon.0) 675 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 23T22:28:46.224227+0000 mon.a (mon.0) 2023-09-23T22:28:46.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 676 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: audit 2023-09-23T22:28:46. 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[110082]: 225176+0000 mon.a (mon.0) 677 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: cluster 2023-09-23T22:28: 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: 45.421543+0000 mgr.y (mgr.24314) 69 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23T22:28:46.205936+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23T22:28:46.206804+0000 mon.a (mon.0) 671 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23T22:28:46.207564+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23T22:28:46.211691+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23T22:28:46.219614+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:46.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23T22:28:46.220632+0000 mon.a (mon.0) 675 : audit [INF] 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: T22:28:46.224227+0000 mon.a (mon.0 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: ) 676 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: -23T22:28:46.225176+0000 mon.a (mon.0) 677 2023-09-23T22:28:46.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:46 smithi052 conmon[100224]: : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:47.684 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:47 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:46. 2023-09-23T22:28:47.684 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:47 smithi052 conmon[100224]: 225873+0000 mgr.y (mgr.24314) 70 : cephadm [INF] Deploying daemon mds.foo.smithi052.thpzxu on smithi052 2023-09-23T22:28:47.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:47 smithi151 conmon[100454]: cephadm 2023-09-23T22:28:46.225873+0000 mgr.y (mgr.24314) 70 : cephadm [INF] 2023-09-23T22:28:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:47 smithi151 conmon[100454]: Deploying daemon mds.foo.smithi052.thpzxu on smithi052 2023-09-23T22:28:47.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:47 smithi052 conmon[110082]: cephadm 2023-09-23T22:28:46.225873+0000 mgr.y 2023-09-23T22:28:47.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:47 smithi052 conmon[110082]: (mgr.24314) 70 : cephadm [INF] Deploying daemon mds.foo.smithi052.thpzxu on smithi052 2023-09-23T22:28:48.376 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:48 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:48] "GET /metrics HTTP/1.1" 200 160663 "" "Prometheus/2.33.4" 2023-09-23T22:28:48.377 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:28:48 smithi052 conmon[151062]: level=info ts=2023-09-23T22:28:48.248Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001954869s 2023-09-23T22:28:48.686 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:48 smithi052 conmon[100224]: cluster 2023-09-23T22:28:47.421864+0000 2023-09-23T22:28:48.686 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:48 smithi052 conmon[100224]: mgr.y (mgr.24314) 71 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:48.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:48 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:28:48.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:48 smithi052 conmon[110082]: 28:47.421864+0000 mgr.y (mgr.24314) 71 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:48 smithi151 conmon[100454]: cluster 2023-09-23T22:28:47.421864+0000 mgr.y (mgr.24314) 71 : cluster [DBG] 2023-09-23T22:28:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:48 smithi151 conmon[100454]: pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:49.591 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T22:28:49.600 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:28:49.600 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:28:49.601 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:28:49.971 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.422224+0000 mgr.y (mgr. 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: 24314) 72 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: audit 2023-09-23T22:28:49.556648+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: audit 2023-09-23T22:28:49.557698+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: audit 2023-09-23T22:28:49.560738+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: audit 2023-09-23T22:28:49.561452+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cephadm 2023-09-23T22:28:49.562060+0000 mgr.y (mgr.24314) 73 : cephadm [INF] Deploying daemon mds.foo.smithi151.rbxgii on smithi151 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.690729+0000 mon.a (mon.0) 682 : cluster [DBG] mds.? [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] up:boot 2023-09-23T22:28:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.690790+0000 mon.a (mon.0) 683 : cluster [INF] daemon mds.foo.smithi052.thpzxu assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.690858+0000 mon.a (mon.0) 684 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.690875+0000 mon.a (mon.0) 685 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.690893+0000 mon.a (mon.0) 686 : cluster [INF] Cluster is now healthy 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.691050+0000 mon.a (mon.0) 687 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: audit 2023-09-23T22:28:49.691183+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.694832+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:creating} 2023-09-23T22:28:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:50 smithi151 conmon[100454]: cluster 2023-09-23T22:28:49.707452+0000 mon.a (mon.0) 690 : cluster [INF] daemon mds.foo.smithi052.thpzxu is now active in filesystem foo as rank 0 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.422224+0000 mgr.y (mgr.24314) 72 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: audit 2023-09-23T22:28:49.556648+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: audit 2023-09-23T22:28:49.557698+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: audit 2023-09-23T22:28:49.560738+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: audit 2023-09-23T22:28:49.561452+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cephadm 2023-09-23T22:28:49.562060+0000 mgr.y (mgr.24314) 73 : cephadm [INF] Deploying daemon mds.foo.smithi151.rbxgii on smithi151 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.690729+0000 mon.a (mon.0) 682 : cluster [DBG] mds.? [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] up:boot 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.690790+0000 mon.a (mon.0) 683 : cluster [INF] daemon mds.foo.smithi052.thpzxu assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.690858+0000 mon.a (mon.0) 684 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.690875+0000 mon.a (mon.0) 685 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.690893+0000 mon.a (mon.0) 686 : cluster [INF] Cluster is now healthy 2023-09-23T22:28:50.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.691050+0000 mon.a (mon.0) 687 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: audit 2023-09-23T22:28:49.691183+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.694832+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:creating} 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[110082]: cluster 2023-09-23T22:28:49.707452+0000 mon.a (mon.0) 690 : cluster [INF] daemon mds.foo.smithi052.thpzxu is now active in filesystem foo as rank 0 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.422224+0000 mgr.y (mgr.24314) 72 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: audit 2023-09-23T22:28:49.556648+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: audit 2023-09-23T22:28:49.557698+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: audit 2023-09-23T22:28:49.560738+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: audit 2023-09-23T22:28:49.561452+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cephadm 2023-09-23T22:28:49.562060+0000 mgr.y (mgr.24314) 73 : cephadm [INF] Deploying daemon mds.foo.smithi151.rbxgii on smithi151 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.690729+0000 mon.a (mon.0) 682 : cluster [DBG] mds.? [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] up:boot 2023-09-23T22:28:50.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.690790+0000 mon.a (mon.0) 683 : cluster [INF] daemon mds.foo.smithi052.thpzxu assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.690858+0000 mon.a (mon.0) 684 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.690875+0000 mon.a (mon.0) 685 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.690893+0000 mon.a (mon.0) 686 : cluster [INF] Cluster is now healthy 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.691050+0000 mon.a (mon.0) 687 : cluster [DBG] fsmap foo:0 1 up:standby 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: audit 2023-09-23T22:28:49.691183+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.694832+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:creating} 2023-09-23T22:28:50.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:50 smithi052 conmon[100224]: cluster 2023-09-23T22:28:49.707452+0000 mon.a (mon.0) 690 : cluster [INF] daemon mds.foo.smithi052.thpzxu is now active in filesystem foo as rank 0 2023-09-23T22:28:51.611 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:51 smithi151 conmon[100454]: cluster 2023-09-23T22:28:50.694781+0000 mon.a ( 2023-09-23T22:28:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:51 smithi151 conmon[100454]: mon.0) 691 : cluster [DBG] mds.? [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] up:active 2023-09-23T22:28:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:51 smithi151 conmon[100454]: cluster 2023-09-23T22:28:50.694836+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 2023-09-23T22:28:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:51 smithi151 conmon[100454]: audit 2023-09-23T22:28:51.606713+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.52:0/802686402' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:28:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[100224]: cluster 2023-09-23T22:28:50. 2023-09-23T22:28:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[100224]: 694781+0000 mon.a (mon.0) 691 : cluster [DBG] mds.? [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] up:active 2023-09-23T22:28:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[100224]: cluster 2023-09-23T22:28:50.694836+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 2023-09-23T22:28:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[100224]: audit 2023-09-23T22:28:51.606713+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.52:0/802686402' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:28:51.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:28:51.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: T22:28:50.694781+0000 mon.a (mon.0) 691 : cluster [DBG] mds.? [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] up:active 2023-09-23T22:28:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: cluster 2023-09-23T22:28:50.694836+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 2023-09-23T22:28:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: audit 2023-09-23T22:28:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: 2023-09-23T22:28: 2023-09-23T22:28:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: 51.606713+0000 mon.c (mon.2) 23 : audit 2023-09-23T22:28:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:51 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/802686402' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:28:52.273 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":19,"active_gid":24314,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6800","nonce":2740388443},{"type":"v1","addr":"172.21.15.52:6801","nonce":2740388443}]},"active_addr":"172.21.15.52:6801/2740388443","active_change":"2023-09-23T22:27:29.395785+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":3070502744}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":1078027830}]}]} 2023-09-23T22:28:52.277 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:28:52.277 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:28:52.278 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:28:52.752 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:53.399 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: cluster 2023-09-23T22:28:51.422716+0000 mgr.y (mgr.24314) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: audit 2023-09-23T22:28:51.992140+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: audit 2023-09-23T22:28:51.993410+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: cluster 2023-09-23T22:28:52.700887+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.151:6832/507517307,v1:172.21.15.151:6833/507517307] up:boot 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: cluster 2023-09-23T22:28:52.701202+0000 mon.a (mon.0) 696 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: audit 2023-09-23T22:28:52.701335+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[100224]: cluster 2023-09-23T22:28:52.704253+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: cluster 2023-09-23T22:28:51.422716+0000 mgr.y (mgr.24314) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: audit 2023-09-23T22:28:51.992140+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:53.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: audit 2023-09-23T22:28:51.993410+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:53.401 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: cluster 2023-09-23T22:28:52.700887+0000 mon.a (mon. 2023-09-23T22:28:53.401 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: 0) 695 : cluster [DBG] mds.? [v2:172.21.15.151:6832/507517307,v1:172.21.15.151:6833/507517307] up:boot 2023-09-23T22:28:53.401 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: cluster 2023-09-23T22:28:52.701202+0000 mon.a (mon.0) 696 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:28:53.401 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: audit 2023-09-23T22:28:52.701335+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:28:53.401 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:52 smithi052 conmon[110082]: cluster 2023-09-23T22:28:52.704253+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: cluster 2023-09-23T22:28:51.422716+0000 mgr.y (mgr.24314) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: audit 2023-09-23T22:28:51.992140+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: audit 2023-09-23T22:28:51.993410+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: cluster 2023-09-23T22:28:52.700887+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.151:6832/507517307,v1:172.21.15.151:6833/507517307] up:boot 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: cluster 2023-09-23T22:28:52.701202+0000 mon.a (mon.0) 696 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: audit 2023-09-23T22:28:52.701335+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: 2023-09-23T22:28:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:52 smithi151 conmon[100454]: cluster 2023-09-23T22:28:52.704253+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:28:54.453 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:54.454 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":60,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:28:44.670574+0000","last_up_change":"2023-09-23T22:27:12.276151+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:28:41.273306+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:28:42.097486+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6803","nonce":858289560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6805","nonce":858289560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6809","nonce":858289560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6807","nonce":858289560}]},"public_addr":"172.21.15.52:6803/858289560","cluster_addr":"172.21.15.52:6805/858289560","heartbeat_back_addr":"172.21.15.52:6809/858289560","heartbeat_front_addr":"172.21.15.52:6807/858289560","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6811","nonce":3743472544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6813","nonce":3743472544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6817","nonce":3743472544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6815","nonce":3743472544}]},"public_addr":"172.21.15.52:6811/3743472544","cluster_addr":"172.21.15.52:6813/3743472544","heartbeat_back_addr":"172.21.15.52:6817/3743472544","heartbeat_front_addr":"172.21.15.52:6815/3743472544","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6819","nonce":1781181972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6821","nonce":1781181972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6825","nonce":1781181972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6823","nonce":1781181972}]},"public_addr":"172.21.15.52:6819/1781181972","cluster_addr":"172.21.15.52:6821/1781181972","heartbeat_back_addr":"172.21.15.52:6825/1781181972","heartbeat_front_addr":"172.21.15.52:6823/1781181972","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6827","nonce":1128061400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6829","nonce":1128061400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6833","nonce":1128061400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6831","nonce":1128061400}]},"public_addr":"172.21.15.52:6827/1128061400","cluster_addr":"172.21.15.52:6829/1128061400","heartbeat_back_addr":"172.21.15.52:6833/1128061400","heartbeat_front_addr":"172.21.15.52:6831/1128061400","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:28:55.138 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:28:55.139 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:28:55.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:28:55 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:28:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:54 smithi151 conmon[100454]: cluster 2023-09-23T22:28:53.423274+0000 mgr.y (mgr.24314) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:54 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:28:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:54 smithi151 conmon[100454]: 28:54.451174+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.52:0/2628523486' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:54 smithi151 conmon[100454]: audit 2023-09-23T22:28:54.460500+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:54 smithi151 conmon[100454]: audit 2023-09-23T22:28:54.629229+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: cluster 2023-09-23T22:28:53.423274+0000 2023-09-23T22:28:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: mgr.y (mgr.24314) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:28:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: -23T22:28:54.451174+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.52:0/2628523486' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:55.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: audit 2023-09-23T22:28: 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: 54.460500+0000 mon.a (mon.0) 699 : audit 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: 23T22:28:54.629229+0000 mon.a (mon. 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[110082]: 0) 700 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: cluster 2023-09-23T22:28: 2023-09-23T22:28:55.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: 53.423274+0000 mgr.y (mgr.24314) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: audit 2023-09-23T22:28:54.451174+0000 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.52:0/2628523486' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: audit 2023-09-23T22:28:54 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: .460500+0000 mon.a (mon.0) 699 : audit 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: audit 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: 2023-09-23T22:28:54. 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: 629229+0000 mon.a (mon.0) 700 : audit 2023-09-23T22:28:55.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:54 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:55.563 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:56.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: cluster 2023-09-23T22:28:55.423787+0000 mgr.y (mgr.24314) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: audit 2023-09-23T22:28:55.565438+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: audit 2023-09-23T22:28:55.566348+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: audit 2023-09-23T22:28:55.567156+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: audit 2023-09-23T22:28:55.572784+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: audit 2023-09-23T22:28:55.580136+0000 mon.a (mon.0) 705 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[100224]: 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[110082]: cluster 2023-09-23T22:28:55.423787+0000 mgr.y (mgr.24314) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[110082]: audit 2023-09-23T22:28:55.565438+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:56.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[110082]: audit 2023-09-23T22:28:55.566348+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:56.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[110082]: audit 2023-09-23T22:28:55.567156+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:56.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[110082]: audit 2023-09-23T22:28:55.572784+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:56.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:56 smithi052 conmon[110082]: audit 2023-09-23T22:28:55.580136+0000 mon.a (mon.0) 705 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:56.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: cluster 2023-09-23T22:28:55.423787+0000 mgr.y (mgr.24314) 76 : 2023-09-23T22:28:56.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:28:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: audit 2023-09-23T22:28:55.565438+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: audit 2023-09-23T22:28:55.566348+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: audit 2023-09-23T22:28:55.567156+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: audit 2023-09-23T22:28:55.572784+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:56 smithi151 conmon[100454]: audit 2023-09-23T22:28:55.580136+0000 mon.a (mon.0) 705 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:28:57.086 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":60,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:28:44.670574+0000","last_up_change":"2023-09-23T22:27:12.276151+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:28:41.273306+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:28:42.097486+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6803","nonce":858289560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6805","nonce":858289560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6809","nonce":858289560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":858289560},{"type":"v1","addr":"172.21.15.52:6807","nonce":858289560}]},"public_addr":"172.21.15.52:6803/858289560","cluster_addr":"172.21.15.52:6805/858289560","heartbeat_back_addr":"172.21.15.52:6809/858289560","heartbeat_front_addr":"172.21.15.52:6807/858289560","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6811","nonce":3743472544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6813","nonce":3743472544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6817","nonce":3743472544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":3743472544},{"type":"v1","addr":"172.21.15.52:6815","nonce":3743472544}]},"public_addr":"172.21.15.52:6811/3743472544","cluster_addr":"172.21.15.52:6813/3743472544","heartbeat_back_addr":"172.21.15.52:6817/3743472544","heartbeat_front_addr":"172.21.15.52:6815/3743472544","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6819","nonce":1781181972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6821","nonce":1781181972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6825","nonce":1781181972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1781181972},{"type":"v1","addr":"172.21.15.52:6823","nonce":1781181972}]},"public_addr":"172.21.15.52:6819/1781181972","cluster_addr":"172.21.15.52:6821/1781181972","heartbeat_back_addr":"172.21.15.52:6825/1781181972","heartbeat_front_addr":"172.21.15.52:6823/1781181972","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6827","nonce":1128061400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6829","nonce":1128061400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6833","nonce":1128061400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":1128061400},{"type":"v1","addr":"172.21.15.52:6831","nonce":1128061400}]},"public_addr":"172.21.15.52:6827/1128061400","cluster_addr":"172.21.15.52:6829/1128061400","heartbeat_back_addr":"172.21.15.52:6833/1128061400","heartbeat_front_addr":"172.21.15.52:6831/1128061400","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:28:57.775 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:28:57.776 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:28:57.776 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:28:57.777 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:28:57.777 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:28:57.778 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:28:57.778 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:28:57.779 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:28:57.824 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:57 smithi052 conmon[110082]: audit 2023-09-23T22:28:57.084858 2023-09-23T22:28:57.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:57 smithi052 conmon[110082]: +0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.52:0/2252951084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:57.825 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:57 smithi052 conmon[100224]: audit 2023-09-23T22:28:57.084858+0000 mon.a 2023-09-23T22:28:57.825 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:57 smithi052 conmon[100224]: (mon.0) 706 : audit [DBG] from='client.? 172.21.15.52:0/2252951084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:57.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:57 smithi151 conmon[100454]: audit 2023-09-23T22:28:57.084858+0000 mon.a (mon.0) 706 : audit [DBG] from='client.? 172.21.15.52:0/2252951084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:28:58.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: cluster 2023-09-23T22:28:57.424171+0000 mgr.y (mgr.24314) 77 : cluster 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: audit 2023-09-23T22:28:58.072432+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: audit 2023-09-23T22:28:58.074967+0000 mon.a (mon.0) 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: 708 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: 23T22:28:58.078247+0000 mon.a ( 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: mon.0) 709 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: -09-23T22:28:58. 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: 113285+0000 mon.a (mon.0) 710 : audit [INF] 2023-09-23T22:28:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:28:58 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:58.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:28:58] "GET /metrics HTTP/1.1" 200 160663 "" "Prometheus/2.33.4" 2023-09-23T22:28:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:28:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: T22:28:57.424171+0000 mgr.y (mgr.24314) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:28:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: audit 2023-09-23T22:28:58. 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: 072432+0000 mon.a (mon.0) 707 : audit [INF] 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: audit 2023 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: -09-23T22:28:58.074967+0000 mon.a (mon 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: .0) 708 : audit [DBG] 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: audit 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: 2023-09-23T22:28:58.078247+0000 2023-09-23T22:28:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: mon.a (mon.0) 709 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: audit 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: 2023-09-23T22:28:58.113285+0000 mon.a (mon.0) 710 : audit 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: cluster 2023-09-23T22:28:57 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: .424171+0000 mgr.y (mgr.24314) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: audit 2023-09-23T22:28:58 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: .072432+0000 mon.a (mon.0) 707 : audit [INF] 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: audit 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: 2023-09-23T22:28:58. 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: 074967+0000 mon.a (mon.0) 708 : audit [DBG] 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: audit 2023-09-23T22:28 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: :58.078247+0000 mon.a (mon.0) 709 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: T22:28:58.113285+0000 mon.a 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: (mon.0) 710 : audit [INF] 2023-09-23T22:28:58.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:28:58 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:28:59.038 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:59.241 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:59.292 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:59.365 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:59.618 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:00.039 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:00.248 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:00 smithi052 conmon[110082]: cluster 2023-09-23T22:28: 2023-09-23T22:29:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:00 smithi052 conmon[110082]: 59.424551+0000 mgr.y (mgr.24314) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:00 smithi052 conmon[110082]: audit 2023-09-23T22:28:59.590879+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:00.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:00 smithi052 conmon[100224]: cluster 2023-09-23T22:28:59.424551+0000 mgr.y 2023-09-23T22:29:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:00 smithi052 conmon[100224]: (mgr.24314) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:00 smithi052 conmon[100224]: audit 2023-09-23T22:28:59.590879+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:00 smithi151 conmon[100454]: cluster 2023-09-23T22:28:59.424551+0000 mgr.y (mgr.24314) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:00 smithi151 conmon[100454]: 2023-09-23T22:29:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:00 smithi151 conmon[100454]: audit 2023-09-23T22:28:59.590879+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:01.721 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:02.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:02 smithi052 conmon[100224]: cluster 2023-09-23T22:29:01.425189+0000 mgr.y (mgr.24314) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:02.733 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:02 smithi052 conmon[110082]: cluster 2023-09-23T22:29:01.425189+0000 mgr.y (mgr.24314) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:02 smithi151 conmon[100454]: cluster 2023-09-23T22:29:01.425189+0000 mgr.y (mgr.24314) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:04 smithi151 conmon[100454]: cluster 2023-09-23T22:29:03.425524+0000 mgr.y (mgr.24314) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:04.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:04 smithi052 conmon[110082]: cluster 2023-09-23T22:29:03.425524+0000 mgr.y (mgr.24314) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:04.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:04 smithi052 conmon[100224]: cluster 2023-09-23T22:29:03.425524+0000 mgr.y (mgr.24314) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:05.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:29:05 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:29:06.070 INFO:teuthology.orchestra.run.smithi052.stdout:128849018917 2023-09-23T22:29:06.071 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:29:06.347 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:06 smithi052 conmon[110082]: cluster 2023-09-23T22:29:05.425982+0000 mgr.y (mgr.24314) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:06.347 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:06 smithi052 conmon[100224]: cluster 2023-09-23T22:29:05.425982+0000 mgr.y (mgr.24314) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:06.406 INFO:teuthology.orchestra.run.smithi052.stdout:34359738422 2023-09-23T22:29:06.407 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:29:06.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:06 smithi151 conmon[100454]: cluster 2023-09-23T22:29:05.425982+0000 mgr.y (mgr.24314) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:29:06.757 INFO:teuthology.orchestra.run.smithi052.stdout:77309411375 2023-09-23T22:29:06.758 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:29:08.152 INFO:teuthology.orchestra.run.smithi052.stdout:180388626462 2023-09-23T22:29:08.153 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:29:08.383 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:08 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:08] "GET /metrics HTTP/1.1" 200 191460 "" "Prometheus/2.33.4" 2023-09-23T22:29:08.634 INFO:teuthology.orchestra.run.smithi052.stdout:55834574898 2023-09-23T22:29:08.634 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:29:08.668 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:08 smithi052 conmon[110082]: cluster 2023-09-23T22:29: 2023-09-23T22:29:08.669 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:08 smithi052 conmon[110082]: 07.426321+0000 mgr.y (mgr.24314) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:08.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:08 smithi052 conmon[100224]: cluster 2023-09-23T22:29:07 2023-09-23T22:29:08.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:08 smithi052 conmon[100224]: .426321+0000 mgr.y (mgr.24314) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:08.776 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:08.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:08 smithi151 conmon[100454]: cluster 2023-09-23T22:29:07.426321+0000 mgr.y (mgr.24314) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:09.862 INFO:teuthology.orchestra.run.smithi052.stdout:154618822690 2023-09-23T22:29:09.863 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:29:10.050 INFO:teuthology.orchestra.run.smithi052.stdout:206158430234 2023-09-23T22:29:10.050 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:29:10.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:10 smithi052 conmon[100224]: cluster 2023-09-23T22:29:09.426816+0000 mgr.y (mgr.24314) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:10.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:10 smithi052 conmon[110082]: cluster 2023-09-23T22:29:09.426816+0000 mgr.y (mgr.24314) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:10.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:10 smithi151 conmon[100454]: cluster 2023-09-23T22:29:09.426816+0000 mgr.y (mgr.24314) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:11.035 INFO:teuthology.orchestra.run.smithi052.stdout:107374182441 2023-09-23T22:29:11.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:29:11.057 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:11.138 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:11.526 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:11.577 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:11.666 INFO:teuthology.orchestra.run.smithi052.stdout:34359738424 2023-09-23T22:29:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:12 smithi052 conmon[110082]: cluster 2023-09-23T22:29:11.427307+0000 mgr.y (mgr.24314) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:12.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:12 smithi052 conmon[110082]: audit 2023-09-23T22:29:11.665809+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.52:0/1098277938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:29:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:12 smithi052 conmon[100224]: cluster 2023-09-23T22:29:11.427307+0000 mgr.y (mgr.24314) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:12 smithi052 conmon[100224]: audit 2023-09-23T22:29:11.665809+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.52:0/1098277938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:29:12.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:12 smithi151 conmon[100454]: cluster 2023-09-23T22:29:11.427307+0000 mgr.y (mgr.24314) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:12 smithi151 conmon[100454]: 2023-09-23T22:29:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:12 smithi151 conmon[100454]: audit 2023-09-23T22:29:11.665809+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.52:0/1098277938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:29:13.748 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:13.774 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738422 got 34359738424 for osd.0 2023-09-23T22:29:13.774 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:14.050 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:14.359 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:14.497 INFO:teuthology.orchestra.run.smithi052.stdout:180388626463 2023-09-23T22:29:14.583 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:14 smithi052 conmon[110082]: cluster 2023-09-23T22:29:13.427618+0000 mgr.y (mgr.24314) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:14.583 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:14 smithi052 conmon[100224]: cluster 2023-09-23T22:29:13.427618+0000 mgr.y (mgr.24314) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:14.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:14 smithi151 conmon[100454]: cluster 2023-09-23T22:29:13.427618+0000 mgr.y (mgr.24314) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:15.133 INFO:teuthology.orchestra.run.smithi052.stdout:128849018919 2023-09-23T22:29:15.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:29:15 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:29:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:15 smithi052 conmon[110082]: audit 2023-09-23T22:29:14.496917+0000 2023-09-23T22:29:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:15 smithi052 conmon[110082]: mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.52:0/3097767546' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:29:15.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:15 smithi052 conmon[110082]: audit 2023-09-23T22:29:15.132929+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.52:0/568701746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:29:15.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:15 smithi052 conmon[100224]: audit 2023-09-23T22:29:14.496917+0000 mon.a (mon.0 2023-09-23T22:29:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:15 smithi052 conmon[100224]: ) 713 : audit [DBG] from='client.? 172.21.15.52:0/3097767546' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:29:15.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:15 smithi052 conmon[100224]: audit 2023-09-23T22:29:15.132929+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.52:0/568701746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:29:15.936 INFO:teuthology.orchestra.run.smithi052.stdout:77309411377 2023-09-23T22:29:15.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:15 smithi151 conmon[100454]: audit 2023-09-23T22:29:14.496917+0000 mon.a (mon.0) 713 : audit [DBG] 2023-09-23T22:29:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:15 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/3097767546' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:29:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:15 smithi151 conmon[100454]: audit 2023-09-23T22:29:15.132929+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.52:0/568701746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:29:16.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[100224]: cluster 2023-09-23T22:29 2023-09-23T22:29:16.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[100224]: :15.428107+0000 mgr.y (mgr.24314) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:16.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[100224]: audit 2023-09-23T22:29:15.936081+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.52:0/3761193132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:29:16.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[110082]: cluster 2023-09-23T22:29:15.428107+0000 mgr.y (mgr.24314) 86 : cluster [DBG] 2023-09-23T22:29:16.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[110082]: pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[110082]: audit 2023-09-23T22:29:15.936081+0000 2023-09-23T22:29:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:16 smithi052 conmon[110082]: mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.52:0/3761193132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:29:16.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:16 smithi151 conmon[100454]: cluster 2023-09-23T22:29:15.428107+0000 mgr.y (mgr.24314) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:16 smithi151 conmon[100454]: 2023-09-23T22:29:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:16 smithi151 conmon[100454]: audit 2023-09-23T22:29:15.936081+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.52:0/3761193132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:29:17.330 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411375 got 77309411377 for osd.2 2023-09-23T22:29:17.331 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:17.360 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626462 got 180388626463 for osd.6 2023-09-23T22:29:17.360 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:17.674 INFO:teuthology.orchestra.run.smithi052.stdout:55834574900 2023-09-23T22:29:18.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:18] "GET /metrics HTTP/1.1" 200 191455 "" "Prometheus/2.33.4" 2023-09-23T22:29:18.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:29:18.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[110082]: 29:17.428410+0000 mgr.y (mgr.24314) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:18.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:29:18.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[110082]: 09-23T22:29:17. 2023-09-23T22:29:18.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[110082]: 674450+0000 mon.a (mon.0) 2023-09-23T22:29:18.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[110082]: 715 : audit [DBG] from='client.? 172.21.15.52:0/1309613610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:29:18.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[100224]: cluster 2023-09- 2023-09-23T22:29:18.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[100224]: 23T22:29:17.428410+0000 mgr.y (mgr.24314) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:18.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:29:18.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[100224]: 09-23T22:29:17.674450+0000 mon.a (mon.0) 2023-09-23T22:29:18.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:18 smithi052 conmon[100224]: 715 : audit [DBG] from='client.? 172.21.15.52:0/1309613610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:29:18.900 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018917 got 128849018919 for osd.4 2023-09-23T22:29:18.900 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:18.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:18 smithi151 conmon[100454]: cluster 2023-09-23T22:29:17.428410+0000 mgr.y (mgr.24314) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:18 smithi151 conmon[100454]: audit 2023-09-23T22:29:17.674450+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.52:0/1309613610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:29:19.414 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574898 got 55834574900 for osd.1 2023-09-23T22:29:19.414 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:19.673 INFO:teuthology.orchestra.run.smithi052.stdout:206158430236 2023-09-23T22:29:20.547 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430234 got 206158430236 for osd.7 2023-09-23T22:29:20.547 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:20 smithi052 conmon[110082]: cluster 2023-09-23T22:29:19.428724+0000 mgr.y ( 2023-09-23T22:29:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:20 smithi052 conmon[110082]: mgr.24314) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:20 smithi052 conmon[110082]: audit 2023-09-23T22:29:19.673392+0000 mon.a (mon.0) 716 : audit [DBG] from='client.? 172.21.15.52:0/3567074967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:29:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:20 smithi052 conmon[100224]: cluster 2023-09-23T22:29:19.428724+0000 mgr.y (mgr.24314) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:20 smithi052 conmon[100224]: audit 2023-09-23T22:29:19.673392+0000 mon.a (mon.0) 716 : audit [DBG] from='client.? 172.21.15.52:0/3567074967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:29:20.878 INFO:teuthology.orchestra.run.smithi052.stdout:154618822692 2023-09-23T22:29:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:20 smithi151 conmon[100454]: cluster 2023-09-23T22:29:19.428724+0000 mgr.y (mgr.24314) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:20 smithi151 conmon[100454]: 2023-09-23T22:29:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:20 smithi151 conmon[100454]: audit 2023-09-23T22:29:19.673392+0000 mon.a (mon.0) 716 : audit [DBG] from='client.? 172.21.15.52:0/3567074967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:29:20.951 INFO:teuthology.orchestra.run.smithi052.stdout:107374182443 2023-09-23T22:29:21.528 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822690 got 154618822692 for osd.5 2023-09-23T22:29:21.528 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:21.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:21 smithi151 conmon[100454]: audit 2023-09-23T22:29:20.877958+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.52:0/1421934961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:29:21.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:21 smithi151 conmon[100454]: audit 2023-09-23T22:29:20.951411+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.52:0/1228731803' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:29:21.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[110082]: audit 2023-09-23T22:29:20.877958 2023-09-23T22:29:21.976 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[110082]: +0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.52:0/1421934961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:29:21.976 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:29:21.976 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[110082]: 29:20.951411+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.52:0/1228731803' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:29:21.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:29:21.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[100224]: :29:20.877958+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.52:0/1421934961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:29:21.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:21 smithi052 conmon[100224]: audit 2023-09-23T22:29:20.951411+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.52:0/1228731803' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:29:21.999 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182441 got 107374182443 for osd.3 2023-09-23T22:29:21.999 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:21.999 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:29:21.999 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:29:22.379 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:22.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:22 smithi052 conmon[110082]: cluster 2023-09-23T22:29:21.429193+0000 mgr.y (mgr.24314) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:22.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:22 smithi052 conmon[100224]: cluster 2023-09-23T22:29:21.429193+0000 mgr.y (mgr.24314) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:22 smithi151 conmon[100454]: cluster 2023-09-23T22:29:21.429193+0000 mgr.y (mgr.24314) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:23.973 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:29:23.974 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:29:24.545 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-09-23T22:29:23.429340+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381560,"kb_used_data":4792,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579656,"statfs":{"total":767960285184,"available":765521567744,"internally_reserved":0,"allocated":4907008,"data_stored":2055400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002365"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.680915+0000","last_change":"2023-09-23T22:28:43.680655+0000","last_active":"2023-09-23T22:28:43.680915+0000","last_peered":"2023-09-23T22:28:43.680915+0000","last_clean":"2023-09-23T22:28:43.680915+0000","last_became_active":"2023-09-23T22:28:43.680336+0000","last_became_peered":"2023-09-23T22:28:43.680336+0000","last_unstale":"2023-09-23T22:28:43.680915+0000","last_undegraded":"2023-09-23T22:28:43.680915+0000","last_fullsized":"2023-09-23T22:28:43.680915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.703751+0000","last_change":"2023-09-23T22:28:42.668130+0000","last_active":"2023-09-23T22:28:49.703751+0000","last_peered":"2023-09-23T22:28:49.703751+0000","last_clean":"2023-09-23T22:28:49.703751+0000","last_became_active":"2023-09-23T22:28:42.667727+0000","last_became_peered":"2023-09-23T22:28:42.667727+0000","last_unstale":"2023-09-23T22:28:49.703751+0000","last_undegraded":"2023-09-23T22:28:49.703751+0000","last_fullsized":"2023-09-23T22:28:49.703751+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839796+0000","last_change":"2023-09-23T22:28:42.687711+0000","last_active":"2023-09-23T22:28:43.839796+0000","last_peered":"2023-09-23T22:28:43.839796+0000","last_clean":"2023-09-23T22:28:43.839796+0000","last_became_active":"2023-09-23T22:28:42.687305+0000","last_became_peered":"2023-09-23T22:28:42.687305+0000","last_unstale":"2023-09-23T22:28:43.839796+0000","last_undegraded":"2023-09-23T22:28:43.839796+0000","last_fullsized":"2023-09-23T22:28:43.839796+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694067+0000","last_change":"2023-09-23T22:28:42.682707+0000","last_active":"2023-09-23T22:28:44.694067+0000","last_peered":"2023-09-23T22:28:44.694067+0000","last_clean":"2023-09-23T22:28:44.694067+0000","last_became_active":"2023-09-23T22:28:42.682433+0000","last_became_peered":"2023-09-23T22:28:42.682433+0000","last_unstale":"2023-09-23T22:28:44.694067+0000","last_undegraded":"2023-09-23T22:28:44.694067+0000","last_fullsized":"2023-09-23T22:28:44.694067+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840021+0000","last_change":"2023-09-23T22:28:43.839102+0000","last_active":"2023-09-23T22:28:43.840021+0000","last_peered":"2023-09-23T22:28:43.840021+0000","last_clean":"2023-09-23T22:28:43.840021+0000","last_became_active":"2023-09-23T22:28:43.838794+0000","last_became_peered":"2023-09-23T22:28:43.838794+0000","last_unstale":"2023-09-23T22:28:43.840021+0000","last_undegraded":"2023-09-23T22:28:43.840021+0000","last_fullsized":"2023-09-23T22:28:43.840021+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.703483+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:28:49.703483+0000","last_peered":"2023-09-23T22:28:49.703483+0000","last_clean":"2023-09-23T22:28:49.703483+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:28:49.703483+0000","last_undegraded":"2023-09-23T22:28:49.703483+0000","last_fullsized":"2023-09-23T22:28:49.703483+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840660+0000","last_change":"2023-09-23T22:28:43.839960+0000","last_active":"2023-09-23T22:28:43.840660+0000","last_peered":"2023-09-23T22:28:43.840660+0000","last_clean":"2023-09-23T22:28:43.840660+0000","last_became_active":"2023-09-23T22:28:43.839640+0000","last_became_peered":"2023-09-23T22:28:43.839640+0000","last_unstale":"2023-09-23T22:28:43.840660+0000","last_undegraded":"2023-09-23T22:28:43.840660+0000","last_fullsized":"2023-09-23T22:28:43.840660+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.702260+0000","last_change":"2023-09-23T22:28:42.676441+0000","last_active":"2023-09-23T22:28:49.702260+0000","last_peered":"2023-09-23T22:28:49.702260+0000","last_clean":"2023-09-23T22:28:49.702260+0000","last_became_active":"2023-09-23T22:28:42.676144+0000","last_became_peered":"2023-09-23T22:28:42.676144+0000","last_unstale":"2023-09-23T22:28:49.702260+0000","last_undegraded":"2023-09-23T22:28:49.702260+0000","last_fullsized":"2023-09-23T22:28:49.702260+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839797+0000","last_change":"2023-09-23T22:28:43.839447+0000","last_active":"2023-09-23T22:28:43.839797+0000","last_peered":"2023-09-23T22:28:43.839797+0000","last_clean":"2023-09-23T22:28:43.839797+0000","last_became_active":"2023-09-23T22:28:43.839139+0000","last_became_peered":"2023-09-23T22:28:43.839139+0000","last_unstale":"2023-09-23T22:28:43.839797+0000","last_undegraded":"2023-09-23T22:28:43.839797+0000","last_fullsized":"2023-09-23T22:28:43.839797+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694068+0000","last_change":"2023-09-23T22:28:42.673254+0000","last_active":"2023-09-23T22:28:44.694068+0000","last_peered":"2023-09-23T22:28:44.694068+0000","last_clean":"2023-09-23T22:28:44.694068+0000","last_became_active":"2023-09-23T22:28:42.672975+0000","last_became_peered":"2023-09-23T22:28:42.672975+0000","last_unstale":"2023-09-23T22:28:44.694068+0000","last_undegraded":"2023-09-23T22:28:44.694068+0000","last_fullsized":"2023-09-23T22:28:44.694068+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.679125+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:28:43.679125+0000","last_peered":"2023-09-23T22:28:43.679125+0000","last_clean":"2023-09-23T22:28:43.679125+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:28:43.679125+0000","last_undegraded":"2023-09-23T22:28:43.679125+0000","last_fullsized":"2023-09-23T22:28:43.679125+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694773+0000","last_change":"2023-09-23T22:28:42.674611+0000","last_active":"2023-09-23T22:28:44.694773+0000","last_peered":"2023-09-23T22:28:44.694773+0000","last_clean":"2023-09-23T22:28:44.694773+0000","last_became_active":"2023-09-23T22:28:42.674252+0000","last_became_peered":"2023-09-23T22:28:42.674252+0000","last_unstale":"2023-09-23T22:28:44.694773+0000","last_undegraded":"2023-09-23T22:28:44.694773+0000","last_fullsized":"2023-09-23T22:28:44.694773+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672465+0000","last_change":"2023-09-23T22:27:50.149464+0000","last_active":"2023-09-23T22:28:43.672465+0000","last_peered":"2023-09-23T22:28:43.672465+0000","last_clean":"2023-09-23T22:28:43.672465+0000","last_became_active":"2023-09-23T22:27:50.149120+0000","last_became_peered":"2023-09-23T22:27:50.149120+0000","last_unstale":"2023-09-23T22:28:43.672465+0000","last_undegraded":"2023-09-23T22:28:43.672465+0000","last_fullsized":"2023-09-23T22:28:43.672465+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838639+0000","last_change":"2023-09-23T22:28:42.862599+0000","last_active":"2023-09-23T22:28:43.838639+0000","last_peered":"2023-09-23T22:28:43.838639+0000","last_clean":"2023-09-23T22:28:43.838639+0000","last_became_active":"2023-09-23T22:28:42.862281+0000","last_became_peered":"2023-09-23T22:28:42.862281+0000","last_unstale":"2023-09-23T22:28:43.838639+0000","last_undegraded":"2023-09-23T22:28:43.838639+0000","last_fullsized":"2023-09-23T22:28:43.838639+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676523+0000","last_change":"2023-09-23T22:27:50.607023+0000","last_active":"2023-09-23T22:28:43.676523+0000","last_peered":"2023-09-23T22:28:43.676523+0000","last_clean":"2023-09-23T22:28:43.676523+0000","last_became_active":"2023-09-23T22:27:50.606684+0000","last_became_peered":"2023-09-23T22:27:50.606684+0000","last_unstale":"2023-09-23T22:28:43.676523+0000","last_undegraded":"2023-09-23T22:28:43.676523+0000","last_fullsized":"2023-09-23T22:28:43.676523+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704843+0000","last_change":"2023-09-23T22:28:42.673817+0000","last_active":"2023-09-23T22:28:49.704843+0000","last_peered":"2023-09-23T22:28:49.704843+0000","last_clean":"2023-09-23T22:28:49.704843+0000","last_became_active":"2023-09-23T22:28:42.673354+0000","last_became_peered":"2023-09-23T22:28:42.673354+0000","last_unstale":"2023-09-23T22:28:49.704843+0000","last_undegraded":"2023-09-23T22:28:49.704843+0000","last_fullsized":"2023-09-23T22:28:49.704843+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.842956+0000","last_change":"2023-09-23T22:28:43.842740+0000","last_active":"2023-09-23T22:28:43.842956+0000","last_peered":"2023-09-23T22:28:43.842956+0000","last_clean":"2023-09-23T22:28:43.842956+0000","last_became_active":"2023-09-23T22:28:43.842371+0000","last_became_peered":"2023-09-23T22:28:43.842371+0000","last_unstale":"2023-09-23T22:28:43.842956+0000","last_undegraded":"2023-09-23T22:28:43.842956+0000","last_fullsized":"2023-09-23T22:28:43.842956+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.678935+0000","last_change":"2023-09-23T22:28:42.680619+0000","last_active":"2023-09-23T22:28:43.678935+0000","last_peered":"2023-09-23T22:28:43.678935+0000","last_clean":"2023-09-23T22:28:43.678935+0000","last_became_active":"2023-09-23T22:28:42.680295+0000","last_became_peered":"2023-09-23T22:28:42.680295+0000","last_unstale":"2023-09-23T22:28:43.678935+0000","last_undegraded":"2023-09-23T22:28:43.678935+0000","last_fullsized":"2023-09-23T22:28:43.678935+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838433+0000","last_change":"2023-09-23T22:27:50.607540+0000","last_active":"2023-09-23T22:28:43.838433+0000","last_peered":"2023-09-23T22:28:43.838433+0000","last_clean":"2023-09-23T22:28:43.838433+0000","last_became_active":"2023-09-23T22:27:50.607232+0000","last_became_peered":"2023-09-23T22:27:50.607232+0000","last_unstale":"2023-09-23T22:28:43.838433+0000","last_undegraded":"2023-09-23T22:28:43.838433+0000","last_fullsized":"2023-09-23T22:28:43.838433+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.681380+0000","last_change":"2023-09-23T22:28:43.679731+0000","last_active":"2023-09-23T22:28:43.681380+0000","last_peered":"2023-09-23T22:28:43.681380+0000","last_clean":"2023-09-23T22:28:43.681380+0000","last_became_active":"2023-09-23T22:28:43.679434+0000","last_became_peered":"2023-09-23T22:28:43.679434+0000","last_unstale":"2023-09-23T22:28:43.681380+0000","last_undegraded":"2023-09-23T22:28:43.681380+0000","last_fullsized":"2023-09-23T22:28:43.681380+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672909+0000","last_change":"2023-09-23T22:28:42.686808+0000","last_active":"2023-09-23T22:28:43.672909+0000","last_peered":"2023-09-23T22:28:43.672909+0000","last_clean":"2023-09-23T22:28:43.672909+0000","last_became_active":"2023-09-23T22:28:42.686432+0000","last_became_peered":"2023-09-23T22:28:42.686432+0000","last_unstale":"2023-09-23T22:28:43.672909+0000","last_undegraded":"2023-09-23T22:28:43.672909+0000","last_fullsized":"2023-09-23T22:28:43.672909+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838267+0000","last_change":"2023-09-23T22:27:50.607516+0000","last_active":"2023-09-23T22:28:43.838267+0000","last_peered":"2023-09-23T22:28:43.838267+0000","last_clean":"2023-09-23T22:28:43.838267+0000","last_became_active":"2023-09-23T22:27:50.607193+0000","last_became_peered":"2023-09-23T22:27:50.607193+0000","last_unstale":"2023-09-23T22:28:43.838267+0000","last_undegraded":"2023-09-23T22:28:43.838267+0000","last_fullsized":"2023-09-23T22:28:43.838267+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841564+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:28:43.841564+0000","last_peered":"2023-09-23T22:28:43.841564+0000","last_clean":"2023-09-23T22:28:43.841564+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:28:43.841564+0000","last_undegraded":"2023-09-23T22:28:43.841564+0000","last_fullsized":"2023-09-23T22:28:43.841564+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838966+0000","last_change":"2023-09-23T22:28:42.670877+0000","last_active":"2023-09-23T22:28:43.838966+0000","last_peered":"2023-09-23T22:28:43.838966+0000","last_clean":"2023-09-23T22:28:43.838966+0000","last_became_active":"2023-09-23T22:28:42.670573+0000","last_became_peered":"2023-09-23T22:28:42.670573+0000","last_unstale":"2023-09-23T22:28:43.838966+0000","last_undegraded":"2023-09-23T22:28:43.838966+0000","last_fullsized":"2023-09-23T22:28:43.838966+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838267+0000","last_change":"2023-09-23T22:27:14.671001+0000","last_active":"2023-09-23T22:28:43.838267+0000","last_peered":"2023-09-23T22:28:43.838267+0000","last_clean":"2023-09-23T22:28:43.838267+0000","last_became_active":"2023-09-23T22:27:14.670462+0000","last_became_peered":"2023-09-23T22:27:14.670462+0000","last_unstale":"2023-09-23T22:28:43.838267+0000","last_undegraded":"2023-09-23T22:28:43.838267+0000","last_fullsized":"2023-09-23T22:28:43.838267+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675151+0000","last_change":"2023-09-23T22:27:50.149615+0000","last_active":"2023-09-23T22:28:43.675151+0000","last_peered":"2023-09-23T22:28:43.675151+0000","last_clean":"2023-09-23T22:28:43.675151+0000","last_became_active":"2023-09-23T22:27:50.149191+0000","last_became_peered":"2023-09-23T22:27:50.149191+0000","last_unstale":"2023-09-23T22:28:43.675151+0000","last_undegraded":"2023-09-23T22:28:43.675151+0000","last_fullsized":"2023-09-23T22:28:43.675151+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683866+0000","last_change":"2023-09-23T22:28:43.683468+0000","last_active":"2023-09-23T22:28:43.683866+0000","last_peered":"2023-09-23T22:28:43.683866+0000","last_clean":"2023-09-23T22:28:43.683866+0000","last_became_active":"2023-09-23T22:28:43.683171+0000","last_became_peered":"2023-09-23T22:28:43.683171+0000","last_unstale":"2023-09-23T22:28:43.683866+0000","last_undegraded":"2023-09-23T22:28:43.683866+0000","last_fullsized":"2023-09-23T22:28:43.683866+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676662+0000","last_change":"2023-09-23T22:28:42.682076+0000","last_active":"2023-09-23T22:28:43.676662+0000","last_peered":"2023-09-23T22:28:43.676662+0000","last_clean":"2023-09-23T22:28:43.676662+0000","last_became_active":"2023-09-23T22:28:42.681758+0000","last_became_peered":"2023-09-23T22:28:42.681758+0000","last_unstale":"2023-09-23T22:28:43.676662+0000","last_undegraded":"2023-09-23T22:28:43.676662+0000","last_fullsized":"2023-09-23T22:28:43.676662+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672554+0000","last_change":"2023-09-23T22:27:50.607355+0000","last_active":"2023-09-23T22:28:43.672554+0000","last_peered":"2023-09-23T22:28:43.672554+0000","last_clean":"2023-09-23T22:28:43.672554+0000","last_became_active":"2023-09-23T22:27:50.607075+0000","last_became_peered":"2023-09-23T22:27:50.607075+0000","last_unstale":"2023-09-23T22:28:43.672554+0000","last_undegraded":"2023-09-23T22:28:43.672554+0000","last_fullsized":"2023-09-23T22:28:43.672554+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.842716+0000","last_change":"2023-09-23T22:28:43.841370+0000","last_active":"2023-09-23T22:28:43.842716+0000","last_peered":"2023-09-23T22:28:43.842716+0000","last_clean":"2023-09-23T22:28:43.842716+0000","last_became_active":"2023-09-23T22:28:43.841110+0000","last_became_peered":"2023-09-23T22:28:43.841110+0000","last_unstale":"2023-09-23T22:28:43.842716+0000","last_undegraded":"2023-09-23T22:28:43.842716+0000","last_fullsized":"2023-09-23T22:28:43.842716+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.677002+0000","last_change":"2023-09-23T22:28:42.673013+0000","last_active":"2023-09-23T22:28:43.677002+0000","last_peered":"2023-09-23T22:28:43.677002+0000","last_clean":"2023-09-23T22:28:43.677002+0000","last_became_active":"2023-09-23T22:28:42.672710+0000","last_became_peered":"2023-09-23T22:28:42.672710+0000","last_unstale":"2023-09-23T22:28:43.677002+0000","last_undegraded":"2023-09-23T22:28:43.677002+0000","last_fullsized":"2023-09-23T22:28:43.677002+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675023+0000","last_change":"2023-09-23T22:27:50.150091+0000","last_active":"2023-09-23T22:28:43.675023+0000","last_peered":"2023-09-23T22:28:43.675023+0000","last_clean":"2023-09-23T22:28:43.675023+0000","last_became_active":"2023-09-23T22:27:50.149716+0000","last_became_peered":"2023-09-23T22:27:50.149716+0000","last_unstale":"2023-09-23T22:28:43.675023+0000","last_undegraded":"2023-09-23T22:28:43.675023+0000","last_fullsized":"2023-09-23T22:28:43.675023+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683879+0000","last_change":"2023-09-23T22:28:43.683463+0000","last_active":"2023-09-23T22:28:43.683879+0000","last_peered":"2023-09-23T22:28:43.683879+0000","last_clean":"2023-09-23T22:28:43.683879+0000","last_became_active":"2023-09-23T22:28:43.683208+0000","last_became_peered":"2023-09-23T22:28:43.683208+0000","last_unstale":"2023-09-23T22:28:43.683879+0000","last_undegraded":"2023-09-23T22:28:43.683879+0000","last_fullsized":"2023-09-23T22:28:43.683879+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675388+0000","last_change":"2023-09-23T22:28:42.672269+0000","last_active":"2023-09-23T22:28:43.675388+0000","last_peered":"2023-09-23T22:28:43.675388+0000","last_clean":"2023-09-23T22:28:43.675388+0000","last_became_active":"2023-09-23T22:28:42.671983+0000","last_became_peered":"2023-09-23T22:28:42.671983+0000","last_unstale":"2023-09-23T22:28:43.675388+0000","last_undegraded":"2023-09-23T22:28:43.675388+0000","last_fullsized":"2023-09-23T22:28:43.675388+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.678825+0000","last_change":"2023-09-23T22:27:50.607380+0000","last_active":"2023-09-23T22:28:43.678825+0000","last_peered":"2023-09-23T22:28:43.678825+0000","last_clean":"2023-09-23T22:28:43.678825+0000","last_became_active":"2023-09-23T22:27:50.607045+0000","last_became_peered":"2023-09-23T22:27:50.607045+0000","last_unstale":"2023-09-23T22:28:43.678825+0000","last_undegraded":"2023-09-23T22:28:43.678825+0000","last_fullsized":"2023-09-23T22:28:43.678825+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841391+0000","last_change":"2023-09-23T22:28:43.839790+0000","last_active":"2023-09-23T22:28:43.841391+0000","last_peered":"2023-09-23T22:28:43.841391+0000","last_clean":"2023-09-23T22:28:43.841391+0000","last_became_active":"2023-09-23T22:28:43.839389+0000","last_became_peered":"2023-09-23T22:28:43.839389+0000","last_unstale":"2023-09-23T22:28:43.841391+0000","last_undegraded":"2023-09-23T22:28:43.841391+0000","last_fullsized":"2023-09-23T22:28:43.841391+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838668+0000","last_change":"2023-09-23T22:28:42.676799+0000","last_active":"2023-09-23T22:28:43.838668+0000","last_peered":"2023-09-23T22:28:43.838668+0000","last_clean":"2023-09-23T22:28:43.838668+0000","last_became_active":"2023-09-23T22:28:42.676112+0000","last_became_peered":"2023-09-23T22:28:42.676112+0000","last_unstale":"2023-09-23T22:28:43.838668+0000","last_undegraded":"2023-09-23T22:28:43.838668+0000","last_fullsized":"2023-09-23T22:28:43.838668+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683882+0000","last_change":"2023-09-23T22:28:43.682852+0000","last_active":"2023-09-23T22:28:43.683882+0000","last_peered":"2023-09-23T22:28:43.683882+0000","last_clean":"2023-09-23T22:28:43.683882+0000","last_became_active":"2023-09-23T22:28:43.682497+0000","last_became_peered":"2023-09-23T22:28:43.682497+0000","last_unstale":"2023-09-23T22:28:43.683882+0000","last_undegraded":"2023-09-23T22:28:43.683882+0000","last_fullsized":"2023-09-23T22:28:43.683882+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676034+0000","last_change":"2023-09-23T22:28:42.663472+0000","last_active":"2023-09-23T22:28:43.676034+0000","last_peered":"2023-09-23T22:28:43.676034+0000","last_clean":"2023-09-23T22:28:43.676034+0000","last_became_active":"2023-09-23T22:28:42.663132+0000","last_became_peered":"2023-09-23T22:28:42.663132+0000","last_unstale":"2023-09-23T22:28:43.676034+0000","last_undegraded":"2023-09-23T22:28:43.676034+0000","last_fullsized":"2023-09-23T22:28:43.676034+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683875+0000","last_change":"2023-09-23T22:28:43.682934+0000","last_active":"2023-09-23T22:28:43.683875+0000","last_peered":"2023-09-23T22:28:43.683875+0000","last_clean":"2023-09-23T22:28:43.683875+0000","last_became_active":"2023-09-23T22:28:43.682593+0000","last_became_peered":"2023-09-23T22:28:43.682593+0000","last_unstale":"2023-09-23T22:28:43.683875+0000","last_undegraded":"2023-09-23T22:28:43.683875+0000","last_fullsized":"2023-09-23T22:28:43.683875+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672549+0000","last_change":"2023-09-23T22:28:42.678925+0000","last_active":"2023-09-23T22:28:43.672549+0000","last_peered":"2023-09-23T22:28:43.672549+0000","last_clean":"2023-09-23T22:28:43.672549+0000","last_became_active":"2023-09-23T22:28:42.678555+0000","last_became_peered":"2023-09-23T22:28:42.678555+0000","last_unstale":"2023-09-23T22:28:43.672549+0000","last_undegraded":"2023-09-23T22:28:43.672549+0000","last_fullsized":"2023-09-23T22:28:43.672549+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840520+0000","last_change":"2023-09-23T22:28:43.839580+0000","last_active":"2023-09-23T22:28:43.840520+0000","last_peered":"2023-09-23T22:28:43.840520+0000","last_clean":"2023-09-23T22:28:43.840520+0000","last_became_active":"2023-09-23T22:28:43.839285+0000","last_became_peered":"2023-09-23T22:28:43.839285+0000","last_unstale":"2023-09-23T22:28:43.840520+0000","last_undegraded":"2023-09-23T22:28:43.840520+0000","last_fullsized":"2023-09-23T22:28:43.840520+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704379+0000","last_change":"2023-09-23T22:28:42.668895+0000","last_active":"2023-09-23T22:28:49.704379+0000","last_peered":"2023-09-23T22:28:49.704379+0000","last_clean":"2023-09-23T22:28:49.704379+0000","last_became_active":"2023-09-23T22:28:42.668572+0000","last_became_peered":"2023-09-23T22:28:42.668572+0000","last_unstale":"2023-09-23T22:28:49.704379+0000","last_undegraded":"2023-09-23T22:28:49.704379+0000","last_fullsized":"2023-09-23T22:28:49.704379+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.679881+0000","last_change":"2023-09-23T22:28:43.678995+0000","last_active":"2023-09-23T22:28:43.679881+0000","last_peered":"2023-09-23T22:28:43.679881+0000","last_clean":"2023-09-23T22:28:43.679881+0000","last_became_active":"2023-09-23T22:28:43.678677+0000","last_became_peered":"2023-09-23T22:28:43.678677+0000","last_unstale":"2023-09-23T22:28:43.679881+0000","last_undegraded":"2023-09-23T22:28:43.679881+0000","last_fullsized":"2023-09-23T22:28:43.679881+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675149+0000","last_change":"2023-09-23T22:28:42.669958+0000","last_active":"2023-09-23T22:28:43.675149+0000","last_peered":"2023-09-23T22:28:43.675149+0000","last_clean":"2023-09-23T22:28:43.675149+0000","last_became_active":"2023-09-23T22:28:42.669668+0000","last_became_peered":"2023-09-23T22:28:42.669668+0000","last_unstale":"2023-09-23T22:28:43.675149+0000","last_undegraded":"2023-09-23T22:28:43.675149+0000","last_fullsized":"2023-09-23T22:28:43.675149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841570+0000","last_change":"2023-09-23T22:28:43.841201+0000","last_active":"2023-09-23T22:28:43.841570+0000","last_peered":"2023-09-23T22:28:43.841570+0000","last_clean":"2023-09-23T22:28:43.841570+0000","last_became_active":"2023-09-23T22:28:43.840936+0000","last_became_peered":"2023-09-23T22:28:43.840936+0000","last_unstale":"2023-09-23T22:28:43.841570+0000","last_undegraded":"2023-09-23T22:28:43.841570+0000","last_fullsized":"2023-09-23T22:28:43.841570+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.695064+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:28:44.695064+0000","last_peered":"2023-09-23T22:28:44.695064+0000","last_clean":"2023-09-23T22:28:44.695064+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:28:44.695064+0000","last_undegraded":"2023-09-23T22:28:44.695064+0000","last_fullsized":"2023-09-23T22:28:44.695064+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840544+0000","last_change":"2023-09-23T22:28:43.839560+0000","last_active":"2023-09-23T22:28:43.840544+0000","last_peered":"2023-09-23T22:28:43.840544+0000","last_clean":"2023-09-23T22:28:43.840544+0000","last_became_active":"2023-09-23T22:28:43.839255+0000","last_became_peered":"2023-09-23T22:28:43.839255+0000","last_unstale":"2023-09-23T22:28:43.840544+0000","last_undegraded":"2023-09-23T22:28:43.840544+0000","last_fullsized":"2023-09-23T22:28:43.840544+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704560+0000","last_change":"2023-09-23T22:28:42.673358+0000","last_active":"2023-09-23T22:28:49.704560+0000","last_peered":"2023-09-23T22:28:49.704560+0000","last_clean":"2023-09-23T22:28:49.704560+0000","last_became_active":"2023-09-23T22:28:42.673029+0000","last_became_peered":"2023-09-23T22:28:42.673029+0000","last_unstale":"2023-09-23T22:28:49.704560+0000","last_undegraded":"2023-09-23T22:28:49.704560+0000","last_fullsized":"2023-09-23T22:28:49.704560+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840230+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:28:43.840230+0000","last_peered":"2023-09-23T22:28:43.840230+0000","last_clean":"2023-09-23T22:28:43.840230+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:28:43.840230+0000","last_undegraded":"2023-09-23T22:28:43.840230+0000","last_fullsized":"2023-09-23T22:28:43.840230+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694435+0000","last_change":"2023-09-23T22:28:42.670634+0000","last_active":"2023-09-23T22:28:44.694435+0000","last_peered":"2023-09-23T22:28:44.694435+0000","last_clean":"2023-09-23T22:28:44.694435+0000","last_became_active":"2023-09-23T22:28:42.670356+0000","last_became_peered":"2023-09-23T22:28:42.670356+0000","last_unstale":"2023-09-23T22:28:44.694435+0000","last_undegraded":"2023-09-23T22:28:44.694435+0000","last_fullsized":"2023-09-23T22:28:44.694435+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839438+0000","last_change":"2023-09-23T22:28:42.863270+0000","last_active":"2023-09-23T22:28:43.839438+0000","last_peered":"2023-09-23T22:28:43.839438+0000","last_clean":"2023-09-23T22:28:43.839438+0000","last_became_active":"2023-09-23T22:28:42.862939+0000","last_became_peered":"2023-09-23T22:28:42.862939+0000","last_unstale":"2023-09-23T22:28:43.839438+0000","last_undegraded":"2023-09-23T22:28:43.839438+0000","last_fullsized":"2023-09-23T22:28:43.839438+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.705969+0000","last_change":"2023-09-23T22:28:42.666505+0000","last_active":"2023-09-23T22:28:49.705969+0000","last_peered":"2023-09-23T22:28:49.705969+0000","last_clean":"2023-09-23T22:28:49.705969+0000","last_became_active":"2023-09-23T22:28:42.666176+0000","last_became_peered":"2023-09-23T22:28:42.666176+0000","last_unstale":"2023-09-23T22:28:49.705969+0000","last_undegraded":"2023-09-23T22:28:49.705969+0000","last_fullsized":"2023-09-23T22:28:49.705969+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841963+0000","last_change":"2023-09-23T22:28:43.841481+0000","last_active":"2023-09-23T22:28:43.841963+0000","last_peered":"2023-09-23T22:28:43.841963+0000","last_clean":"2023-09-23T22:28:43.841963+0000","last_became_active":"2023-09-23T22:28:43.841146+0000","last_became_peered":"2023-09-23T22:28:43.841146+0000","last_unstale":"2023-09-23T22:28:43.841963+0000","last_undegraded":"2023-09-23T22:28:43.841963+0000","last_fullsized":"2023-09-23T22:28:43.841963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838424+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:28:43.838424+0000","last_peered":"2023-09-23T22:28:43.838424+0000","last_clean":"2023-09-23T22:28:43.838424+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:28:43.838424+0000","last_undegraded":"2023-09-23T22:28:43.838424+0000","last_fullsized":"2023-09-23T22:28:43.838424+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840313+0000","last_change":"2023-09-23T22:28:43.839847+0000","last_active":"2023-09-23T22:28:43.840313+0000","last_peered":"2023-09-23T22:28:43.840313+0000","last_clean":"2023-09-23T22:28:43.840313+0000","last_became_active":"2023-09-23T22:28:43.839519+0000","last_became_peered":"2023-09-23T22:28:43.839519+0000","last_unstale":"2023-09-23T22:28:43.840313+0000","last_undegraded":"2023-09-23T22:28:43.840313+0000","last_fullsized":"2023-09-23T22:28:43.840313+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672322+0000","last_change":"2023-09-23T22:28:42.666721+0000","last_active":"2023-09-23T22:28:43.672322+0000","last_peered":"2023-09-23T22:28:43.672322+0000","last_clean":"2023-09-23T22:28:43.672322+0000","last_became_active":"2023-09-23T22:28:42.666400+0000","last_became_peered":"2023-09-23T22:28:42.666400+0000","last_unstale":"2023-09-23T22:28:43.672322+0000","last_undegraded":"2023-09-23T22:28:43.672322+0000","last_fullsized":"2023-09-23T22:28:43.672322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839004+0000","last_change":"2023-09-23T22:28:42.863332+0000","last_active":"2023-09-23T22:28:43.839004+0000","last_peered":"2023-09-23T22:28:43.839004+0000","last_clean":"2023-09-23T22:28:43.839004+0000","last_became_active":"2023-09-23T22:28:42.863049+0000","last_became_peered":"2023-09-23T22:28:42.863049+0000","last_unstale":"2023-09-23T22:28:43.839004+0000","last_undegraded":"2023-09-23T22:28:43.839004+0000","last_fullsized":"2023-09-23T22:28:43.839004+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704523+0000","last_change":"2023-09-23T22:28:42.680816+0000","last_active":"2023-09-23T22:28:49.704523+0000","last_peered":"2023-09-23T22:28:49.704523+0000","last_clean":"2023-09-23T22:28:49.704523+0000","last_became_active":"2023-09-23T22:28:42.680489+0000","last_became_peered":"2023-09-23T22:28:42.680489+0000","last_unstale":"2023-09-23T22:28:49.704523+0000","last_undegraded":"2023-09-23T22:28:49.704523+0000","last_fullsized":"2023-09-23T22:28:49.704523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683983+0000","last_change":"2023-09-23T22:28:43.683746+0000","last_active":"2023-09-23T22:28:43.683983+0000","last_peered":"2023-09-23T22:28:43.683983+0000","last_clean":"2023-09-23T22:28:43.683983+0000","last_became_active":"2023-09-23T22:28:43.683291+0000","last_became_peered":"2023-09-23T22:28:43.683291+0000","last_unstale":"2023-09-23T22:28:43.683983+0000","last_undegraded":"2023-09-23T22:28:43.683983+0000","last_fullsized":"2023-09-23T22:28:43.683983+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.705770+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:28:49.705770+0000","last_peered":"2023-09-23T22:28:49.705770+0000","last_clean":"2023-09-23T22:28:49.705770+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:28:49.705770+0000","last_undegraded":"2023-09-23T22:28:49.705770+0000","last_fullsized":"2023-09-23T22:28:49.705770+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683840+0000","last_change":"2023-09-23T22:28:43.682855+0000","last_active":"2023-09-23T22:28:43.683840+0000","last_peered":"2023-09-23T22:28:43.683840+0000","last_clean":"2023-09-23T22:28:43.683840+0000","last_became_active":"2023-09-23T22:28:43.682568+0000","last_became_peered":"2023-09-23T22:28:43.682568+0000","last_unstale":"2023-09-23T22:28:43.683840+0000","last_undegraded":"2023-09-23T22:28:43.683840+0000","last_fullsized":"2023-09-23T22:28:43.683840+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704604+0000","last_change":"2023-09-23T22:28:42.679399+0000","last_active":"2023-09-23T22:28:49.704604+0000","last_peered":"2023-09-23T22:28:49.704604+0000","last_clean":"2023-09-23T22:28:49.704604+0000","last_became_active":"2023-09-23T22:28:42.679100+0000","last_became_peered":"2023-09-23T22:28:42.679100+0000","last_unstale":"2023-09-23T22:28:49.704604+0000","last_undegraded":"2023-09-23T22:28:49.704604+0000","last_fullsized":"2023-09-23T22:28:49.704604+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839868+0000","last_change":"2023-09-23T22:28:42.863696+0000","last_active":"2023-09-23T22:28:43.839868+0000","last_peered":"2023-09-23T22:28:43.839868+0000","last_clean":"2023-09-23T22:28:43.839868+0000","last_became_active":"2023-09-23T22:28:42.863397+0000","last_became_peered":"2023-09-23T22:28:42.863397+0000","last_unstale":"2023-09-23T22:28:43.839868+0000","last_undegraded":"2023-09-23T22:28:43.839868+0000","last_fullsized":"2023-09-23T22:28:43.839868+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675609+0000","last_change":"2023-09-23T22:28:42.675839+0000","last_active":"2023-09-23T22:28:43.675609+0000","last_peered":"2023-09-23T22:28:43.675609+0000","last_clean":"2023-09-23T22:28:43.675609+0000","last_became_active":"2023-09-23T22:28:42.675494+0000","last_became_peered":"2023-09-23T22:28:42.675494+0000","last_unstale":"2023-09-23T22:28:43.675609+0000","last_undegraded":"2023-09-23T22:28:43.675609+0000","last_fullsized":"2023-09-23T22:28:43.675609+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838793+0000","last_change":"2023-09-23T22:28:42.863635+0000","last_active":"2023-09-23T22:28:43.838793+0000","last_peered":"2023-09-23T22:28:43.838793+0000","last_clean":"2023-09-23T22:28:43.838793+0000","last_became_active":"2023-09-23T22:28:42.863315+0000","last_became_peered":"2023-09-23T22:28:42.863315+0000","last_unstale":"2023-09-23T22:28:43.838793+0000","last_undegraded":"2023-09-23T22:28:43.838793+0000","last_fullsized":"2023-09-23T22:28:43.838793+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.673281+0000","last_change":"2023-09-23T22:28:42.670242+0000","last_active":"2023-09-23T22:28:43.673281+0000","last_peered":"2023-09-23T22:28:43.673281+0000","last_clean":"2023-09-23T22:28:43.673281+0000","last_became_active":"2023-09-23T22:28:42.668777+0000","last_became_peered":"2023-09-23T22:28:42.668777+0000","last_unstale":"2023-09-23T22:28:43.673281+0000","last_undegraded":"2023-09-23T22:28:43.673281+0000","last_fullsized":"2023-09-23T22:28:43.673281+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.680732+0000","last_change":"2023-09-23T22:28:43.680206+0000","last_active":"2023-09-23T22:28:43.680732+0000","last_peered":"2023-09-23T22:28:43.680732+0000","last_clean":"2023-09-23T22:28:43.680732+0000","last_became_active":"2023-09-23T22:28:43.679901+0000","last_became_peered":"2023-09-23T22:28:43.679901+0000","last_unstale":"2023-09-23T22:28:43.680732+0000","last_undegraded":"2023-09-23T22:28:43.680732+0000","last_fullsized":"2023-09-23T22:28:43.680732+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838929+0000","last_change":"2023-09-23T22:28:42.676985+0000","last_active":"2023-09-23T22:28:43.838929+0000","last_peered":"2023-09-23T22:28:43.838929+0000","last_clean":"2023-09-23T22:28:43.838929+0000","last_became_active":"2023-09-23T22:28:42.676516+0000","last_became_peered":"2023-09-23T22:28:42.676516+0000","last_unstale":"2023-09-23T22:28:43.838929+0000","last_undegraded":"2023-09-23T22:28:43.838929+0000","last_fullsized":"2023-09-23T22:28:43.838929+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.842591+0000","last_change":"2023-09-23T22:28:43.842223+0000","last_active":"2023-09-23T22:28:43.842591+0000","last_peered":"2023-09-23T22:28:43.842591+0000","last_clean":"2023-09-23T22:28:43.842591+0000","last_became_active":"2023-09-23T22:28:43.841902+0000","last_became_peered":"2023-09-23T22:28:43.841902+0000","last_unstale":"2023-09-23T22:28:43.842591+0000","last_undegraded":"2023-09-23T22:28:43.842591+0000","last_fullsized":"2023-09-23T22:28:43.842591+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676444+0000","last_change":"2023-09-23T22:28:42.681628+0000","last_active":"2023-09-23T22:28:43.676444+0000","last_peered":"2023-09-23T22:28:43.676444+0000","last_clean":"2023-09-23T22:28:43.676444+0000","last_became_active":"2023-09-23T22:28:42.681301+0000","last_became_peered":"2023-09-23T22:28:42.681301+0000","last_unstale":"2023-09-23T22:28:43.676444+0000","last_undegraded":"2023-09-23T22:28:43.676444+0000","last_fullsized":"2023-09-23T22:28:43.676444+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839285+0000","last_change":"2023-09-23T22:28:42.687696+0000","last_active":"2023-09-23T22:28:43.839285+0000","last_peered":"2023-09-23T22:28:43.839285+0000","last_clean":"2023-09-23T22:28:43.839285+0000","last_became_active":"2023-09-23T22:28:42.687402+0000","last_became_peered":"2023-09-23T22:28:42.687402+0000","last_unstale":"2023-09-23T22:28:43.839285+0000","last_undegraded":"2023-09-23T22:28:43.839285+0000","last_fullsized":"2023-09-23T22:28:43.839285+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.673418+0000","last_change":"2023-09-23T22:28:42.672628+0000","last_active":"2023-09-23T22:28:44.673418+0000","last_peered":"2023-09-23T22:28:44.673418+0000","last_clean":"2023-09-23T22:28:44.673418+0000","last_became_active":"2023-09-23T22:28:42.672274+0000","last_became_peered":"2023-09-23T22:28:42.672274+0000","last_unstale":"2023-09-23T22:28:44.673418+0000","last_undegraded":"2023-09-23T22:28:44.673418+0000","last_fullsized":"2023-09-23T22:28:44.673418+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430237,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68500000000000005,"5min":1.032,"15min":1.032},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.503,"15min":0.498},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.749,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.41099999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.75900000000000001,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72199999999999998,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.52100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.52900000000000003,"15min":0.52100000000000002},"min":{"1min":0.36199999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.8440000000000001,"5min":1.8440000000000001,"15min":1.8440000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82799999999999996,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.69699999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56799999999999995,"15min":0.57199999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71099999999999997,"5min":1.087,"15min":1.087},"last":0.624}]},{"osd":4,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47099999999999997,"15min":0.46800000000000003},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.69499999999999995,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.496,"15min":0.49299999999999999},"min":{"1min":0.36799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.68500000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74299999999999999,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.45500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.751,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.73199999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.40300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.69799999999999995,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.56799999999999995,"15min":0.56200000000000006},"min":{"1min":0.42599999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.755}]}]},{"osd":6,"up_from":42,"seq":180388626465,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":257745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65400000000000003,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.62},{"interface":"front","average":{"1min":0.443,"5min":0.51900000000000002,"15min":0.53200000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.72499999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.56599999999999995,"15min":0.58399999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.58799999999999997,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.499},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.54400000000000004,"15min":0.55400000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68100000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.66400000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.66500000000000004,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.69599999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72899999999999998,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.56899999999999995,"15min":0.58099999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66500000000000004,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.42699999999999999,"15min":0.42499999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.629,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.42199999999999999,"15min":0.41899999999999998},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.59099999999999997,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63600000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.47099999999999997,"15min":0.46200000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68999999999999995,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:29:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.50800000000000001,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.74299999999999999,"5min":1.357,"15min":1.357},"last":0.52100000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574901,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.378,"15min":0.35799999999999998},"min":{"1min":0.252,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.61299999999999999,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.44},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.374,"15min":0.35299999999999998},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":0.56899999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.28299999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.40200000000000002,"15min":0.39600000000000002},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.48799999999999999,"15min":0.5},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.46700000000000003,"15min":0.48599999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69299999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.47699999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.61399999999999999,"15min":0.626},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81699999999999995,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63600000000000001,"15min":0.63900000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82499999999999996,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.60599999999999998,"15min":0.60499999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.63800000000000001,"15min":0.64000000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.94199999999999995,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.77300000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.70299999999999996,"15min":0.72299999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85499999999999998,"5min":2.524,"15min":2.524},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63200000000000001,"15min":0.64000000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.75900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67300000000000004,"15min":0.68500000000000005},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85499999999999998,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.70699999999999996,"15min":0.71799999999999997},"min":{"1min":0.48899999999999999,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.86799999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.76800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738426,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.376,"15min":0.35199999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.311},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.38100000000000001,"15min":0.35099999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.28100000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.40200000000000002,"15min":0.40000000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.41499999999999998,"15min":0.40799999999999997},"min":{"1min":0.308,"5min":0.255,"15min":0.255},"max":{"1min":0.623,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.437}]},{"osd":3,"last update":"Sat Sep 23 22:28:53 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.46100000000000002,"15min":0.46700000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.54700000000000004,"5min":0.754,"15min":0.754},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.443,"5min":0.499,"15min":0.50800000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.58199999999999996,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.627},"min":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.625,"15min":0.62},"min":{"1min":0.41299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93000000000000005,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.82799999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.63700000000000001,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 23 22:28:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.62,"15min":0.623},"min":{"1min":0.44800000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.65600000000000003,"15min":0.66000000000000003},"min":{"1min":0.46600000000000003,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.623}]},{"osd":7,"last update":"Sat Sep 23 22:29:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.877,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.498},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.64900000000000002,"15min":0.64800000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.46300000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36599999999999999,"15min":0.35099999999999998},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53900000000000003,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.374,"15min":0.34899999999999998},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.54400000000000004,"5min":0.629,"15min":0.629},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27000000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48299999999999998,"5min":0.625,"15min":0.625},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.443,"15min":0.45700000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57199999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.443,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59799999999999998,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:29:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.627,"15min":0.64100000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84399999999999997,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.66600000000000004,"15min":0.68400000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82299999999999995,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 22:28:33 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63600000000000001,"15min":0.63700000000000001},"min":{"1min":0.48799999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.84499999999999997,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78400000000000003,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.627,"15min":0.63400000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.754,"5min":1.044,"15min":1.044},"last":0.497},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68000000000000005,"15min":0.68799999999999994},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.83799999999999997,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:29:15 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.629,"15min":0.629},"min":{"1min":0.46999999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.81399999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.625,"5min":0.66100000000000003,"15min":0.66700000000000004},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76900000000000002,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.55900000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182444,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.42099999999999999,"15min":0.438},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.434,"15min":0.46400000000000002},"min":{"1min":0.26000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.35499999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.441,"15min":0.45000000000000001},"min":{"1min":0.28000000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.308},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.48299999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.27300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.499,"15min":0.52700000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.49199999999999999,"15min":0.51800000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70199999999999996,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.38600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:29:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.61399999999999999,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.71499999999999997,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61899999999999999,"15min":0.63900000000000001},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.80200000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.56200000000000006}]},{"osd":5,"last update":"Sat Sep 23 22:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58899999999999997,"15min":0.58999999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.60399999999999998,"15min":0.59699999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.65000000000000002,"15min":0.65900000000000003},"min":{"1min":0.46100000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.92500000000000004,"5min":1.046,"15min":1.046},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.623,"15min":0.63500000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.76700000000000002,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:29:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.628,"15min":0.623},"min":{"1min":0.501,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.5149999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.624},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.46500000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83199999999999996,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":257821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.53500000000000003,"15min":0.55500000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81100000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.58599999999999997,"15min":0.61199999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.82599999999999996,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.626}]},{"osd":1,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.55800000000000005,"15min":0.58699999999999997},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.78800000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.59599999999999997,"15min":0.624},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79600000000000004,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.58299999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.57299999999999995,"15min":0.60099999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.499,"5min":0.57499999999999996,"15min":0.59899999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.82799999999999996,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59699999999999998,"15min":0.61699999999999999},"min":{"1min":0.41199999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.59199999999999997,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.83399999999999996,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:28:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.44500000000000001,"15min":0.44400000000000001},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.66900000000000004,"5min":0.755,"15min":0.755},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.44500000000000001,"15min":0.437},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.44},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.46400000000000002,"15min":0.45400000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70699999999999996,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:29:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.52500000000000002,"15min":0.51900000000000002},"min":{"1min":0.42699999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67800000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.50700000000000001,"15min":0.498},"min":{"1min":0.42399999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.496}]}]},{"osd":5,"up_from":36,"seq":154618822693,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.52400000000000002,"15min":0.52900000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64300000000000002,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.52600000000000002,"15min":0.52300000000000002},"min":{"1min":0.38200000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.58299999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66200000000000003,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.58799999999999997,"15min":0.60199999999999998},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67100000000000004,"5min":1.823,"15min":1.823},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69199999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.47699999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.747,"5min":1.278,"15min":1.278},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.76100000000000001,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.66800000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.32100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.432},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.626}]},{"osd":6,"last update":"Sat Sep 23 22:28:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.45300000000000001,"15min":0.441},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.47499999999999998,"15min":0.46000000000000002},"min":{"1min":0.36299999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.76300000000000001,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:29:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.52600000000000002,"15min":0.51500000000000001},"min":{"1min":0.36099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80000000000000004,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.52600000000000002,"15min":0.51400000000000001},"min":{"1min":0.433,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.753,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.61899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:29:24.550 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:29:24.916 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:24.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:24 smithi151 conmon[100454]: cluster 2023-09-23T22:29:23.429503+0000 mgr.y (mgr.24314) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:24 smithi151 conmon[100454]: audit 2023-09-23T22:29:23.961804+0000 mgr.y (mgr.24314) 2023-09-23T22:29:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:24 smithi151 conmon[100454]: 91 : audit [DBG] from='client.24644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:29:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:24 smithi052 conmon[110082]: cluster 2023-09-23T22:29:23.429503+0000 mgr.y (mgr.24314) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:24 smithi052 conmon[110082]: audit 2023-09-23T22:29:23.961804+0000 mgr.y (mgr.24314) 91 : audit [DBG] from='client.24644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:29:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:24 smithi052 conmon[100224]: cluster 2023-09-23T22:29:23. 2023-09-23T22:29:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:24 smithi052 conmon[100224]: 429503+0000 mgr.y (mgr.24314) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:24 smithi052 conmon[100224]: audit 2023-09-23T22:29:23.961804+0000 mgr.y (mgr.24314) 91 : audit [DBG] from='client.24644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:29:25.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:29:25 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:29:26.165 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:26 smithi052 conmon[110082]: cluster 2023-09-23T22:29:25.429990+0000 mgr.y (mgr.24314) 92 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:26.165 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:26 smithi052 conmon[100224]: cluster 2023-09-23T22:29:25.429990+0000 mgr.y (mgr.24314) 92 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:26.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:26 smithi151 conmon[100454]: cluster 2023-09-23T22:29:25.429990+0000 mgr.y (mgr.24314) 92 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:26.543 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:29:26.545 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:29:27.128 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":70,"stamp":"2023-09-23T22:29:25.429677+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381560,"kb_used_data":4792,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579656,"statfs":{"total":767960285184,"available":765521567744,"internally_reserved":0,"allocated":4907008,"data_stored":2055400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.680915+0000","last_change":"2023-09-23T22:28:43.680655+0000","last_active":"2023-09-23T22:28:43.680915+0000","last_peered":"2023-09-23T22:28:43.680915+0000","last_clean":"2023-09-23T22:28:43.680915+0000","last_became_active":"2023-09-23T22:28:43.680336+0000","last_became_peered":"2023-09-23T22:28:43.680336+0000","last_unstale":"2023-09-23T22:28:43.680915+0000","last_undegraded":"2023-09-23T22:28:43.680915+0000","last_fullsized":"2023-09-23T22:28:43.680915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.703751+0000","last_change":"2023-09-23T22:28:42.668130+0000","last_active":"2023-09-23T22:28:49.703751+0000","last_peered":"2023-09-23T22:28:49.703751+0000","last_clean":"2023-09-23T22:28:49.703751+0000","last_became_active":"2023-09-23T22:28:42.667727+0000","last_became_peered":"2023-09-23T22:28:42.667727+0000","last_unstale":"2023-09-23T22:28:49.703751+0000","last_undegraded":"2023-09-23T22:28:49.703751+0000","last_fullsized":"2023-09-23T22:28:49.703751+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839796+0000","last_change":"2023-09-23T22:28:42.687711+0000","last_active":"2023-09-23T22:28:43.839796+0000","last_peered":"2023-09-23T22:28:43.839796+0000","last_clean":"2023-09-23T22:28:43.839796+0000","last_became_active":"2023-09-23T22:28:42.687305+0000","last_became_peered":"2023-09-23T22:28:42.687305+0000","last_unstale":"2023-09-23T22:28:43.839796+0000","last_undegraded":"2023-09-23T22:28:43.839796+0000","last_fullsized":"2023-09-23T22:28:43.839796+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694067+0000","last_change":"2023-09-23T22:28:42.682707+0000","last_active":"2023-09-23T22:28:44.694067+0000","last_peered":"2023-09-23T22:28:44.694067+0000","last_clean":"2023-09-23T22:28:44.694067+0000","last_became_active":"2023-09-23T22:28:42.682433+0000","last_became_peered":"2023-09-23T22:28:42.682433+0000","last_unstale":"2023-09-23T22:28:44.694067+0000","last_undegraded":"2023-09-23T22:28:44.694067+0000","last_fullsized":"2023-09-23T22:28:44.694067+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840021+0000","last_change":"2023-09-23T22:28:43.839102+0000","last_active":"2023-09-23T22:28:43.840021+0000","last_peered":"2023-09-23T22:28:43.840021+0000","last_clean":"2023-09-23T22:28:43.840021+0000","last_became_active":"2023-09-23T22:28:43.838794+0000","last_became_peered":"2023-09-23T22:28:43.838794+0000","last_unstale":"2023-09-23T22:28:43.840021+0000","last_undegraded":"2023-09-23T22:28:43.840021+0000","last_fullsized":"2023-09-23T22:28:43.840021+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.703483+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:28:49.703483+0000","last_peered":"2023-09-23T22:28:49.703483+0000","last_clean":"2023-09-23T22:28:49.703483+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:28:49.703483+0000","last_undegraded":"2023-09-23T22:28:49.703483+0000","last_fullsized":"2023-09-23T22:28:49.703483+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840660+0000","last_change":"2023-09-23T22:28:43.839960+0000","last_active":"2023-09-23T22:28:43.840660+0000","last_peered":"2023-09-23T22:28:43.840660+0000","last_clean":"2023-09-23T22:28:43.840660+0000","last_became_active":"2023-09-23T22:28:43.839640+0000","last_became_peered":"2023-09-23T22:28:43.839640+0000","last_unstale":"2023-09-23T22:28:43.840660+0000","last_undegraded":"2023-09-23T22:28:43.840660+0000","last_fullsized":"2023-09-23T22:28:43.840660+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.702260+0000","last_change":"2023-09-23T22:28:42.676441+0000","last_active":"2023-09-23T22:28:49.702260+0000","last_peered":"2023-09-23T22:28:49.702260+0000","last_clean":"2023-09-23T22:28:49.702260+0000","last_became_active":"2023-09-23T22:28:42.676144+0000","last_became_peered":"2023-09-23T22:28:42.676144+0000","last_unstale":"2023-09-23T22:28:49.702260+0000","last_undegraded":"2023-09-23T22:28:49.702260+0000","last_fullsized":"2023-09-23T22:28:49.702260+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839797+0000","last_change":"2023-09-23T22:28:43.839447+0000","last_active":"2023-09-23T22:28:43.839797+0000","last_peered":"2023-09-23T22:28:43.839797+0000","last_clean":"2023-09-23T22:28:43.839797+0000","last_became_active":"2023-09-23T22:28:43.839139+0000","last_became_peered":"2023-09-23T22:28:43.839139+0000","last_unstale":"2023-09-23T22:28:43.839797+0000","last_undegraded":"2023-09-23T22:28:43.839797+0000","last_fullsized":"2023-09-23T22:28:43.839797+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694068+0000","last_change":"2023-09-23T22:28:42.673254+0000","last_active":"2023-09-23T22:28:44.694068+0000","last_peered":"2023-09-23T22:28:44.694068+0000","last_clean":"2023-09-23T22:28:44.694068+0000","last_became_active":"2023-09-23T22:28:42.672975+0000","last_became_peered":"2023-09-23T22:28:42.672975+0000","last_unstale":"2023-09-23T22:28:44.694068+0000","last_undegraded":"2023-09-23T22:28:44.694068+0000","last_fullsized":"2023-09-23T22:28:44.694068+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.679125+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:28:43.679125+0000","last_peered":"2023-09-23T22:28:43.679125+0000","last_clean":"2023-09-23T22:28:43.679125+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:28:43.679125+0000","last_undegraded":"2023-09-23T22:28:43.679125+0000","last_fullsized":"2023-09-23T22:28:43.679125+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694773+0000","last_change":"2023-09-23T22:28:42.674611+0000","last_active":"2023-09-23T22:28:44.694773+0000","last_peered":"2023-09-23T22:28:44.694773+0000","last_clean":"2023-09-23T22:28:44.694773+0000","last_became_active":"2023-09-23T22:28:42.674252+0000","last_became_peered":"2023-09-23T22:28:42.674252+0000","last_unstale":"2023-09-23T22:28:44.694773+0000","last_undegraded":"2023-09-23T22:28:44.694773+0000","last_fullsized":"2023-09-23T22:28:44.694773+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672465+0000","last_change":"2023-09-23T22:27:50.149464+0000","last_active":"2023-09-23T22:28:43.672465+0000","last_peered":"2023-09-23T22:28:43.672465+0000","last_clean":"2023-09-23T22:28:43.672465+0000","last_became_active":"2023-09-23T22:27:50.149120+0000","last_became_peered":"2023-09-23T22:27:50.149120+0000","last_unstale":"2023-09-23T22:28:43.672465+0000","last_undegraded":"2023-09-23T22:28:43.672465+0000","last_fullsized":"2023-09-23T22:28:43.672465+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838639+0000","last_change":"2023-09-23T22:28:42.862599+0000","last_active":"2023-09-23T22:28:43.838639+0000","last_peered":"2023-09-23T22:28:43.838639+0000","last_clean":"2023-09-23T22:28:43.838639+0000","last_became_active":"2023-09-23T22:28:42.862281+0000","last_became_peered":"2023-09-23T22:28:42.862281+0000","last_unstale":"2023-09-23T22:28:43.838639+0000","last_undegraded":"2023-09-23T22:28:43.838639+0000","last_fullsized":"2023-09-23T22:28:43.838639+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676523+0000","last_change":"2023-09-23T22:27:50.607023+0000","last_active":"2023-09-23T22:28:43.676523+0000","last_peered":"2023-09-23T22:28:43.676523+0000","last_clean":"2023-09-23T22:28:43.676523+0000","last_became_active":"2023-09-23T22:27:50.606684+0000","last_became_peered":"2023-09-23T22:27:50.606684+0000","last_unstale":"2023-09-23T22:28:43.676523+0000","last_undegraded":"2023-09-23T22:28:43.676523+0000","last_fullsized":"2023-09-23T22:28:43.676523+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704843+0000","last_change":"2023-09-23T22:28:42.673817+0000","last_active":"2023-09-23T22:28:49.704843+0000","last_peered":"2023-09-23T22:28:49.704843+0000","last_clean":"2023-09-23T22:28:49.704843+0000","last_became_active":"2023-09-23T22:28:42.673354+0000","last_became_peered":"2023-09-23T22:28:42.673354+0000","last_unstale":"2023-09-23T22:28:49.704843+0000","last_undegraded":"2023-09-23T22:28:49.704843+0000","last_fullsized":"2023-09-23T22:28:49.704843+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.842956+0000","last_change":"2023-09-23T22:28:43.842740+0000","last_active":"2023-09-23T22:28:43.842956+0000","last_peered":"2023-09-23T22:28:43.842956+0000","last_clean":"2023-09-23T22:28:43.842956+0000","last_became_active":"2023-09-23T22:28:43.842371+0000","last_became_peered":"2023-09-23T22:28:43.842371+0000","last_unstale":"2023-09-23T22:28:43.842956+0000","last_undegraded":"2023-09-23T22:28:43.842956+0000","last_fullsized":"2023-09-23T22:28:43.842956+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.678935+0000","last_change":"2023-09-23T22:28:42.680619+0000","last_active":"2023-09-23T22:28:43.678935+0000","last_peered":"2023-09-23T22:28:43.678935+0000","last_clean":"2023-09-23T22:28:43.678935+0000","last_became_active":"2023-09-23T22:28:42.680295+0000","last_became_peered":"2023-09-23T22:28:42.680295+0000","last_unstale":"2023-09-23T22:28:43.678935+0000","last_undegraded":"2023-09-23T22:28:43.678935+0000","last_fullsized":"2023-09-23T22:28:43.678935+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838433+0000","last_change":"2023-09-23T22:27:50.607540+0000","last_active":"2023-09-23T22:28:43.838433+0000","last_peered":"2023-09-23T22:28:43.838433+0000","last_clean":"2023-09-23T22:28:43.838433+0000","last_became_active":"2023-09-23T22:27:50.607232+0000","last_became_peered":"2023-09-23T22:27:50.607232+0000","last_unstale":"2023-09-23T22:28:43.838433+0000","last_undegraded":"2023-09-23T22:28:43.838433+0000","last_fullsized":"2023-09-23T22:28:43.838433+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.681380+0000","last_change":"2023-09-23T22:28:43.679731+0000","last_active":"2023-09-23T22:28:43.681380+0000","last_peered":"2023-09-23T22:28:43.681380+0000","last_clean":"2023-09-23T22:28:43.681380+0000","last_became_active":"2023-09-23T22:28:43.679434+0000","last_became_peered":"2023-09-23T22:28:43.679434+0000","last_unstale":"2023-09-23T22:28:43.681380+0000","last_undegraded":"2023-09-23T22:28:43.681380+0000","last_fullsized":"2023-09-23T22:28:43.681380+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672909+0000","last_change":"2023-09-23T22:28:42.686808+0000","last_active":"2023-09-23T22:28:43.672909+0000","last_peered":"2023-09-23T22:28:43.672909+0000","last_clean":"2023-09-23T22:28:43.672909+0000","last_became_active":"2023-09-23T22:28:42.686432+0000","last_became_peered":"2023-09-23T22:28:42.686432+0000","last_unstale":"2023-09-23T22:28:43.672909+0000","last_undegraded":"2023-09-23T22:28:43.672909+0000","last_fullsized":"2023-09-23T22:28:43.672909+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838267+0000","last_change":"2023-09-23T22:27:50.607516+0000","last_active":"2023-09-23T22:28:43.838267+0000","last_peered":"2023-09-23T22:28:43.838267+0000","last_clean":"2023-09-23T22:28:43.838267+0000","last_became_active":"2023-09-23T22:27:50.607193+0000","last_became_peered":"2023-09-23T22:27:50.607193+0000","last_unstale":"2023-09-23T22:28:43.838267+0000","last_undegraded":"2023-09-23T22:28:43.838267+0000","last_fullsized":"2023-09-23T22:28:43.838267+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841564+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:28:43.841564+0000","last_peered":"2023-09-23T22:28:43.841564+0000","last_clean":"2023-09-23T22:28:43.841564+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:28:43.841564+0000","last_undegraded":"2023-09-23T22:28:43.841564+0000","last_fullsized":"2023-09-23T22:28:43.841564+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838966+0000","last_change":"2023-09-23T22:28:42.670877+0000","last_active":"2023-09-23T22:28:43.838966+0000","last_peered":"2023-09-23T22:28:43.838966+0000","last_clean":"2023-09-23T22:28:43.838966+0000","last_became_active":"2023-09-23T22:28:42.670573+0000","last_became_peered":"2023-09-23T22:28:42.670573+0000","last_unstale":"2023-09-23T22:28:43.838966+0000","last_undegraded":"2023-09-23T22:28:43.838966+0000","last_fullsized":"2023-09-23T22:28:43.838966+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838267+0000","last_change":"2023-09-23T22:27:14.671001+0000","last_active":"2023-09-23T22:28:43.838267+0000","last_peered":"2023-09-23T22:28:43.838267+0000","last_clean":"2023-09-23T22:28:43.838267+0000","last_became_active":"2023-09-23T22:27:14.670462+0000","last_became_peered":"2023-09-23T22:27:14.670462+0000","last_unstale":"2023-09-23T22:28:43.838267+0000","last_undegraded":"2023-09-23T22:28:43.838267+0000","last_fullsized":"2023-09-23T22:28:43.838267+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675151+0000","last_change":"2023-09-23T22:27:50.149615+0000","last_active":"2023-09-23T22:28:43.675151+0000","last_peered":"2023-09-23T22:28:43.675151+0000","last_clean":"2023-09-23T22:28:43.675151+0000","last_became_active":"2023-09-23T22:27:50.149191+0000","last_became_peered":"2023-09-23T22:27:50.149191+0000","last_unstale":"2023-09-23T22:28:43.675151+0000","last_undegraded":"2023-09-23T22:28:43.675151+0000","last_fullsized":"2023-09-23T22:28:43.675151+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683866+0000","last_change":"2023-09-23T22:28:43.683468+0000","last_active":"2023-09-23T22:28:43.683866+0000","last_peered":"2023-09-23T22:28:43.683866+0000","last_clean":"2023-09-23T22:28:43.683866+0000","last_became_active":"2023-09-23T22:28:43.683171+0000","last_became_peered":"2023-09-23T22:28:43.683171+0000","last_unstale":"2023-09-23T22:28:43.683866+0000","last_undegraded":"2023-09-23T22:28:43.683866+0000","last_fullsized":"2023-09-23T22:28:43.683866+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676662+0000","last_change":"2023-09-23T22:28:42.682076+0000","last_active":"2023-09-23T22:28:43.676662+0000","last_peered":"2023-09-23T22:28:43.676662+0000","last_clean":"2023-09-23T22:28:43.676662+0000","last_became_active":"2023-09-23T22:28:42.681758+0000","last_became_peered":"2023-09-23T22:28:42.681758+0000","last_unstale":"2023-09-23T22:28:43.676662+0000","last_undegraded":"2023-09-23T22:28:43.676662+0000","last_fullsized":"2023-09-23T22:28:43.676662+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672554+0000","last_change":"2023-09-23T22:27:50.607355+0000","last_active":"2023-09-23T22:28:43.672554+0000","last_peered":"2023-09-23T22:28:43.672554+0000","last_clean":"2023-09-23T22:28:43.672554+0000","last_became_active":"2023-09-23T22:27:50.607075+0000","last_became_peered":"2023-09-23T22:27:50.607075+0000","last_unstale":"2023-09-23T22:28:43.672554+0000","last_undegraded":"2023-09-23T22:28:43.672554+0000","last_fullsized":"2023-09-23T22:28:43.672554+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.842716+0000","last_change":"2023-09-23T22:28:43.841370+0000","last_active":"2023-09-23T22:28:43.842716+0000","last_peered":"2023-09-23T22:28:43.842716+0000","last_clean":"2023-09-23T22:28:43.842716+0000","last_became_active":"2023-09-23T22:28:43.841110+0000","last_became_peered":"2023-09-23T22:28:43.841110+0000","last_unstale":"2023-09-23T22:28:43.842716+0000","last_undegraded":"2023-09-23T22:28:43.842716+0000","last_fullsized":"2023-09-23T22:28:43.842716+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.677002+0000","last_change":"2023-09-23T22:28:42.673013+0000","last_active":"2023-09-23T22:28:43.677002+0000","last_peered":"2023-09-23T22:28:43.677002+0000","last_clean":"2023-09-23T22:28:43.677002+0000","last_became_active":"2023-09-23T22:28:42.672710+0000","last_became_peered":"2023-09-23T22:28:42.672710+0000","last_unstale":"2023-09-23T22:28:43.677002+0000","last_undegraded":"2023-09-23T22:28:43.677002+0000","last_fullsized":"2023-09-23T22:28:43.677002+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675023+0000","last_change":"2023-09-23T22:27:50.150091+0000","last_active":"2023-09-23T22:28:43.675023+0000","last_peered":"2023-09-23T22:28:43.675023+0000","last_clean":"2023-09-23T22:28:43.675023+0000","last_became_active":"2023-09-23T22:27:50.149716+0000","last_became_peered":"2023-09-23T22:27:50.149716+0000","last_unstale":"2023-09-23T22:28:43.675023+0000","last_undegraded":"2023-09-23T22:28:43.675023+0000","last_fullsized":"2023-09-23T22:28:43.675023+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683879+0000","last_change":"2023-09-23T22:28:43.683463+0000","last_active":"2023-09-23T22:28:43.683879+0000","last_peered":"2023-09-23T22:28:43.683879+0000","last_clean":"2023-09-23T22:28:43.683879+0000","last_became_active":"2023-09-23T22:28:43.683208+0000","last_became_peered":"2023-09-23T22:28:43.683208+0000","last_unstale":"2023-09-23T22:28:43.683879+0000","last_undegraded":"2023-09-23T22:28:43.683879+0000","last_fullsized":"2023-09-23T22:28:43.683879+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675388+0000","last_change":"2023-09-23T22:28:42.672269+0000","last_active":"2023-09-23T22:28:43.675388+0000","last_peered":"2023-09-23T22:28:43.675388+0000","last_clean":"2023-09-23T22:28:43.675388+0000","last_became_active":"2023-09-23T22:28:42.671983+0000","last_became_peered":"2023-09-23T22:28:42.671983+0000","last_unstale":"2023-09-23T22:28:43.675388+0000","last_undegraded":"2023-09-23T22:28:43.675388+0000","last_fullsized":"2023-09-23T22:28:43.675388+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.678825+0000","last_change":"2023-09-23T22:27:50.607380+0000","last_active":"2023-09-23T22:28:43.678825+0000","last_peered":"2023-09-23T22:28:43.678825+0000","last_clean":"2023-09-23T22:28:43.678825+0000","last_became_active":"2023-09-23T22:27:50.607045+0000","last_became_peered":"2023-09-23T22:27:50.607045+0000","last_unstale":"2023-09-23T22:28:43.678825+0000","last_undegraded":"2023-09-23T22:28:43.678825+0000","last_fullsized":"2023-09-23T22:28:43.678825+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841391+0000","last_change":"2023-09-23T22:28:43.839790+0000","last_active":"2023-09-23T22:28:43.841391+0000","last_peered":"2023-09-23T22:28:43.841391+0000","last_clean":"2023-09-23T22:28:43.841391+0000","last_became_active":"2023-09-23T22:28:43.839389+0000","last_became_peered":"2023-09-23T22:28:43.839389+0000","last_unstale":"2023-09-23T22:28:43.841391+0000","last_undegraded":"2023-09-23T22:28:43.841391+0000","last_fullsized":"2023-09-23T22:28:43.841391+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838668+0000","last_change":"2023-09-23T22:28:42.676799+0000","last_active":"2023-09-23T22:28:43.838668+0000","last_peered":"2023-09-23T22:28:43.838668+0000","last_clean":"2023-09-23T22:28:43.838668+0000","last_became_active":"2023-09-23T22:28:42.676112+0000","last_became_peered":"2023-09-23T22:28:42.676112+0000","last_unstale":"2023-09-23T22:28:43.838668+0000","last_undegraded":"2023-09-23T22:28:43.838668+0000","last_fullsized":"2023-09-23T22:28:43.838668+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683882+0000","last_change":"2023-09-23T22:28:43.682852+0000","last_active":"2023-09-23T22:28:43.683882+0000","last_peered":"2023-09-23T22:28:43.683882+0000","last_clean":"2023-09-23T22:28:43.683882+0000","last_became_active":"2023-09-23T22:28:43.682497+0000","last_became_peered":"2023-09-23T22:28:43.682497+0000","last_unstale":"2023-09-23T22:28:43.683882+0000","last_undegraded":"2023-09-23T22:28:43.683882+0000","last_fullsized":"2023-09-23T22:28:43.683882+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676034+0000","last_change":"2023-09-23T22:28:42.663472+0000","last_active":"2023-09-23T22:28:43.676034+0000","last_peered":"2023-09-23T22:28:43.676034+0000","last_clean":"2023-09-23T22:28:43.676034+0000","last_became_active":"2023-09-23T22:28:42.663132+0000","last_became_peered":"2023-09-23T22:28:42.663132+0000","last_unstale":"2023-09-23T22:28:43.676034+0000","last_undegraded":"2023-09-23T22:28:43.676034+0000","last_fullsized":"2023-09-23T22:28:43.676034+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683875+0000","last_change":"2023-09-23T22:28:43.682934+0000","last_active":"2023-09-23T22:28:43.683875+0000","last_peered":"2023-09-23T22:28:43.683875+0000","last_clean":"2023-09-23T22:28:43.683875+0000","last_became_active":"2023-09-23T22:28:43.682593+0000","last_became_peered":"2023-09-23T22:28:43.682593+0000","last_unstale":"2023-09-23T22:28:43.683875+0000","last_undegraded":"2023-09-23T22:28:43.683875+0000","last_fullsized":"2023-09-23T22:28:43.683875+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672549+0000","last_change":"2023-09-23T22:28:42.678925+0000","last_active":"2023-09-23T22:28:43.672549+0000","last_peered":"2023-09-23T22:28:43.672549+0000","last_clean":"2023-09-23T22:28:43.672549+0000","last_became_active":"2023-09-23T22:28:42.678555+0000","last_became_peered":"2023-09-23T22:28:42.678555+0000","last_unstale":"2023-09-23T22:28:43.672549+0000","last_undegraded":"2023-09-23T22:28:43.672549+0000","last_fullsized":"2023-09-23T22:28:43.672549+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840520+0000","last_change":"2023-09-23T22:28:43.839580+0000","last_active":"2023-09-23T22:28:43.840520+0000","last_peered":"2023-09-23T22:28:43.840520+0000","last_clean":"2023-09-23T22:28:43.840520+0000","last_became_active":"2023-09-23T22:28:43.839285+0000","last_became_peered":"2023-09-23T22:28:43.839285+0000","last_unstale":"2023-09-23T22:28:43.840520+0000","last_undegraded":"2023-09-23T22:28:43.840520+0000","last_fullsized":"2023-09-23T22:28:43.840520+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704379+0000","last_change":"2023-09-23T22:28:42.668895+0000","last_active":"2023-09-23T22:28:49.704379+0000","last_peered":"2023-09-23T22:28:49.704379+0000","last_clean":"2023-09-23T22:28:49.704379+0000","last_became_active":"2023-09-23T22:28:42.668572+0000","last_became_peered":"2023-09-23T22:28:42.668572+0000","last_unstale":"2023-09-23T22:28:49.704379+0000","last_undegraded":"2023-09-23T22:28:49.704379+0000","last_fullsized":"2023-09-23T22:28:49.704379+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.679881+0000","last_change":"2023-09-23T22:28:43.678995+0000","last_active":"2023-09-23T22:28:43.679881+0000","last_peered":"2023-09-23T22:28:43.679881+0000","last_clean":"2023-09-23T22:28:43.679881+0000","last_became_active":"2023-09-23T22:28:43.678677+0000","last_became_peered":"2023-09-23T22:28:43.678677+0000","last_unstale":"2023-09-23T22:28:43.679881+0000","last_undegraded":"2023-09-23T22:28:43.679881+0000","last_fullsized":"2023-09-23T22:28:43.679881+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675149+0000","last_change":"2023-09-23T22:28:42.669958+0000","last_active":"2023-09-23T22:28:43.675149+0000","last_peered":"2023-09-23T22:28:43.675149+0000","last_clean":"2023-09-23T22:28:43.675149+0000","last_became_active":"2023-09-23T22:28:42.669668+0000","last_became_peered":"2023-09-23T22:28:42.669668+0000","last_unstale":"2023-09-23T22:28:43.675149+0000","last_undegraded":"2023-09-23T22:28:43.675149+0000","last_fullsized":"2023-09-23T22:28:43.675149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841570+0000","last_change":"2023-09-23T22:28:43.841201+0000","last_active":"2023-09-23T22:28:43.841570+0000","last_peered":"2023-09-23T22:28:43.841570+0000","last_clean":"2023-09-23T22:28:43.841570+0000","last_became_active":"2023-09-23T22:28:43.840936+0000","last_became_peered":"2023-09-23T22:28:43.840936+0000","last_unstale":"2023-09-23T22:28:43.841570+0000","last_undegraded":"2023-09-23T22:28:43.841570+0000","last_fullsized":"2023-09-23T22:28:43.841570+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.695064+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:28:44.695064+0000","last_peered":"2023-09-23T22:28:44.695064+0000","last_clean":"2023-09-23T22:28:44.695064+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:28:44.695064+0000","last_undegraded":"2023-09-23T22:28:44.695064+0000","last_fullsized":"2023-09-23T22:28:44.695064+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840544+0000","last_change":"2023-09-23T22:28:43.839560+0000","last_active":"2023-09-23T22:28:43.840544+0000","last_peered":"2023-09-23T22:28:43.840544+0000","last_clean":"2023-09-23T22:28:43.840544+0000","last_became_active":"2023-09-23T22:28:43.839255+0000","last_became_peered":"2023-09-23T22:28:43.839255+0000","last_unstale":"2023-09-23T22:28:43.840544+0000","last_undegraded":"2023-09-23T22:28:43.840544+0000","last_fullsized":"2023-09-23T22:28:43.840544+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704560+0000","last_change":"2023-09-23T22:28:42.673358+0000","last_active":"2023-09-23T22:28:49.704560+0000","last_peered":"2023-09-23T22:28:49.704560+0000","last_clean":"2023-09-23T22:28:49.704560+0000","last_became_active":"2023-09-23T22:28:42.673029+0000","last_became_peered":"2023-09-23T22:28:42.673029+0000","last_unstale":"2023-09-23T22:28:49.704560+0000","last_undegraded":"2023-09-23T22:28:49.704560+0000","last_fullsized":"2023-09-23T22:28:49.704560+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840230+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:28:43.840230+0000","last_peered":"2023-09-23T22:28:43.840230+0000","last_clean":"2023-09-23T22:28:43.840230+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:28:43.840230+0000","last_undegraded":"2023-09-23T22:28:43.840230+0000","last_fullsized":"2023-09-23T22:28:43.840230+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.694435+0000","last_change":"2023-09-23T22:28:42.670634+0000","last_active":"2023-09-23T22:28:44.694435+0000","last_peered":"2023-09-23T22:28:44.694435+0000","last_clean":"2023-09-23T22:28:44.694435+0000","last_became_active":"2023-09-23T22:28:42.670356+0000","last_became_peered":"2023-09-23T22:28:42.670356+0000","last_unstale":"2023-09-23T22:28:44.694435+0000","last_undegraded":"2023-09-23T22:28:44.694435+0000","last_fullsized":"2023-09-23T22:28:44.694435+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839438+0000","last_change":"2023-09-23T22:28:42.863270+0000","last_active":"2023-09-23T22:28:43.839438+0000","last_peered":"2023-09-23T22:28:43.839438+0000","last_clean":"2023-09-23T22:28:43.839438+0000","last_became_active":"2023-09-23T22:28:42.862939+0000","last_became_peered":"2023-09-23T22:28:42.862939+0000","last_unstale":"2023-09-23T22:28:43.839438+0000","last_undegraded":"2023-09-23T22:28:43.839438+0000","last_fullsized":"2023-09-23T22:28:43.839438+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.705969+0000","last_change":"2023-09-23T22:28:42.666505+0000","last_active":"2023-09-23T22:28:49.705969+0000","last_peered":"2023-09-23T22:28:49.705969+0000","last_clean":"2023-09-23T22:28:49.705969+0000","last_became_active":"2023-09-23T22:28:42.666176+0000","last_became_peered":"2023-09-23T22:28:42.666176+0000","last_unstale":"2023-09-23T22:28:49.705969+0000","last_undegraded":"2023-09-23T22:28:49.705969+0000","last_fullsized":"2023-09-23T22:28:49.705969+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.841963+0000","last_change":"2023-09-23T22:28:43.841481+0000","last_active":"2023-09-23T22:28:43.841963+0000","last_peered":"2023-09-23T22:28:43.841963+0000","last_clean":"2023-09-23T22:28:43.841963+0000","last_became_active":"2023-09-23T22:28:43.841146+0000","last_became_peered":"2023-09-23T22:28:43.841146+0000","last_unstale":"2023-09-23T22:28:43.841963+0000","last_undegraded":"2023-09-23T22:28:43.841963+0000","last_fullsized":"2023-09-23T22:28:43.841963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838424+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:28:43.838424+0000","last_peered":"2023-09-23T22:28:43.838424+0000","last_clean":"2023-09-23T22:28:43.838424+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:28:43.838424+0000","last_undegraded":"2023-09-23T22:28:43.838424+0000","last_fullsized":"2023-09-23T22:28:43.838424+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.840313+0000","last_change":"2023-09-23T22:28:43.839847+0000","last_active":"2023-09-23T22:28:43.840313+0000","last_peered":"2023-09-23T22:28:43.840313+0000","last_clean":"2023-09-23T22:28:43.840313+0000","last_became_active":"2023-09-23T22:28:43.839519+0000","last_became_peered":"2023-09-23T22:28:43.839519+0000","last_unstale":"2023-09-23T22:28:43.840313+0000","last_undegraded":"2023-09-23T22:28:43.840313+0000","last_fullsized":"2023-09-23T22:28:43.840313+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.672322+0000","last_change":"2023-09-23T22:28:42.666721+0000","last_active":"2023-09-23T22:28:43.672322+0000","last_peered":"2023-09-23T22:28:43.672322+0000","last_clean":"2023-09-23T22:28:43.672322+0000","last_became_active":"2023-09-23T22:28:42.666400+0000","last_became_peered":"2023-09-23T22:28:42.666400+0000","last_unstale":"2023-09-23T22:28:43.672322+0000","last_undegraded":"2023-09-23T22:28:43.672322+0000","last_fullsized":"2023-09-23T22:28:43.672322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839004+0000","last_change":"2023-09-23T22:28:42.863332+0000","last_active":"2023-09-23T22:28:43.839004+0000","last_peered":"2023-09-23T22:28:43.839004+0000","last_clean":"2023-09-23T22:28:43.839004+0000","last_became_active":"2023-09-23T22:28:42.863049+0000","last_became_peered":"2023-09-23T22:28:42.863049+0000","last_unstale":"2023-09-23T22:28:43.839004+0000","last_undegraded":"2023-09-23T22:28:43.839004+0000","last_fullsized":"2023-09-23T22:28:43.839004+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704523+0000","last_change":"2023-09-23T22:28:42.680816+0000","last_active":"2023-09-23T22:28:49.704523+0000","last_peered":"2023-09-23T22:28:49.704523+0000","last_clean":"2023-09-23T22:28:49.704523+0000","last_became_active":"2023-09-23T22:28:42.680489+0000","last_became_peered":"2023-09-23T22:28:42.680489+0000","last_unstale":"2023-09-23T22:28:49.704523+0000","last_undegraded":"2023-09-23T22:28:49.704523+0000","last_fullsized":"2023-09-23T22:28:49.704523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683983+0000","last_change":"2023-09-23T22:28:43.683746+0000","last_active":"2023-09-23T22:28:43.683983+0000","last_peered":"2023-09-23T22:28:43.683983+0000","last_clean":"2023-09-23T22:28:43.683983+0000","last_became_active":"2023-09-23T22:28:43.683291+0000","last_became_peered":"2023-09-23T22:28:43.683291+0000","last_unstale":"2023-09-23T22:28:43.683983+0000","last_undegraded":"2023-09-23T22:28:43.683983+0000","last_fullsized":"2023-09-23T22:28:43.683983+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.705770+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:28:49.705770+0000","last_peered":"2023-09-23T22:28:49.705770+0000","last_clean":"2023-09-23T22:28:49.705770+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:28:49.705770+0000","last_undegraded":"2023-09-23T22:28:49.705770+0000","last_fullsized":"2023-09-23T22:28:49.705770+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.683840+0000","last_change":"2023-09-23T22:28:43.682855+0000","last_active":"2023-09-23T22:28:43.683840+0000","last_peered":"2023-09-23T22:28:43.683840+0000","last_clean":"2023-09-23T22:28:43.683840+0000","last_became_active":"2023-09-23T22:28:43.682568+0000","last_became_peered":"2023-09-23T22:28:43.682568+0000","last_unstale":"2023-09-23T22:28:43.683840+0000","last_undegraded":"2023-09-23T22:28:43.683840+0000","last_fullsized":"2023-09-23T22:28:43.683840+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:49.704604+0000","last_change":"2023-09-23T22:28:42.679399+0000","last_active":"2023-09-23T22:28:49.704604+0000","last_peered":"2023-09-23T22:28:49.704604+0000","last_clean":"2023-09-23T22:28:49.704604+0000","last_became_active":"2023-09-23T22:28:42.679100+0000","last_became_peered":"2023-09-23T22:28:42.679100+0000","last_unstale":"2023-09-23T22:28:49.704604+0000","last_undegraded":"2023-09-23T22:28:49.704604+0000","last_fullsized":"2023-09-23T22:28:49.704604+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839868+0000","last_change":"2023-09-23T22:28:42.863696+0000","last_active":"2023-09-23T22:28:43.839868+0000","last_peered":"2023-09-23T22:28:43.839868+0000","last_clean":"2023-09-23T22:28:43.839868+0000","last_became_active":"2023-09-23T22:28:42.863397+0000","last_became_peered":"2023-09-23T22:28:42.863397+0000","last_unstale":"2023-09-23T22:28:43.839868+0000","last_undegraded":"2023-09-23T22:28:43.839868+0000","last_fullsized":"2023-09-23T22:28:43.839868+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.675609+0000","last_change":"2023-09-23T22:28:42.675839+0000","last_active":"2023-09-23T22:28:43.675609+0000","last_peered":"2023-09-23T22:28:43.675609+0000","last_clean":"2023-09-23T22:28:43.675609+0000","last_became_active":"2023-09-23T22:28:42.675494+0000","last_became_peered":"2023-09-23T22:28:42.675494+0000","last_unstale":"2023-09-23T22:28:43.675609+0000","last_undegraded":"2023-09-23T22:28:43.675609+0000","last_fullsized":"2023-09-23T22:28:43.675609+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838793+0000","last_change":"2023-09-23T22:28:42.863635+0000","last_active":"2023-09-23T22:28:43.838793+0000","last_peered":"2023-09-23T22:28:43.838793+0000","last_clean":"2023-09-23T22:28:43.838793+0000","last_became_active":"2023-09-23T22:28:42.863315+0000","last_became_peered":"2023-09-23T22:28:42.863315+0000","last_unstale":"2023-09-23T22:28:43.838793+0000","last_undegraded":"2023-09-23T22:28:43.838793+0000","last_fullsized":"2023-09-23T22:28:43.838793+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.673281+0000","last_change":"2023-09-23T22:28:42.670242+0000","last_active":"2023-09-23T22:28:43.673281+0000","last_peered":"2023-09-23T22:28:43.673281+0000","last_clean":"2023-09-23T22:28:43.673281+0000","last_became_active":"2023-09-23T22:28:42.668777+0000","last_became_peered":"2023-09-23T22:28:42.668777+0000","last_unstale":"2023-09-23T22:28:43.673281+0000","last_undegraded":"2023-09-23T22:28:43.673281+0000","last_fullsized":"2023-09-23T22:28:43.673281+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.680732+0000","last_change":"2023-09-23T22:28:43.680206+0000","last_active":"2023-09-23T22:28:43.680732+0000","last_peered":"2023-09-23T22:28:43.680732+0000","last_clean":"2023-09-23T22:28:43.680732+0000","last_became_active":"2023-09-23T22:28:43.679901+0000","last_became_peered":"2023-09-23T22:28:43.679901+0000","last_unstale":"2023-09-23T22:28:43.680732+0000","last_undegraded":"2023-09-23T22:28:43.680732+0000","last_fullsized":"2023-09-23T22:28:43.680732+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.838929+0000","last_change":"2023-09-23T22:28:42.676985+0000","last_active":"2023-09-23T22:28:43.838929+0000","last_peered":"2023-09-23T22:28:43.838929+0000","last_clean":"2023-09-23T22:28:43.838929+0000","last_became_active":"2023-09-23T22:28:42.676516+0000","last_became_peered":"2023-09-23T22:28:42.676516+0000","last_unstale":"2023-09-23T22:28:43.838929+0000","last_undegraded":"2023-09-23T22:28:43.838929+0000","last_fullsized":"2023-09-23T22:28:43.838929+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.842591+0000","last_change":"2023-09-23T22:28:43.842223+0000","last_active":"2023-09-23T22:28:43.842591+0000","last_peered":"2023-09-23T22:28:43.842591+0000","last_clean":"2023-09-23T22:28:43.842591+0000","last_became_active":"2023-09-23T22:28:43.841902+0000","last_became_peered":"2023-09-23T22:28:43.841902+0000","last_unstale":"2023-09-23T22:28:43.842591+0000","last_undegraded":"2023-09-23T22:28:43.842591+0000","last_fullsized":"2023-09-23T22:28:43.842591+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.676444+0000","last_change":"2023-09-23T22:28:42.681628+0000","last_active":"2023-09-23T22:28:43.676444+0000","last_peered":"2023-09-23T22:28:43.676444+0000","last_clean":"2023-09-23T22:28:43.676444+0000","last_became_active":"2023-09-23T22:28:42.681301+0000","last_became_peered":"2023-09-23T22:28:42.681301+0000","last_unstale":"2023-09-23T22:28:43.676444+0000","last_undegraded":"2023-09-23T22:28:43.676444+0000","last_fullsized":"2023-09-23T22:28:43.676444+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-09-23T22:28:43.839285+0000","last_change":"2023-09-23T22:28:42.687696+0000","last_active":"2023-09-23T22:28:43.839285+0000","last_peered":"2023-09-23T22:28:43.839285+0000","last_clean":"2023-09-23T22:28:43.839285+0000","last_became_active":"2023-09-23T22:28:42.687402+0000","last_became_peered":"2023-09-23T22:28:42.687402+0000","last_unstale":"2023-09-23T22:28:43.839285+0000","last_undegraded":"2023-09-23T22:28:43.839285+0000","last_fullsized":"2023-09-23T22:28:43.839285+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-09-23T22:28:44.673418+0000","last_change":"2023-09-23T22:28:42.672628+0000","last_active":"2023-09-23T22:28:44.673418+0000","last_peered":"2023-09-23T22:28:44.673418+0000","last_clean":"2023-09-23T22:28:44.673418+0000","last_became_active":"2023-09-23T22:28:42.672274+0000","last_became_peered":"2023-09-23T22:28:42.672274+0000","last_unstale":"2023-09-23T22:28:44.673418+0000","last_undegraded":"2023-09-23T22:28:44.673418+0000","last_fullsized":"2023-09-23T22:28:44.673418+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430238,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68500000000000005,"5min":1.032,"15min":1.032},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.503,"15min":0.498},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.749,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.77000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.41099999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.75900000000000001,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.62},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72199999999999998,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.71899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.52900000000000003,"15min":0.52100000000000002},"min":{"1min":0.36199999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.8440000000000001,"5min":1.8440000000000001,"15min":1.8440000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82799999999999996,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.74399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56799999999999995,"15min":0.57199999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71099999999999997,"5min":1.087,"15min":1.087},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.47099999999999997,"15min":0.46800000000000003},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.69499999999999995,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.496,"15min":0.49299999999999999},"min":{"1min":0.36799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.68500000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.67000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74299999999999999,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.45500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.751,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.40300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.69799999999999995,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.56799999999999995,"15min":0.56200000000000006},"min":{"1min":0.42599999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.57899999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626466,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":257745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65400000000000003,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.443,"5min":0.51900000000000002,"15min":0.53200000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.72499999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.56599999999999995,"15min":0.58399999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.58799999999999997,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.54400000000000004,"15min":0.55400000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68100000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.66500000000000004,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.69599999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.67100000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72899999999999998,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.56899999999999995,"15min":0.58099999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66500000000000004,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.57799999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.42699999999999999,"15min":0.42499999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.629,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.442,"5min":0.42199999999999999,"15min":0.41899999999999998},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.59099999999999997,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 22:28:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63600000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.47099999999999997,"15min":0.46200000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68999999999999995,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:29:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.50800000000000001,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.74299999999999999,"5min":1.357,"15min":1.357},"last":0.499}]}]},{"osd":1,"up_from":13,"seq":55834574902,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.378,"15min":0.35799999999999998},"min":{"1min":0.252,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.61299999999999999,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.246},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.374,"15min":0.35299999999999998},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":0.56899999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.28299999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.308},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.40200000000000002,"15min":0.39600000000000002},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.48799999999999999,"15min":0.5},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.46700000000000003,"15min":0.48599999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69299999999999995,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.27300000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.61399999999999999,"15min":0.626},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81699999999999995,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.62},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63600000000000001,"15min":0.63900000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82499999999999996,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.626}]},{"osd":5,"last update":"Sat Sep 23 22:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.60599999999999998,"15min":0.60499999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.499},{"interface":"front","average":{"1min":0.628,"5min":0.63800000000000001,"15min":0.64000000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.94199999999999995,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:28:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.70299999999999996,"15min":0.72299999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.85499999999999998,"5min":2.524,"15min":2.524},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63200000000000001,"15min":0.64000000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67300000000000004,"15min":0.68500000000000005},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85499999999999998,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.70699999999999996,"15min":0.71799999999999997},"min":{"1min":0.48899999999999999,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.86799999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.56000000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738427,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.376,"15min":0.35199999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.38100000000000001,"15min":0.35099999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.40200000000000002,"15min":0.40000000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.41499999999999998,"15min":0.40799999999999997},"min":{"1min":0.308,"5min":0.255,"15min":0.255},"max":{"1min":0.623,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:28:53 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.46100000000000002,"15min":0.46700000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.54700000000000004,"5min":0.754,"15min":0.754},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.443,"5min":0.499,"15min":0.50800000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.58199999999999996,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.627},"min":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.5},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.625,"15min":0.62},"min":{"1min":0.41299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93000000000000005,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.82799999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.442},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.63700000000000001,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:28:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.62,"15min":0.623},"min":{"1min":0.44800000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.65600000000000003,"15min":0.66000000000000003},"min":{"1min":0.46600000000000003,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.59099999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:29:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.877,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.64900000000000002,"15min":0.64800000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.60199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.46300000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36599999999999999,"15min":0.35099999999999998},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53900000000000003,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.29599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.374,"15min":0.34899999999999998},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.54400000000000004,"5min":0.629,"15min":0.629},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.376,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27000000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48299999999999998,"5min":0.625,"15min":0.625},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.443,"15min":0.45700000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57199999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.443,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59799999999999998,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:29:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.627,"15min":0.64100000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84399999999999997,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.66600000000000004,"15min":0.68400000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82299999999999995,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:28:33 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63600000000000001,"15min":0.63700000000000001},"min":{"1min":0.48799999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.84499999999999997,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78400000000000003,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.627,"15min":0.63400000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.754,"5min":1.044,"15min":1.044},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68000000000000005,"15min":0.68799999999999994},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.83799999999999997,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 22:29:15 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.629,"15min":0.629},"min":{"1min":0.46999999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.81399999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.66100000000000003,"15min":0.66700000000000004},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76900000000000002,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.58399999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182445,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.42099999999999999,"15min":0.438},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.434,"15min":0.46400000000000002},"min":{"1min":0.26000000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.373}]},{"osd":1,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.441,"15min":0.45000000000000001},"min":{"1min":0.28000000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.48299999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.499,"15min":0.52700000000000002},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.49199999999999999,"15min":0.51800000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70199999999999996,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:29:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.61399999999999999,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.71499999999999997,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61899999999999999,"15min":0.63900000000000001},"min":{"1min":0.38600000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.80200000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58899999999999997,"15min":0.58999999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.60399999999999998,"15min":0.59699999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.68200000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.65000000000000002,"15min":0.65900000000000003},"min":{"1min":0.46100000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.92500000000000004,"5min":1.046,"15min":1.046},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.623,"15min":0.63500000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.76700000000000002,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:29:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.628,"15min":0.623},"min":{"1min":0.501,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.5149999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.46500000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83199999999999996,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.626}]}]},{"osd":4,"up_from":30,"seq":128849018922,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":257821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.53500000000000003,"15min":0.55500000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81100000000000005,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.58599999999999997,"15min":0.61199999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.82599999999999996,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.67200000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.55800000000000005,"15min":0.58699999999999997},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.78800000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.59599999999999997,"15min":0.624},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79600000000000004,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.59199999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.57299999999999995,"15min":0.60099999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.57499999999999996,"15min":0.59899999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.82799999999999996,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.64100000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.59699999999999998,"15min":0.61699999999999999},"min":{"1min":0.41199999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.59199999999999997,"15min":0.61899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.83399999999999996,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.65500000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:28:39 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.44500000000000001,"15min":0.44400000000000001},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.66900000000000004,"5min":0.755,"15min":0.755},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.44500000000000001,"15min":0.437},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.46400000000000002,"15min":0.45400000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70699999999999996,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.39300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:29:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.52500000000000002,"15min":0.51900000000000002},"min":{"1min":0.42699999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67800000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.50700000000000001,"15min":0.498},"min":{"1min":0.42399999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57699999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822694,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.52400000000000002,"15min":0.52900000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64300000000000002,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.52600000000000002,"15min":0.52300000000000002},"min":{"1min":0.38200000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66200000000000003,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.60799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.58799999999999997,"15min":0.60199999999999998},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67100000000000004,"5min":1.823,"15min":1.823},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69199999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.47699999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.747,"5min":1.278,"15min":1.278},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.76100000000000001,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.72999999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.32100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.432},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.68600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:28:53 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.45300000000000001,"15min":0.441},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.47499999999999998,"15min":0.46000000000000002},"min":{"1min":0.36299999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.76300000000000001,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.76900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:29:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.52600000000000002,"15min":0.51500000000000001},"min":{"1min":0.36099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80000000000000004,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.52600000000000002,"15min":0.51400000000000001},"min":{"1min":0.433,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.753,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.79000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:29:27.133 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:29:27.133 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:29:27.133 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:29:27.133 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:29:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:27 smithi151 conmon[100454]: audit 2023-09-23T22:29:26.533352+0000 mgr.y (mgr.24314) 93 : audit [DBG] from='client.14748 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:29:27.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:27 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:29:27.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:27 smithi052 conmon[110082]: :29:26.533352+0000 mgr.y (mgr.24314) 93 : audit [DBG] from='client.14748 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:29:27.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:27 smithi052 conmon[100224]: audit 2023-09-23T22:29:26. 2023-09-23T22:29:27.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:27 smithi052 conmon[100224]: 533352+0000 mgr.y (mgr.24314) 93 : audit [DBG] from='client.14748 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:29:27.494 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:28.419 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:28 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:28] "GET /metrics HTTP/1.1" 200 191455 "" "Prometheus/2.33.4" 2023-09-23T22:29:28.420 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:28 smithi052 conmon[110082]: cluster 2023-09-23T22:29:27.430278+0000 mgr.y (mgr.24314) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:28.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:28 smithi052 conmon[100224]: cluster 2023-09-23T22:29:27.430278+0000 mgr.y (mgr.24314) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:28.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:28 smithi151 conmon[100454]: cluster 2023-09-23T22:29:27.430278+0000 mgr.y (mgr.24314) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:29.092 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:29:29.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:29 smithi151 conmon[100454]: audit 2023-09-23T22:29:29.092360+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.52:0/2519296619' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:29:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:29 smithi052 conmon[110082]: audit 2023-09-23T22:29:29 2023-09-23T22:29:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:29 smithi052 conmon[110082]: .092360+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.52:0/2519296619' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:29:29.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:29 smithi052 conmon[100224]: audit 2023-09-23T22:29: 2023-09-23T22:29:29.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:29 smithi052 conmon[100224]: 29.092360+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.52:0/2519296619' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:29:29.593 INFO:teuthology.orchestra.run.smithi052.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:29:29.593 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:29:29.593 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:29:29.602 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T22:29:29.602 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T22:29:29.610 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T22:29:29.610 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:29:29.611 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T22:29:29.614 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T22:29:29.615 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T22:29:29.615 DEBUG:teuthology.orchestra.run.smithi052:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:29:29.615 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:29:29.615 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T22:29:29.616 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi052.front.sepia.ceph.com 2023-09-23T22:29:29.616 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T22:29:29.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:29:29.634 INFO:teuthology.orchestra.run.smithi052.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T22:29:29.635 DEBUG:teuthology.orchestra.run.smithi052:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:29:29.692 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T22:29:29.692 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T22:29:29.748 INFO:tasks.workunit:timeout=3h 2023-09-23T22:29:29.748 INFO:tasks.workunit:cleanup=True 2023-09-23T22:29:29.749 DEBUG:teuthology.orchestra.run.smithi052:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:29:29.806 INFO:tasks.workunit.client.0.smithi052.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T22:29:30.386 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[110082]: audit 2023-09-23T22:29:29.417731+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T22:29:30.386 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[110082]: audit 2023-09-23T22:29:29.417978+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T22:29:30.386 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[110082]: audit 2023-09-23T22:29:29.418179+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T22:29:30.386 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[110082]: cluster 2023-09-23T22:29:29.430801+0000 mgr.y (mgr.24314) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:30.386 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[100224]: audit 2023-09-23T22:29:29.417731+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T22:29:30.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[100224]: audit 2023-09-23T22:29:29.417978+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T22:29:30.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[100224]: audit 2023-09-23T22:29:29.418179+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T22:29:30.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:30 smithi052 conmon[100224]: cluster 2023-09-23T22:29:29.430801+0000 mgr.y (mgr.24314) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:30.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: audit 2023-09-23T22:29:29.417731+0000 mon.a (mon.0) 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: 719 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: audit 2023-09-23T22:29:29.417978+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: T22:29:29.418179+0000 mon.a (mon 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: .0) 721 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: cluster 2023-09-23T22:29:29.430801+0000 2023-09-23T22:29:30.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:30 smithi151 conmon[100454]: mgr.y (mgr.24314) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:31.430 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100224]: audit 2023-09-23T22:29:30.165917+0000 2023-09-23T22:29:31.431 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100224]: mon.a (mon.0) 722 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T22:29:31.431 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100224]: audit 2023-09-23T22:29:30.165975+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T22:29:31.431 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100224]: audit 2023-09-23T22:29:30.166023+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T22:29:31.431 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100224]: cluster 2023-09-23T22:29:30.166085+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:29:31.431 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100670]: [23/Sep/2023:22:29:31] ENGINE Bus STOPPING 2023-09-23T22:29:31.431 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: 22:29:30.165917+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: audit 2023-09-23T22:29 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: :30.165975+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: audit 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: 2023-09-23 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: T22:29:30.166023+0000 mon.a (mon.0) 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: 724 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: cluster 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: 2023-09-23T22:29: 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: 30.166085+0000 mon.a (mon 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: .0) 725 : cluster [DBG] 2023-09-23T22:29:31.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[110082]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:29:31.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:31 smithi151 conmon[100454]: audit 2023-09-23T22:29:30.165917+0000 mon.a (mon.0) 722 : audit [INF] 2023-09-23T22:29:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:31 smithi151 conmon[100454]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T22:29:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:31 smithi151 conmon[100454]: audit 2023-09-23T22:29:30.165975+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T22:29:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:31 smithi151 conmon[100454]: audit 2023-09-23T22:29:30.166023+0000 mon.a (mon.0 2023-09-23T22:29:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:31 smithi151 conmon[100454]: ) 724 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T22:29:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:31 smithi151 conmon[100454]: cluster 2023-09-23T22:29:30.166085+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:29:31.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100670]: [23/Sep/2023:22:29:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:29:31.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100670]: [23/Sep/2023:22:29:31] ENGINE Bus STOPPED 2023-09-23T22:29:31.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100670]: [23/Sep/2023:22:29:31] ENGINE Bus STARTING 2023-09-23T22:29:31.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100670]: [23/Sep/2023:22:29:31] ENGINE Serving on http://:::9283 2023-09-23T22:29:31.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:31 smithi052 conmon[100670]: [23/Sep/2023:22:29:31] ENGINE Bus STARTED 2023-09-23T22:29:31.960 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T22:29:32.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: cluster 2023-09-23T22:29:31.167392 2023-09-23T22:29:32.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: +0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:29:32.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit 2023-09-23T22:29:31. 2023-09-23T22:29:32.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 413768+0000 mon.a (mon.0) 727 : audit [INF] from='client.? 172.21.15.52:0/1266060210' entity='client.admin' 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: cluster 2023 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: -09-23T22:29: 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 31.431152+0000 mgr.y (mgr.24314) 96 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit 2023 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: -09-23T22:29: 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 31.464973+0000 mon.a (mon.0) 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 728 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 09-23T22:29:31 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: .466378+0000 mon.a (mon.0) 729 : 2023-09-23T22:29:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 09-23T22:29:31 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: .466981+0000 mon.a (mon.0) 730 : 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 09-23T22:29:31. 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: 471554+0000 mon.a (mon.0) 731 : audit 2023-09-23T22:29:32.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[100224]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: cluster 2023-09-23T22:29: 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: 31.167392+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: 22:29:31.413768+0000 mon.a (mon 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: .0) 727 : audit [INF] from='client.? 172.21.15.52:0/1266060210' entity='client.admin' 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: cluster 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: 2023-09-23T22:29:31 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: .431152+0000 mgr.y (mgr.24314) 96 : 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:29:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: :29:31.464973+0000 mon.a (mon. 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: 0) 728 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: audit 2023 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: -09-23T22:29:31. 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: 466378+0000 mon.a (mon.0) 729 : 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: :29:31.466981+0000 mon.a (mon.0 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: ) 730 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: 09-23T22:29:31.471554 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: +0000 mon.a (mon.0) 731 : audit 2023-09-23T22:29:32.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:32 smithi052 conmon[110082]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:32.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: cluster 2023-09-23T22:29:31.167392+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: audit 2023-09-23T22:29:31.413768+0000 mon.a (mon.0) 727 : audit [INF] from='client.? 172.21.15.52:0/1266060210' entity='client.admin' 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: cluster 2023-09-23T22:29:31.431152+0000 mgr.y (mgr.24314) 96 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: audit 2023-09-23T22:29:31.464973+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: audit 2023-09-23T22:29:31.466378+0000 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: mon.a (mon.0) 729 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: audit 2023-09-23T22:29:31 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: .466981+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:29:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:32 smithi151 conmon[100454]: 23T22:29:31.471554+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:34.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100670]: [23/Sep/2023:22:29:34] ENGINE Bus STOPPING 2023-09-23T22:29:34.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100670]: [23/Sep/2023:22:29:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:29:34.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100670]: [23/Sep/2023:22:29:34] ENGINE Bus STOPPED 2023-09-23T22:29:34.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100670]: [23/Sep/2023:22:29:34] ENGINE Bus STARTING 2023-09-23T22:29:34.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100670]: [23/Sep/2023:22:29:34] ENGINE Serving on http://:::9283 2023-09-23T22:29:34.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100670]: [23/Sep/2023:22:29:34] ENGINE Bus STARTED 2023-09-23T22:29:34.922 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T22:29:35.177 INFO:tasks.workunit.client.0.smithi052.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T22:29:35.179 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:29:35.179 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T22:29:35.386 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100224]: cluster 2023-09-23T22:29:33.431482+0000 mgr.y (mgr.24314) 97 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:35.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100224]: audit 2023-09-23T22:29:33.999556+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.52:0/3696258194' entity='client.admin' 2023-09-23T22:29:35.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[100224]: audit 2023-09-23T22:29:34.051796+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:35.387 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[110082]: cluster 2023-09-23T22:29:33.431482+0000 mgr.y (mgr.24314) 97 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:35.387 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[110082]: audit 2023-09-23T22:29:33.999556+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.52:0/3696258194' entity='client.admin' 2023-09-23T22:29:35.387 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:34 smithi052 conmon[110082]: audit 2023-09-23T22:29:34.051796+0000 mon.a (mon.0) 733 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:35.406 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T22:29:35.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:29:35 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:29:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:34 smithi151 conmon[100454]: cluster 2023-09-23T22:29:33.431482+0000 mgr.y (mgr.24314) 97 : cluster [DBG] 2023-09-23T22:29:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:34 smithi151 conmon[100454]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:34 smithi151 conmon[100454]: audit 2023-09-23T22:29:33.999556+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.52:0/3696258194' entity='client.admin' 2023-09-23T22:29:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:34 smithi151 conmon[100454]: audit 2023-09-23T22:29:34.051796+0000 mon.a (mon.0) 2023-09-23T22:29:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:34 smithi151 conmon[100454]: 733 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:36.037 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T22:29:36.088 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T22:29:36.127 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:29:36.129 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:29:36.129 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T22:29:36.186 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:29:36.190 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T22:29:36.190 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T22:29:36.209 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T22:29:36.210 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T22:29:36.210 DEBUG:teuthology.orchestra.run.smithi052:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T22:29:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[110082]: audit 2023-09-23T22:29:35. 2023-09-23T22:29:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[110082]: 010369+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[110082]: audit 2023-09-23T22:29:35.011093+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[110082]: audit 2023-09-23T22:29:35.015311+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:36.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:29:36.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[100224]: 29:35.010369+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:36.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[100224]: audit 2023-09-23T22:29:35.011093+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:36.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:36 smithi052 conmon[100224]: audit 2023-09-23T22:29:35.015311+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:36.274 INFO:tasks.workunit.client.0.smithi052.stdout:Running main() from gmock_main.cc 2023-09-23T22:29:36.274 INFO:tasks.workunit.client.0.smithi052.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T22:29:36.275 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] Global test environment set-up. 2023-09-23T22:29:36.275 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T22:29:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:36 smithi151 conmon[100454]: audit 2023-09-23T22:29:35.010369+0000 mon.a (mon.0) 734 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:36 smithi151 conmon[100454]: 2023-09-23T22:29:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:36 smithi151 conmon[100454]: audit 2023-09-23T22:29:35.011093+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:36 smithi151 conmon[100454]: audit 2023-09-23T22:29 2023-09-23T22:29:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:36 smithi151 conmon[100454]: :35.015311+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:37.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:37 smithi151 conmon[100454]: cluster 2023-09-23T22:29:35.431985+0000 mgr.y (mgr.24314) 98 : cluster [DBG] 2023-09-23T22:29:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:37 smithi151 conmon[100454]: pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:37 smithi052 conmon[110082]: cluster 2023-09-23T22:29:35.431985+0000 mgr.y ( 2023-09-23T22:29:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:37 smithi052 conmon[110082]: mgr.24314) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:37 smithi052 conmon[100224]: cluster 2023-09-23T22: 2023-09-23T22:29:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:37 smithi052 conmon[100224]: 29:35.431985+0000 mgr.y (mgr.24314) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:38.166 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T22:29:38.181 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_all_features (16 ms) 2023-09-23T22:29:38.181 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T22:29:38.394 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.copyup (212 ms) 2023-09-23T22:29:38.394 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T22:29:38.409 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.sparse_copyup (16 ms) 2023-09-23T22:29:38.409 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T22:29:38.417 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-09-23T22:29:38.417 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T22:29:38.427 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-09-23T22:29:38.428 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T22:29:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:38 smithi151 conmon[100454]: cluster 2023-09-23T22:29:37.156467+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:29:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:38 smithi151 conmon[100454]: audit 2023-09-23T22:29:37.159608+0000 mon.c (mon.2 2023-09-23T22:29:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:38 smithi151 conmon[100454]: ) 27 : audit [INF] from='client.? 172.21.15.52:0/636348626' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:29:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:38 smithi151 conmon[100454]: audit 2023-09-23T22:29:37.160127+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:29:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:38 smithi151 conmon[100454]: cluster 2023-09-23T22:29: 2023-09-23T22:29:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:38 smithi151 conmon[100454]: 37.432327+0000 mgr.y (mgr.24314) 99 : cluster [DBG] pgmap v79: 105 pgs: 32 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:38.463 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.directory_methods (36 ms) 2023-09-23T22:29:38.463 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.create 2023-09-23T22:29:38.477 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.create (13 ms) 2023-09-23T22:29:38.477 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T22:29:38.478 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T22:29:38.478 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T22:29:38.480 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T22:29:38.480 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T22:29:38.482 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-23T22:29:38.482 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T22:29:38.483 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T22:29:38.484 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T22:29:38.485 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-09-23T22:29:38.485 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T22:29:38.488 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-09-23T22:29:38.488 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T22:29:38.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:38] "GET /metrics HTTP/1.1" 200 191467 "" "Prometheus/2.33.4" 2023-09-23T22:29:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[110082]: cluster 2023-09-23T22:29:37.156467+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:29:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[110082]: audit 2023-09-23T22:29:37.159608+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.52:0/636348626' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:29:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[110082]: audit 2023-09-23T22:29:37.160127+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:29:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:29:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[110082]: :29:37.432327+0000 mgr.y (mgr.24314) 99 : cluster [DBG] pgmap v79: 105 pgs: 32 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: cluster 2023-09-23T22:29:37.156467+0000 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: audit 2023-09-23T22:29:37.159608+0000 mon.c (mon. 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: 2) 27 : audit [INF] from='client.? 172.21.15.52:0/636348626' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: audit 2023-09-23T22:29:37.160127+0000 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:29:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: 2023-09-23T22:29:38.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:29:38.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: 09-23T22:29:37.432327+0000 mgr.y (mgr 2023-09-23T22:29:38.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:38 smithi052 conmon[100224]: .24314) 99 : cluster [DBG] pgmap v79: 105 pgs: 32 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:38.495 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-09-23T22:29:38.495 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T22:29:38.499 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-09-23T22:29:38.499 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T22:29:38.516 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.protection_status (18 ms) 2023-09-23T22:29:38.517 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T22:29:38.524 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-09-23T22:29:38.524 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T22:29:38.573 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.parents_v1 (49 ms) 2023-09-23T22:29:38.573 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T22:29:38.604 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2023-09-23T22:29:38.604 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T22:29:38.625 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-09-23T22:29:38.625 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T22:29:38.630 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-09-23T22:29:38.630 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T22:29:38.639 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-09-23T22:29:38.639 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T22:29:38.641 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-09-23T22:29:38.641 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T22:29:38.646 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2023-09-23T22:29:38.646 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T22:29:38.650 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-09-23T22:29:38.650 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T22:29:38.651 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T22:29:38.651 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T22:29:38.654 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-09-23T22:29:38.654 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T22:29:38.659 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2023-09-23T22:29:38.660 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T22:29:38.666 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.flags (6 ms) 2023-09-23T22:29:38.666 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T22:29:38.692 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.metadata (26 ms) 2023-09-23T22:29:38.692 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T22:29:38.699 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-09-23T22:29:38.699 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T22:29:38.734 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2023-09-23T22:29:38.734 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T22:29:38.749 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-09-23T22:29:38.750 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T22:29:39.034 INFO:teuthology.orchestra.run.smithi052.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:39.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: audit 2023-09-23T22:29:38.161251+0000 mon.a (mon.0) 739 : audit [INF] 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: cluster 2023-09-23T22:29:38.161400+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: audit 2023-09-23T22:29:39.034163 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: +0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: audit 2023-09-23T22:29:39.042152+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: audit 2023-09-23T22:29:39.043999 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: +0000 mon.a (mon.0) 743 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: audit 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: 2023-09-23T22:29: 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: 39.044839+0000 mon.a (mon.0) 744 : audit 2023-09-23T22:29:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:39.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:29:39.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: 29:39.050444+0000 mon.a (mon.0) 745 2023-09-23T22:29:39.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:39 smithi151 conmon[100454]: : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: audit 2023-09-23T22:29:38.161251+0000 mon.a (mon.0) 739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T22:29:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: cluster 2023-09-23T22:29:38.161400+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:29:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: audit 2023-09-23T22:29:39.034163+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: audit 2023-09-23T22:29:39.042152+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: audit 2023-09-23T22:29:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: 2023-09-23T22:29:39.043999+0000 mon.a (mon.0) 743 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: audit 2023-09-23T22:29:39.044839+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[110082]: audit 2023-09-23T22:29:39.050444+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: audit 2023-09-23T22:29:38.161251+0000 mon.a (mon.0) 739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-164382-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: cluster 2023-09-23T22:29:38.161400+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: audit 2023-09-23T22:29:39.034163+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: audit 2023-09-23T22:29:39.042152+0000 mon.a (mon.0) 742 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: audit 2023-09-23T22:29:39.043999+0000 mon.a (mon.0) 743 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: audit 2023-09-23T22:29:39.044839+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:39 smithi052 conmon[100224]: audit 2023-09-23T22:29:39.050444+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:29:39.928 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:29:40.165 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:29:40.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[100224]: 29:39.028818+0000 mgr.y (mgr.24314) 100 : audit [DBG] from='client.24680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:29:40.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[100224]: cephadm 2023-09-23T22:29:39.029657+0000 mgr.y (mgr.24314) 101 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:40.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[100224]: cephadm 2023-09-23T22:29:39.056235+0000 mgr.y (mgr.24314) 102 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:40.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[100224]: cluster 2023-09-23T22:29:39.161293+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:29:40.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[100224]: cluster 2023-09-23T22:29:39.432693+0000 mgr.y (mgr.24314) 103 : cluster [DBG] pgmap v82: 105 pgs: 32 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: audit 2023-09-23T22:29:39 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: .028818+0000 mgr.y (mgr.24314) 100 : audit [DBG] from='client.24680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: cephadm 2023- 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: 09-23T22:29:39.029657+0000 mgr.y (mgr.24314) 101 : cephadm [INF] 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: cephadm 2023-09-23T22:29:40.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: 2023-09-23T22:29:39.056235+0000 mgr.y ( 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: mgr.24314) 102 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: 23T22:29:39.161293+0000 mon.a (mon 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: .0) 746 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: T22:29:39.432693+0000 mgr.y (mgr. 2023-09-23T22:29:40.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:40 smithi052 conmon[110082]: 24314) 103 : cluster [DBG] pgmap v82: 105 pgs: 32 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:40.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: audit 2023-09-23T22:29:39.028818+0000 mgr.y (mgr. 2023-09-23T22:29:40.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: 24314) 100 : audit [DBG] from='client.24680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: cephadm 2023-09-23T22:29:39.029657+0000 mgr.y (mgr.24314) 101 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: cephadm 2023-09 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: -23T22:29:39.056235+0000 mgr.y (mgr.24314) 102 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: cluster 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: 2023-09-23T22:29:39 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: .161293+0000 mon.a (mon.0) 746 : 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: cluster 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: 2023-09-23T22:29: 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: 39.432693+0000 mgr.y (mgr.24314) 2023-09-23T22:29:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:40 smithi151 conmon[100454]: 103 : cluster [DBG] pgmap v82: 105 pgs: 32 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:41.422 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_image_status (2672 ms) 2023-09-23T22:29:41.422 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T22:29:41.503 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_image_map (82 ms) 2023-09-23T22:29:41.503 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T22:29:41.512 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-09-23T22:29:41.512 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T22:29:41.525 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_snapshot (13 ms) 2023-09-23T22:29:41.525 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T22:29:41.527 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-09-23T22:29:41.527 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T22:29:41.529 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T22:29:41.529 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T22:29:41.532 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T22:29:41.532 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T22:29:41.539 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-09-23T22:29:41.539 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T22:29:41.542 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-09-23T22:29:41.542 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T22:29:41.551 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T22:29:41.551 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T22:29:41.551 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T22:29:41.551 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T22:29:41.552 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T22:29:41.552 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T22:29:41.555 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-09-23T22:29:41.555 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T22:29:41.560 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-09-23T22:29:41.560 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T22:29:41.562 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-09-23T22:29:41.562 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T22:29:41.566 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-09-23T22:29:41.566 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T22:29:41.568 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T22:29:41.568 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T22:29:41.570 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T22:29:41.570 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T22:29:41.572 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T22:29:41.572 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T22:29:41.575 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T22:29:41.575 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T22:29:41.578 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2023-09-23T22:29:41.578 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T22:29:41.580 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-09-23T22:29:41.580 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T22:29:41.583 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T22:29:41.584 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T22:29:41.601 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2023-09-23T22:29:41.602 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T22:29:41.803 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (201 ms) 2023-09-23T22:29:41.803 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T22:29:41.806 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T22:29:41.806 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T22:29:41.809 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T22:29:41.809 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T22:29:41.816 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-09-23T22:29:41.817 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T22:29:41.825 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T22:29:41.825 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T22:29:41.848 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.clone_parent (23 ms) 2023-09-23T22:29:41.848 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T22:29:41.856 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-09-23T22:29:41.856 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T22:29:41.869 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-09-23T22:29:41.869 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T22:29:41.876 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-23T22:29:41.876 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T22:29:41.887 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.migration (11 ms) 2023-09-23T22:29:41.887 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T22:29:41.897 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2023-09-23T22:29:41.897 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T22:29:42.374 INFO:teuthology.orchestra.run.smithi052.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:29:42.374 INFO:teuthology.orchestra.run.smithi052.stdout:alertmanager.a smithi052 *:9093,9094 running (64s) 46s ago 98s 14.6M - 0.23.0 ba2b418f427c f6ec125d25a1 2023-09-23T22:29:42.374 INFO:teuthology.orchestra.run.smithi052.stdout:grafana.a smithi151 *:3000 running (78s) 44s ago 78s 50.3M - 8.3.5 dad864ee21e9 6cfce387f58f 2023-09-23T22:29:42.374 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi052.thpzxu smithi052 running (53s) 46s ago 52s 17.4M - 16.2.14-37-g2e1f1015 1e6a756174a8 d8befa844e73 2023-09-23T22:29:42.374 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi151.rbxgii smithi151 running (50s) 44s ago 50s 13.4M - 16.2.14-37-g2e1f1015 1e6a756174a8 61e8f44cb3b6 2023-09-23T22:29:42.374 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.x smithi151 *:8443 running (5m) 44s ago 5m 392M - 16.2.14-37-g2e1f1015 1e6a756174a8 14a110bae095 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.y smithi052 *:9283 running (7m) 46s ago 7m 446M - 16.2.14-37-g2e1f1015 1e6a756174a8 cf5f0eb823e6 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a smithi052 running (7m) 46s ago 7m 103M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 da0d22fd61de 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b smithi151 running (5m) 44s ago 5m 104M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 4a3cec32371a 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c smithi052 running (5m) 46s ago 5m 102M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 58c863a9b169 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.a smithi052 *:9100 running (112s) 46s ago 112s 11.5M - 1.3.1 1dbe0e931976 319ab89d6824 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.b smithi151 *:9100 running (108s) 44s ago 108s 15.8M - 1.3.1 1dbe0e931976 ab706ae51a5d 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 smithi052 running (5m) 46s ago 5m 46.1M 4037M 16.2.14-37-g2e1f1015 1e6a756174a8 667e33a7d512 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 smithi052 running (4m) 46s ago 4m 49.7M 4037M 16.2.14-37-g2e1f1015 1e6a756174a8 cb0928f7abc9 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 smithi052 running (4m) 46s ago 4m 46.2M 4037M 16.2.14-37-g2e1f1015 1e6a756174a8 6d0289679661 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 smithi052 running (4m) 46s ago 4m 46.6M 4037M 16.2.14-37-g2e1f1015 1e6a756174a8 1744ec2d869b 2023-09-23T22:29:42.375 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 smithi151 running (3m) 44s ago 3m 47.3M 4292M 16.2.14-37-g2e1f1015 1e6a756174a8 b2126e632b7b 2023-09-23T22:29:42.376 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 smithi151 running (3m) 44s ago 3m 47.1M 4292M 16.2.14-37-g2e1f1015 1e6a756174a8 46585800d682 2023-09-23T22:29:42.376 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 smithi151 running (2m) 44s ago 2m 44.9M 4292M 16.2.14-37-g2e1f1015 1e6a756174a8 17e250c1eb43 2023-09-23T22:29:42.376 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 smithi151 running (2m) 44s ago 2m 48.2M 4292M 16.2.14-37-g2e1f1015 1e6a756174a8 73447cd68609 2023-09-23T22:29:42.376 INFO:teuthology.orchestra.run.smithi052.stdout:prometheus.a smithi151 *:9095 running (60s) 44s ago 117s 43.3M - 2.33.4 514e6a882f6e 73d582b307e4 2023-09-23T22:29:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:42 smithi052 conmon[100224]: cluster 2023-09-23T22:29:41.433348+0000 mgr.y ( 2023-09-23T22:29:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:42 smithi052 conmon[100224]: mgr.24314) 104 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.06k op/s 2023-09-23T22:29:42.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:42 smithi052 conmon[110082]: cluster 2023-09-23T22:29:41 2023-09-23T22:29:42.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:42 smithi052 conmon[110082]: .433348+0000 mgr.y (mgr.24314) 104 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.06k op/s 2023-09-23T22:29:42.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:42 smithi151 conmon[100454]: cluster 2023-09-23T22:29:41.433348+0000 mgr.y (mgr 2023-09-23T22:29:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:42 smithi151 conmon[100454]: .24314) 104 : cluster [DBG] pgmap v83: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.06k op/s 2023-09-23T22:29:43.352 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T22:29:43.502 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1605 ms) 2023-09-23T22:29:43.503 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T22:29:43.537 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.sparsify (35 ms) 2023-09-23T22:29:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:43 smithi052 conmon[110082]: audit 2023-09-23T22:29:42.357241+0000 mgr.y (mgr.24314) 105 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:29:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:43 smithi052 conmon[110082]: cluster 2023-09-23T22:29:42.488857+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:29:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:43 smithi052 conmon[100224]: audit 2023-09-23T22:29:42.357241+0000 mgr.y (mgr.24314) 105 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:29:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:43 smithi052 conmon[100224]: cluster 2023-09-23T22:29:42.488857+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:29:43.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:43 smithi151 conmon[100454]: audit 2023-09-23T22:29:42.357241+0000 mgr.y (mgr.24314) 105 : 2023-09-23T22:29:43.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:43 smithi151 conmon[100454]: audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:29:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:43 smithi151 conmon[100454]: cluster 2023-09-23T22:29:42.488857+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:29:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:43 smithi151 conmon[100454]: 2023-09-23T22:29:44.512 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] 70 tests from TestClsRbd (5372 ms total) 2023-09-23T22:29:44.512 INFO:tasks.workunit.client.0.smithi052.stdout: 2023-09-23T22:29:44.513 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] Global test environment tear-down 2023-09-23T22:29:44.513 INFO:tasks.workunit.client.0.smithi052.stdout:[==========] 70 tests from 1 test suite ran. (8239 ms total) 2023-09-23T22:29:44.513 INFO:tasks.workunit.client.0.smithi052.stdout:[ PASSED ] 70 tests. 2023-09-23T22:29:44.515 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T22:29:44.515 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T22:29:44.547 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T22:29:44.547 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:29:44.846 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:44 smithi052 conmon[100224]: cluster 2023-09-23T22:29:43.433727+0000 2023-09-23T22:29:44.846 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:44 smithi052 conmon[100224]: mgr.y (mgr.24314) 106 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.06k op/s 2023-09-23T22:29:44.847 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:44 smithi052 conmon[100224]: cluster 2023-09-23T22:29:43.499261+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:29:44.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:44 smithi052 conmon[110082]: cluster 2023-09-23T22:29:43.433727+0000 mgr.y ( 2023-09-23T22:29:44.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:44 smithi052 conmon[110082]: mgr.24314) 106 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.06k op/s 2023-09-23T22:29:44.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:44 smithi052 conmon[110082]: cluster 2023-09-23T22:29:43.499261+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:29:44.848 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:44.848 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:29:44.880 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:29:44.880 DEBUG:teuthology.orchestra.run.smithi052:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:29:44.944 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:29:44.944 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T22:29:44.944 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T22:29:44.945 DEBUG:teuthology.parallel:result is None 2023-09-23T22:29:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:44 smithi151 conmon[100454]: cluster 2023-09-23T22:29:43.433727+0000 mgr.y (mgr.24314) 106 : cluster [DBG] pgmap v85: 105 pgs: 105 active+clean; 2.6 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.1 MiB/s wr, 1.06k op/s 2023-09-23T22:29:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:44 smithi151 conmon[100454]: cluster 2023-09-23T22:29:43. 2023-09-23T22:29:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:44 smithi151 conmon[100454]: 499261+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:29:45.269 INFO:teuthology.orchestra.run.smithi052.stdout:wait for minority of mons to upgrade 2023-09-23T22:29:45.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:29:45 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:29:45.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:45 smithi052 conmon[100224]: cluster 2023-09-23T22:29:44.504025+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:29:45.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:45 smithi052 conmon[110082]: cluster 2023-09-23T22:29:44.504025+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:29:45.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:45 smithi151 conmon[100454]: cluster 2023-09-23T22:29:44.504025+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:29:46.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T22:29:46.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:46 smithi052 conmon[110082]: cluster 2023-09-23T22:29:45.434249+0000 mgr.y (mgr.24314) 107 : cluster [DBG] 2023-09-23T22:29:46.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:46 smithi052 conmon[110082]: pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:46.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:46 smithi052 conmon[100224]: cluster 2023-09-23T22:29:45. 2023-09-23T22:29:46.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:46 smithi052 conmon[100224]: 434249+0000 mgr.y (mgr.24314) 107 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:46.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:46 smithi151 conmon[100454]: cluster 2023-09-23T22:29:45.434249+0000 mgr.y (mgr.24314) 107 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:48.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:48] "GET /metrics HTTP/1.1" 200 192137 "" "Prometheus/2.33.4" 2023-09-23T22:29:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:48 smithi151 conmon[100454]: cluster 2023-09-23T22:29:47.434572+0000 mgr.y (mgr.24314) 108 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:48 smithi151 conmon[100454]: audit 2023-09-23T22:29:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:48 smithi151 conmon[100454]: 2023-09-23T22:29:48.419222+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.52:0/2488500212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[110082]: cluster 2023-09-23T22:29 2023-09-23T22:29:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[110082]: :47.434572+0000 mgr.y (mgr.24314) 108 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[110082]: audit 2023-09-23T22:29:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[110082]: 2023-09-23T22:29:48.419222+0000 2023-09-23T22:29:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[110082]: mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.52:0/2488500212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[110082]: 2023-09-23T22:29:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[100224]: cluster 2023-09-23T22:29:47.434572 2023-09-23T22:29:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 108 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:48.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[100224]: audit 2023-09-23T22:29:48. 2023-09-23T22:29:48.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:48 smithi052 conmon[100224]: 419222+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.52:0/2488500212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:50.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:50 smithi151 conmon[100454]: cluster 2023-09-23T22:29:49.434885+0000 mgr.y (mgr.24314) 109 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:50 smithi052 conmon[110082]: cluster 2023-09-23T22:29:49.434885+0000 2023-09-23T22:29:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:50 smithi052 conmon[110082]: mgr.y (mgr.24314) 109 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:50 smithi052 conmon[100224]: cluster 2023-09-23T22:29:49.434885+0000 mgr.y (mgr 2023-09-23T22:29:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:50 smithi052 conmon[100224]: .24314) 109 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:51.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:51 smithi151 conmon[100454]: audit 2023-09-23T22:29:50.917173+0000 mon.a (mon.0) 750 : audit [DBG] from='client.? 172.21.15.52:0/3894710530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:51.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:51 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:29:51.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:51 smithi052 conmon[110082]: :29:50.917173+0000 mon.a (mon.0) 750 : audit [DBG] from='client.? 172.21.15.52:0/3894710530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:51 smithi052 conmon[100224]: audit 2023-09-23T22:29:50.917173+0000 mon.a (mon.0) 2023-09-23T22:29:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:51 smithi052 conmon[100224]: 750 : audit [DBG] from='client.? 172.21.15.52:0/3894710530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:52.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:52 smithi151 conmon[100454]: cluster 2023-09-23T22:29:51.435528+0000 mgr.y (mgr.24314) 110 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:52 smithi052 conmon[110082]: cluster 2023-09-23T22:29:51.435528+0000 mgr.y ( 2023-09-23T22:29:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:52 smithi052 conmon[110082]: mgr.24314) 110 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:52.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:52 smithi052 conmon[100224]: cluster 2023-09-23T22:29:51 2023-09-23T22:29:52.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:52 smithi052 conmon[100224]: .435528+0000 mgr.y (mgr.24314) 110 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:53.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:53 smithi151 conmon[100454]: audit 2023-09-23T22:29:53.401384+0000 mon.a (mon.0) 751 : audit [DBG] from='client.? 172.21.15.52:0/2849396219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:53 smithi052 conmon[110082]: audit 2023-09-23T22:29:53.401384+0000 mon.a (mon.0) 751 : audit [DBG] from='client.? 172.21.15.52:0/2849396219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:53.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:53 smithi052 conmon[100224]: audit 2023-09-23T22:29:53.401384+0000 mon.a (mon.0) 751 : audit [DBG] from='client.? 172.21.15.52:0/2849396219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:54.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:54 smithi151 conmon[100454]: cluster 2023-09-23T22:29:53.435880+0000 mgr.y (mgr.24314) 111 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:54.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:54 smithi052 conmon[110082]: cluster 2023-09-23T22:29:53. 2023-09-23T22:29:54.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:54 smithi052 conmon[110082]: 435880+0000 mgr.y (mgr.24314) 111 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:54.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:54 smithi052 conmon[100224]: cluster 2023-09-23T22:29:53.435880+0000 2023-09-23T22:29:54.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:54 smithi052 conmon[100224]: mgr.y (mgr.24314) 111 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:55.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:29:55 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:29:56.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:56 smithi151 conmon[100454]: cluster 2023-09-23T22:29:55.436387+0000 mgr.y (mgr.24314) 112 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:56 smithi151 conmon[100454]: audit 2023-09-23T22:29:55.881024+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.52:0/128192012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[110082]: cluster 2023-09-23T22:29:55. 2023-09-23T22:29:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[110082]: 436387+0000 mgr.y (mgr.24314) 112 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:29:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[110082]: 22:29:55.881024+0000 mon.a (mon.0) 752 : audit [DBG] 2023-09-23T22:29:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/128192012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[100224]: cluster 2023-09-23T22:29:55 2023-09-23T22:29:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[100224]: .436387+0000 mgr.y (mgr.24314) 112 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:29:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:56 smithi052 conmon[100224]: 22:29:55.881024+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.52:0/128192012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:58.490 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:29:58] "GET /metrics HTTP/1.1" 200 192137 "" "Prometheus/2.33.4" 2023-09-23T22:29:58.490 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[110082]: cluster 2023-09-23T22:29:57.436715 2023-09-23T22:29:58.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 113 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:58.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[110082]: audit 2023-09-23T22:29:58.288022+0000 mon.c (mon.2) 29 : audit 2023-09-23T22:29:58.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/461313084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:58.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[100224]: cluster 2023-09-23T22:29:57.436715+0000 mgr.y (mgr.24314) 113 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:58.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:29:58 smithi052 conmon[100224]: audit 2023-09-23T22:29:58.288022+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.52:0/461313084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:29:58.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:58 smithi151 conmon[100454]: cluster 2023-09-23T22:29:57.436715+0000 mgr.y (mgr.24314) 113 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:29:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:29:58 smithi151 conmon[100454]: audit 2023-09-23T22:29:58.288022+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.52:0/461313084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[110082]: cluster 2023-09-23T22:29:59. 2023-09-23T22:30:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[110082]: 437636+0000 mgr.y (mgr.24314) 114 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:30:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[110082]: 23T22:30:00.000119+0000 mon.a (mon 2023-09-23T22:30:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[110082]: .0) 753 : cluster [INF] overall HEALTH_OK 2023-09-23T22:30:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[100224]: cluster 2023-09-23T22:29:59.437636 2023-09-23T22:30:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 114 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[100224]: cluster 2023-09-23T22:30:00.000119+0000 mon.a (mon.0) 2023-09-23T22:30:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:00 smithi052 conmon[100224]: 753 : cluster [INF] overall HEALTH_OK 2023-09-23T22:30:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:00 smithi151 conmon[100454]: cluster 2023-09-23T22:29:59.437636+0000 mgr.y (mgr.24314) 114 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:00 smithi151 conmon[100454]: cluster 2023-09-23T22:30:00.000119+0000 mon.a (mon.0) 753 : cluster [INF] overall HEALTH_OK 2023-09-23T22:30:01.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:01 smithi151 conmon[100454]: audit 2023-09-23T22:30:00.705673+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.52:0/3472723575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:01.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:01 smithi052 conmon[100224]: audit 2023-09-23T22:30:00.705673+0000 mon.c (mon. 2023-09-23T22:30:01.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:01 smithi052 conmon[100224]: 2) 30 : audit [DBG] from='client.? 172.21.15.52:0/3472723575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:01.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:01 smithi052 conmon[110082]: audit 2023-09-23T22:30:00.705673+0000 mon.c 2023-09-23T22:30:01.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:01 smithi052 conmon[110082]: (mon.2) 30 : audit [DBG] from='client.? 172.21.15.52:0/3472723575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:02 smithi151 conmon[100454]: cluster 2023-09-23T22:30:01.438235+0000 mgr.y (mgr.24314) 115 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:02 smithi052 conmon[100224]: cluster 2023-09-23T22:30:01.438235 2023-09-23T22:30:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:02 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 115 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:02 smithi052 conmon[110082]: cluster 2023-09-23T22:30:01. 2023-09-23T22:30:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:02 smithi052 conmon[110082]: 438235+0000 mgr.y (mgr.24314) 115 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:03.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:03 smithi151 conmon[100454]: audit 2023-09-23T22:30:03.101369+0000 mon.a (mon.0) 754 : audit [DBG] 2023-09-23T22:30:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:03 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/3606520817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:03 smithi052 conmon[110082]: audit 2023-09-23T22:30:03.101369+0000 2023-09-23T22:30:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:03 smithi052 conmon[110082]: mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.52:0/3606520817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:03 smithi052 conmon[100224]: audit 2023-09-23T22:30:03 2023-09-23T22:30:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:03 smithi052 conmon[100224]: .101369+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.52:0/3606520817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:04 smithi151 conmon[100454]: cluster 2023-09-23T22:30:03.438606+0000 mgr.y (mgr. 2023-09-23T22:30:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:04 smithi151 conmon[100454]: 24314) 116 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:04 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:30:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:04 smithi052 conmon[110082]: 30:03.438606+0000 mgr.y (mgr.24314) 116 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:04 smithi052 conmon[100224]: cluster 2023-09-23T22:30:03.438606+0000 2023-09-23T22:30:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:04 smithi052 conmon[100224]: mgr.y (mgr.24314) 116 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:05.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:05 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:30:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:06 smithi151 conmon[100454]: cluster 2023-09-23T22:30:05.439038+0000 mgr.y (mgr.24314) 117 : cluster [DBG] 2023-09-23T22:30:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:06 smithi151 conmon[100454]: pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:06 smithi151 conmon[100454]: audit 2023-09-23T22:30:05.480001+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.52:0/1382073035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:06.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:06 smithi052 conmon[100224]: cluster 2023-09-23T22:30:05.439038 2023-09-23T22:30:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:06 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 117 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:06 smithi052 conmon[100224]: audit 2023-09-23T22:30:05.480001+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.52:0/1382073035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:06 smithi052 conmon[110082]: cluster 2023-09-23T22:30:05.439038+0000 mgr.y (mgr.24314) 2023-09-23T22:30:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:06 smithi052 conmon[110082]: 117 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:06 smithi052 conmon[110082]: audit 2023-09-23T22:30:05.480001+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.52:0/1382073035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:08.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:08] "GET /metrics HTTP/1.1" 200 192257 "" "Prometheus/2.33.4" 2023-09-23T22:30:08.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[100224]: cluster 2023-09-23T22:30:07.439496+0000 mgr.y (mgr.24314) 118 2023-09-23T22:30:08.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[100224]: : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:08.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[100224]: audit 2023-09-23T22:30:07.853898+0000 mon.a (mon.0) 756 : audit [DBG] from='client.? 172.21.15.52:0/4071995124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:08.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[110082]: cluster 2023-09-23T22:30:07 2023-09-23T22:30:08.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[110082]: .439496+0000 mgr.y (mgr.24314) 118 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:08.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:30:08.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[110082]: 09-23T22:30:07. 2023-09-23T22:30:08.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[110082]: 853898+0000 mon.a (mon.0) 2023-09-23T22:30:08.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:08 smithi052 conmon[110082]: 756 : audit [DBG] from='client.? 172.21.15.52:0/4071995124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:08 smithi151 conmon[100454]: cluster 2023-09-23T22:30:07.439496+0000 mgr.y (mgr.24314) 118 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:08 smithi151 conmon[100454]: 2023-09-23T22:30:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:08 smithi151 conmon[100454]: audit 2023-09-23T22:30:07.853898+0000 mon.a (mon.0) 756 : audit [DBG] from='client.? 172.21.15.52:0/4071995124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:10.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:10 smithi151 conmon[100454]: cluster 2023-09-23T22:30:09.439884+0000 mgr.y (mgr.24314) 119 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:10 smithi151 conmon[100454]: 2023-09-23T22:30:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:10 smithi151 conmon[100454]: audit 2023-09-23T22:30:10.222363+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.52:0/2457750605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[110082]: cluster 2023-09-23T22:30:09.439884 2023-09-23T22:30:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 119 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:30:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[110082]: :30:10.222363+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.52:0/2457750605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[110082]: 2023-09-23T22:30:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[100224]: cluster 2023-09-23T22:30:09. 2023-09-23T22:30:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[100224]: 439884+0000 mgr.y (mgr.24314) 119 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[100224]: audit 2023-09-23T22:30: 2023-09-23T22:30:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[100224]: 10.222363+0000 mon.a (mon.0) 757 : audit [DBG] 2023-09-23T22:30:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:10 smithi052 conmon[100224]: from='client.? 172.21.15.52:0/2457750605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:12 smithi151 conmon[100454]: cluster 2023-09-23T22:30:11.440679+0000 mgr.y (mgr.24314) 120 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:12 smithi052 conmon[110082]: cluster 2023-09-23T22:30:11.440679 2023-09-23T22:30:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:12 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 120 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:12.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:12 smithi052 conmon[100224]: cluster 2023-09-23T22:30:11.440679+0000 mgr.y (mgr 2023-09-23T22:30:12.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:12 smithi052 conmon[100224]: .24314) 120 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:13.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:13 smithi151 conmon[100454]: audit 2023-09-23T22:30:12.591658+0000 mon.c (mon.2) 31 : audit [DBG] 2023-09-23T22:30:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:13 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/1565779415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:13.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:13 smithi052 conmon[110082]: audit 2023-09-23T22:30: 2023-09-23T22:30:13.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:13 smithi052 conmon[110082]: 12.591658+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.52:0/1565779415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:13.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:13 smithi052 conmon[100224]: audit 2023-09-23T22:30:12.591658+0000 mon.c (mon.2) 31 : 2023-09-23T22:30:13.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:13 smithi052 conmon[100224]: audit [DBG] from='client.? 172.21.15.52:0/1565779415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:14.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:14 smithi151 conmon[100454]: cluster 2023-09-23T22:30:13.441108+0000 mgr.y (mgr.24314) 121 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:14.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:14 smithi151 conmon[100454]: 2023-09-23T22:30:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:14 smithi052 conmon[110082]: cluster 2023-09-23T22:30:13.441108+0000 mgr.y (mgr.24314) 121 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:14.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:14 smithi052 conmon[100224]: cluster 2023-09-23T22:30:13.441108+0000 mgr.y (mgr.24314) 121 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:15.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:30:15.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: audit 2023-09-23T22:30:14. 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 528377+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: cephadm 2023 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: -09-23T22:30:14.528793+0000 mgr.y (mgr.24314) 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 122 : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: cephadm 2023 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: -09-23T22:30:14. 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 528881+0000 mgr.y (mgr.24314) 123 : cephadm [INF] 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:30:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 09-23T22:30:14 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: .529849+0000 mon.a (mon. 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 0) 759 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 09-23T22:30:14.534613 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: +0000 mon.a (mon.0) 760 : audit 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: cephadm 2023 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: -09-23T22:30: 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 14.535329+0000 mgr.y (mgr.24314 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: ) 124 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:30:15.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: cephadm 2023 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: -09-23T22:30:14 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: .834999+0000 mgr.y (mgr.24314 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: ) 125 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi151 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: audit 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: 2023-09-23T22:30 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: :14.951987+0000 mon.a (mon 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: .0) 761 : audit [DBG] 2023-09-23T22:30:15.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:15 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/2420580823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:15.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: audit 2023-09-23T22:30:14.528377+0000 mon.a (mon.0 2023-09-23T22:30:15.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: ) 758 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:15.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:30:14.528793+0000 mgr.y (mgr.24314) 122 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: : cephadm [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:30:14.528881+0000 mgr.y 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: (mgr.24314) 123 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: T22:30:14.529849+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: audit 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: 2023-09-23T22:30:14.534613+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:30:14.535329+0000 mgr.y ( 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: mgr.24314) 124 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:30:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: 2023-09-23T22:30:14.834999+0000 mgr.y (mgr.24314) 125 : cephadm [INF] 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi151 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[100224]: 30:14.951987+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.52:0/2420580823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: audit 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: 2023-09-23T22:30:14.528377+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: cephadm 2023-09-23T22 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: :30:14.528793+0000 mgr.y (mgr.24314) 122 : cephadm 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: [INF] Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: cephadm 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: 2023-09-23T22:30:14.528881+0000 mgr.y (mgr.24314) 123 : cephadm 2023-09-23T22:30:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: audit 2023-09-23T22:30 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: :14.529849+0000 mon.a (mon.0 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: ) 759 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: audit 2023-09-23T22:30:14.534613+0000 mon.a 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: (mon.0) 760 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: cephadm 2023-09-23T 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: 22:30:14.535329+0000 mgr.y (mgr.24314) 124 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: cephadm 2023-09 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: -23T22:30:14.834999+0000 mgr.y (mgr.24314) 125 : 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi151 2023-09-23T22:30:15.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:30:15.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: 23T22:30:14.951987+0000 mon.a (mon.0) 761 : audit 2023-09-23T22:30:15.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:15 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/2420580823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:16.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:16 smithi151 conmon[100454]: cluster 2023-09-23T22:30:15.441673+0000 mgr.y (mgr.24314) 126 2023-09-23T22:30:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:16 smithi151 conmon[100454]: : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:16 smithi052 conmon[110082]: cluster 2023-09-23T22:30:15.441673+0000 mgr.y (mgr 2023-09-23T22:30:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:16 smithi052 conmon[110082]: .24314) 126 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:16.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:16 smithi052 conmon[100224]: cluster 2023-09-23T22:30:15.441673 2023-09-23T22:30:16.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:16 smithi052 conmon[100224]: +0000 mgr.y (mgr.24314) 126 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:17.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:17 smithi151 conmon[100454]: audit 2023-09-23T22:30:17.320379 2023-09-23T22:30:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:17 smithi151 conmon[100454]: +0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.52:0/3166481927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:17 smithi052 conmon[110082]: audit 2023-09-23T22:30:17.320379+0000 mon.c ( 2023-09-23T22:30:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:17 smithi052 conmon[110082]: mon.2) 32 : audit [DBG] from='client.? 172.21.15.52:0/3166481927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:17.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:17 smithi052 conmon[100224]: audit 2023-09-23T22:30:17.320379+0000 mon.c 2023-09-23T22:30:17.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:17 smithi052 conmon[100224]: (mon.2) 32 : audit [DBG] from='client.? 172.21.15.52:0/3166481927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:18.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:18 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:18] "GET /metrics HTTP/1.1" 200 192255 "" "Prometheus/2.33.4" 2023-09-23T22:30:18.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:18 smithi151 conmon[100454]: cluster 2023-09-23T22:30:17.442144+0000 mgr.y (mgr.24314) 127 2023-09-23T22:30:18.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:18 smithi151 conmon[100454]: : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:18 smithi052 conmon[110082]: cluster 2023-09-23T22:30:17.442144+0000 mgr.y (mgr.24314) 127 : 2023-09-23T22:30:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:18 smithi052 conmon[110082]: cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:18 smithi052 conmon[100224]: cluster 2023-09-23T22:30:17.442144+0000 mgr.y (mgr.24314) 127 : cluster 2023-09-23T22:30:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:18 smithi052 conmon[100224]: [DBG] pgmap v104: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:20 smithi151 conmon[100454]: cluster 2023-09-23T22:30:19.442439+0000 mgr.y (mgr.24314) 128 : 2023-09-23T22:30:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:20 smithi151 conmon[100454]: cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:20 smithi151 conmon[100454]: audit 2023-09-23T22:30:19.689083+0000 mon.a (mon.0) 762 : audit [DBG] from='client.? 172.21.15.52:0/1871010870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[110082]: cluster 2023-09-23T22:30:19.442439+0000 mgr.y (mgr.24314) 128 : cluster 2023-09-23T22:30:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[110082]: [DBG] pgmap v105: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[110082]: audit 2023-09-23T22:30:19.689083+0000 mon.a (mon.0) 762 : audit [DBG] from='client.? 172.21.15.52:0/1871010870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[100224]: cluster 2023-09-23T22:30:19.442439+0000 mgr.y 2023-09-23T22:30:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[100224]: (mgr.24314) 128 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[100224]: audit 2023-09-23T22:30:19.689083+0000 mon.a (mon.0) 2023-09-23T22:30:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:20 smithi052 conmon[100224]: 762 : audit [DBG] from='client.? 172.21.15.52:0/1871010870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:22.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:22 smithi151 conmon[100454]: cluster 2023-09-23T22:30:21.443077+0000 mgr.y (mgr.24314) 129 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:22 smithi151 conmon[100454]: audit 2023-09-23T22:30:22.057465+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.52:0/392835667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[110082]: cluster 2023-09-23T22:30:21.443077+0000 mgr.y (mgr.24314) 129 : cluster 2023-09-23T22:30:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[110082]: [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[110082]: audit 2023-09-23T22:30:22.057465+0000 mon.c (mon.2) 33 : 2023-09-23T22:30:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[110082]: audit [DBG] from='client.? 172.21.15.52:0/392835667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[100224]: cluster 2023-09-23T22:30:21.443077+0000 2023-09-23T22:30:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[100224]: mgr.y (mgr.24314) 129 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:22 smithi052 conmon[100224]: audit 2023-09-23T22:30:22.057465+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.52:0/392835667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:24.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:24 smithi151 conmon[100454]: cluster 2023-09-23T22:30:23.443677+0000 mgr.y (mgr.24314) 2023-09-23T22:30:24.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:24 smithi151 conmon[100454]: 130 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:24 smithi151 conmon[100454]: audit 2023-09-23T22:30:24.427305+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.52:0/1921759974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[110082]: cluster 2023-09-23T22:30:23.443677+0000 mgr.y ( 2023-09-23T22:30:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[110082]: mgr.24314) 130 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[110082]: audit 2023-09-23T22:30:24.427305 2023-09-23T22:30:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[110082]: +0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.52:0/1921759974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[100224]: cluster 2023-09-23T22:30:23.443677+0000 2023-09-23T22:30:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[100224]: mgr.y (mgr.24314) 130 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:30:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:24 smithi052 conmon[100224]: 22:30:24.427305+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.52:0/1921759974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:25.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:25 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:30:26.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:26 smithi151 conmon[100454]: cluster 2023-09-23T22:30:25.444397+0000 mgr.y (mgr.24314) 2023-09-23T22:30:26.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:26 smithi151 conmon[100454]: 131 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:26.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:26 smithi052 conmon[110082]: cluster 2023-09-23T22:30:25.444397+0000 mgr.y (mgr.24314) 131 2023-09-23T22:30:26.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:26 smithi052 conmon[110082]: : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:26.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:26 smithi052 conmon[100224]: cluster 2023-09-23T22:30:25.444397+0000 2023-09-23T22:30:26.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:26 smithi052 conmon[100224]: mgr.y (mgr.24314) 131 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:27 smithi151 conmon[100454]: audit 2023-09-23T22:30:26.801521+0000 mon.a (mon.0 2023-09-23T22:30:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:27 smithi151 conmon[100454]: ) 764 : audit [DBG] from='client.? 172.21.15.52:0/4179874620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:27.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:27 smithi052 conmon[110082]: audit 2023-09-23T22:30:26.801521+0000 mon.a (mon.0) 764 : 2023-09-23T22:30:27.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:27 smithi052 conmon[110082]: audit [DBG] from='client.? 172.21.15.52:0/4179874620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:27.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:27 smithi052 conmon[100224]: audit 2023-09-23T22:30:26.801521+0000 mon.a ( 2023-09-23T22:30:27.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:27 smithi052 conmon[100224]: mon.0) 764 : audit [DBG] from='client.? 172.21.15.52:0/4179874620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:28.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:28 smithi151 conmon[100454]: cluster 2023-09-23T22:30:27.445023+0000 mgr.y (mgr.24314) 132 : cluster 2023-09-23T22:30:28.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:28 smithi151 conmon[100454]: [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:28.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:28 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:28] "GET /metrics HTTP/1.1" 200 192255 "" "Prometheus/2.33.4" 2023-09-23T22:30:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:28 smithi052 conmon[100224]: cluster 2023-09-23T22:30:27.445023+0000 mgr.y (mgr.24314) 2023-09-23T22:30:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:28 smithi052 conmon[100224]: 132 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:28 smithi052 conmon[110082]: cluster 2023-09-23T22:30:27.445023+0000 mgr.y (mgr.24314) 132 : 2023-09-23T22:30:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:28 smithi052 conmon[110082]: cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:29.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:29 smithi151 conmon[100454]: audit 2023-09-23T22:30:29.167090+0000 mon.a (mon.0) 2023-09-23T22:30:29.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:29 smithi151 conmon[100454]: 765 : audit [DBG] from='client.? 172.21.15.52:0/2036230710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:29.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:29 smithi052 conmon[100224]: audit 2023-09-23T22:30:29.167090+0000 mon.a 2023-09-23T22:30:29.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:29 smithi052 conmon[100224]: (mon.0) 765 : audit [DBG] from='client.? 172.21.15.52:0/2036230710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:29 smithi052 conmon[110082]: audit 2023-09-23T22:30:29.167090+0000 mon.a (mon. 2023-09-23T22:30:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:29 smithi052 conmon[110082]: 0) 765 : audit [DBG] from='client.? 172.21.15.52:0/2036230710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:30.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:30 smithi151 conmon[100454]: cluster 2023-09-23T22:30:29.445528+0000 mgr.y (mgr.24314 2023-09-23T22:30:30.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:30 smithi151 conmon[100454]: ) 133 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:30.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:30 smithi052 conmon[110082]: cluster 2023-09-23T22:30:29.445528+0000 mgr.y (mgr.24314) 133 2023-09-23T22:30:30.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:30 smithi052 conmon[110082]: : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:30.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:30 smithi052 conmon[100224]: cluster 2023-09-23T22:30:29.445528+0000 mgr.y (mgr.24314) 133 2023-09-23T22:30:30.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:30 smithi052 conmon[100224]: : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:32.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:32 smithi151 conmon[100454]: cluster 2023-09-23T22:30:31.446044+0000 mgr.y (mgr.24314 2023-09-23T22:30:32.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:32 smithi151 conmon[100454]: ) 134 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:32 smithi151 conmon[100454]: audit 2023-09-23T22:30:31.540010+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.52:0/112966897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[110082]: cluster 2023-09-23T22:30:31.446044+0000 mgr.y (mgr.24314) 134 2023-09-23T22:30:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[110082]: : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[110082]: audit 2023-09-23T22:30:31.540010+0000 mon.a (mon.0 2023-09-23T22:30:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[110082]: ) 766 : audit [DBG] from='client.? 172.21.15.52:0/112966897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[100224]: cluster 2023-09-23T22:30:31.446044+0000 2023-09-23T22:30:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[100224]: mgr.y (mgr.24314) 134 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:30:32.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:32 smithi052 conmon[100224]: 09-23T22:30:31.540010+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.52:0/112966897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:34 smithi151 conmon[100454]: cluster 2023-09-23T22:30:33.446389+0000 mgr.y (mgr.24314) 135 2023-09-23T22:30:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:34 smithi151 conmon[100454]: : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:34 smithi151 conmon[100454]: audit 2023-09-23T22:30:33.915102+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.52:0/1685041731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:34 smithi052 conmon[110082]: cluster 2023-09-23T22:30:33.446389+0000 2023-09-23T22:30:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:34 smithi052 conmon[110082]: mgr.y (mgr.24314) 135 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:34 smithi052 conmon[110082]: audit 2023-09-23T22:30:33 2023-09-23T22:30:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:34 smithi052 conmon[110082]: .915102+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.52:0/1685041731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:34 smithi052 conmon[100224]: cluster 2023-09-23T22:30:33.446389+0000 mgr.y (mgr.24314) 135 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:34 smithi052 conmon[100224]: audit 2023-09-23T22:30:33.915102+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.52:0/1685041731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:35.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:35 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:30:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:36 smithi151 conmon[100454]: cluster 2023-09-23T22:30:35.446977+0000 mgr.y (mgr.24314) 2023-09-23T22:30:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:36 smithi151 conmon[100454]: 136 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:36.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:36 smithi052 conmon[110082]: cluster 2023-09-23T22:30:35 2023-09-23T22:30:36.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:36 smithi052 conmon[110082]: .446977+0000 mgr.y (mgr.24314) 136 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:36.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:36 smithi052 conmon[100224]: cluster 2023-09-23T22:30:35.446977+0000 2023-09-23T22:30:36.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:36 smithi052 conmon[100224]: mgr.y (mgr.24314) 136 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:37.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:37 smithi151 conmon[100454]: audit 2023-09-23T22:30:36.288207+0000 mon.a (mon. 2023-09-23T22:30:37.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:37 smithi151 conmon[100454]: 0) 767 : audit [DBG] from='client.? 172.21.15.52:0/3280149698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:37 smithi052 conmon[110082]: audit 2023-09-23T22:30:36.288207+0000 mon.a ( 2023-09-23T22:30:37.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:37 smithi052 conmon[110082]: mon.0) 767 : audit [DBG] from='client.? 172.21.15.52:0/3280149698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:37 smithi052 conmon[100224]: audit 2023-09-23T22:30:36.288207+0000 mon.a (mon.0) 767 : 2023-09-23T22:30:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:37 smithi052 conmon[100224]: audit [DBG] from='client.? 172.21.15.52:0/3280149698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:38 smithi151 conmon[100454]: cluster 2023-09-23T22:30:37.447497+0000 mgr.y (mgr.24314) 137 : cluster [DBG] 2023-09-23T22:30:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:38 smithi151 conmon[100454]: pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:38.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:38 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:38] "GET /metrics HTTP/1.1" 200 192256 "" "Prometheus/2.33.4" 2023-09-23T22:30:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:38 smithi052 conmon[100224]: cluster 2023-09-23T22:30:37.447497+0000 mgr.y (mgr.24314 2023-09-23T22:30:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:38 smithi052 conmon[100224]: ) 137 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:38 smithi052 conmon[110082]: cluster 2023-09-23T22:30:37.447497+0000 2023-09-23T22:30:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:38 smithi052 conmon[110082]: mgr.y (mgr.24314) 137 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:39.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:39 smithi151 conmon[100454]: audit 2023-09-23T22:30:38.656585+0000 mon.c (mon.2) 35 : audit 2023-09-23T22:30:39.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:39 smithi151 conmon[100454]: [DBG] from='client.? 172.21.15.52:0/1701359503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:39 smithi052 conmon[110082]: audit 2023-09-23T22:30:38.656585+0000 mon.c (mon 2023-09-23T22:30:39.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:39 smithi052 conmon[110082]: .2) 35 : audit [DBG] from='client.? 172.21.15.52:0/1701359503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:39.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:39 smithi052 conmon[100224]: audit 2023-09-23T22:30:38.656585+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.52:0/1701359503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:40.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:40 smithi151 conmon[100454]: cluster 2023-09-23T22:30:39.447942+0000 mgr.y (mgr. 2023-09-23T22:30:40.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:40 smithi151 conmon[100454]: 24314) 138 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:40 smithi052 conmon[110082]: cluster 2023-09-23T22:30:39.447942+0000 mgr.y 2023-09-23T22:30:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:40 smithi052 conmon[110082]: (mgr.24314) 138 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:40.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:40 smithi052 conmon[100224]: cluster 2023-09-23T22:30:39.447942+0000 mgr.y (mgr.24314) 138 : cluster [DBG] 2023-09-23T22:30:40.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:40 smithi052 conmon[100224]: pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:41.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:41 smithi151 conmon[100454]: audit 2023-09-23T22:30:41.025349+0000 mon.a (mon.0) 768 : audit [DBG] 2023-09-23T22:30:41.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:41 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/2277367512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:41.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:41 smithi052 conmon[110082]: audit 2023-09-23T22:30:41.025349+0000 mon.a (mon.0) 768 : audit 2023-09-23T22:30:41.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:41 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/2277367512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:41 smithi052 conmon[100224]: audit 2023-09-23T22:30: 2023-09-23T22:30:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:41 smithi052 conmon[100224]: 41.025349+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.52:0/2277367512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:42.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:42 smithi151 conmon[100454]: cluster 2023-09-23T22:30:41.448566+0000 mgr.y (mgr.24314) 139 : cluster 2023-09-23T22:30:42.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:42 smithi151 conmon[100454]: [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:42.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:42 smithi052 conmon[110082]: cluster 2023-09-23T22:30:41.448566 2023-09-23T22:30:42.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:42 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:42 smithi052 conmon[100224]: cluster 2023-09-23T22:30:41.448566+0000 mgr.y (mgr. 2023-09-23T22:30:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:42 smithi052 conmon[100224]: 24314) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:43.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:43 smithi151 conmon[100454]: audit 2023-09-23T22:30:43.391441+0000 mon.c (mon.2) 2023-09-23T22:30:43.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:43 smithi151 conmon[100454]: 36 : audit [DBG] from='client.? 172.21.15.52:0/84979283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:43 smithi052 conmon[110082]: audit 2023-09-23T22:30:43.391441+0000 mon.c ( 2023-09-23T22:30:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:43 smithi052 conmon[110082]: mon.2) 36 : audit [DBG] from='client.? 172.21.15.52:0/84979283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:43 smithi052 conmon[100224]: audit 2023-09-23T22:30:43.391441+0000 mon.c (mon 2023-09-23T22:30:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:43 smithi052 conmon[100224]: .2) 36 : audit [DBG] from='client.? 172.21.15.52:0/84979283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:44 smithi151 conmon[100454]: cluster 2023-09-23T22:30:43.448910+0000 mgr.y (mgr.24314) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:44.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:44 smithi052 conmon[110082]: cluster 2023-09-23T22:30:43.448910+0000 mgr.y (mgr.24314) 140 : 2023-09-23T22:30:44.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:44 smithi052 conmon[110082]: cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:44.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:44 smithi052 conmon[100224]: cluster 2023-09-23T22:30:43 2023-09-23T22:30:44.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:44 smithi052 conmon[100224]: .448910+0000 mgr.y (mgr.24314) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:45.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:45 smithi151 conmon[102744]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:30:46.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:46 smithi151 conmon[100454]: cluster 2023-09-23T22:30:45.449457+0000 mgr.y (mgr.24314) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:46 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:30:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:46 smithi151 conmon[100454]: 30:45.757653+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.52:0/3563138272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:46.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[110082]: cluster 2023-09-23T22:30:45.449457+0000 mgr.y 2023-09-23T22:30:46.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[110082]: (mgr.24314) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:46.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:30:46.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[110082]: 30:45.757653+0000 mon.c (mon.2) 37 : audit [DBG] 2023-09-23T22:30:46.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/3563138272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[100224]: cluster 2023-09-23T22:30:45. 2023-09-23T22:30:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[100224]: 449457+0000 mgr.y (mgr.24314) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:30:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[100224]: -23T22:30:45.757653+0000 mon.c (mon.2) 37 2023-09-23T22:30:46.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:46 smithi052 conmon[100224]: : audit [DBG] from='client.? 172.21.15.52:0/3563138272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:48.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:48] "GET /metrics HTTP/1.1" 200 192256 "" "Prometheus/2.33.4" 2023-09-23T22:30:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:48 smithi151 conmon[100454]: cluster 2023-09-23T22:30:47.450039+0000 mgr.y (mgr.24314) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:48 smithi151 conmon[100454]: audit 2023-09-23T22:30:48.125043+0000 mon.a (mon.0) 769 : audit [DBG] from='client.? 172.21.15.52:0/1647138095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[110082]: cluster 2023-09-23T22:30:47. 2023-09-23T22:30:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[110082]: 450039+0000 mgr.y (mgr.24314) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:30:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[110082]: 09-23T22:30:48.125043+0000 mon.a ( 2023-09-23T22:30:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[110082]: mon.0) 769 : audit [DBG] from='client.? 172.21.15.52:0/1647138095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[100224]: cluster 2023-09-23T22:30: 2023-09-23T22:30:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[100224]: 47.450039+0000 mgr.y (mgr.24314) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:30:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[100224]: 23T22:30:48.125043+0000 mon.a (mon.0) 769 : audit 2023-09-23T22:30:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:48 smithi052 conmon[100224]: [DBG] from='client.? 172.21.15.52:0/1647138095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:50 smithi151 conmon[100454]: cluster 2023-09-23T22:30:49.450435+0000 mgr.y (mgr.24314) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:50 smithi151 conmon[100454]: audit 2023-09-23T22:30:50.493966+0000 2023-09-23T22:30:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:50 smithi151 conmon[100454]: mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.52:0/2511442166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[100224]: cluster 2023-09-23T22:30:49.450435+0000 mgr.y (mgr.24314) 143 : cluster 2023-09-23T22:30:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[100224]: [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[100224]: audit 2023-09-23T22:30:50.493966+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.52:0/2511442166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[110082]: cluster 2023-09-23T22:30:49.450435+0000 mgr.y (mgr 2023-09-23T22:30:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[110082]: .24314) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[110082]: audit 2023-09-23T22:30:50.493966+0000 mon.c (mon.2) 2023-09-23T22:30:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:50 smithi052 conmon[110082]: 38 : audit [DBG] from='client.? 172.21.15.52:0/2511442166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:52.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:52 smithi151 conmon[100454]: cluster 2023-09-23T22:30:51.451158+0000 mgr.y (mgr.24314) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:52 smithi052 conmon[110082]: cluster 2023-09-23T22:30:51.451158+0000 2023-09-23T22:30:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:52 smithi052 conmon[110082]: mgr.y (mgr.24314) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:52.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:52 smithi052 conmon[100224]: cluster 2023-09-23T22:30:51.451158+0000 mgr.y (mgr.24314) 144 : cluster [DBG] 2023-09-23T22:30:52.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:52 smithi052 conmon[100224]: pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:54.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: cephadm 2023-09-23T22:30:52.795393+0000 mgr.y (mgr.24314) 145 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T22:30:54.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: audit 2023-09-23T22:30:52.799937+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:54.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: audit 2023-09-23T22:30:52.800739+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:30:54.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: audit 2023-09-23T22:30:52.801535+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:30:54.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:30:54.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: T22:30:52.802318+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:54.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: cephadm 2023-09-23T22:30:52.802934+0000 mgr.y (mgr. 2023-09-23T22:30:54.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: 24314) 146 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:30:54.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: audit 2023-09-23T22:30:52 2023-09-23T22:30:54.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:53 smithi151 conmon[100454]: .865015+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.52:0/350094527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: cephadm 2023-09-23T22:30:52.795393+0000 mgr.y (mgr 2023-09-23T22:30:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: .24314) 145 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T22:30:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: audit 2023-09-23T22:30:52.799937+0000 mon.a (mon.0) 770 : audit 2023-09-23T22:30:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: audit 2023 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: -09-23T22:30:52.800739+0000 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: mon.a (mon.0) 771 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: audit 2023-09-23T22:30:52.801535+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: audit 2023-09-23T22:30:52.802318+0000 mon.a 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: (mon.0) 773 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: cephadm 2023-09-23T22:30:52.802934+0000 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: mgr.y (mgr.24314) 146 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: -23T22:30:52.865015+0000 mon.c (mon.2) 39 2023-09-23T22:30:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[110082]: : audit [DBG] from='client.? 172.21.15.52:0/350094527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: cephadm 2023-09-23T22:30:52.795393+0000 mgr.y (mgr.24314) 145 : 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: audit 2023-09-23T22:30:52.799937+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: audit 2023-09-23T22:30: 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: 52.800739+0000 mon.a (mon.0) 771 : 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: 09-23T22:30:52 2023-09-23T22:30:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: .801535+0000 mon.a (mon.0) 772 : audit 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: -23T22:30:52.802318 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: +0000 mon.a (mon.0) 773 : audit [DBG] 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: cephadm 2023-09- 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: 23T22:30:52.802934+0000 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: mgr.y (mgr.24314) 146 : cephadm [INF] 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: Deploying daemon mgr.x on smithi151 2023-09-23T22:30:54.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:30:54.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: T22:30:52.865015+0000 2023-09-23T22:30:54.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: mon.c (mon.2) 39 : audit [DBG] 2023-09-23T22:30:54.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:53 smithi052 conmon[100224]: from='client.? 172.21.15.52:0/350094527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:54.444 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:54 smithi151 systemd[1]: Stopping Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:30:54.445 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:54 smithi151 bash[137681]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-mgr.x found: no such container 2023-09-23T22:30:55.140 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:54 smithi151 conmon[100454]: cluster 2023-09-23T22:30:53.451554+0000 mgr.y (mgr.24314) 147 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:55.140 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:54 smithi151 bash[137798]: 14a110bae0953f543bca13c4341d725fb2fe4a825cce52651685c12ca9b275d9 2023-09-23T22:30:55.140 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:54 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:30:55.140 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T22:30:55.140 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 systemd[1]: Stopped Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:30:55.140 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 systemd[1]: Starting Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:30:55.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:54 smithi052 conmon[100224]: cluster 2023-09-23T22:30:53.451554+0000 2023-09-23T22:30:55.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:54 smithi052 conmon[100224]: mgr.y (mgr.24314) 147 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:54 smithi052 conmon[110082]: cluster 2023-09-23T22:30:53.451554 2023-09-23T22:30:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:54 smithi052 conmon[110082]: +0000 mgr.y (mgr.24314) 147 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:55.815 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 conmon[138201]: debug 2023-09-23T22:30:55.688+0000 7fd1bb39a700 1 -- 172.21.15.151:0/3251617048 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a61fb331e0 con 0x55a61fb0b000 2023-09-23T22:30:55.816 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 bash[138160]: b06bad11a511208244a8719701fe1f54990450e1dc369428d8664632df43052d 2023-09-23T22:30:55.816 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 systemd[1]: Started Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:30:56.119 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 conmon[138201]: debug 2023-09-23T22:30:55.814+0000 7fd1c54e7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:30:56.119 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:55 smithi151 conmon[138201]: debug 2023-09-23T22:30:55.897+0000 7fd1c54e7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:30:56.120 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[138201]: debug 2023-09-23T22:30:56.005+0000 7fd1c54e7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:30:56.120 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[138201]: debug 2023-09-23T22:30:56.119+0000 7fd1c54e7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:30:56.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[138201]: debug 2023-09-23T22:30:56.316+0000 7fd1c54e7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: audit 2023-09-23T22:30:55.232394+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.52:0/572485080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: cluster 2023-09-23T22:30:55.452033+0000 mgr.y (mgr.24314) 148 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: audit 2023-09-23T22:30 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: :55.744818+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: audit 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: 2023-09-23T22:30:55.745816+0000 mon.a (mon. 2023-09-23T22:30:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[100454]: 0) 775 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[110082]: audit 2023-09-23T22:30:55.232394+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.52:0/572485080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[110082]: cluster 2023-09-23T22:30:55.452033+0000 mgr.y (mgr.24314) 148 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[110082]: audit 2023-09-23T22:30:55.744818+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[110082]: audit 2023-09-23T22:30:55.745816+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[100224]: audit 2023-09-23T22:30 2023-09-23T22:30:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[100224]: :55.232394+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.52:0/572485080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[100224]: cluster 2023-09-23T22:30:55.452033+0000 mgr.y (mgr.24314) 148 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[100224]: audit 2023-09-23T22:30:55.744818+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:56 smithi052 conmon[100224]: audit 2023-09-23T22:30:55.745816+0000 mon.a (mon.0) 775 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:57.062 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[138201]: debug 2023-09-23T22:30:56.724+0000 7fd1c54e7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:30:57.062 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[138201]: debug 2023-09-23T22:30:56.865+0000 7fd1c54e7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:30:57.062 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:56 smithi151 conmon[138201]: debug 2023-09-23T22:30:56.960+0000 7fd1c54e7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:30:57.062 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:57 smithi151 conmon[138201]: debug 2023-09-23T22:30:57.061+0000 7fd1c54e7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:30:57.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:57 smithi151 conmon[138201]: debug 2023-09-23T22:30:57.551+0000 7fd1c54e7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:30:58.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:30:58] "GET /metrics HTTP/1.1" 200 192256 "" "Prometheus/2.33.4" 2023-09-23T22:30:58.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[138201]: debug 2023-09-23T22:30:58.282+0000 7fd1c54e7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:30:58.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[138201]: debug 2023-09-23T22:30:58.379+0000 7fd1c54e7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: cluster 2023-09-23T22:30:57.452422+0000 mgr.y 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: (mgr.24314) 149 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: audit 2023-09-23T22:30:57.605138+0000 mon.c (mon.2) 41 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: : audit [DBG] from='client.? 172.21.15.52:0/1029939706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: 09-23T22:30:57.961211+0000 mon.a (mon.0) 776 : audit 2023-09-23T22:30:58.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:58 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: cluster 2023-09-23T22:30:57.452422+0000 mgr.y (mgr 2023-09-23T22:30:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: .24314) 149 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: audit 2023-09-23T22:30:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: 2023-09-23T22:30:57.605138+0000 mon.c ( 2023-09-23T22:30:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: mon.2) 41 : audit [DBG] 2023-09-23T22:30:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/1029939706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: -23T22:30:57.961211+0000 mon.a (mon.0 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[110082]: ) 776 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: cluster 2023-09-23T22:30:57.452422+0000 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: mgr.y (mgr.24314) 149 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: T22:30:57.605138+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.52:0/1029939706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: audit 2023-09-23T22:30:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: 2023-09-23T22:30:57.961211+0000 2023-09-23T22:30:58.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: mon.a (mon.0) 776 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:58.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:58 smithi052 conmon[100224]: 2023-09-23T22:30:59.328 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: debug 2023-09-23T22:30:59.036+0000 7fb1420c2700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T22:30:59.328 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: ignoring --setuser ceph since I am not root 2023-09-23T22:30:59.328 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: ignoring --setgroup ceph since I am not root 2023-09-23T22:30:59.328 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: debug 2023-09-23T22:30:59.083+0000 7f1d5ba42700 1 -- 172.21.15.52:0/3483880091 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e6b09104e0 con 0x55e6b08e9000 2023-09-23T22:30:59.329 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: debug 2023-09-23T22:30:59.229+0000 7f1d6bc09680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:30:59.329 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: debug 2023-09-23T22:30:59.327+0000 7f1d6bc09680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:30:59.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[138201]: debug 2023-09-23T22:30:59.227+0000 7fd1c54e7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:30:59.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[138201]: debug 2023-09-23T22:30:59.319+0000 7fd1c54e7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:30:59.783 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[138201]: debug 2023-09-23T22:30:59.496+0000 7fd1c54e7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:30:59.783 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023-09-23T22:30:58.691903+0000 mon.a ( 2023-09-23T22:30:59.783 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: mon.0) 777 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:59.783 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023-09-23T22:30:59.013489+0000 mon.a (mon.0) 778 : 2023-09-23T22:30:59.783 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:59.783 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 09-23T22:30:59.014465+0000 mon.a (mon.0) 779 : audit 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023-09-23T22:30: 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 59.019820+0000 mon.a (mon.0) 780 : audit 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: :30:59.026242+0000 mon.a (mon.0) 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 781 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:59.784 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 23T22:30:59.031478+0000 mon.a (mon 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: .0) 782 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: cluster 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 2023-09-23T22:30:59.038016+0000 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: mon.a (mon.0) 783 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: audit 2023-09-23T22:30:59. 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: 038321+0000 mon.a (mon.0) 784 : audit 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: cluster 2023-09-23T22:30 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: :59.038397+0000 mon.a (mon.0) 785 2023-09-23T22:30:59.785 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[100454]: : cluster [DBG] mgrmap e20: x(active, starting, since 0.00635419s) 2023-09-23T22:30:59.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100670]: debug 2023-09-23T22:30:59.789+0000 7f1d6bc09680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:30:59.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30: 2023-09-23T22:30:59.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: 58.691903+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: 2023-09-23T22:30:59.013489 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: +0000 mon.a (mon.0) 778 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.014465+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.019820+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:59.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.026242+0000 mon.a (mon.0) 781 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.031478+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: cluster 2023-09-23T22:30:59.038016+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.038321+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[100224]: cluster 2023-09-23T22:30:59.038397+0000 mon.a (mon.0) 785 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00635419s) 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: :30:58.691903+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:59.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: :30:59.013489+0000 mon.a (mon.0) 778 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023-09-23T22:30 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: :59.014465+0000 mon.a (mon.0) 779 : 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: :30:59.019820+0000 mon.a (mon.0 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: ) 780 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:30:59.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: 23T22:30:59.026242+0000 mon.a ( 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: mon.0) 781 : audit [DBG] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: :30:59.031478+0000 mon.a (mon.0 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: ) 782 : audit [INF] from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: 23T22:30:59.038016+0000 mon.a 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: (mon.0) 783 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: audit 2023 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: -09-23T22:30:59.038321 2023-09-23T22:30:59.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: +0000 mon.a (mon.0) 784 : audit [INF] 2023-09-23T22:31:00.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: from='mgr.24314 172.21.15.52:0/652079322' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:31:00.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: cluster 2023-09-23T22:30 2023-09-23T22:31:00.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: :59.038397+0000 mon.a (mon.0) 785 2023-09-23T22:31:00.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:30:59 smithi052 conmon[110082]: : cluster [DBG] mgrmap e20: x(active, starting, since 0.00635419s) 2023-09-23T22:31:00.163 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:30:59 smithi151 conmon[138201]: debug 2023-09-23T22:30:59.782+0000 7fd1c54e7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:31:00.163 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:00 smithi151 conmon[138201]: debug 2023-09-23T22:31:00.012+0000 7fd1c54e7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:31:00.692 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:00 smithi151 conmon[138201]: debug 2023-09-23T22:31:00.162+0000 7fd1c54e7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:31:00.692 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:00 smithi151 conmon[138201]: debug 2023-09-23T22:31:00.430+0000 7fd1c54e7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:31:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:00 smithi151 conmon[100454]: audit 2023-09-23T22:30:59.970150+0000 mon.a (mon.0) 2023-09-23T22:31:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:00 smithi151 conmon[100454]: 786 : audit [DBG] from='client.? 172.21.15.52:0/2955655496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:00.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:00 smithi052 conmon[100224]: audit 2023-09-23T22:30:59.970150+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.52:0/2955655496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:00 smithi052 conmon[110082]: audit 2023-09-23T22:30:59 2023-09-23T22:31:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:00 smithi052 conmon[110082]: .970150+0000 mon.a (mon.0) 786 : audit [DBG] from='client.? 172.21.15.52:0/2955655496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:00.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:00 smithi052 conmon[100670]: debug 2023-09-23T22:31:00.725+0000 7f1d6bc09680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:31:00.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:00 smithi052 conmon[100670]: debug 2023-09-23T22:31:00.821+0000 7f1d6bc09680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:31:01.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:01 smithi151 conmon[138201]: debug 2023-09-23T22:31:01.143+0000 7fd1c54e7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:31:01.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:01 smithi151 conmon[138201]: debug 2023-09-23T22:31:01.276+0000 7fd1c54e7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:31:01.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:01 smithi052 conmon[100670]: debug 2023-09-23T22:31:01.071+0000 7f1d6bc09680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:31:01.811 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:01 smithi151 conmon[138201]: debug 2023-09-23T22:31:01.474+0000 7fd1c54e7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:31:01.811 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:01 smithi151 conmon[138201]: debug 2023-09-23T22:31:01.589+0000 7fd1c54e7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:31:01.812 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:01 smithi151 conmon[138201]: debug 2023-09-23T22:31:01.810+0000 7fd1c54e7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:31:02.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[110082]: audit 2023-09-23T22:31:02.328185+0000 mon.a ( 2023-09-23T22:31:02.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[110082]: mon.0) 787 : audit [DBG] from='client.? 172.21.15.52:0/648860617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:02.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100224]: audit 2023-09-23T22:31:02. 2023-09-23T22:31:02.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100224]: 328185+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.52:0/648860617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:02.481 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100670]: debug 2023-09-23T22:31:02.216+0000 7f1d6bc09680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:31:02.482 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100670]: debug 2023-09-23T22:31:02.383+0000 7f1d6bc09680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:31:02.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.328185+0000 mon.a (mon.0) 787 : audit [DBG] from='client.? 172.21.15.52:0/648860617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:02.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100670]: debug 2023-09-23T22:31:02.480+0000 7f1d6bc09680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:31:02.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100670]: debug 2023-09-23T22:31:02.591+0000 7f1d6bc09680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:31:02.948 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.681+0000 7fd1c54e7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:31:02.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE Bus STARTING 2023-09-23T22:31:02.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: CherryPy Checker: 2023-09-23T22:31:02.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: The Application mounted at '' has an empty config. 2023-09-23T22:31:02.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: 2023-09-23T22:31:02.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE Serving on http://:::9283 2023-09-23T22:31:02.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE Bus STARTED 2023-09-23T22:31:02.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE Bus STOPPING 2023-09-23T22:31:02.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:31:02.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE Bus STOPPED 2023-09-23T22:31:02.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: [23/Sep/2023:22:31:02] ENGINE Bus STARTING 2023-09-23T22:31:03.192 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:02 smithi052 conmon[100670]: debug 2023-09-23T22:31:02.814+0000 7f1d6bc09680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:31:03.381 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd126961700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.381 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd126961700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.381 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd126961700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd126961700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd126961700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd12395b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd12395b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd12395b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd12395b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:02 smithi151 conmon[138201]: debug 2023-09-23T22:31:02.952+0000 7fd12395b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[138201]: [23/Sep/2023:22:31:03] ENGINE Serving on http://:::9283 2023-09-23T22:31:03.382 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[138201]: [23/Sep/2023:22:31:03] ENGINE Bus STARTED 2023-09-23T22:31:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100670]: debug 2023-09-23T22:31:03.190+0000 7f1d6bc09680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:31:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100670]: debug 2023-09-23T22:31:03.316+0000 7f1d6bc09680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:31:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: cluster 2023-09-23T 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 22:31:02.696028+0000 mon.a (mon.0) 788 : cluster [INF] Active manager daemon x restarted 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02.696417+0000 mon.a 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.0) 789 : cluster 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [INF] Activating manager daemon x 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02.697680+0000 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 31 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: : audit [DBG] from='mgr.? 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02.701049+0000 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.a (mon.0) 790 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:31:03.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: cluster 2023 2023-09-23T22:31:03.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31:02.701341+0000 2023-09-23T22:31:03.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.a (mon.0) 791 2023-09-23T22:31:03.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: : cluster [DBG] mgrmap e21: x(active, starting, since 0.00508993s) 2023-09-23T22:31:03.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31:02.705509 2023-09-23T22:31:03.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: +0000 mon.b (mon.1) 32 2023-09-23T22:31:03.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:31:03.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:03.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -23T22:31:02.705663+0000 2023-09-23T22:31:03.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 33 : 2023-09-23T22:31:03.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:03.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31: 2023-09-23T22:31:03.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 02.705792+0000 mon.b (mon 2023-09-23T22:31:03.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: .1) 34 : audit [DBG] 2023-09-23T22:31:03.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:03.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:03.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 31:02.705916+0000 mon.b 2023-09-23T22:31:03.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.1) 35 : audit 2023-09-23T22:31:03.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:03.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22 2023-09-23T22:31:03.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :31:02.706138+0000 2023-09-23T22:31:03.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 36 : 2023-09-23T22:31:03.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:31:03.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31: 2023-09-23T22:31:03.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 02.706362+0000 mon.b ( 2023-09-23T22:31:03.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.1) 37 : audit 2023-09-23T22:31:03.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:31:03.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:03.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 31:02.706659+0000 mon.b (mon.1) 38 2023-09-23T22:31:03.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: cluster 2023-09-23T22:31:02. 2023-09-23T22:31:03.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 696028+0000 mon.a (mon.0) 788 : cluster [INF] Active manager daemon x restarted 2023-09-23T22:31:03.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: cluster 2023-09-23T22:31:02.696417+0000 mon.a (mon.0) 789 : cluster [INF] Activating manager daemon x 2023-09-23T22:31:03.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:03.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: -23T22:31:02.697680+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:31:03.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: cluster 2023-09-23T22 2023-09-23T22:31:03.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :31:02.701049+0000 mon.a (mon.0) 2023-09-23T22:31:03.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 790 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:31:03.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:31:03.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: T22:31:02.701341+0000 mon.a (mon.0) 2023-09-23T22:31:03.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 791 : cluster [DBG] mgrmap e21: x(active, starting, since 0.00508993s) 2023-09-23T22:31:03.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31 2023-09-23T22:31:03.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :02.705509+0000 mon.b (mon.1) 32 : audit 2023-09-23T22:31:03.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:31:03.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:03.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :31:02.705663+0000 mon.b (mon.1 2023-09-23T22:31:03.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: ) 33 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:03.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:31:03.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 22:31:02.705792+0000 mon.b (mon.1) 2023-09-23T22:31:03.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 34 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:03.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:03.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :31:02.705916+0000 mon.b (mon.1) 35 2023-09-23T22:31:03.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:03.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:31:03.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 31:02.706138+0000 mon.b (mon.1) 36 2023-09-23T22:31:03.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:31:03.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:02.706362 2023-09-23T22:31:03.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: +0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:31:03.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31:03.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:03.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: -23T22:31:02.706659+0000 mon.b (mon.1) 38 : audit 2023-09-23T22:31:03.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:31:03.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31 2023-09-23T22:31:03.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :02.706891+0000 mon.b (mon.1) 39 : audit 2023-09-23T22:31:03.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:31:03.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:02. 2023-09-23T22:31:03.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 707147+0000 mon.b (mon.1) 40 : audit [DBG] 2023-09-23T22:31:03.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:31:03.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:03.511 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 02.706891+0000 mon.b ( 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.1) 39 : audit 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31: 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 02.707147+0000 mon.b (mon 2023-09-23T22:31:03.512 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: .1) 40 : audit [DBG] 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 31:02.707371+0000 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 41 : 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 31:02.707602+0000 mon.b 2023-09-23T22:31:03.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.1) 42 : audit 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :02.707922+0000 mon.b 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.1) 43 : audit 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :02.708258+0000 mon.b 2023-09-23T22:31:03.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.1) 44 : audit 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :02.708598+0000 mon.b 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.1) 45 : audit 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :02.708939+0000 mon.b ( 2023-09-23T22:31:03.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.1) 46 : audit 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023- 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: .709600+0000 mon.b (mon. 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 1) 47 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023- 2023-09-23T22:31:03.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: .709943+0000 mon.b (mon.1) 48 : 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31: 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 02.710459+0000 mon.b (mon.1 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: ) 49 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -23T22:31:02. 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 710640+0000 mon.b (mon.1) 50 : audit 2023-09-23T22:31:03.517 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :02.711799+0000 mon.b ( 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.1) 51 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:02. 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 712061+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: cluster 2023-09-23T22:31:02.855950 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: +0000 mon.a (mon.0) 792 : cluster [INF] Manager daemon x is now available 2023-09-23T22:31:03.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: :02.875572+0000 mon.a (mon.0) 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 793 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 22:31:02.880166+0000 mon.b ( 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.1) 53 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02.882062+0000 mon.b (mon 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: .1) 54 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:03.519 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02.891451+0000 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 55 : audit [INF] 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:02.891935+0000 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.a (mon.0) 794 : audit [INF] 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:02.901456+0000 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 56 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 23T22:31:02.910382+0000 mon.b ( 2023-09-23T22:31:03.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.1) 57 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 09-23T22:31:02.910698+0000 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31:02.707371+0000 mon.b ( 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: mon.1) 41 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: -23T22:31:02.707602+0000 mon.b ( 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: mon.1) 42 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: T22:31:02.707922+0000 mon.b (mon.1 2023-09-23T22:31:03.521 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: ) 43 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 22:31:02.708258+0000 mon.b (mon.1) 44 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :31:02.708598+0000 mon.b (mon.1) 45 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 31:02.708939+0000 mon.b (mon.1) 46 : audit 2023-09-23T22:31:03.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31: 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 02.709600+0000 mon.b (mon.1) 47 : audit [DBG] 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:02. 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 709943+0000 mon.b (mon.1) 48 : audit [DBG] 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31:02.710459+0000 mon.b (mon. 2023-09-23T22:31:03.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 1) 49 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: T22:31:02.710640+0000 mon.b (mon.1 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: ) 50 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 31:02.711799+0000 mon.b (mon.1) 51 : 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31: 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 02.712061+0000 mon.b (mon.1) 52 : audit 2023-09-23T22:31:03.524 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:31:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: cluster 2023-09-23T22:31: 2023-09-23T22:31:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 02.855950+0000 mon.a (mon.0) 792 : cluster 2023-09-23T22:31:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [INF] Manager daemon x is now available 2023-09-23T22:31:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31 2023-09-23T22:31:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :02.875572+0000 mon.a (mon.0) 793 : audit 2023-09-23T22:31:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.a (mon.0) 795 : audit [INF] 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:02.932080+0000 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.b (mon.1) 58 : audit [INF] 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:31:03.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:02.933913+0000 mon.a 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.0) 796 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -09-23T22:31:02.934887+0000 mon.a ( 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.0) 797 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: -23T22:31:03.167760+0000 2023-09-23T22:31:03.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: mon.a (mon.0) 798 : audit [INF] 2023-09-23T22:31:03.528 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.528 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: audit 2023-09-23T22:31:03.528 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:03.172269+0000 mon.a 2023-09-23T22:31:03.528 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: (mon.0) 799 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.528 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[110082]: 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31: 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 02.880166+0000 mon.b (mon.1) 53 : audit [DBG] 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31:02.882062+0000 mon.b 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: (mon.1) 54 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 23T22:31:02.891451+0000 mon.b (mon 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: .1) 55 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:03.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 22:31:02.891935+0000 mon.a (mon. 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 0) 794 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :31:02.901456+0000 mon.b (mon.1) 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 56 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:03.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: :31:02.910382+0000 mon.b (mon.1) 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 57 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: -23T22:31:02.910698+0000 mon.a 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: (mon.0) 795 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:02.932080+0000 mon.b (mon.1) 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 58 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:31:03.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31:02.933913+0000 mon.a (mon 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: .0) 796 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31: 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 02.934887+0000 mon.a (mon.0) 797 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:31:03.532 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09-23T22:31:03.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 2023-09-23T22:31:03.167760+0000 2023-09-23T22:31:03.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: mon.a (mon.0) 798 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:31:03.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 23T22:31:03.172269+0000 mon.a (mon. 2023-09-23T22:31:03.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:03 smithi052 conmon[100224]: 0) 799 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: cluster 2023-09-23T22:31:02 2023-09-23T22:31:03.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: .696028+0000 mon.a (mon.0) 788 : cluster [INF] Active manager daemon x restarted 2023-09-23T22:31:03.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: cluster 2023-09-23T22:31:03.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22:31:02.696417+0000 mon.a 2023-09-23T22:31:03.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: (mon.0) 789 : cluster [INF] 2023-09-23T22:31:03.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: Activating manager daemon x 2023-09-23T22:31:03.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:03.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: -09-23T22:31: 2023-09-23T22:31:03.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 02.697680+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:31:03.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22:31:03.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: cluster 2023-09-23T22 2023-09-23T22:31:03.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: :31:02.701049+0000 mon.a (mon 2023-09-23T22:31:03.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: .0) 790 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:31:03.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:31:03.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: -09-23T22:31:02.701341 2023-09-23T22:31:03.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: +0000 mon.a (mon.0) 791 : cluster 2023-09-23T22:31:03.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: [DBG] mgrmap e21: x(active, starting, since 0.00508993s) 2023-09-23T22:31:03.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:03.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.705509+0000 mon.b (mon.1 2023-09-23T22:31:03.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: ) 32 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:31:03.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31 2023-09-23T22:31:03.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: :02.705663+0000 mon.b (mon.1) 33 : audit 2023-09-23T22:31:03.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:03.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31: 2023-09-23T22:31:03.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 02.705792+0000 mon.b (mon.1) 34 : 2023-09-23T22:31:03.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:03.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31 2023-09-23T22:31:03.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: :02.705916+0000 mon.b (mon.1) 2023-09-23T22:31:03.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 35 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:03.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:31:03.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 22:31:02.706138+0000 mon.b (mon.1 2023-09-23T22:31:03.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: ) 36 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:31:03.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:03.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.706362+0000 mon.b (mon.1) 37 : 2023-09-23T22:31:03.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:31:03.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02. 2023-09-23T22:31:03.708 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 706659+0000 mon.b (mon.1) 38 : audit [DBG] 2023-09-23T22:31:03.708 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:31:03.709 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:03.709 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22:31:02.706891+0000 mon.b (mon. 2023-09-23T22:31:03.709 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 1) 39 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:31:03.709 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:03.709 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.707147+0000 mon.b (mon.1) 40 2023-09-23T22:31:03.710 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:31:03.710 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:31:03.710 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 22:31:02.707371+0000 mon.b (mon.1) 2023-09-23T22:31:03.711 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 41 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:31:03.711 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:31:03.711 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: :31:02.707602+0000 mon.b (mon.1) 2023-09-23T22:31:03.711 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 42 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:31:03.712 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02 2023-09-23T22:31:03.712 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: .707922+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:31:03.712 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.708258+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:31:03.713 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.708598+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:31:03.713 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.708939+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:31:03.713 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.709600+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:31:03.713 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.709943+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:31:03.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:03.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 23T22:31:02.710459+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:31:03.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:31:03.715 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 22:31:02.710640+0000 2023-09-23T22:31:03.715 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: mon.b (mon.1) 50 : 2023-09-23T22:31:03.715 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:31:03.716 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:03.716 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22:31 2023-09-23T22:31:03.716 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: :02.711799+0000 mon.b (mon 2023-09-23T22:31:03.716 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: .1) 51 : audit [DBG] 2023-09-23T22:31:03.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:31:03.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:03.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 23T22:31:02.712061 2023-09-23T22:31:03.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: +0000 mon.b (mon.1) 2023-09-23T22:31:03.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 52 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:31:03.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: cluster 2023-09-23T22:31:03.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22: 2023-09-23T22:31:03.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.855950+0000 mon.a 2023-09-23T22:31:03.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: (mon.0) 792 : cluster 2023-09-23T22:31:03.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: [INF] Manager daemon x is now available 2023-09-23T22:31:03.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:03.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 09-23T22:31: 2023-09-23T22:31:03.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 02.875572+0000 mon.a (mon. 2023-09-23T22:31:03.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 0) 793 : audit [INF] 2023-09-23T22:31:03.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:03.721 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 23T22:31:02. 2023-09-23T22:31:03.721 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 880166+0000 mon.b (mon.1) 2023-09-23T22:31:03.721 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 53 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:03.721 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:03.721 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22: 2023-09-23T22:31:03.722 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.882062+0000 2023-09-23T22:31:03.722 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: mon.b (mon.1) 54 : audit 2023-09-23T22:31:03.722 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:03.722 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:03.722 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22: 2023-09-23T22:31:03.723 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.891451+0000 mon.b ( 2023-09-23T22:31:03.723 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: mon.1) 55 : audit 2023-09-23T22:31:03.723 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:03.723 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:03.723 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 09-23T22:31:02 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: .891935+0000 mon.a (mon.0 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: ) 794 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22: 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.901456+0000 mon.b ( 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: mon.1) 56 : audit [DBG] 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:03.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:03.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: -23T22:31:02. 2023-09-23T22:31:03.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 910382+0000 mon.b (mon.1 2023-09-23T22:31:03.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: ) 57 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:31:03.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:03.725 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 31:02.910698+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:31:03.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:02.932080 2023-09-23T22:31:03.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: +0000 mon.b (mon.1) 58 : audit [INF] 2023-09-23T22:31:03.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:31:03.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31:03.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 2023-09-23T22:31:02.933913+0000 mon.a (mon. 2023-09-23T22:31:03.726 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: 0) 796 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.727 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:03.727 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: -09-23T22:31:02.934887 2023-09-23T22:31:03.727 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: +0000 mon.a (mon.0) 797 : audit [INF] 2023-09-23T22:31:03.727 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:31:03.727 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23T22:31 2023-09-23T22:31:03.727 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: :03.167760+0000 mon.a (mon.0) 798 2023-09-23T22:31:03.728 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:03.728 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:31:03.728 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: T22:31:03.172269+0000 mon.a (mon 2023-09-23T22:31:03.728 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:03 smithi151 conmon[100454]: .0) 799 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:04.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100670]: debug 2023-09-23T22:31:04.212+0000 7f1d6bc09680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:31:04.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100670]: debug 2023-09-23T22:31:04.314+0000 7f1d6bc09680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:31:04.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100670]: debug 2023-09-23T22:31:04.432+0000 7f1d6bc09680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:31:04.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:03.262887 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: +0000 mgr.x (mgr.24787) 1 : cephadm [INF] Deploying cephadm binary to smithi151 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:03.270662+0000 mgr.x (mgr.24787) 2 : cephadm [INF] Deploying cephadm binary to smithi052 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: cluster 2023-09-23T22:31:03.703234+0000 mon.a ( 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: mon.0) 800 : cluster [DBG] mgrmap e22: x(active, since 1.00698s) 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: :31:03.876697+0000 mon.a (mon.0) 801 : 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: -23T22:31:03.876820+0000 mon.a 2023-09-23T22:31:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: (mon.0) 802 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: 23T22:31:03.880777+0000 mon.b (mon.1 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: ) 59 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: -09-23T22:31:03.881216+0000 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100224]: mon.a (mon.0) 803 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:03.262887 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: +0000 mgr.x (mgr.24787) 1 : cephadm [INF] Deploying cephadm binary to smithi151 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:03.270662+0000 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: mgr.x (mgr.24787) 2 : cephadm [INF] Deploying cephadm binary to smithi052 2023-09-23T22:31:04.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: cluster 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: 2023-09-23T22:31:03.703234+0000 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: mon.a (mon.0) 800 : cluster [DBG] mgrmap e22: x(active, since 1.00698s) 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: 31:03.876697+0000 mon.a (mon.0) 801 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: cluster 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: 2023-09-23T22:31:03.876820+0000 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: mon.a (mon.0) 802 : cluster [DBG] 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:31:04.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: audit 2023-09-23T22:31 2023-09-23T22:31:04.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: :03.880777+0000 mon.b (mon.1) 59 2023-09-23T22:31:04.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:04.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:31:04.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: 31:03.881216+0000 mon.a (mon.0) 803 : audit [INF] 2023-09-23T22:31:04.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:04.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:03. 2023-09-23T22:31:04.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 262887+0000 mgr.x (mgr.24787) 1 : cephadm [INF] Deploying cephadm binary to smithi151 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 2023-09-23T22:31:03.270662+0000 mgr.x (mgr.24787 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: ) 2 : cephadm [INF] Deploying cephadm binary to smithi052 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: cluster 2023- 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 09-23T22:31:03.703234+0000 mon.a (mon. 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 0) 800 : cluster [DBG] mgrmap e22: x(active, since 1.00698s) 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 09-23T22:31:03.876697+0000 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: mon.a (mon.0) 801 : audit 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: -09-23T22:31:03.876820 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: +0000 mon.a (mon.0) 802 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 09-23T22:31:03.880777+0000 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: mon.b (mon.1) 59 : audit 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: audit 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: 2023-09-23T22:31:03.881216 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: +0000 mon.a (mon.0) 803 : 2023-09-23T22:31:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:04 smithi151 conmon[100454]: audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:04.953 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100670]: debug 2023-09-23T22:31:04.651+0000 7f1d6bc09680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:31:05.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:04 smithi052 conmon[100670]: debug 2023-09-23T22:31:04.953+0000 7f1d6bc09680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:31:05.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: debug 2023-09-23T22:31:05.146+0000 7f1d6bc09680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:31:05.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[138201]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:05] "GET /metrics HTTP/1.1" 200 31038 "" "Prometheus/2.33.4" 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: cluster 2023-09-23T22:31:03.716292+0000 mgr.x (mgr. 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: 24787) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:04.206256+0000 mgr.x (mgr.24787) 4 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Bus STARTING 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:04.326449+0000 mgr.x (mgr.24787) 5 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Serving on http://172.21.15.151:8765 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:04.476805+0000 mgr.x (mgr.24787) 6 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Serving on https://172.21.15.151:7150 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:04.476895+0000 mgr.x (mgr.24787) 7 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Bus STARTED 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: audit 2023-09-23T22:31:04.721737+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.52:0/4215236489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: audit 2023-09-23T22:31:04.877707+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:05 smithi151 conmon[100454]: cluster 2023-09-23T22:31:04.877818+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:31:05.686 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: debug 2023-09-23T22:31:05.337+0000 7f1d6bc09680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:31:05.686 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: debug 2023-09-23T22:31:05.578+0000 7f1d6bc09680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:31:05.686 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: debug 2023-09-23T22:31:05.685+0000 7f1d6bc09680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:31:05.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: cluster 2023-09-23T22:31:03. 2023-09-23T22:31:05.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 716292+0000 mgr.x (mgr.24787) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:05.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: cephadm 2023- 2023-09-23T22:31:05.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 09-23T22:31:04.206256+0000 mgr.x (mgr.24787) 4 : cephadm 2023-09-23T22:31:05.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: [INF] [23/Sep/2023:22:31:04] ENGINE Bus STARTING 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 2023-09-23T22:31:04.326449+0000 mgr.x (mgr 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: .24787) 5 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Serving on http://172.21.15.151:8765 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: cephadm 2023-09-23T22 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: :31:04.476805+0000 mgr.x (mgr.24787) 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 6 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Serving on https://172.21.15.151:7150 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 31:04.476895+0000 mgr.x (mgr. 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 24787) 7 : cephadm [INF] 2023-09-23T22:31:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: [23/Sep/2023:22:31:04] ENGINE Bus STARTED 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: audit 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 2023-09-23T22:31:04.721737+0000 mon.c ( 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: mon.2) 42 : audit [DBG] from='client.? 172.21.15.52:0/4215236489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: -23T22:31:04.877707+0000 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: mon.a (mon.0) 804 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: 23T22:31:04.877818+0000 mon.a ( 2023-09-23T22:31:05.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[110082]: mon.0) 805 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: cluster 2023-09-23T22:31:03.716292+0000 mgr.x (mgr.24787 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: ) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:04.206256+0000 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: mgr.x (mgr.24787) 4 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Bus STARTING 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: cephadm 2023 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: -09-23T22:31:04.326449+0000 mgr.x 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: (mgr.24787) 5 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Serving on http://172.21.15.151:8765 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: cephadm 2023-09 2023-09-23T22:31:05.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: -23T22:31:04.476805+0000 mgr.x (mgr.24787) 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: 6 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Serving on https://172.21.15.151:7150 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: 31:04.476895+0000 mgr.x (mgr.24787 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: ) 7 : cephadm [INF] [23/Sep/2023:22:31:04] ENGINE Bus STARTED 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: audit 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: 2023-09-23T22:31 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: :04.721737+0000 mon.c (mon.2) 42 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: : audit [DBG] from='client.? 172.21.15.52:0/4215236489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:05.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: -23T22:31:04.877707+0000 mon.a 2023-09-23T22:31:05.692 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: (mon.0) 804 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:05.692 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: cluster 2023-09-23T 2023-09-23T22:31:05.692 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: 22:31:04.877818+0000 mon.a (mon.0) 805 : cluster 2023-09-23T22:31:05.692 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100224]: [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:31:05.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: [23/Sep/2023:22:31:05] ENGINE Bus STARTING 2023-09-23T22:31:05.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: CherryPy Checker: 2023-09-23T22:31:05.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: The Application mounted at '' has an empty config. 2023-09-23T22:31:05.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: 2023-09-23T22:31:05.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: [23/Sep/2023:22:31:05] ENGINE Serving on http://:::9283 2023-09-23T22:31:05.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:05 smithi052 conmon[100670]: [23/Sep/2023:22:31:05] ENGINE Bus STARTED 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: cluster 2023-09-23T22:31 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: :04.709782+0000 mgr.x (mgr.24787) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: cluster 2023-09-23T22:31:05.387434+0000 mon.a (mon.0) 806 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: cluster 2023-09-23T22:31:05.699600+0000 mon.a (mon.0) 807 : cluster [DBG] Standby manager daemon y started 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: audit 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 2023-09-23T22:31:05.701560+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: audit 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 2023-09-23T22 2023-09-23T22:31:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: :31:05.702208+0000 mon.c ( 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: mon.2) 44 : audit [DBG] 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: audit 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 2023-09-23T22:31:05.705498+0000 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: mon.c (mon.2) 45 : audit [DBG] 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: audit 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 2023-09-23T22:31: 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 05.707266+0000 mon.c (mon.2 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: ) 46 : audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: audit 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 2023-09-23T22:31 2023-09-23T22:31:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: :05.932220+0000 mon.a (mon. 2023-09-23T22:31:06.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 0) 808 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:06.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:06 smithi151 conmon[100454]: 2023-09-23T22:31:06.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: cluster 2023-09-23T22:31:04 2023-09-23T22:31:06.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: .709782+0000 mgr.x (mgr.24787) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:06.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: cluster 2023-09-23T22:31:06.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 2023-09-23T22:31:05.387434+0000 mon.a (mon.0) 806 : 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: cluster [DBG] mgrmap e23: x(active, since 2s) 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: cluster 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 2023-09-23T22:31:05. 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 699600+0000 mon.a (mon.0) 807 : cluster [DBG] 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: Standby manager daemon y started 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: :31:05.701560+0000 mon.c (mon.2 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: ) 43 : audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 09-23T22:31:05.702208+0000 mon.c 2023-09-23T22:31:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: (mon.2) 44 : audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: audit 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 2023-09-23T22:31:05.705498+0000 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: mon.c (mon.2) 45 : audit [DBG] 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: audit 2023-09-23T22:31: 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 05.707266+0000 mon.c (mon.2) 46 : 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 22:31:05.932220+0000 mon.a (mon. 2023-09-23T22:31:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[110082]: 0) 808 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:06.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: cluster 2023-09-23T22:31:04.709782+0000 2023-09-23T22:31:06.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: mgr.x (mgr.24787) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: cluster 2023-09-23T22:31:05.387434+0000 mon.a (mon.0) 806 : cluster [DBG] 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: mgrmap e23: x(active, since 2s) 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: cluster 2023-09-23T22:31:05.699600+0000 mon.a (mon.0) 807 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: : cluster [DBG] Standby manager daemon y started 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: audit 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 2023-09-23T22:31:05.701560+0000 mon.c (mon.2) 43 : audit 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: audit 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 2023-09-23T22:31:05.702208+0000 mon.c (mon.2) 44 2023-09-23T22:31:06.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: : audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: audit 2023-09-23T22:31:05.705498+0000 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: mon.c (mon.2) 45 : audit [DBG] from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: audit 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 2023-09- 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 23T22:31:05.707266+0000 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: mon.c (mon.2) 46 : audit [DBG] 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: from='mgr.? 172.21.15.52:0/2919889339' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:31:06.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 23T22:31:05.932220+0000 mon.a (mon.0) 2023-09-23T22:31:06.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:06 smithi052 conmon[100224]: 808 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:07.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:07 smithi151 conmon[100454]: cluster 2023-09-23T22:31:06.400906+0000 mon.a (mon.0) 809 : cluster [DBG] mgrmap e24: x(active, since 3s), standbys: y 2023-09-23T22:31:07.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:07 smithi151 conmon[100454]: audit 2023-09-23T22:31:06.403196+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:31:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:07 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.142739+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.52:0/1840526197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:07 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.237467+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:07 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.244606+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[100224]: cluster 2023-09-23T22:31:06.400906+0000 2023-09-23T22:31:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[100224]: mon.a (mon.0) 809 : cluster [DBG] mgrmap e24: x(active, since 3s), standbys: y 2023-09-23T22:31:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[100224]: audit 2023-09-23T22:31:06.403196+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:31:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[100224]: audit 2023-09-23T22:31:07.142739+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.52:0/1840526197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[100224]: audit 2023-09-23T22:31:07.237467+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[100224]: audit 2023-09-23T22:31:07.244606+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: 23T22:31:06.400906+0000 mon.a (mon.0) 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: 809 : cluster [DBG] mgrmap e24: x(active, since 3s), standbys: y 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: -09-23T22:31: 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: 06.403196+0000 mon.b ( 2023-09-23T22:31:07.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: mon.1) 60 : audit 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: -09-23T22:31:07. 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: 142739+0000 mon.a (mon 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: .0) 810 : audit 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/1840526197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: -09-23T22:31:07 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: .237467+0000 mon.a ( 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: mon.0) 811 : audit 2023-09-23T22:31:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:07.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:07.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: -09-23T22:31:07. 2023-09-23T22:31:07.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: 244606+0000 mon.a (mon 2023-09-23T22:31:07.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: .0) 812 : audit 2023-09-23T22:31:07.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:07 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:31:08.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: cluster 2023-09-23T22:31:06.710096+0000 mgr.x 2023-09-23T22:31:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: (mgr.24787) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: cluster 2023-09-23T22:31:07.409357+0000 mon.a (mon.0) 813 : cluster 2023-09-23T22:31:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: [DBG] mgrmap e25: x(active, since 4s), standbys: y 2023-09-23T22:31:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: -23T22:31:07.742027+0000 mon.a (mon.0) 814 : 2023-09-23T22:31:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: 22:31:07.750004+0000 mon.a (mon. 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: 0) 815 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: -09-23T22:31:07.940445+0000 mon.a 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: (mon.0) 816 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: audit 2023-09-23T22:31:08.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: 2023-09-23T22:31:07. 2023-09-23T22:31:08.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: 946295+0000 mon.a (mon.0) 817 : audit [INF] 2023-09-23T22:31:08.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: cluster 2023-09-23T22:31:06.710096+0000 2023-09-23T22:31:08.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: mgr.x (mgr.24787) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:08.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: 09-23T22:31:07.409357+0000 mon.a (mon.0) 813 : cluster 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: [DBG] mgrmap e25: x(active, since 4s), standbys: y 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: audit 2023-09-23T22:31:07 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: .742027+0000 mon.a (mon.0) 814 : audit 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: audit 2023-09-23T22:31 2023-09-23T22:31:08.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: :07.750004+0000 mon.a (mon.0) 815 : 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: 22:31:07.940445+0000 mon.a (mon.0 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: ) 816 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: 22:31:07.946295+0000 mon.a (mon.0) 2023-09-23T22:31:08.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:08 smithi052 conmon[110082]: 817 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: cluster 2023-09-23T22:31:06.710096+0000 mgr.x (mgr.24787 2023-09-23T22:31:08.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: ) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: cluster 2023-09-23T22:31:07.409357+0000 mon.a (mon.0) 813 : cluster [DBG] mgrmap e25: x(active, since 4s), standbys: y 2023-09-23T22:31:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.742027+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.750004+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.940445+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:08 smithi151 conmon[100454]: audit 2023-09-23T22:31:07.946295+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:09 smithi151 conmon[100454]: audit 2023-09-23T22:31:08.435233+0000 2023-09-23T22:31:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:09 smithi151 conmon[100454]: mon.a (mon.0) 818 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:09 smithi151 conmon[100454]: audit 2023-09-23T22:31:08.440802+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:09 smithi151 conmon[100454]: audit 2023-09-23T22:31:09.284407+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[100224]: audit 2023-09-23T22:31:08.435233+0000 mon.a ( 2023-09-23T22:31:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[100224]: mon.0) 818 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[100224]: audit 2023-09-23T22:31:08.440802+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[100224]: audit 2023-09-23T22:31:09.284407+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: 22:31:08.435233+0000 mon.a (mon.0) 818 : audit [INF] 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: -09-23T22:31: 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: 08.440802+0000 mon.a (mon 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: .0) 819 : audit [INF] 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: audit 2023-09-23T22:31:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: 2023-09-23T22 2023-09-23T22:31:09.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: :31:09.284407+0000 2023-09-23T22:31:09.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: mon.a (mon.0) 820 : 2023-09-23T22:31:09.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:09 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:10 smithi151 conmon[100454]: cluster 2023-09-23T22:31:08.710435+0000 mgr.x (mgr.24787) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:10 smithi151 conmon[100454]: audit 2023-09-23T22:31:09.546037+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.52:0/1023459130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:10.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:10 smithi052 conmon[110082]: cluster 2023-09-23T22:31:08.710435+0000 mgr.x (mgr.24787) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:10.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:10 smithi052 conmon[110082]: audit 2023-09-23T22:31:09.546037+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.52:0/1023459130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:10.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:10 smithi052 conmon[100224]: cluster 2023-09-23T22:31:08.710435+0000 mgr.x (mgr.24787) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 3.0 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:10.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:10 smithi052 conmon[100224]: audit 2023-09-23T22:31:09.546037+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.52:0/1023459130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:12.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: cluster 2023-09-23T22:31:10.710893+0000 2023-09-23T22:31:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: mgr.x (mgr.24787) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 233 KiB/s wr, 31 op/s 2023-09-23T22:31:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: audit 2023-09-23T22:31:10.839280+0000 mon.a (mon.0) 821 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: audit 2023-09-23T22:31:10.839554+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: audit 2023-09-23T22:31:10.839827+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: audit 2023-09-23T22:31:10.840016+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:12 smithi151 conmon[100454]: audit 2023-09-23T22:31:11.735398+0000 mon.a (mon.0) 822 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: cluster 2023-09-23T22:31:10.710893+0000 mgr.x (mgr 2023-09-23T22:31:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: .24787) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 233 KiB/s wr, 31 op/s 2023-09-23T22:31:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: audit 2023-09-23T22:31:10.839280+0000 mon.a (mon.0) 821 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: audit 2023-09-23T22:31:10.839554+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: audit 2023-09-23T22:31:10.839827+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: audit 2023-09-23T22:31:10.840016+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[110082]: audit 2023-09-23T22:31:11.735398+0000 mon.a (mon.0) 822 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: cluster 2023-09-23T22:31:10.710893+0000 mgr.x 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: (mgr.24787) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 233 KiB/s wr, 31 op/s 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: audit 2023-09-23T22:31:10.839280+0000 mon.a (mon.0) 821 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: audit 2023-09-23T22:31:10.839554+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: audit 2023-09-23T22:31:10.839827+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: audit 2023-09-23T22:31:10.840016+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:12 smithi052 conmon[100224]: audit 2023-09-23T22:31:11.735398+0000 mon.a (mon.0) 822 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:11.737079+0000 mon.b (mon.1) 64 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:11.944269+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.52:0/1416230318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:12.235830+0000 mon.b (mon.1) 65 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:12.239242+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:12.239401+0000 mon.b (mon.1) 2023-09-23T22:31:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 67 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:12.239587+0000 mon.b (mon.1) 68 : audit [DBG] 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:12.241781+0000 mon.a (mon. 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 0) 823 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31: 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 12.245386+0000 mon.c (mon.2) 49 : audit 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: :12.245665+0000 mon.a (mon.0) 824 : 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:31:13.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: :31:12.246348+0000 mon.b (mon.1) 69 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 31:12.246503+0000 mon.b (mon.1) 70 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: :12.246675+0000 mon.b (mon.1) 71 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 23T22:31:13.137249+0000 mon.c (mon 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: .2) 50 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -23T22:31:13.140170+0000 mon.a (mon 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: .0) 825 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 23T22:31:13.143776+0000 mon.a ( 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.0) 826 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -23T22:31:13.145862+0000 mon.b ( 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.1) 72 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -23T22:31:13.146174+0000 mon.a ( 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.0) 827 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:13.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -23T22:31:13.146745+0000 mon.b ( 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.1) 73 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 09-23T22:31:13.147037+0000 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.a (mon.0) 828 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -09-23T22:31:13.148360+0000 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.b (mon.1) 74 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -09-23T22:31:13.148704+0000 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.a (mon.0) 829 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:13.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 2023-09-23T22:31: 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 13.149453+0000 mon.b (mon.1 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: ) 75 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 23T22:31:13.149875+0000 mon.a (mon.0 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: ) 830 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 2023-09-23T22:31:13.150426 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: +0000 mon.b (mon.1) 76 : audit [INF] 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 22:31:13.150771+0000 mon.a (mon.0 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: ) 831 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: :31:13.151479+0000 mon.b (mon.1) 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 77 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 22:31:13.151769+0000 mon.a (mon. 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 0) 832 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: T22:31:13.152490+0000 mon.b (mon. 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 1) 78 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22:31:13.152778+0000 mon.a ( 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: mon.0) 833 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:31:13.705 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 22:31:13.152980+0000 mon.b (mon. 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 1) 79 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: :31:13.153309+0000 mon.a (mon.0 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: ) 834 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -09-23T22:31:13.158297 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: +0000 mon.a (mon.0) 835 : audit 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -09-23T22:31:13. 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 162482+0000 mon.a (mon.0) 836 2023-09-23T22:31:13.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: -09-23T22:31:13. 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 164201+0000 mon.b (mon.1) 80 : audit 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: 09-23T22:31:13.165035+0000 mon.b (mon.1 2023-09-23T22:31:13.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:13 smithi151 conmon[100454]: ) 81 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:31:13.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:31:13.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: T22:31:11.737079+0000 mon.b (mon.1) 64 : audit [INF] 2023-09-23T22:31:13.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:13.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:13.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: -23T22:31:11.944269+0000 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.c (mon.2) 48 : 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit [DBG] from='client.? 172.21.15.52:0/1416230318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: -09-23T22:31:12.235830 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: +0000 mon.b (mon.1) 65 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:12.239242 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: +0000 mon.b (mon.1) 66 : audit 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31 2023-09-23T22:31:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: :12.239401+0000 mon.b ( 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.1) 67 : audit 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 22:31:12.239587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:12.241781 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: +0000 mon.a (mon.0) 823 : audit [INF] 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:12.245386+0000 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.c (mon.2) 49 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 23T22:31:12.245665+0000 mon.a (mon 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: .0) 824 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 12.246348+0000 mon.b (mon.1) 69 : 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: T22:31:12.246503+0000 mon.b (mon. 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 1) 70 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 12.246675+0000 mon.b (mon.1) 71 : 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13. 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 137249+0000 mon.c (mon.2) 50 : audit 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:11.737079+0000 2023-09-23T22:31:13.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.b (mon.1) 64 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:11.944269+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.52:0/1416230318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -23T22:31:12.235830+0000 mon.b (mon 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .1) 65 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 09-23T22:31:12.239242+0000 mon.b ( 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.1) 66 : audit [DBG] 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :12.239401+0000 mon.b (mon 2023-09-23T22:31:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .1) 67 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 09-23T22:31:12.239587+0000 mon.b (mon. 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 1) 68 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31:12.241781 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: +0000 mon.a (mon.0) 823 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 23T22:31:12.245386+0000 mon.c ( 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.2) 49 : audit [INF] 2023-09-23T22:31:13.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :31:12.245665+0000 mon.a ( 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.0) 824 : audit [INF] 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31:12 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .246348+0000 mon.b (mon.1) 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 69 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:31:13.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31:12. 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 246503+0000 mon.b (mon. 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 1) 70 : audit [DBG] 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 22:31:12.246675+0000 mon.b 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: (mon.1) 71 : 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31: 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 13.137249+0000 mon.c (mon. 2023-09-23T22:31:13.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2) 50 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31:13.140170 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: +0000 mon.a (mon.0) 825 : audit 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31:13 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .143776+0000 mon.a (mon.0) 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 826 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 22:31:13.145862+0000 mon.b (mon.1 2023-09-23T22:31:13.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: ) 72 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :31:13.146174+0000 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.a (mon.0) 827 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :13.146745+0000 mon.b 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: (mon.1) 73 : audit 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.147037+0000 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.a (mon.0) 828 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.148360+0000 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.b (mon.1) 74 : 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31:13 2023-09-23T22:31:13.754 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .148704+0000 mon.a (mon. 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 0) 829 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.149453+0000 mon.b 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: (mon.1) 75 : audit 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31:13 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .149875+0000 mon.a (mon. 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 0) 830 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31:13.755 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31: 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 13.150426+0000 mon.b (mon 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 13.140170+0000 mon.a (mon.0) 825 : 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13.143776+0000 mon.a 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: (mon.0) 826 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 31:13.145862+0000 mon.b (mon.1) 72 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13.146174+0000 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.a (mon.0) 827 : audit [INF] 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:13.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 22:31:13.146745+0000 mon.b (mon. 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 1) 73 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 13.147037+0000 mon.a (mon.0) 828 : audit 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31: 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 13.148360+0000 mon.b (mon.1) 74 : audit 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 22:31:13.148704+0000 mon.a (mon.0 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: ) 829 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.758 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: .149453+0000 mon.b (mon.1) 75 : audit 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 09-23T22:31:13.149875+0000 mon.a ( 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.0) 830 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 13.150426+0000 mon.b (mon.1) 76 : audit 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.759 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13.150771 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: +0000 mon.a (mon.0) 831 : audit [INF] 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 23T22:31:13.151479+0000 mon.b (mon 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: .1) 77 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 13.151769+0000 mon.a (mon.0) 832 : 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: T22:31:13.152490+0000 mon.b (mon. 2023-09-23T22:31:13.760 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 1) 78 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: .152778+0000 mon.a (mon.0) 833 : 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: -09-23T22:31:13.152980+0000 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.b (mon.1) 79 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 23T22:31:13.153309+0000 mon.a (mon.0 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: ) 834 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:13.761 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: -23T22:31:13.158297+0000 mon.a ( 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.0) 835 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: -23T22:31:13.162482+0000 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: mon.a (mon.0) 836 : audit 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 31:13.164201+0000 mon.b (mon.1) 2023-09-23T22:31:13.762 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: 80 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: -09-23T22:31:13.165035 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: +0000 mon.b (mon.1) 81 : audit 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[110082]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .1) 76 : audit [INF] 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31 2023-09-23T22:31:13.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :13.150771+0000 mon.a ( 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.0) 831 : audit 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.151479+0000 mon.b 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: (mon.1) 77 : audit 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.151769+0000 mon.a 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: (mon.0) 832 : audit 2023-09-23T22:31:13.764 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.152490+0000 mon.b 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: (mon.1) 78 : audit 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 31:13.152778+0000 mon.a ( 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.0) 833 : audit 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31: 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 13.152980+0000 mon.b (mon 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .1) 79 : audit [INF] 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 09-23T22:31: 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 13.153309+0000 mon.a (mon 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .0) 834 : audit [INF] 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :13.158297+0000 mon.a ( 2023-09-23T22:31:13.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.0) 835 : audit 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: -09-23T22:31 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :13.162482+0000 mon.a ( 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: mon.0) 836 : audit [INF] 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: :13.164201+0000 mon.b (mon 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .1) 80 : audit [DBG] 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: audit 2023-09-23T22:31:13.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 2023-09-23T22:31: 2023-09-23T22:31:13.768 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: 13.165035+0000 mon.b (mon 2023-09-23T22:31:13.768 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: .1) 81 : audit [INF] 2023-09-23T22:31:13.768 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:13 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:31:14.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cluster 2023-09-23T22:31:12. 2023-09-23T22:31:14.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 711229+0000 mgr.x (mgr.24787) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cephadm 2023-09-23T 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 22:31:13.153920+0000 mgr.x (mgr.24787) 13 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 2501M 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cephadm 2023-09-23T 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 22:31:13.154266+0000 mgr.x (mgr. 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 24787) 14 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 2500M 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cephadm 2023-09 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: -23T22:31:13.166307+0000 mgr.x (mgr.24787 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: ) 15 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 2023-09-23T22:31:13.166530+0000 mgr.x (mgr. 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 24787) 16 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:31:14.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 2023-09-23T22:31:13.430790+0000 mgr.x (mgr.24787) 17 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:13.431011+0000 mgr.x 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: (mgr.24787) 18 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: audit 2023-09-23T22:31:14 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: .128577+0000 mon.a (mon.0) 837 : audit [INF] 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: audit 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 2023-09-23T22:31:14.135225+0000 mon.a (mon.0) 838 : 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:31:14.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: T22:31:14.228755+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: audit 2023-09-23T22:31:14.233572+0000 mon.a (mon. 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: 0) 840 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: -09-23T22:31:14.237967+0000 mon.a (mon.0) 841 : audit [INF] 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cluster 2023-09-23T22:31:12.711229+0000 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: mgr.x (mgr.24787) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 2023-09-23T22:31:13.153920+0000 mgr.x (mgr.24787) 13 : 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm [INF] Adjusting osd_memory_target on smithi052 to 2501M 2023-09-23T22:31:14.641 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:13.154266 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: +0000 mgr.x (mgr.24787) 14 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 2500M 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 2023-09-23T22:31:13.166307+0000 mgr.x (mgr. 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 24787) 15 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 2023-09-23T22:31:13.166530+0000 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: mgr.x (mgr.24787) 16 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm 2023- 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 09-23T22:31:13.430790+0000 mgr.x (mgr. 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 24787) 17 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:31:14.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: cephadm 2023-09- 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 23T22:31:13.431011+0000 mgr.x (mgr. 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 24787) 18 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 23T22:31:14.128577+0000 mon.a ( 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: mon.0) 837 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: -09-23T22:31:14.135225+0000 mon.a 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: (mon.0) 838 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: -09-23T22:31:14.228755+0000 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: mon.a (mon.0) 839 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: audit 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: 2023-09-23T22:31:14.233572 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: +0000 mon.a (mon.0) 840 : audit [INF] 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: -09-23T22:31:14.237967+0000 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: mon.a (mon.0) 841 : audit [INF] 2023-09-23T22:31:14.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:14 smithi052 conmon[110082]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cluster 2023-09-23T22:31:12.711229+0000 mgr.x (mgr.24787) 12 : 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 24 op/s 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.153920+0000 mgr.x (mgr.24787) 13 : cephadm [INF] Adjusting osd_memory_target on smithi052 to 2501M 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.154266+0000 mgr.x (mgr.24787) 14 : cephadm [INF] Adjusting osd_memory_target on smithi151 to 2500M 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.166307+0000 mgr.x (mgr.24787) 15 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.166530+0000 mgr.x (mgr.24787) 16 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: 2023-09-23T22:31:13.430790+0000 mgr.x (mgr.24787) 17 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.431011+0000 mgr.x 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: (mgr.24787) 18 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:31:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: audit 2023-09-23T22:31:14.128577 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: +0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: audit 2023-09-23T22:31:14 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: .135225+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: 31:14.228755+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: audit 2023-09-23T22:31: 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: 14.233572+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:31:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:14 smithi151 conmon[100454]: :31:14.237967+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:14.993 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:14 smithi052 systemd[1]: Stopping Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:14.993 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:14 smithi052 bash[170993]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-node-exporter.a found: no such container 2023-09-23T22:31:15.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[138201]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:15] "GET /metrics HTTP/1.1" 200 31038 "" "Prometheus/2.33.4" 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.695026+0000 mgr.x (mgr.24787) 19 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: cephadm 2023-09-23T22 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: :31:13.695512+0000 mgr.x (mgr.24787) 20 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.908978+0000 mgr.x (mgr.24787) 21 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:13.959930+0000 mgr.x (mgr.24787) 22 : cephadm 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:31:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: :31:14.340754+0000 mon.a (mon.0 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: ) 842 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: 09-23T22:31:14.344182 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: +0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: 31:14.347211+0000 mon.a (mon.0) 844 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: T22:31:14.350250+0000 mon.a ( 2023-09-23T22:31:15.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: mon.0) 845 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: cephadm 2023-09- 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: 23T22:31:14.351314+0000 mgr.x (mgr.24787) 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: cephadm 2023-09- 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: 23T22:31:14.351626+0000 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: mgr.x (mgr.24787) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi052 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: -23T22:31:14.363039+0000 mon.a (mon 2023-09-23T22:31:15.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:15 smithi151 conmon[100454]: .0) 846 : audit [DBG] from='client.? 172.21.15.52:0/813446943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:15.662 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:15 smithi052 bash[171108]: 319ab89d682474b3091df72c468c6f1b87efb5353a58b18b03587f07a9587f3c 2023-09-23T22:31:15.662 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:15 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:31:15.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:13.695026+0000 mgr.x (mgr.24787 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: ) 19 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:13.695512+0000 mgr.x (mgr.24787) 20 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:13.908978+0000 mgr.x (mgr.24787) 21 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:13.959930+0000 mgr.x (mgr.24787) 22 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: audit 2023-09-23T22:31:14.340754+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: audit 2023-09-23T22:31:14.344182+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: audit 2023-09-23T22:31:14.347211+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: audit 2023-09-23T22:31:14.350250+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:14.351314+0000 mgr.x (mgr.24787) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:14.351626+0000 mgr.x (mgr.24787) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi052 2023-09-23T22:31:15.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[100224]: -09-23T22:31:14.363039+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.52:0/813446943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:13.695026+0000 mgr.x (mgr.24787) 19 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:13.695512+0000 mgr.x (mgr.24787) 20 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 2023-09-23T22:31:13.908978+0000 mgr.x (mgr.24787) 21 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: cephadm 2023- 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 09-23T22:31:13 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: .959930+0000 mgr.x (mgr.24787 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: ) 22 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:31:15.664 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: audit 2023-09-23T22:31:14.340754+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: audit 2023-09-23T22:31:14.344182+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: audit 2023-09-23T22:31:14. 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 347211+0000 mon.a (mon.0) 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 844 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: -09-23T22:31:14. 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 350250+0000 mon.a (mon.0) 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 845 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:15.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 14.351314+0000 mgr.x (mgr. 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 24787) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: cephadm 2023 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: -09-23T22:31:14. 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 351626+0000 mgr.x (mgr.24787) 24 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi052 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: 09-23T22:31:14.363039+0000 mon.a (mon.0 2023-09-23T22:31:15.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:15 smithi052 conmon[110082]: ) 846 : audit [DBG] from='client.? 172.21.15.52:0/813446943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:15.993 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:15 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T22:31:15.993 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:15 smithi052 systemd[1]: Stopped Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:15.993 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:15 smithi052 systemd[1]: Starting Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:16.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[110082]: cluster 2023-09-23T22:31:14.711711+0000 mgr.x 2023-09-23T22:31:16.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[110082]: (mgr.24787) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-09-23T22:31:16.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[110082]: audit 2023-09-23T22:31:16.283318+0000 mon.a ( 2023-09-23T22:31:16.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[110082]: mon.0) 847 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:16.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[110082]: audit 2023-09-23T22:31:16.289708 2023-09-23T22:31:16.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[110082]: +0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:16.494 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.235Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T22:31:16.494 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.235Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T22:31:16.494 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.235Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T22:31:16.494 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.235Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T22:31:16.495 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T22:31:16.496 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T22:31:16.497 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.236Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[171475]: ts=2023-09-23T22:31:16.237Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T22:31:16.498 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 bash[171434]: 3bd65f3ab9c988b957ee13f59898c0f7e3c8445aee18c2724eae4b1a9eacc3e0 2023-09-23T22:31:16.499 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 22:31:16 smithi052 systemd[1]: Started Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: cluster 2023-09-23T22:31:14.711711+0000 mgr.x ( 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: mgr.24787) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: audit 2023-09-23T22:31:16. 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: 283318+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: 09-23T22:31:16 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: .289708+0000 mon.a (mon.0) 848 : 2023-09-23T22:31:16.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:16 smithi052 conmon[100224]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:16.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:16 smithi151 conmon[100454]: cluster 2023-09-23T22:31:14.711711+0000 mgr.x (mgr.24787) 25 : cluster [DBG] 2023-09-23T22:31:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:16 smithi151 conmon[100454]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-09-23T22:31:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:16 smithi151 conmon[100454]: audit 2023-09-23T22:31:16.283318+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:16 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:31:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:16 smithi151 conmon[100454]: -23T22:31:16.289708+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:17.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:17 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:16.320436+0000 mgr.x (mgr.24787) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:31:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:17 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:17 smithi151 conmon[100454]: 2023-09-23T22:31:16.332486+0000 mgr.x (mgr.24787) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi052 2023-09-23T22:31:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:17 smithi151 conmon[100454]: audit 2023-09-23T22:31:16.805848+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.52:0/207776757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:31:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: 31:16.320436+0000 mgr.x (mgr.24787) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:31:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: cephadm 2023-09 2023-09-23T22:31:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: -23T22:31:16.332486+0000 mgr.x (mgr 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: .24787) 27 : cephadm 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: [INF] Reconfiguring daemon alertmanager.a on smithi052 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: audit 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: 2023-09-23T22:31 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: :16.805848+0000 mon.c (mon 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: .2) 51 : audit [DBG] 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/207776757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:16.320436+0000 mgr.x (mgr 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[100224]: .24787) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:31:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:16.332486+0000 mgr.x (mgr.24787) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi052 2023-09-23T22:31:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:17 smithi052 conmon[100224]: audit 2023-09-23T22:31:16.805848+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.52:0/207776757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:18.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:31:18.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[110082]: cluster 2023-09-23T22:31 2023-09-23T22:31:18.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[110082]: :16.712130+0000 mgr.x (mgr.24787) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[110082]: 23T22:31:17.902157+0000 mon.b (mon.1 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[110082]: ) 82 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[100224]: cluster 2023-09-23T22:31:16.712130+0000 mgr.x (mgr.24787 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[100224]: ) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[100224]: audit 2023-09-23T22:31:17.902157+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:18.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:18 smithi052 conmon[100224]: 2023-09-23T22:31:18.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:18 smithi151 conmon[100454]: cluster 2023-09-23T22:31:16.712130+0000 mgr.x (mgr.24787) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-09-23T22:31:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:18 smithi151 conmon[100454]: audit 2023-09-23T22:31:17.902157+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:19.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:19 smithi151 conmon[100454]: audit 2023-09-23T22:31:19.239246+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.52:0/3309831866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:19 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:31:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:19 smithi052 conmon[110082]: :31:19.239246+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.52:0/3309831866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:19.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:19 smithi052 conmon[100224]: audit 2023-09-23T22:31:19. 2023-09-23T22:31:19.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:19 smithi052 conmon[100224]: 239246+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.52:0/3309831866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:20.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:20 smithi151 conmon[100454]: cluster 2023-09-23T22:31:18.712681+0000 mgr.x (mgr.24787) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T22:31:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:20 smithi052 conmon[110082]: cluster 2023-09-23T22:31:18.712681+0000 mgr.x (mgr.24787) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T22:31:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:20 smithi052 conmon[100224]: cluster 2023-09-23T22:31:18.712681+0000 mgr.x (mgr.24787) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T22:31:22.686 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[100224]: cluster 2023-09-23T22:31:20.713463+0000 mgr.x (mgr 2023-09-23T22:31:22.686 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[100224]: .24787) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T22:31:22.686 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[100224]: audit 2023-09-23T22:31:21.661174+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.52:0/3828830602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:22.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[110082]: cluster 2023-09-23T22:31: 2023-09-23T22:31:22.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[110082]: 20.713463+0000 mgr.x (mgr.24787) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T22:31:22.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:31:22.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[110082]: :31:21.661174+0000 mon.a (mon.0) 2023-09-23T22:31:22.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[110082]: 850 : audit [DBG] from='client.? 172.21.15.52:0/3828830602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:22.687 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:22 smithi052 systemd[1]: Stopping Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:22.687 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:22 smithi052 bash[172006]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-alertmanager.a found: no such container 2023-09-23T22:31:22.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:22 smithi151 conmon[133561]: ts=2023-09-23T22:31:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.52:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:31:22.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:22 smithi151 conmon[100454]: cluster 2023-09-23T22:31:20.713463+0000 mgr.x (mgr.24787) 30 : cluster 2023-09-23T22:31:22.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:22 smithi151 conmon[100454]: [DBG] pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-09-23T22:31:22.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:22 smithi151 conmon[100454]: audit 2023-09-23T22:31:21.661174+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.52:0/3828830602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:22.993 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:22 smithi052 conmon[151062]: level=info ts=2023-09-23T22:31:22.685Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:31:23.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 bash[172122]: f6ec125d25a1efdb37354e69173ef9460c7537a44b41332a79c4c44916b082ea 2023-09-23T22:31:23.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:31:23.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 systemd[1]: Stopped Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:23.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 systemd[1]: Starting Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:24.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.946Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-09-23T22:31:24.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.946Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-09-23T22:31:24.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.948Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.52 port=9094 2023-09-23T22:31:24.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.949Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:31:24.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 bash[172450]: a82e31979683aee0a1ff291c99740c816d2ab6fcde10e183b5a25365db947a32 2023-09-23T22:31:24.244 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 systemd[1]: Started Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:24.244 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.985Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:31:24.244 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.985Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:31:24.244 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.988Z caller=main.go:518 msg=Listening address=:9093 2023-09-23T22:31:24.244 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:23 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:23.988Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-09-23T22:31:24.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:24 smithi151 conmon[100454]: cluster 2023-09-23T22:31:22.713870+0000 mgr.x (mgr.24787) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T22:31:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:24 smithi151 conmon[100454]: audit 2023-09-23T22:31:23.998172+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:24 smithi151 conmon[100454]: audit 2023-09-23T22:31:24.004296+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:24 smithi151 conmon[100454]: audit 2023-09-23T22:31:24.071070+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.52:0/88658873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:24.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[100224]: cluster 2023-09-23T22:31:22.713870+0000 mgr.x (mgr 2023-09-23T22:31:24.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[100224]: .24787) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T22:31:24.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[100224]: audit 2023-09-23T22:31:23.998172+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:24.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[100224]: audit 2023-09-23T22:31:24.004296+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:24.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[100224]: audit 2023-09-23T22:31:24.071070+0000 mon.c (mon 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[100224]: .2) 52 : audit [DBG] from='client.? 172.21.15.52:0/88658873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: 31:22.713870+0000 mgr.x (mgr.24787) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: -09-23T22:31:23.998172 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: +0000 mon.a (mon.0) 851 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: -09-23T22:31:24 2023-09-23T22:31:24.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: .004296+0000 mon.a (mon. 2023-09-23T22:31:24.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: 0) 852 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:24.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: audit 2023-09-23T22:31:24.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: 2023-09-23T22:31: 2023-09-23T22:31:24.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: 24.071070+0000 mon.c (mon 2023-09-23T22:31:24.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: .2) 52 : audit [DBG] 2023-09-23T22:31:24.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:24 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/88658873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:25.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:25 smithi151 conmon[138201]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:25] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.33.4" 2023-09-23T22:31:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:25 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:24.015535+0000 mgr.x (mgr 2023-09-23T22:31:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:25 smithi151 conmon[100454]: .24787) 32 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:31:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:25 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:24.236246+0000 mgr.x (mgr.24787) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:31:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:24. 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[100224]: 015535+0000 mgr.x (mgr.24787) 32 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:24.236246+0000 mgr.x (mgr.24787 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[100224]: ) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:24.015535+0000 mgr.x 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[110082]: (mgr.24787) 32 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:24.236246+0000 2023-09-23T22:31:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[110082]: mgr.x (mgr.24787) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:31:26.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:25 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:25.949Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000022261s 2023-09-23T22:31:26.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:26 smithi151 conmon[100454]: cluster 2023-09-23T22:31:24.714557+0000 mgr.x (mgr 2023-09-23T22:31:26.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:26 smithi151 conmon[100454]: .24787) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T22:31:26.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:26 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:31:26.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:26 smithi052 conmon[110082]: :31:24.714557+0000 mgr.x (mgr.24787) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T22:31:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:26 smithi052 conmon[100224]: cluster 2023-09-23T22:31:24.714557+0000 mgr.x (mgr 2023-09-23T22:31:26.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:26 smithi052 conmon[100224]: .24787) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T22:31:27.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:27 smithi151 conmon[100454]: audit 2023-09-23T22:31:26.473604 2023-09-23T22:31:27.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:27 smithi151 conmon[100454]: +0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.52:0/3846481040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:27 smithi052 conmon[110082]: audit 2023-09-23T22:31:26.473604+0000 mon.c 2023-09-23T22:31:27.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:27 smithi052 conmon[110082]: (mon.2) 53 : audit [DBG] from='client.? 172.21.15.52:0/3846481040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:27.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:27 smithi052 conmon[100224]: audit 2023-09-23T22:31:26.473604+0000 2023-09-23T22:31:27.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:27 smithi052 conmon[100224]: mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.52:0/3846481040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:28.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:31:28 smithi052 conmon[100670]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-09-23T22:31:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:28 smithi052 conmon[100224]: cluster 2023-09-23T22:31:26.714867+0000 mgr.x (mgr.24787) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:28.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:28 smithi052 conmon[110082]: cluster 2023-09-23T22:31:26 2023-09-23T22:31:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:28 smithi052 conmon[110082]: .714867+0000 mgr.x (mgr.24787) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:28.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:28 smithi151 conmon[100454]: cluster 2023-09-23T22:31:26.714867+0000 mgr.x (mgr. 2023-09-23T22:31:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:28 smithi151 conmon[100454]: 24787) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:29 smithi151 conmon[100454]: audit 2023-09-23T22:31:28.874947+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.52:0/3876703107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:29.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:29 smithi052 conmon[110082]: audit 2023-09-23T22:31:28.874947+0000 mon.a (mon.0 2023-09-23T22:31:29.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:29 smithi052 conmon[110082]: ) 853 : audit [DBG] from='client.? 172.21.15.52:0/3876703107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:29 smithi052 conmon[100224]: audit 2023-09-23T22:31:28.874947+0000 mon.a ( 2023-09-23T22:31:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:29 smithi052 conmon[100224]: mon.0) 853 : audit [DBG] from='client.? 172.21.15.52:0/3876703107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:30.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:30 smithi151 conmon[100454]: cluster 2023-09-23T22:31:28.715319+0000 mgr.x (mgr.24787) 36 : cluster [DBG] 2023-09-23T22:31:30.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:30 smithi151 conmon[100454]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:30 smithi052 conmon[110082]: cluster 2023-09-23T22:31: 2023-09-23T22:31:30.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:30 smithi052 conmon[110082]: 28.715319+0000 mgr.x (mgr.24787) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:30.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:30 smithi052 conmon[100224]: cluster 2023-09-23T22:31:28.715319 2023-09-23T22:31:30.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:30 smithi052 conmon[100224]: +0000 mgr.x (mgr.24787) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:31.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:31 smithi151 conmon[100454]: audit 2023-09-23T22:31:31.280345+0000 mon.c (mon.2) 54 : audit 2023-09-23T22:31:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:31 smithi151 conmon[100454]: [DBG] from='client.? 172.21.15.52:0/3088460345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:31 smithi052 conmon[110082]: audit 2023-09-23T22:31:31 2023-09-23T22:31:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:31 smithi052 conmon[110082]: .280345+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.52:0/3088460345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:31 smithi052 conmon[100224]: audit 2023-09-23T22:31:31.280345+0000 mon.c ( 2023-09-23T22:31:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:31 smithi052 conmon[100224]: mon.2) 54 : audit [DBG] from='client.? 172.21.15.52:0/3088460345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:32.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:32 smithi151 conmon[133561]: ts=2023-09-23T22:31:32.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.52:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:31:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:32 smithi151 conmon[100454]: cluster 2023-09-23T22:31:30.715950+0000 mgr.x (mgr.24787) 37 : cluster 2023-09-23T22:31:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:32 smithi151 conmon[100454]: [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:32.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:32 smithi052 conmon[110082]: cluster 2023-09-23T22:31:30.715950+0000 2023-09-23T22:31:32.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:32 smithi052 conmon[110082]: mgr.x (mgr.24787) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:32.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:32 smithi052 conmon[100224]: cluster 2023-09-23T22:31:30.715950+0000 mgr.x (mgr.24787) 37 : cluster 2023-09-23T22:31:32.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:32 smithi052 conmon[100224]: [DBG] pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:33.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:33 smithi151 conmon[100454]: audit 2023-09-23T22:31:32.902369+0000 mon.b (mon.1) 83 : 2023-09-23T22:31:33.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:33 smithi151 conmon[100454]: audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:33.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:33 smithi052 conmon[110082]: audit 2023-09-23T22:31:32.902369+0000 mon.b (mon. 2023-09-23T22:31:33.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:33 smithi052 conmon[110082]: 1) 83 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:33.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:33 smithi052 conmon[100224]: audit 2023-09-23T22:31:32. 2023-09-23T22:31:33.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:33 smithi052 conmon[100224]: 902369+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:34.243 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 22:31:33 smithi052 conmon[172520]: level=info ts=2023-09-23T22:31:33.952Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002879517s 2023-09-23T22:31:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[100454]: cluster 2023-09-23T22:31:32.716393+0000 mgr.x (mgr.24787) 2023-09-23T22:31:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[100454]: 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[100454]: audit 2023-09-23T22:31:33.687597+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.52:0/4137170267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:34.700 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 systemd[1]: Stopping Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:34.700 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 bash[143585]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-prometheus.a found: no such container 2023-09-23T22:31:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[100224]: cluster 2023-09-23T22:31:32.716393+0000 2023-09-23T22:31:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[100224]: mgr.x (mgr.24787) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[100224]: :31:33.687597+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.52:0/4137170267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[110082]: cluster 2023-09-23T22:31:32 2023-09-23T22:31:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[110082]: .716393+0000 mgr.x (mgr.24787) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[110082]: audit 2023-09-23T22:31:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[110082]: 2023-09-23T22:31:33. 2023-09-23T22:31:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[110082]: 687597+0000 mon.a (mon.0) 854 : audit 2023-09-23T22:31:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:34 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/4137170267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.734Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.734Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.734Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.734Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.734Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.734Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T22:31:35.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.735Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:31:35.190 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.735Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T22:31:35.190 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.735Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:31:35.190 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.735Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:31:35.190 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.735Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T22:31:35.190 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:34 smithi151 conmon[133561]: ts=2023-09-23T22:31:34.735Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T22:31:35.190 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 bash[143703]: 73d582b307e40d22ae8960aa76a02a5082f6fbbd2bb503a3e9d30629e6f5ac7b 2023-09-23T22:31:35.448 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:31:35.448 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 systemd[1]: Stopped Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:35.449 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 systemd[1]: Starting Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:36.198 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 bash[144039]: 1da9848cdc25f3f868568f701f788a0ce8573b4a90ccf5cc7263dfd4d9d67637 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.944Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.944Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.944Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi151 (none))" 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.944Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.944Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.946Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.946Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.947Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 systemd[1]: Started Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.950Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:31:36.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.950Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.839µs 2023-09-23T22:31:36.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:35 smithi151 conmon[144080]: ts=2023-09-23T22:31:35.950Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:31:36.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.116Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T22:31:36.696 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: cluster 2023-09-23T22:31:34.716886+0000 mgr.x (mgr 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: .24787) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: audit 2023-09-23T22:31:35.976438+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: 09-23T22:31:35.981820+0000 mon.a (mon. 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: 0) 856 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: audit 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: 2023-09-23T22:31:36.089519+0000 mon.a 2023-09-23T22:31:36.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[100454]: (mon.0) 857 : audit [DBG] from='client.? 172.21.15.52:0/3906340407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:36.697 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.344Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.344Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.344Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.844µs wal_replay_duration=394.039286ms total_replay_duration=394.091542ms 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.347Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.347Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.347Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.366Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.737998ms db_storage=957ns remote_storage=1.397µs web_handler=376ns query_engine=624ns scrape=371.691µs scrape_sd=102.377µs notify=17.189µs notify_sd=15.33µs rules=17.995909ms 2023-09-23T22:31:36.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:36 smithi151 conmon[144080]: ts=2023-09-23T22:31:36.366Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T22:31:36.698 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:36 smithi151 systemd[1]: Stopping Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:36.698 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:36 smithi151 bash[144234]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-node-exporter.b found: no such container 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: cluster 2023-09-23T22:31:34.716886+0000 mgr.x (mgr.24787) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: audit 2023-09-23T22:31:35.976438+0000 mon.a (mon.0) 855 : audit [INF] 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: audit 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: 2023-09-23T22:31:35. 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: 981820+0000 mon.a (mon.0 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: ) 856 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: audit 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: 2023-09-23T22:31 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: :36.089519+0000 mon.a (mon.0) 857 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[100224]: : audit [DBG] from='client.? 172.21.15.52:0/3906340407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: cluster 2023-09-23T22:31 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: :34.716886+0000 mgr.x (mgr.24787) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: audit 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: 2023-09 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: -23T22:31:35. 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: 976438+0000 mon.a (mon.0 2023-09-23T22:31:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: ) 855 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: -09-23T22:31:35.981820 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: +0000 mon.a (mon.0) 856 : 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: -23T22:31:36.089519+0000 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: mon.a (mon.0) 857 : 2023-09-23T22:31:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:36 smithi052 conmon[110082]: audit [DBG] from='client.? 172.21.15.52:0/3906340407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:37.419 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 bash[144350]: ab706ae51a5db211cac3634070ce70e71ce4e24c51d300df631d8b381d24a317 2023-09-23T22:31:37.419 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:31:37.419 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T22:31:37.419 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 systemd[1]: Stopped Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:37.419 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 systemd[1]: Starting Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:37.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:36.024530 2023-09-23T22:31:37.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[100454]: +0000 mgr.x (mgr.24787) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:31:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:36.024992+0000 mgr.x (mgr.24787) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi151 2023-09-23T22:31:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:36. 2023-09-23T22:31:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[110082]: 024530+0000 mgr.x (mgr.24787) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:31:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[110082]: cephadm 2023-09-23T22:31: 2023-09-23T22:31:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[110082]: 36.024992+0000 mgr.x (mgr. 2023-09-23T22:31:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[110082]: 24787) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi151 2023-09-23T22:31:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[110082]: 2023-09-23T22:31:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:36.024530+0000 mgr.x ( 2023-09-23T22:31:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[100224]: mgr.24787) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:31:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:36.024992+0000 mgr.x (mgr.24787) 41 : 2023-09-23T22:31:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:37 smithi052 conmon[100224]: cephadm [INF] Reconfiguring daemon node-exporter.b on smithi151 2023-09-23T22:31:38.199 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.857Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-09-23T22:31:38.199 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.857Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-09-23T22:31:38.199 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.857Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-09-23T22:31:38.199 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.857Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=arp 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=bcache 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=bonding 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=btrfs 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=conntrack 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=cpu 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=diskstats 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=dmi 2023-09-23T22:31:38.200 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=edac 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=entropy 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=filefd 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=filesystem 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=hwmon 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=infiniband 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=ipvs 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=loadavg 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=mdadm 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=meminfo 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=netclass 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=netdev 2023-09-23T22:31:38.201 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=netstat 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=nfs 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=nfsd 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=nvme 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=os 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=pressure 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=rapl 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=schedstat 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=sockstat 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=softnet 2023-09-23T22:31:38.202 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=stat 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=tapestats 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=textfile 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=time 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=uname 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=vmstat 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=xfs 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:115 level=info collector=zfs 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 conmon[144720]: ts=2023-09-23T22:31:37.858Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-09-23T22:31:38.203 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 bash[144679]: 7e4657a95489b7700a5038813c86d137828b91ad552fadfbd81b4ede8454653e 2023-09-23T22:31:38.204 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 22:31:37 smithi151 systemd[1]: Started Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:38 smithi151 conmon[100454]: cluster 2023-09-23T22:31:36.717176+0000 mgr.x (mgr.24787) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:38 smithi151 conmon[100454]: audit 2023-09-23T22:31:37.907031+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:38 smithi151 conmon[100454]: audit 2023-09-23T22:31:37.913712+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:38.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: cluster 2023-09-23T22:31:36. 2023-09-23T22:31:38.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: 717176+0000 mgr.x (mgr.24787) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:38.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:38.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: 09-23T22:31:37.907031+0000 mon.a (mon.0) 858 : audit 2023-09-23T22:31:38.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: audit 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: 2023-09-23T22:31:37.913712+0000 mon.a (mon 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[110082]: .0) 859 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: cluster 2023-09-23T22:31: 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: 36.717176+0000 mgr.x (mgr.24787) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: audit 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: 2023-09-23T22:31:37.907031+0000 mon.a (mon.0) 858 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: -23T22:31:37.913712+0000 mon.a ( 2023-09-23T22:31:38.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:38 smithi052 conmon[100224]: mon.0) 859 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:39.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:39 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:37.941720+0000 mgr.x (mgr.24787) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:31:39.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:39 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:37.952619+0000 2023-09-23T22:31:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:39 smithi151 conmon[100454]: mgr.x (mgr.24787) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi151 2023-09-23T22:31:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:39 smithi151 conmon[100454]: audit 2023-09-23T22:31:38.492219+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.52:0/2176209957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:39.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:37.941720+0000 mgr.x ( 2023-09-23T22:31:39.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: mgr.24787) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:31:39.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:37.952619+0000 mgr.x (mgr.24787) 44 : 2023-09-23T22:31:39.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: cephadm [INF] Reconfiguring daemon grafana.a on smithi151 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: -23T22:31:38.492219+0000 mon.a (mon.0) 860 : audit [DBG] 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/2176209957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:37.941720+0000 mgr.x (mgr. 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[100224]: 24787) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:37.952619+0000 mgr.x (mgr.24787 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[100224]: ) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi151 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[100224]: audit 2023-09-23T22:31:38 2023-09-23T22:31:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:39 smithi052 conmon[100224]: .492219+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.52:0/2176209957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:40.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:40 smithi151 conmon[100454]: cluster 2023-09-23T22:31:38.717591 2023-09-23T22:31:40.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:40 smithi151 conmon[100454]: +0000 mgr.x (mgr.24787) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:40.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:40 smithi052 conmon[110082]: cluster 2023-09-23T22:31:38.717591+0000 2023-09-23T22:31:40.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:40 smithi052 conmon[110082]: mgr.x (mgr.24787) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:40 smithi052 conmon[100224]: cluster 2023-09-23T22:31:38.717591+0000 mgr.x (mgr.24787 2023-09-23T22:31:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:40 smithi052 conmon[100224]: ) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:41.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:41 smithi151 conmon[100454]: audit 2023-09-23T22:31:40.895650 2023-09-23T22:31:41.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:41 smithi151 conmon[100454]: +0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.52:0/4100801128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:41.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:41 smithi052 conmon[110082]: audit 2023-09-23T22:31:40.895650+0000 mon.a 2023-09-23T22:31:41.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:41 smithi052 conmon[110082]: (mon.0) 861 : audit [DBG] from='client.? 172.21.15.52:0/4100801128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:41.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:41 smithi052 conmon[100224]: audit 2023-09-23T22:31:40.895650+0000 mon.a (mon.0) 861 : audit [DBG] 2023-09-23T22:31:41.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:41 smithi052 conmon[100224]: from='client.? 172.21.15.52:0/4100801128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:42.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:42 smithi151 conmon[100454]: cluster 2023-09-23T22:31:40.717961 2023-09-23T22:31:42.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:42 smithi151 conmon[100454]: +0000 mgr.x (mgr.24787) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:42 smithi052 conmon[100224]: cluster 2023-09-23T22:31:40.717961+0000 2023-09-23T22:31:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:42 smithi052 conmon[100224]: mgr.x (mgr.24787) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:42.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:42 smithi052 conmon[110082]: cluster 2023-09-23T22:31:40.717961+0000 mgr.x (mgr.24787) 46 : cluster 2023-09-23T22:31:42.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:42 smithi052 conmon[110082]: [DBG] pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:43.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:43 smithi151 conmon[100454]: audit 2023-09-23T22:31:43.296488+0000 mon.c (mon.2) 55 : 2023-09-23T22:31:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:43 smithi151 conmon[100454]: audit [DBG] from='client.? 172.21.15.52:0/2444008850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:43 smithi052 conmon[110082]: audit 2023-09-23T22:31:43.296488+0000 mon.c (mon.2) 55 : audit 2023-09-23T22:31:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:43 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/2444008850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:43 smithi052 conmon[100224]: audit 2023-09-23T22:31:43.296488+0000 mon.c 2023-09-23T22:31:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:43 smithi052 conmon[100224]: (mon.2) 55 : audit [DBG] from='client.? 172.21.15.52:0/2444008850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:44.439 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:44 smithi151 conmon[100454]: cluster 2023-09-23T22:31:42.718337+0000 mgr.x (mgr.24787) 47 2023-09-23T22:31:44.439 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:44 smithi151 conmon[100454]: : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:44.439 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:43 smithi151 conmon[138201]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:43] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.33.4" 2023-09-23T22:31:44.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:44 smithi052 conmon[110082]: cluster 2023-09-23T22:31:42.718337+0000 mgr.x (mgr. 2023-09-23T22:31:44.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:44 smithi052 conmon[110082]: 24787) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:44.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:44 smithi052 conmon[100224]: cluster 2023-09-23T22:31:42.718337+0000 mgr.x (mgr. 2023-09-23T22:31:44.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:44 smithi052 conmon[100224]: 24787) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:46.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:46 smithi151 conmon[100454]: cluster 2023-09-23T22:31:44.718822+0000 mgr.x (mgr.24787 2023-09-23T22:31:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:46 smithi151 conmon[100454]: ) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:46 smithi151 conmon[100454]: audit 2023-09-23T22:31:45.699259+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.52:0/678185465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[110082]: cluster 2023-09-23T22:31:44.718822+0000 mgr.x 2023-09-23T22:31:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[110082]: (mgr.24787) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:31:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[110082]: T22:31:45.699259+0000 mon.c (mon.2) 56 : audit [DBG] 2023-09-23T22:31:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[110082]: from='client.? 172.21.15.52:0/678185465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[100224]: cluster 2023-09-23T22:31:44. 2023-09-23T22:31:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[100224]: 718822+0000 mgr.x (mgr.24787) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[100224]: audit 2023-09-23T22:31:45.699259+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.52:0/678185465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:46 smithi052 conmon[100224]: 2023-09-23T22:31:48.698 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[144080]: ts=2023-09-23T22:31:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:31:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[100454]: cluster 2023-09-23T22:31:46.719154+0000 mgr.x (mgr.24787) 2023-09-23T22:31:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[100454]: 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[100454]: audit 2023-09-23T22:31:47.902601+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[100454]: audit 2023-09-23T22:31:48.107008 2023-09-23T22:31:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[100454]: +0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.52:0/3298121176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:48 smithi151 conmon[100454]: 2023-09-23T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[100224]: cluster 2023-09-23T22:31:46.719154+0000 mgr.x (mgr.24787) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[100224]: -09-23T22:31:47.902601+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[100224]: -09-23T22:31:48.107008+0000 mon.a (mon 2023-09-23T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[100224]: .0) 862 : audit [DBG] from='client.? 172.21.15.52:0/3298121176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: cluster 2023-09-23T22:31:46.719154+0000 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: mgr.x (mgr.24787) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: 22:31:47.902601+0000 mon.b (mon.1) 84 : audit [DBG] 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: audit 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: 2023-09-23T22:31:48.107008+0000 mon.a 2023-09-23T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:48 smithi052 conmon[110082]: (mon.0) 862 : audit [DBG] from='client.? 172.21.15.52:0/3298121176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:50 smithi151 conmon[100454]: cluster 2023-09-23T22:31:48.719628+0000 mgr.x (mgr.24787) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:50 smithi052 conmon[110082]: cluster 2023-09-23T22:31:48.719628+0000 mgr.x ( 2023-09-23T22:31:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:50 smithi052 conmon[110082]: mgr.24787) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:50.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:50 smithi052 conmon[100224]: cluster 2023-09-23T22:31: 2023-09-23T22:31:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:50 smithi052 conmon[100224]: 48.719628+0000 mgr.x (mgr.24787) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:51.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:51 smithi052 conmon[110082]: audit 2023-09-23T22:31:50.515984+0000 2023-09-23T22:31:51.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:51 smithi052 conmon[110082]: mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.52:0/48615413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:51 smithi052 conmon[100224]: audit 2023-09-23T22:31:50. 2023-09-23T22:31:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:51 smithi052 conmon[100224]: 515984+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.52:0/48615413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:51.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:51 smithi151 conmon[100454]: audit 2023-09-23T22:31:50.515984+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.52:0/48615413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:52 smithi052 conmon[110082]: cluster 2023-09-23T22:31:50.720136+0000 mgr.x 2023-09-23T22:31:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:52 smithi052 conmon[110082]: (mgr.24787) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:52 smithi052 conmon[100224]: cluster 2023-09-23T22:31:50.720136+0000 mgr.x (mgr.24787 2023-09-23T22:31:52.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:52 smithi052 conmon[100224]: ) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:52.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:52 smithi151 conmon[100454]: cluster 2023-09-23T22:31:50.720136+0000 mgr.x (mgr.24787) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:53.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:53 smithi052 conmon[110082]: audit 2023-09-23T22:31:52.918293+0000 mon.c ( 2023-09-23T22:31:53.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:53 smithi052 conmon[110082]: mon.2) 58 : audit [DBG] from='client.? 172.21.15.52:0/2657048436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:53.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:53 smithi052 conmon[100224]: audit 2023-09-23T22:31:52. 2023-09-23T22:31:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:53 smithi052 conmon[100224]: 918293+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.52:0/2657048436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:53.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:53 smithi151 conmon[100454]: audit 2023-09-23T22:31:52.918293+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.52:0/2657048436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:53 smithi151 conmon[100454]: 2023-09-23T22:31:54.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:53 smithi151 conmon[138201]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:31:53] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.33.4" 2023-09-23T22:31:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:54 smithi052 conmon[110082]: cluster 2023-09-23T22:31:52.720561+0000 mgr.x (mgr.24787) 52 : 2023-09-23T22:31:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:54 smithi052 conmon[110082]: cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:54.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:54 smithi052 conmon[100224]: cluster 2023-09-23T22:31:52.720561+0000 mgr.x (mgr 2023-09-23T22:31:54.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:54 smithi052 conmon[100224]: .24787) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:54 smithi151 conmon[100454]: cluster 2023-09-23T22:31:52.720561+0000 mgr.x (mgr.24787) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:54 smithi151 conmon[100454]: 2023-09-23T22:31:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:56 smithi151 conmon[100454]: cluster 2023-09-23T22:31:54.721089+0000 mgr.x (mgr.24787) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:56 smithi151 conmon[100454]: audit 2023-09-23T22:31:55.320531+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.52:0/2521478777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: cluster 2023-09-23T22:31:54.721089 2023-09-23T22:31:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: +0000 mgr.x (mgr.24787) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: audit 2023-09-23T22:31:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: 2023-09 2023-09-23T22:31:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: -23T22:31:55.320531 2023-09-23T22:31:56.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: +0000 mon.a (mon.0) 863 2023-09-23T22:31:56.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[110082]: : audit [DBG] from='client.? 172.21.15.52:0/2521478777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:31:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[100224]: T22:31:54.721089+0000 mgr.x (mgr.24787) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:56 smithi052 conmon[100224]: -23T22:31:55.320531+0000 mon.a (mon.0) 863 : audit [DBG] from='client.? 172.21.15.52:0/2521478777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:56.766 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:56 smithi151 systemd[1]: Stopping Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:56.766 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:56 smithi151 bash[145132]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-grafana.a found: no such container 2023-09-23T22:31:57.026 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:56 smithi151 conmon[131789]: t=2023-09-23T22:31:56+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-09-23T22:31:57.026 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:56 smithi151 bash[145247]: 6cfce387f58fdecf283a8af4f7ca3025a0b3b38189c94e7f1d8bdd7377c092f8 2023-09-23T22:31:57.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:57 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@grafana.a.service: Succeeded. 2023-09-23T22:31:57.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:57 smithi151 systemd[1]: Stopped Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:57.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:57 smithi151 systemd[1]: Starting Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:31:58.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STOPPING 2023-09-23T22:31:58.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:31:58.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STOPPED 2023-09-23T22:31:58.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STARTING 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Serving on http://:::9283 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STARTED 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STOPPING 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: cluster 2023-09-23T22:31:56.721474+0000 mgr.x (mgr. 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 24787) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:57.725747+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.52:0/2479962323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:58.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.058786+0000 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: mon.a (mon.0) 864 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 2023-09-23T22: 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 31:58.063516+0000 mon.a (mon.0) 865 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 23T22:31:58.116831+0000 mon.b (mon 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: .1) 85 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: -09-23T22:31:58.118282+0000 2023-09-23T22:31:58.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: mon.b (mon.1) 86 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 2023-09-23T22:31:58.122373+0000 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: mon.a (mon.0) 866 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: :58.134871+0000 mon.b (mon.1) 87 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: T22:31:58.138654+0000 mon.b (mon. 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 1) 88 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 2023-09-23T22:31 2023-09-23T22:31:58.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: :58.143069+0000 mon.a (mon.0) 867 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 2023-09-23T22:31:58.152736 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: +0000 mon.b (mon.1) 89 : audit [DBG] 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 23T22:31:58.156032+0000 mon.b (mon 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: .1) 90 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 2023-09-23T22:31:58 2023-09-23T22:31:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: .160774+0000 mon.a (mon.0) 868 : audit [INF] 2023-09-23T22:31:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:31:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 23T22:31:58.172043+0000 mon.b (mon. 2023-09-23T22:31:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[100454]: 1) 91 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:58.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 bash[145588]: 8803df2494ffc3cb481e394160a62d1230fdbbfa3f8d47e2c90b5183004a9ce8 2023-09-23T22:31:58.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-09-23T22:31:58.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-09-23T22:31:58.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="App mode production" logger=settings 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 systemd[1]: Started Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=547.409µs 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-09-23T22:31:58.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-09-23T22:31:58.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-09-23T22:31:58.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-09-23T22:31:58.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T22:31:58.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: cluster 2023-09-23T22:31:56.721474+0000 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: mgr.x (mgr.24787) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 23T22:31:57.725747+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.52:0/2479962323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 2023-09-23T22:31:58.058786+0000 mon.a (mon. 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 0) 864 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: -09-23T22:31:58.063516+0000 mon.a 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: (mon.0) 865 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 2023-09-23T22:31:58.116831+0000 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: mon.b (mon.1) 85 : audit 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 09-23T22:31:58.118282+0000 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: mon.b (mon.1) 86 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: .122373+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: .134871+0000 mon.b (mon.1) 87 : audit [DBG] 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.138654 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: +0000 mon.b (mon.1) 88 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: :58.143069+0000 mon.a (mon.0) 867 : 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: T22:31:58.152736+0000 mon.b (mon.1) 89 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:31:58.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 31:58.156032+0000 mon.b ( 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: mon.1) 90 : audit [INF] 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 2023-09-23T22:31:58.160774+0000 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: mon.a (mon.0) 868 : audit 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: 09-23T22:31:58.172043+0000 mon.b ( 2023-09-23T22:31:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: mon.1) 91 : audit [DBG] 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: cluster 2023-09-23T22:31:56.721474+0000 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: mgr.x (mgr.24787) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31:57.725747+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.52:0/2479962323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.058786+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: :31:58.063516+0000 mon.a 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: (mon.0) 865 : audit 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 2023-09-23T22:31: 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 58.116831+0000 mon.b (mon. 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 1) 85 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 2023-09-23T22:31 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: :58.118282+0000 mon.b (mon.1) 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 86 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 2023-09 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: -23T22:31:58.122373 2023-09-23T22:31:58.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: +0000 mon.a (mon.0) 866 : audit 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: -09-23T22:31: 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 58.134871+0000 mon.b (mon.1) 87 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: -09-23T22:31:58.138654 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: +0000 mon.b (mon.1) 88 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 09-23T22:31:58.143069+0000 mon.a (mon. 2023-09-23T22:31:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 0) 867 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31: 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 58.152736+0000 mon.b (mon.1) 89 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.156032+0000 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: mon.b (mon.1) 90 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: -23T22:31:58. 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 160774+0000 mon.a (mon.0) 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 868 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 23T22:31:58.172043+0000 mon.b (mon.1) 2023-09-23T22:31:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:58 smithi052 conmon[100224]: 91 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:58.659 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-09-23T22:31:58.659 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-09-23T22:31:58.660 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[145629]: t=2023-09-23T22:31:58+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STOPPED 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STARTING 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Serving on http://:::9283 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STARTED 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STOPPING 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STOPPED 2023-09-23T22:31:58.660 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STARTING 2023-09-23T22:31:58.661 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Serving on http://:::9283 2023-09-23T22:31:58.661 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[138201]: [23/Sep/2023:22:31:58] ENGINE Bus STARTED 2023-09-23T22:31:58.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:31:58 smithi151 conmon[144080]: ts=2023-09-23T22:31:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:31:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.117322+0000 mgr.x (mgr.24787 2023-09-23T22:31:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: ) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:31:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.118561+0000 mgr.x (mgr.24787) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.135290+0000 mgr.x (mgr.24787) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.139034+0000 mgr.x (mgr.24787) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.153166+0000 mgr.x (mgr.24787) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: audit 2023-09-23T22:31:58.156364+0000 mgr.x (mgr.24787) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:58.174218+0000 mgr.x (mgr.24787) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:58.174745+0000 mgr.x (mgr.24787) 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: audit 2023-09-23T22:31:58 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: .117322+0000 mgr.x (mgr.24787) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.118561+0000 mgr.x (mgr.24787) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:31:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.135290+0000 mgr.x (mgr.24787) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: audit 2023-09-23T22:31:58.139034 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: +0000 mgr.x (mgr.24787) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: 23T22:31:58.153166+0000 mgr.x (mgr.24787) 59 : audit 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: audit 2023 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: -09-23T22:31:58.156364+0000 mgr.x 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: (mgr.24787) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: cephadm 2023-09-23T22 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: :31:58.174218+0000 mgr.x (mgr.24787) 61 : 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:31:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: 2023-09-23T22:31:58.174745+0000 mgr.x (mgr.24787) 2023-09-23T22:31:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:31:59 smithi052 conmon[110082]: 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:31:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.117322+0000 mgr.x (mgr.24787) 55 : audit 2023-09-23T22:31:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.118561+0000 mgr.x (mgr.24787) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi052.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.135290+0000 mgr.x (mgr.24787) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: 09-23T22:31:58.139034+0000 mgr.x (mgr.24787) 58 : 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151:9095"}]: dispatch 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit 2023-09-23T22:31:58 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: .153166+0000 mgr.x (mgr.24787) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: audit 2023-09-23T22:31:58.156364 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: +0000 mgr.x (mgr.24787) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151:3000"}]: dispatch 2023-09-23T22:31:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:59.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: 2023-09-23T22:31:58.174218+0000 mgr.x ( 2023-09-23T22:31:59.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: mgr.24787) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:31:59.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: cephadm 2023 2023-09-23T22:31:59.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: -09-23T22:31:58.174745+0000 mgr.x (mgr 2023-09-23T22:31:59.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:31:59 smithi151 conmon[100454]: .24787) 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:32:00.722 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: cluster 2023-09-23T22:31:58.721926+0000 mgr.x (mgr.24787) 63 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:00.722 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:59.541607+0000 mgr.x (mgr.24787) 64 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T22:32:00.722 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: audit 2023-09-23T22:31:59.546213+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: audit 2023-09-23T22:31:59.548104+0000 mon.b (mon.1) 92 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: audit 2023-09-23T22:31:59.548449+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: audit 2023-09-23T22:31:59.549596+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: audit 2023-09-23T22:31:59.550459+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: cephadm 2023-09-23T22:31:59.551550+0000 mgr.x (mgr.24787) 65 : cephadm [INF] Deploying daemon mgr.y on smithi052 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[100224]: audit 2023-09-23T22:32:00.153276+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.52:0/2893827121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: cluster 2023-09-23T22:31:58.721926+0000 mgr.x (mgr.24787) 63 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:59.541607+0000 mgr.x (mgr.24787) 64 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: audit 2023-09-23T22:31:59.546213+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:00.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: audit 2023-09-23T22:31:59.548104+0000 mon.b ( 2023-09-23T22:32:00.724 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: mon.1) 92 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:00.724 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: audit 2023-09-23T22:31:59.548449+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:00.724 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: audit 2023-09-23T22:31:59.549596+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:32:00.724 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: audit 2023-09-23T22:31:59.550459+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:00.724 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: cephadm 2023-09-23T22:31:59.551550+0000 mgr.x (mgr.24787) 65 : cephadm [INF] Deploying daemon mgr.y on smithi052 2023-09-23T22:32:00.724 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:00 smithi052 conmon[110082]: audit 2023-09-23T22:32:00.153276+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.52:0/2893827121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: cluster 2023-09-23T22:31:58.721926+0000 mgr.x (mgr.24787) 63 : cluster [DBG] 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: cephadm 2023-09-23T22:31:59.541607+0000 mgr.x (mgr.24787) 64 : cephadm [INF] Upgrade: Updating mgr.y 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: 09-23T22:31:59.546213+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: audit 2023-09-23T22:31:59.548104+0000 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: mon.b (mon.1) 92 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: -09-23T22:31:59.548449+0000 mon.a (mon. 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: 0) 870 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: -23T22:31:59.549596+0000 mon.b (mon.1) 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: 93 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: T22:31:59.550459+0000 mon.b (mon.1) 94 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: cephadm 2023-09-23 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: T22:31:59.551550+0000 mgr.x (mgr.24787) 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: 65 : cephadm [INF] Deploying daemon mgr.y on smithi052 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: -23T22:32:00.153276+0000 mon.c (mon.2 2023-09-23T22:32:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:00 smithi151 conmon[100454]: ) 60 : audit [DBG] from='client.? 172.21.15.52:0/2893827121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:00.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:00 smithi052 systemd[1]: Stopping Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:01.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:01 smithi052 bash[173546]: cf5f0eb823e68cc6547018e792249d21e9e25d7a3d57e4cb95c274ff56052b1b 2023-09-23T22:32:01.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:01 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:32:01.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:01 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T22:32:01.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:01 smithi052 systemd[1]: Stopped Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:02.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:02 smithi052 systemd[1]: Starting Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:02.896 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:02 smithi052 conmon[110082]: cluster 2023-09-23T 2023-09-23T22:32:02.896 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:02 smithi052 conmon[110082]: 22:32:00.722344+0000 mgr.x (mgr.24787) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:02.896 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:02 smithi052 ceph-mgr[173992]: -- 172.21.15.52:0/1174521969 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557fd01b11e0 con 0x557fd0188800 2023-09-23T22:32:02.897 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:02 smithi052 bash[173905]: d2a437c50b3c7cc5c568cb740c4eff1739ae030917bfa8c1531a4e4041de3641 2023-09-23T22:32:02.897 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:02 smithi052 systemd[1]: Started Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:02.897 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:02 smithi052 conmon[173965]: 2023-09-23T22:32:02.796+0000 7f97b3a00200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:32:02.897 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:02 smithi052 conmon[100224]: cluster 2023-09-23T 2023-09-23T22:32:02.897 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:02 smithi052 conmon[100224]: 22:32:00.722344+0000 mgr.x (mgr.24787) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:02 smithi151 conmon[100454]: cluster 2023-09-23T22:32:00.722344+0000 mgr.x (mgr.24787) 2023-09-23T22:32:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:02 smithi151 conmon[100454]: 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:03.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:02 smithi052 conmon[173965]: 2023-09-23T22:32:02.895+0000 7f97b3a00200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:03.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[173965]: 2023-09-23T22:32:03.010+0000 7f97b3a00200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:32:03.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[173965]: 2023-09-23T22:32:03.142+0000 7f97b3a00200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:32:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[173965]: 2023-09-23T22:32:03.374+0000 7f97b3a00200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: audit 2023-09-23T22:32: 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: 02.597703+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.52:0/2471233627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: audit 2023-09-23T22:32:02. 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: 721058+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: audit 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: 2023-09-23T22:32 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: :02.727037+0000 mon.a (mon.0 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: ) 873 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: -09-23T22 2023-09-23T22:32:03.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: :32:02.773469+0000 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: mon.b (mon.1) 95 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: -09-23T22:32:02. 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: 904399+0000 mon.b (mon.1) 96 : audit [DBG] 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[100224]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: audit 2023-09-23T22:32:02. 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: 597703+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.52:0/2471233627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: 23T22:32:02.721058+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:03.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: audit 2023-09-23T22:32:02.727037+0000 mon.a 2023-09-23T22:32:03.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: (mon.0) 873 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:03.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: audit 2023-09-23T22:32: 2023-09-23T22:32:03.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: 02.773469+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:03.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: 2023-09-23T22:32:03.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:03.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[110082]: 32:02.904399+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:03 smithi151 conmon[100454]: audit 2023-09-23T22:32:02.597703+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.52:0/2471233627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:03 smithi151 conmon[100454]: audit 2023-09-23T22:32:02.721058+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:03 smithi151 conmon[100454]: audit 2023-09-23T22:32:02.727037+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:03 smithi151 conmon[100454]: audit 2023-09-23T22:32:02.773469+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:03 smithi151 conmon[100454]: audit 2023-09-23T22:32:02.904399+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:04.156 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:03 smithi052 conmon[173965]: 2023-09-23T22:32:03.885+0000 7f97b3a00200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:32:04.156 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[173965]: 2023-09-23T22:32:04.045+0000 7f97b3a00200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:32:04.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:03 smithi151 conmon[138201]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:32:03] "GET /metrics HTTP/1.1" 200 33860 "" "Prometheus/2.33.4" 2023-09-23T22:32:04.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[173965]: 2023-09-23T22:32:04.155+0000 7f97b3a00200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:32:04.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[173965]: 2023-09-23T22:32:04.282+0000 7f97b3a00200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:32:04.856 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:04.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[110082]: 02.722708+0000 mgr.x (mgr.24787) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:04.857 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[100224]: cluster 2023-09-23T22:32:02.722708+0000 mgr.x (mgr.24787 2023-09-23T22:32:04.857 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[100224]: ) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:04 smithi151 conmon[100454]: cluster 2023-09-23T22:32:02.722708+0000 mgr.x 2023-09-23T22:32:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:04 smithi151 conmon[100454]: (mgr.24787) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:05.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:04 smithi052 conmon[173965]: 2023-09-23T22:32:04.855+0000 7f97b3a00200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:32:05.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:05 smithi151 conmon[100454]: audit 2023-09-23T22:32:05.012739+0000 mon.a (mon.0) 874 : audit [DBG] from='client.? 172.21.15.52:0/2354001402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:05.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:05 smithi052 conmon[173965]: 2023-09-23T22:32:05.694+0000 7f97b3a00200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:32:05.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:05 smithi052 conmon[173965]: 2023-09-23T22:32:05.799+0000 7f97b3a00200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:32:05.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:05 smithi052 conmon[110082]: audit 2023-09-23T22:32:05.012739+0000 mon.a (mon. 2023-09-23T22:32:05.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:05 smithi052 conmon[110082]: 0) 874 : audit [DBG] from='client.? 172.21.15.52:0/2354001402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:05 smithi052 conmon[100224]: audit 2023-09-23T22:32:05.012739 2023-09-23T22:32:05.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:05 smithi052 conmon[100224]: +0000 mon.a (mon.0) 874 : audit [DBG] from='client.? 172.21.15.52:0/2354001402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:06.859 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[173965]: 2023-09-23T22:32:06.844+0000 7f97b3a00200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:06.859 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: cluster 2023-09-23T22:32:04 2023-09-23T22:32:06.859 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: .723163+0000 mgr.x (mgr.24787) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: 09-23T22:32:05.599839+0000 mon.a (mon.0) 875 : audit 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: 32:05.606118+0000 mon.a (mon.0) 876 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: T22:32:05.901068+0000 mon.a (mon. 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: 0) 877 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: -23T22:32:05.906289+0000 mon.a (mon 2023-09-23T22:32:06.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: .0) 878 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: -23T22:32:06.296078+0000 mon.a ( 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: mon.0) 879 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: audit 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: 2023-09-23T22:32:06.301228 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: +0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[110082]: 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: cluster 2023-09-23T22:32:04.723163+0000 mgr.x (mgr.24787) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: audit 2023-09-23T22:32:05.599839+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: audit 2023-09-23T22:32:05.606118+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: audit 2023-09-23T22:32:05.901068+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: audit 2023-09-23T22:32:05.906289+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: audit 2023-09-23T22:32:06.296078+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[100224]: audit 2023-09-23T22:32:06.301228+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: cluster 2023-09-23T22:32:04.723163+0000 mgr.x (mgr.24787) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: audit 2023-09-23T22:32:05. 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: 599839+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: audit 2023-09-23T22:32:05.606118+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: audit 2023-09-23T22:32:05.901068+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: audit 2023-09-23T22:32:05.906289+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: audit 2023-09-23T22:32:06.296078+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:06 smithi151 conmon[100454]: audit 2023-09-23T22:32:06.301228+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:07.224 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:06 smithi052 conmon[173965]: 2023-09-23T22:32:06.971+0000 7f97b3a00200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:32:07.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[173965]: 2023-09-23T22:32:07.233+0000 7f97b3a00200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:07.864 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[173965]: 2023-09-23T22:32:07.599+0000 7f97b3a00200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:32:07.865 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[110082]: audit 2023-09-23T22:32:07.472908+0000 mon.a ( 2023-09-23T22:32:07.865 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[110082]: mon.0) 881 : audit [DBG] from='client.? 172.21.15.52:0/3930083146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:07.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[100224]: audit 2023-09-23T22:32:07.472908+0000 2023-09-23T22:32:07.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[100224]: mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.52:0/3930083146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:07.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:07 smithi151 conmon[100454]: audit 2023-09-23T22:32:07.472908+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.52:0/3930083146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:08.116 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:07 smithi052 conmon[173965]: 2023-09-23T22:32:07.863+0000 7f97b3a00200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:32:08.117 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:08 smithi052 conmon[173965]: 2023-09-23T22:32:08.059+0000 7f97b3a00200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:32:08.712 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:08 smithi052 conmon[173965]: 2023-09-23T22:32:08.434+0000 7f97b3a00200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:32:08.712 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:08 smithi052 conmon[110082]: cluster 2023-09-23T22:32:06.723641+0000 mgr.x (mgr.24787) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:08.712 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:08 smithi052 conmon[100224]: cluster 2023-09-23T22:32:06.723641+0000 mgr.x 2023-09-23T22:32:08.712 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:08 smithi052 conmon[100224]: (mgr.24787) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:08.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:08 smithi151 conmon[100454]: cluster 2023-09-23T22:32:06.723641+0000 mgr.x (mgr.24787) 69 : cluster 2023-09-23T22:32:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:08 smithi151 conmon[100454]: [DBG] pgmap v37: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:08.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:08 smithi151 conmon[144080]: ts=2023-09-23T22:32:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:32:09.571 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:09 smithi052 conmon[173965]: 2023-09-23T22:32:09.290+0000 7f97b3a00200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:32:09.572 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:09 smithi052 conmon[173965]: 2023-09-23T22:32:09.470+0000 7f97b3a00200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:32:09.875 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: debug 2023-09-23T22:32:09.544+0000 7fd1b8b95700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T22:32:09.875 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: ignoring --setuser ceph since I am not root 2023-09-23T22:32:09.875 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: ignoring --setgroup ceph since I am not root 2023-09-23T22:32:09.875 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: debug 2023-09-23T22:32:09.593+0000 7f81c6ba2700 1 -- 172.21.15.151:0/259727002 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555fdc0711e0 con 0x555fdc049000 2023-09-23T22:32:09.876 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: debug 2023-09-23T22:32:09.708+0000 7f81d08d1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:32:09.876 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: debug 2023-09-23T22:32:09.786+0000 7f81d08d1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:09.876 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: debug 2023-09-23T22:32:09.874+0000 7f81d08d1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:32:09.884 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:09 smithi052 conmon[173965]: 2023-09-23T22:32:09.739+0000 7f97b3a00200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:32:09.884 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:09 smithi052 conmon[173965]: 2023-09-23T22:32:09.882+0000 7f97b3a00200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:10.138 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:09 smithi151 conmon[138201]: debug 2023-09-23T22:32:09.968+0000 7f81d08d1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:32:10.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[138201]: debug 2023-09-23T22:32:10.138+0000 7f81d08d1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T22:32:09.378244+0000 mon.a (mon.0) 882 : audit [INF] 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T22:32:09.385939+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: -23T22:32:09.387344+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: 22:32:09.388276+0000 mon.b (mon.1) 98 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: 23T22:32:09.394359+0000 mon.a ( 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: mon.0) 884 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: -23T22:32:09.532322+0000 mon.b 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: (mon.1) 99 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: 32:09.537593+0000 mon.b (mon.1) 100 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: 2023-09-23T22:32:09.537886+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: cluster 2023-09-23T22:32:09.543492+0000 mon.a ( 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: mon.0) 886 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T22:32:09.543750+0000 mon.a (mon.0) 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: 887 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:32:10.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: cluster 2023-09 2023-09-23T22:32:10.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: -23T22:32:09.543823+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00550363s) 2023-09-23T22:32:10.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: audit 2023-09-23T22:32:09.876086+0000 mon.c (mon 2023-09-23T22:32:10.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[100454]: .2) 61 : audit [DBG] from='client.? 172.21.15.52:0/4139940492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[173965]: 2023-09-23T22:32:10.167+0000 7f97b3a00200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09-23T22:32:09.378244+0000 mon.a ( 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: mon.0) 882 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09-23T22:32 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: :09.385939+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 32:09.387344+0000 mon.b (mon.1 2023-09-23T22:32:10.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: ) 97 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09-23T22:32:09 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: .388276+0000 mon.b (mon.1) 98 : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 2023-09-23T22:32:09.394359+0000 mon.a ( 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: mon.0) 884 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 32:09.532322+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 23T22:32:09.537593+0000 mon.b (mon.1) 100 : audit [INF] 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:32:10.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 09-23T22:32:09.537886+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: T22:32:09.543492+0000 mon.a (mon.0) 886 : cluster [DBG] 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: -23T22:32:09.543750+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 23T22:32:09.543823+0000 mon.a (mon.0) 888 : cluster [DBG] 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: mgrmap e26: y(active, starting, since 0.00550363s) 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 09-23T22:32:09.876086+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.52:0/4139940492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[110082]: 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32: 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 09.378244+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 2023-09-23T22:32:09.385939+0000 mon.a (mon.0 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: ) 883 : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 2023-09-23T22:32:09.387344+0000 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: mon.b (mon.1) 97 : audit 2023-09-23T22:32:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: -09-23T22:32:09.388276+0000 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: mon.b (mon.1) 98 : audit 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 2023-09-23T22:32:09.394359 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: +0000 mon.a (mon.0) 884 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 ' entity='mgr.x' 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 09-23T22:32:09.532322+0000 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: mon.b (mon.1) 99 : audit 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: [DBG] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32:10.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 2023-09-23T22:32:09.537593 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: +0000 mon.b (mon.1) 100 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: : audit [INF] from='mgr.24787 172.21.15.151:0/1926817417' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 2023-09-23T22:32:09 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: .537886+0000 mon.a (mon.0 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: ) 885 : audit [INF] from='mgr.24787 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 09-23T22:32:09.543492+0000 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: mon.a (mon.0) 886 : 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:10.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 09-23T22:32:09.543750+0000 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: mon.a (mon.0) 887 : 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit [INF] from='mgr.24787 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: cluster 2023- 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 09-23T22:32:09.543823+0000 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: mon.a (mon.0) 888 : cluster 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: [DBG] mgrmap e26: y(active, starting, since 0.00550363s) 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: audit 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: 2023-09-23T22 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: :32:09.876086+0000 mon.c 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: (mon.2) 61 : audit 2023-09-23T22:32:10.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:10 smithi052 conmon[100224]: [DBG] from='client.? 172.21.15.52:0/4139940492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:10.818 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[138201]: debug 2023-09-23T22:32:10.501+0000 7f81d08d1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:32:10.819 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[138201]: debug 2023-09-23T22:32:10.631+0000 7f81d08d1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:32:10.819 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[138201]: debug 2023-09-23T22:32:10.722+0000 7f81d08d1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:32:11.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: cluster 2023-09-23T22:32:11.258024+0000 mon.a (mon 2023-09-23T22:32:11.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: .0) 889 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:32:11.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: cluster 2023-09-23T22:32:11.258347+0000 mon.a 2023-09-23T22:32:11.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: (mon.0) 890 : cluster [INF] Activating manager daemon y 2023-09-23T22:32:11.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -23T22:32:11.263772+0000 mon.a (mon 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: .0) 891 : cluster [DBG] 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -23T22:32:11.264167+0000 mon.a (mon 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: .0) 892 : cluster [DBG] mgrmap e27: y(active, starting, since 0.00594032s) 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2023-09-23T22 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: :32:11.264303+0000 mon.a (mon 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: .0) 893 : audit [DBG] from='mgr.? 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:32:11.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11.265679+0000 mon.c ( 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.2) 62 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32: 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 11.265970+0000 mon.c (mon. 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2) 63 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 22:32:11.266406+0000 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.c (mon.2) 64 : 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11. 2023-09-23T22:32:11.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 266664+0000 mon.c (mon.2) 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 65 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.266936+0000 mon.c 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: (mon.2) 66 : audit 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.267259+0000 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.c (mon.2) 67 : 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11.267510+0000 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.c (mon.2) 68 : 2023-09-23T22:32:11.552 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:32:11.553 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.553 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11.267778 2023-09-23T22:32:11.553 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: +0000 mon.c (mon.2) 69 2023-09-23T22:32:11.553 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:10 smithi151 conmon[138201]: debug 2023-09-23T22:32:10.818+0000 7f81d08d1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:32:11.553 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[138201]: debug 2023-09-23T22:32:11.241+0000 7f81d08d1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:32:11.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:32:11.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11.268039 2023-09-23T22:32:11.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: +0000 mon.c (mon.2) 70 2023-09-23T22:32:11.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:32:11.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.268356+0000 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.c (mon.2) 71 : 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11.268689+0000 mon.c (mon. 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2) 72 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.268923+0000 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.c (mon.2) 73 : 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: -09-23T22:32:11.269167+0000 2023-09-23T22:32:11.555 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: mon.c (mon.2) 74 : 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.269620+0000 mon.c 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: (mon.2) 75 : audit 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.269889+0000 mon.c 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: (mon.2) 76 : audit 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.270234+0000 mon.c 2023-09-23T22:32:11.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: (mon.2) 77 : audit [DBG] 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.270543+0000 mon.c 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: (mon.2) 78 : audit [DBG] 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023- 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 09-23T22:32:11.271029 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: +0000 mon.c (mon.2) 79 : audit [DBG] 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2023-09-23T22:32 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: :11.271290+0000 mon.c (mon.2) 80 2023-09-23T22:32:11.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2023-09-23T22:32 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: :11.272737+0000 mon.c (mon.2 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: ) 81 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: 2023-09-23T22:32 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: :11.273057+0000 mon.c (mon 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[100224]: .2) 82 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:11.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: cluster 2023-09-23T22:32:11.258024+0000 mon.a (mon.0 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: ) 889 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: cluster 2023-09-23T22:32:11.258347+0000 mon.a (mon.0) 890 : cluster [INF] Activating manager daemon y 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: cluster 2023-09-23T22:32:11.263772+0000 mon.a (mon.0) 891 : 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: cluster 2023-09-23T22:32:11. 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 264167+0000 mon.a (mon.0) 892 : cluster [DBG] 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: mgrmap e27: y(active, starting, since 0.00594032s) 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 2023-09-23T22:32:11.264303+0000 mon.a (mon.0 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: ) 893 : audit [DBG] from='mgr.? 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32 2023-09-23T22:32:11.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: :11.265679+0000 mon.c (mon.2) 62 : 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32: 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 11.265970+0000 mon.c (mon.2) 63 : audit 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .266406+0000 mon.c (mon.2) 64 : audit 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .266664+0000 mon.c (mon.2) 65 : audit 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:32:11.560 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11. 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 266936+0000 mon.c (mon.2) 66 : audit [DBG] 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 2023-09-23T22:32:11.267259+0000 mon.c (mon 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .2) 67 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: :32:11.267510+0000 mon.c (mon.2) 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 68 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 32:11.267778+0000 mon.c (mon.2) 69 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:32:11.561 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .268039+0000 mon.c (mon.2) 70 : audit 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .268356+0000 mon.c (mon.2) 71 : audit 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .268689+0000 mon.c (mon.2) 72 : audit 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .268923+0000 mon.c (mon.2) 73 : audit 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:32:11.562 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .269167+0000 mon.c (mon.2) 74 : audit 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .269620+0000 mon.c (mon.2) 75 : audit 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 2023-09-23T22:32:11.269889+0000 mon.c ( 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: mon.2) 76 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: -23T22:32:11.270234+0000 mon.c ( 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: mon.2) 77 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:32:11.563 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: -23T22:32:11.270543+0000 mon.c (mon 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: .2) 78 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 22:32:11.271029+0000 mon.c (mon. 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 2) 79 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: T22:32:11.271290+0000 mon.c (mon.2 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: ) 80 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: :32:11.272737+0000 mon.c (mon.2) 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 81 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:32:11.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: 32:11.273057+0000 mon.c (mon.2) 82 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[100454]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.243+0000 7f97b3a00200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: CherryPy Checker: 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: The Application mounted at '' has an empty config. 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Serving on http://:::9283 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Bus STARTED 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Bus STOPPING 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Bus STOPPED 2023-09-23T22:32:11.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.548+0000 7f9710e9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.548+0000 7f9710e9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.548+0000 7f9710e9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.548+0000 7f9710e9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.548+0000 7f9710e9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: cluster 2023-09-23T22:32:11.258024+0000 2023-09-23T22:32:11.566 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.a (mon.0) 889 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: cluster 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.258347+0000 mon.a (mon.0) 890 : cluster 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [INF] Activating manager daemon y 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: cluster 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.263772+0000 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.a (mon.0) 891 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 11.264167+0000 mon.a (mon.0) 892 : cluster 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [DBG] mgrmap e27: y(active, starting, since 0.00594032s) 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 22:32:11.264303+0000 mon.a (mon.0 2023-09-23T22:32:11.567 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: ) 893 : audit [DBG] from='mgr.? 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: -23T22:32:11.265679+0000 mon.c ( 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.2) 62 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.265970+0000 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.c (mon.2) 63 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.266406+0000 mon.c 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: (mon.2) 64 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32: 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 11.266664+0000 mon.c (mon.2) 65 : audit 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32: 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 11.266936+0000 mon.c (mon.2) 66 : audit 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: -09-23T22:32:11.267259+0000 mon.c ( 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.2) 67 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: .267510+0000 mon.c (mon.2) 68 : audit 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:32:11.569 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 32:11.267778+0000 mon.c (mon.2) 69 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.268039+0000 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.c (mon.2) 70 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32: 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 11.268356+0000 mon.c (mon.2) 71 : 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.268689+0000 2023-09-23T22:32:11.570 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.c (mon.2) 72 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11. 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 268923+0000 mon.c (mon.2) 73 : audit [DBG] 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 23T22:32:11.269167+0000 mon.c (mon. 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2) 74 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32: 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 11.269620+0000 mon.c (mon.2) 75 : audit 2023-09-23T22:32:11.571 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.269889+0000 mon.c 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: (mon.2) 76 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.270234+0000 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: mon.c (mon.2) 77 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11. 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 270543+0000 mon.c (mon.2) 78 : audit [DBG] 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32:11 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: .271029+0000 mon.c (mon.2) 79 : audit 2023-09-23T22:32:11.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22:32 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: :11.271290+0000 mon.c (mon.2) 80 : 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: :32:11.272737+0000 mon.c (mon.2) 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 81 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: T22:32:11.273057+0000 mon.c (mon. 2023-09-23T22:32:11.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[110082]: 2) 82 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:11.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.553+0000 7f970ce94700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.553+0000 7f970ce94700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.553+0000 7f970ce94700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.553+0000 7f970ce94700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: 2023-09-23T22:32:11.553+0000 7f970ce94700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:11.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Serving on http://:::9283 2023-09-23T22:32:11.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:11 smithi052 conmon[173965]: [23/Sep/2023:22:32:11] ENGINE Bus STARTED 2023-09-23T22:32:12.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:11 smithi151 conmon[138201]: debug 2023-09-23T22:32:11.930+0000 7f81d08d1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:32:12.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[138201]: debug 2023-09-23T22:32:12.019+0000 7f81d08d1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: cluster 2023-09-23T22:32:11.432839+0000 mon.a (mon.0) 894 : cluster 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: [INF] Manager daemon y is now available 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.455606+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.457531+0000 mon.c (mon. 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: 2) 84 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.482797+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.491305+0000 mon.c (mon.2) 86 : audit 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:32:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: 23T22:32:11.491716+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09-23T22:32:11.516300+0000 mon.c (mon. 2023-09-23T22:32:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: 2) 87 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: -09-23T22:32:11.517180+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: audit 2023-09-23T22:32:12.339326+0000 mon.c ( 2023-09-23T22:32:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[100454]: mon.2) 88 : audit [DBG] from='client.? 172.21.15.52:0/3897775266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:12.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: cluster 2023-09-23T22:32:11.432839+0000 mon.a (mon.0) 894 : cluster [INF] Manager daemon y is now available 2023-09-23T22:32:12.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.455606+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:12.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.457531+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.482797+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.491305+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.491716+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.516300+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:11.517180+0000 mon.a (mon.0) 896 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[100224]: audit 2023-09-23T22:32:12.339326+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.52:0/3897775266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: cluster 2023-09-23T22:32:11.432839+0000 mon.a (mon.0) 894 : cluster [INF] Manager daemon y is now available 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.455606+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:12.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.457531+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.482797+0000 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: mon.c (mon.2) 85 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.491305+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09-23T22:32:11.491716+0000 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: mon.a (mon.0) 895 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: 2023-09-23T22:32:11.516300+0000 mon.c 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: (mon.2) 87 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: -09-23T22:32:11.517180+0000 mon.a ( 2023-09-23T22:32:12.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: mon.0) 896 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:12.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:12.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: -23T22:32:12.339326+0000 mon.c 2023-09-23T22:32:12.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: (mon.2) 88 : audit [DBG] from='client.? 172.21.15.52:0/3897775266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:12.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:12 smithi052 conmon[110082]: 2023-09-23T22:32:13.189 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[138201]: debug 2023-09-23T22:32:12.898+0000 7f81d08d1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:13.189 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:12 smithi151 conmon[138201]: debug 2023-09-23T22:32:12.998+0000 7f81d08d1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:32:13.190 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[138201]: debug 2023-09-23T22:32:13.189+0000 7f81d08d1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:13.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:11.874573+0000 mgr.y (mgr.15174) 1 : cephadm [INF] [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:32:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:11.978478+0000 mgr.y (mgr.15174) 2 : cephadm [INF] [23/Sep/2023:22:32:11] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:32:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:12.109435+0000 mgr.y (mgr.15174) 3 : cephadm [INF] [23/Sep/2023:22:32:12] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:32:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:12.109636+0000 mgr.y (mgr.15174) 4 : cephadm [INF] [23/Sep/2023:22:32:12] ENGINE Bus STARTED 2023-09-23T22:32:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[100454]: cluster 2023-09-23T22:32:12.389491+0000 mon.a (mon.0) 897 : cluster [DBG] mgrmap e28: y(active, since 1.13126s) 2023-09-23T22:32:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[100454]: cluster 2023-09-23T22:32:12.395952+0000 mgr.y (mgr.15174) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:13.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:11.874573+0000 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: mgr.y (mgr.15174) 1 : cephadm [INF] [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:11.978478+0000 mgr.y (mgr.15174) 2 : cephadm [INF] [23/Sep/2023:22:32:11] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:12.109435+0000 mgr.y (mgr.15174) 3 : cephadm [INF] [23/Sep/2023:22:32:12] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:12.109636+0000 mgr.y (mgr.15174) 4 : cephadm [INF] [23/Sep/2023:22:32:12] ENGINE Bus STARTED 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: cluster 2023-09-23T22:32:12.389491+0000 mon.a ( 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: mon.0) 897 : cluster [DBG] mgrmap e28: y(active, since 1.13126s) 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: cluster 2023-09-23T22:32: 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[100224]: 12.395952+0000 mgr.y (mgr.15174) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:11.874573+0000 mgr.y 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: (mgr.15174) 1 : cephadm [INF] [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:11.978478 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: +0000 mgr.y (mgr.15174) 2 : cephadm [INF] [23/Sep/2023:22:32:11] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: 2023-09-23T22:32:12.109435+0000 mgr.y (mgr.15174) 3 : cephadm 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: [INF] [23/Sep/2023:22:32:12] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: cephadm 2023-09-23T22 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: :32:12.109636+0000 mgr.y (mgr.15174) 4 : cephadm [INF] [23/Sep/2023:22:32:12] ENGINE Bus STARTED 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: cluster 2023-09-23T22:32:12.389491 2023-09-23T22:32:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: +0000 mon.a (mon.0) 897 : cluster [DBG] mgrmap e28: y(active, since 1.13126s) 2023-09-23T22:32:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: cluster 2023-09-23T22:32:12. 2023-09-23T22:32:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:13 smithi052 conmon[110082]: 395952+0000 mgr.y (mgr.15174) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:13.785 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[138201]: debug 2023-09-23T22:32:13.533+0000 7f81d08d1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:32:14.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[138201]: debug 2023-09-23T22:32:13.784+0000 7f81d08d1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:32:14.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:13 smithi151 conmon[138201]: debug 2023-09-23T22:32:13.957+0000 7f81d08d1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:32:14.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:14 smithi151 conmon[138201]: debug 2023-09-23T22:32:14.249+0000 7f81d08d1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:32:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:14 smithi151 conmon[100454]: cluster 2023-09-23T22:32:13.268863+0000 mgr.y (mgr. 2023-09-23T22:32:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:14 smithi151 conmon[100454]: 15174) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:14 smithi151 conmon[100454]: cluster 2023-09-23T22:32:13.394298+0000 mon.a (mon.0) 898 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-09-23T22:32:14.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[110082]: cluster 2023-09-23T22:32:13.268863+0000 mgr.y 2023-09-23T22:32:14.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[110082]: (mgr.15174) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:14.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:14.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[110082]: 13.394298+0000 mon.a (mon.0) 898 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-09-23T22:32:14.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[100224]: cluster 2023-09-23T22:32:13.268863+0000 mgr.y (mgr. 2023-09-23T22:32:14.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[100224]: 15174) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:14.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:14 smithi052 conmon[100224]: cluster 2023-09-23T22:32:13.394298+0000 mon.a (mon.0) 898 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-09-23T22:32:15.394 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[138201]: debug 2023-09-23T22:32:15.035+0000 7f81d08d1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:32:15.394 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[138201]: debug 2023-09-23T22:32:15.182+0000 7f81d08d1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:32:15.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[138201]: debug 2023-09-23T22:32:15.394+0000 7f81d08d1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[138201]: debug 2023-09-23T22:32:15.534+0000 7f81d08d1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09-23T22:32:14.508998+0000 mon.a 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: (mon.0) 899 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09-23T22:32:14.513936+0000 mon.a (mon. 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: 0) 900 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09-23T22:32:14. 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: 761340+0000 mon.c (mon.2) 89 : audit [DBG] 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/3595621769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09-23T22:32:14. 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: 846032+0000 mon.a (mon.0) 901 : audit [INF] 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09-23T22:32 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: :14.851105+0000 mon.a (mon.0) 902 : 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: 22:32:15.199663+0000 mon.a (mon.0) 903 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: -23T22:32:15.205211+0000 mon.a ( 2023-09-23T22:32:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[100454]: mon.0) 904 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.782 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: :14.508998+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32:14.513936+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32:14.761340+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.52:0/3595621769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32:14.846032+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32:14.851105+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: 2023-09-23T22:32:15.199663+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: audit 2023-09-23T22:32: 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: 15.205211+0000 mon.a (mon.0) 904 : audit 2023-09-23T22:32:15.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[110082]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:14.508998+0000 mon.a ( 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: mon.0) 899 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:14.513936+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:14.761340+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.52:0/3595621769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:14.846032+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:14.851105+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:15.199663+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:15.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:15 smithi052 conmon[100224]: audit 2023-09-23T22:32:15.205211+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:15 smithi151 conmon[138201]: debug 2023-09-23T22:32:15.766+0000 7f81d08d1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:32:16.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:32:16.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: :32:15.269239+0000 mgr.y (mgr.15174) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:16.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: audit 2023-09-23T22:32:16.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: 2023-09- 2023-09-23T22:32:16.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: 23T22:32:15.643760 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: +0000 mon.a (mon.0) 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: 905 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: 09-23T22:32:15.649404+0000 mon.a 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[110082]: (mon.0) 906 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[100224]: cluster 2023-09-23T22:32:15.269239+0000 mgr.y (mgr 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[100224]: .15174) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[100224]: audit 2023-09-23T22:32:15.643760+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:16 smithi052 conmon[100224]: audit 2023-09-23T22:32:15.649404+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[100454]: cluster 2023-09-23T22:32:15.269239+0000 mgr.y (mgr.15174) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[100454]: audit 2023-09-23T22:32:15.643760+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[100454]: audit 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[100454]: 2023-09-23T22:32:15.649404+0000 mon.a (mon.0) 906 : audit 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[100454]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: debug 2023-09-23T22:32:16.639+0000 7f81d08d1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:32:16.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: [23/Sep/2023:22:32:16] ENGINE Bus STARTING 2023-09-23T22:32:16.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: CherryPy Checker: 2023-09-23T22:32:16.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: The Application mounted at '' has an empty config. 2023-09-23T22:32:16.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: 2023-09-23T22:32:16.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: [23/Sep/2023:22:32:16] ENGINE Serving on http://:::9283 2023-09-23T22:32:16.950 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:16 smithi151 conmon[138201]: [23/Sep/2023:22:32:16] ENGINE Bus STARTED 2023-09-23T22:32:17.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: cluster 2023-09-23T22:32:16. 2023-09-23T22:32:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: 653062+0000 mon.a (mon.0) 907 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: audit 2023-09-23T22:32:16.654581+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: audit 2023-09-23T22:32:16.655498+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: audit 2023-09-23T22:32:16.657898+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: audit 2023-09-23T22:32:16.658229+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[100224]: audit 2023-09-23T22:32:17.167734+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.52:0/820283003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: cluster 2023-09-23T22:32:16.653062+0000 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: mon.a (mon.0) 907 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: audit 2023-09-23T22:32:16.654581+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: audit 2023-09-23T22:32:16.655498+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: audit 2023-09-23T22:32:16.657898+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: audit 2023-09-23T22:32:16.658229+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:17 smithi052 conmon[110082]: audit 2023-09-23T22:32:17.167734+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.52:0/820283003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:17 smithi151 conmon[100454]: cluster 2023-09-23T22:32:16.653062+0000 mon.a (mon.0) 907 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:17 smithi151 conmon[100454]: audit 2023-09-23T22:32:16.654581+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:17 smithi151 conmon[100454]: audit 2023-09-23T22:32:16.655498+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:17 smithi151 conmon[100454]: audit 2023-09-23T22:32:16.657898+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:17 smithi151 conmon[100454]: audit 2023-09-23T22:32:16.658229+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.151:0/3784175595' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:17 smithi151 conmon[100454]: audit 2023-09-23T22:32:17.167734+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.52:0/820283003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:18.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: cluster 2023-09-23T22:32:17.269587 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: +0000 mgr.y (mgr.15174) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: cluster 2023-09-23T22:32:17.652045+0000 mon.a (mon.0) 909 : cluster [DBG] mgrmap e30: y(active, since 6s), standbys: x 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: audit 2023-09-23T22:32:17.654411+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: audit 2023-09-23T22:32:18.084394+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: audit 2023-09-23T22:32:18 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: .089629+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: audit 2023-09-23T22:32:18.091450+0000 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: mon.c (mon.2) 91 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: audit 2023-09-23T22:32:18.092291+0000 mon.c 2023-09-23T22:32:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:18 smithi151 conmon[100454]: (mon.2) 92 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: cluster 2023-09-23T22:32:17.269587+0000 2023-09-23T22:32:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: mgr.y (mgr.15174) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: cluster 2023-09-23T22:32:17.652045+0000 mon.a (mon.0) 909 : cluster [DBG] mgrmap e30: y(active, since 6s), standbys: x 2023-09-23T22:32:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: audit 2023-09-23T22:32:17.654411+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: audit 2023-09-23T22:32:18.084394+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: audit 2023-09-23T22:32:18.089629+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: audit 2023-09-23T22:32:18.091450+0000 mon.c (mon 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: .2) 91 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[100224]: 32:18.092291+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: cluster 2023-09-23T22:32:17.269587+0000 2023-09-23T22:32:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: mgr.y (mgr.15174) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: 17.652045+0000 mon.a (mon.0) 909 : cluster [DBG] mgrmap e30: y(active, since 6s), standbys: x 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: audit 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: 2023-09-23T22:32:17.654411+0000 mon.c (mon.2) 90 : audit 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: audit 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: 2023-09-23T22:32:18.084394+0000 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: mon.a (mon.0) 910 : audit [INF] 2023-09-23T22:32:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: audit 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: 2023-09-23T22:32:18.089629+0000 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: mon.a (mon.0) 911 : audit [INF] 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: audit 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: 2023-09-23T22:32:18.091450+0000 mon.c 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: (mon.2) 91 : audit [DBG] 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: audit 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: 2023-09-23T22:32:18.092291+0000 2023-09-23T22:32:18.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: mon.c (mon.2) 92 : audit [INF] 2023-09-23T22:32:18.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:18 smithi052 conmon[110082]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:18.093421+0000 mgr.y (mgr.15174) 9 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:18.093627+0000 mgr.y (mgr.15174) 10 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 2023-09-23T22:32:18.357913+0000 mgr.y (mgr.15174) 11 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: cephadm 2023-09-23 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: T22:32:18.358085+0000 mgr.y (mgr. 2023-09-23T22:32:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 15174) 12 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: cephadm 2023-09-23T 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 22:32:18.622193+0000 mgr.y (mgr.15174 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: ) 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: cephadm 2023-09-23T 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 22:32:18.622731+0000 mgr.y (mgr.15174 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: ) 14 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 22:32:19.156101+0000 mon.a (mon.0 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: ) 912 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 22:32:19.158907+0000 mon.a (mon.0 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: ) 913 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: audit 2023-09-23T22 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: :32:19.163010+0000 mon.a (mon.0) 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 914 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: audit 2023-09-23T22:32: 2023-09-23T22:32:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 19.166594+0000 mon.a (mon.0) 915 : audit 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: audit 2023-09-23T22:32: 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 19.170941+0000 mon.a (mon.0) 916 : audit 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: audit 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: 2023-09-23T22:32:19.562309+0000 mon.a (mon 2023-09-23T22:32:19.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:19 smithi151 conmon[100454]: .0) 917 : audit [DBG] from='client.? 172.21.15.52:0/1342249894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm 2023-09-23T22:32 2023-09-23T22:32:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: :18.093421+0000 mgr.y (mgr.15174) 9 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 32:18.093627+0000 mgr.y (mgr. 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 15174) 10 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm 2023-09-23T22:32 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: :18.357913+0000 mgr.y (mgr.15174) 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 11 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm 2023 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: -09-23T22:32:18 2023-09-23T22:32:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: .358085+0000 mgr.y (mgr.15174) 12 : 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm 2023-09-23T 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 22:32:18.622193+0000 mgr.y (mgr.15174 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: ) 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 2023-09-23T22:32: 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 18.622731+0000 mgr.y (mgr.15174) 14 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 09-23T22:32:19.156101+0000 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: mon.a (mon.0) 912 : audit 2023-09-23T22:32:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: -09-23T22:32: 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 19.158907+0000 mon.a (mon.0) 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 913 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: -23T22:32:19.163010+0000 mon.a 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: (mon.0) 914 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: audit 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 2023-09-23T22:32:19.166594+0000 mon.a 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: (mon.0) 915 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 2023-09-23T22:32:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: audit 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: 2023-09-23T22:32:19.170941+0000 mon.a 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: (mon.0) 916 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: -09-23T22:32:19.562309 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: +0000 mon.a (mon.0) 917 : audit 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[110082]: [DBG] from='client.? 172.21.15.52:0/1342249894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:18.093421+0000 mgr.y (mgr.15174) 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 9 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:18.093627+0000 mgr.y (mgr.15174) 10 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:32:19.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 2023-09-23T22:32:18.357913+0000 mgr.y (mgr.15174) 11 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: cephadm 2023-09 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: -23T22:32:18.358085 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: +0000 mgr.y (mgr.15174) 12 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: cephadm 2023 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: -09-23T22:32: 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 18.622193+0000 mgr.y (mgr. 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 15174) 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: cephadm 2023 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: -09-23T22:32:18 2023-09-23T22:32:19.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: .622731+0000 mgr.y (mgr. 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 15174) 14 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: -09-23T22:32: 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 19.156101+0000 mon.a (mon. 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 0) 912 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: audit 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 2023-09-23T 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 22:32:19.158907+0000 mon.a 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: (mon.0) 913 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:19.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: -09-23T22:32:19. 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 163010+0000 mon.a (mon.0 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: ) 914 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: audit 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 2023-09- 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 23T22:32:19.166594 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: +0000 mon.a (mon.0) 915 : audit [INF] 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: 23T22:32:19.170941+0000 mon.a ( 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: mon.0) 916 : audit [INF] 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:20.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: -09-23T22:32:19 2023-09-23T22:32:20.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: .562309+0000 mon.a (mon.0 2023-09-23T22:32:20.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:19 smithi052 conmon[100224]: ) 917 : audit [DBG] from='client.? 172.21.15.52:0/1342249894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:18.886796+0000 mgr.y (mgr.15174) 15 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:18.887106+0000 mgr.y (mgr.15174) 16 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:20 smithi151 conmon[100454]: cluster 2023-09-23T22:32:19.270223+0000 mgr.y (mgr.15174) 17 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:19.304394+0000 mgr.y (mgr.15174) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:32:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:20 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:19.558882+0000 mgr.y (mgr.15174) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:32:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:18.886796+0000 2023-09-23T22:32:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: mgr.y (mgr.15174) 15 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: cephadm 2023-09- 2023-09-23T22:32:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: 23T22:32:18.887106+0000 mgr.y (mgr.15174) 16 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: cluster 2023-09-23T22:32:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: 2023-09-23T22:32:19.270223+0000 mgr.y 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: (mgr.15174) 17 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: cephadm 2023-09-23T 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: 22:32:19.304394+0000 mgr.y (mgr.15174) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[110082]: 32:19.558882+0000 mgr.y (mgr.15174) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:18.886796 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: +0000 mgr.y (mgr.15174) 15 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:18.887106+0000 mgr.y (mgr.15174) 16 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: cluster 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: 2023-09-23 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: T22:32:19.270223+0000 mgr.y ( 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: mgr.15174) 17 : cluster [DBG] 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: 2023-09-23T22:32:19. 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: 304394+0000 mgr.y (mgr.15174) 18 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: cephadm 2023 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: -09-23T22:32:19.558882+0000 mgr.y (mgr.15174) 2023-09-23T22:32:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:20 smithi052 conmon[100224]: 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:32:22.297 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 systemd[1]: Stopping Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:22.297 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 bash[149546]: Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-prometheus.a found: no such container 2023-09-23T22:32:22.655 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.296Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:32:22.655 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.296Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.296Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.296Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.296Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.296Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.297Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.297Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.297Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.297Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.297Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T22:32:22.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[144080]: ts=2023-09-23T22:32:22.297Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T22:32:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[100454]: cluster 2023-09-23T22:32:21.270749+0000 mgr.y ( 2023-09-23T22:32:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[100454]: mgr.15174) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:22 smithi151 conmon[100454]: audit 2023-09-23T22:32:21.972402+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.52:0/3639123676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:22.950 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 bash[149660]: 1da9848cdc25f3f868568f701f788a0ce8573b4a90ccf5cc7263dfd4d9d67637 2023-09-23T22:32:22.950 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:32:22.950 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 systemd[1]: Stopped Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:22.950 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:22 smithi151 systemd[1]: Starting Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[110082]: cluster 2023-09-23T22:32:21.270749+0000 mgr.y (mgr. 2023-09-23T22:32:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[110082]: 15174) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[110082]: audit 2023-09-23T22:32:21.972402+0000 mon.c (mon 2023-09-23T22:32:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[110082]: .2) 93 : audit [DBG] from='client.? 172.21.15.52:0/3639123676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[100224]: cluster 2023-09-23T22:32:21.270749+0000 mgr.y ( 2023-09-23T22:32:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[100224]: mgr.15174) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:32:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:22 smithi052 conmon[100224]: :32:21.972402+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.52:0/3639123676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:23.663 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.477Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-09-23T22:32:23.663 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.477Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-09-23T22:32:23.663 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.477Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi151 (none))" 2023-09-23T22:32:23.663 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.477Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:32:23.663 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.477Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:32:23.663 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.478Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.479Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.480Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.481Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.481Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.152µs 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.481Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 bash[149998]: 88e6208a51e9128327b44820aba7ce1f8e6a1465f4263a4f9852406741b7bc46 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 systemd[1]: Started Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:23.664 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.662Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T22:32:23.942 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.903Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T22:32:23.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:23 smithi052 conmon[173965]: [23/Sep/2023:22:32:23] ENGINE Bus STOPPING 2023-09-23T22:32:23.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:23 smithi052 conmon[173965]: [23/Sep/2023:22:32:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:32:23.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:23 smithi052 conmon[173965]: [23/Sep/2023:22:32:23] ENGINE Bus STOPPED 2023-09-23T22:32:23.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:23 smithi052 conmon[173965]: [23/Sep/2023:22:32:23] ENGINE Bus STARTING 2023-09-23T22:32:23.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:23 smithi052 conmon[173965]: [23/Sep/2023:22:32:23] ENGINE Serving on http://:::9283 2023-09-23T22:32:23.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:23 smithi052 conmon[173965]: [23/Sep/2023:22:32:23] ENGINE Bus STARTED 2023-09-23T22:32:24.198 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.942Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.942Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.942Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=49.566µs wal_replay_duration=460.660909ms total_replay_duration=460.731829ms 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.946Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.946Z caller=main.go:947 level=info msg="TSDB started" 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.946Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.961Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.874905ms db_storage=824ns remote_storage=1.18µs web_handler=433ns query_engine=703ns scrape=462.792µs scrape_sd=109.688µs notify=10.159µs notify_sd=8.197µs rules=15.034547ms 2023-09-23T22:32:24.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:23 smithi151 conmon[150039]: ts=2023-09-23T22:32:23.961Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-09-23T22:32:24.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: cluster 2023-09-23T22:32:23.271404+0000 mgr.y (mgr.15174) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.534715+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.541127+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.593856+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.594532+0000 mgr.y (mgr.15174) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.595685+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.596175+0000 mgr.y (mgr.15174) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.601519+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:23.613793+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:24.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:24 smithi151 conmon[100454]: audit 2023-09-23T22:32:24.378604+0000 mon.a (mon.0) 921 : audit [DBG] from='client.? 172.21.15.52:0/1334773642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 23.271404+0000 mgr.y (mgr.15174) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09-23T22:32:23 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: .534715+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 23T22:32:23.541127+0000 mon.a (mon 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: .0) 919 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: T22:32:23.593856+0000 mon.c (mon.2) 94 : audit 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:32:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: :32:23.594532+0000 mgr.y (mgr.15174) 22 : audit [DBG] 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 09-23T22:32:23.595685+0000 mon.c (mon.2 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: ) 95 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09-23T22:32: 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 23.596175+0000 mgr.y (mgr.15174) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: T22:32:23.601519+0000 mon.a (mon. 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 0) 920 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: 2023-09-23T22:32 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: :23.613793+0000 mon.c (mon.2 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: ) 96 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: -09-23T22:32:24.378604+0000 mon.a (mon.0 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[110082]: ) 921 : audit [DBG] from='client.? 172.21.15.52:0/1334773642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: cluster 2023-09-23T22:32:23.271404+0000 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: mgr.y (mgr.15174) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:23. 2023-09-23T22:32:24.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 534715+0000 mon.a (mon.0) 918 : audit [INF] 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: -09-23T22:32:23. 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 541127+0000 mon.a (mon.0 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: ) 919 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23T22:32: 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 23.593856+0000 mon.c (mon 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: .2) 94 : audit [DBG] 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23T22:32: 2023-09-23T22:32:24.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 23.594532+0000 mgr.y (mgr 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: .15174) 22 : audit [DBG] 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23T22:32 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: :23.595685+0000 mon.c ( 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: mon.2) 95 : audit 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23T22: 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 32:23.596175+0000 mgr.y ( 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: mgr.15174) 23 : audit 2023-09-23T22:32:24.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23T22:32: 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 23.601519+0000 mon.a (mon 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: .0) 920 : audit [INF] 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: T22:32:23.613793+0000 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: mon.c (mon.2) 96 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 2023-09-23T22:32: 2023-09-23T22:32:24.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: 24.378604+0000 mon.a (mon 2023-09-23T22:32:25.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: .0) 921 : audit [DBG] 2023-09-23T22:32:25.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:24 smithi052 conmon[100224]: from='client.? 172.21.15.52:0/1334773642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:25.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: audit 2023-09-23T22:32:24.985107+0000 mon.c 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: (mon.2) 97 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: audit 2023-09-23T22:32:24.985581+0000 mon.a (mon.0) 922 : audit 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: audit 2023-09-23T22:32: 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: 24.986971+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: T22:32:24.988080+0000 mon.c (mon.2 2023-09-23T22:32:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:25 smithi151 conmon[100454]: ) 99 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: audit 2023-09-23T22:32:24.985107+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: audit 2023-09-23T22:32:24.985581+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: 09-23T22:32:24.986971+0000 mon.c (mon.2) 98 : audit 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: audit 2023-09-23T22:32:24 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: .988080+0000 mon.c (mon.2) 99 : audit 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.985107+0000 mon.c (mon 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: .2) 97 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.985581+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:32:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: audit 2023-09-23T22:32:24.986971+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:32:25.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: audit 2023-09-23 2023-09-23T22:32:25.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: T22:32:24.988080+0000 mon.c (mon.2) 99 : audit 2023-09-23T22:32:25.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:25 smithi052 conmon[100224]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:26.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:26 smithi151 systemd[1]: Stopping Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:26.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:26 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:24.984595+0000 mgr.y (mgr.15174) 24 : cephadm 2023-09-23T22:32:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:26 smithi151 conmon[100454]: [INF] Upgrade: Updating mgr.x 2023-09-23T22:32:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:26 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:24.989200+0000 mgr.y (mgr.15174) 25 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:32:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:26 smithi151 conmon[100454]: cluster 2023-09-23T22:32:25.271880+0000 mgr.y (mgr.15174) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:26 smithi151 conmon[100454]: audit 2023-09-23T22:32:26.483085+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:26.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:24.984595+0000 2023-09-23T22:32:26.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: mgr.y (mgr.15174) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:24.989200+0000 mgr.y (mgr.15174) 25 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: cluster 2023-09 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: -23T22:32:25.271880+0000 mgr.y (mgr. 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: 15174) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: audit 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: 2023-09-23T22:32:26 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: .483085+0000 mon.c (mon.2) 100 : audit 2023-09-23T22:32:26.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[100224]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: cephadm 2023-09-23T 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 22:32:24.984595+0000 mgr.y (mgr.15174) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 2023-09- 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 23T22:32:24. 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 989200+0000 mgr.y (mgr.15174 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: ) 25 : cephadm [INF] Deploying daemon mgr.x on smithi151 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: cluster 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 2023-09- 2023-09-23T22:32:26.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 23T22:32:25. 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 271880+0000 mgr.y (mgr. 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 15174) 26 : cluster [DBG] 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: audit 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: 32:26.483085+0000 mon.c 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: (mon.2) 100 : audit 2023-09-23T22:32:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:26 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:27.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:27 smithi151 bash[150672]: b06bad11a511208244a8719701fe1f54990450e1dc369428d8664632df43052d 2023-09-23T22:32:27.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:27 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:32:27.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:27 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T22:32:27.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:27 smithi151 systemd[1]: Stopped Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:27.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:27 smithi151 conmon[100454]: audit 2023-09-23T22:32:26.789685+0000 mon.a (mon 2023-09-23T22:32:27.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:27 smithi151 conmon[100454]: .0) 923 : audit [DBG] from='client.? 172.21.15.52:0/3318090375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:27.917 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:27 smithi151 systemd[1]: Starting Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:27.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:27 smithi052 conmon[110082]: audit 2023-09-23T22:32:26.789685+0000 2023-09-23T22:32:27.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:27 smithi052 conmon[110082]: mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.52:0/3318090375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:27 smithi052 conmon[100224]: audit 2023-09-23T22:32:26.789685+0000 2023-09-23T22:32:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:27 smithi052 conmon[100224]: mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.52:0/3318090375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:28.368 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 bash[151027]: 648eee1c1677c61330a2cff6cc15292080bd6771420e5154f216bdd17a0ce104 2023-09-23T22:32:28.369 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 systemd[1]: Started Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:28.369 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 ceph-mgr[151092]: -- 172.21.15.151:0/2948459308 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562f4abab1e0 con 0x562f4ab82c00 2023-09-23T22:32:28.369 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[151068]: 2023-09-23T22:32:28.185+0000 7f9ee5e25200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:32:28.369 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[151068]: 2023-09-23T22:32:28.270+0000 7f9ee5e25200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:28.369 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[151068]: 2023-09-23T22:32:28.366+0000 7f9ee5e25200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:32:28.680 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[151068]: 2023-09-23T22:32:28.482+0000 7f9ee5e25200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:32:28.680 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[151068]: 2023-09-23T22:32:28.678+0000 7f9ee5e25200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:32:28.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: cluster 2023-09-23T22:32:27.272406 2023-09-23T22:32:28.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: +0000 mgr.y (mgr.15174) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:28.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: audit 2023-09-23T22:32:28.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: 2023-09-23T22:32:28.120611+0000 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: mon.a (mon.0) 924 : audit 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: 22:32:28.126615+0000 mon.a (mon.0) 925 : 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: audit 2023-09-23T22:32 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: :28.161159+0000 mon.c (mon.2) 101 2023-09-23T22:32:28.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:28 smithi151 conmon[100454]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: 27.272406+0000 mgr.y (mgr.15174) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: audit 2023-09-23T22:32:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: 2023-09-23T 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: 22:32:28.120611+0000 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: mon.a (mon.0) 924 : audit 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: -09-23T22:32:28.126615 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: +0000 mon.a (mon.0) 925 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: -09-23T22:32:28.161159 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: +0000 mon.c (mon.2) 101 : 2023-09-23T22:32:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[110082]: audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: cluster 2023-09-23T22:32:27.272406+0000 mgr.y 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: (mgr.15174) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: audit 2023-09-23T22:32 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: :28.120611+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: audit 2023-09-23T22:32:28.126615+0000 mon.a (mon 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: .0) 925 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: audit 2023-09-23T22:32:28.161159+0000 mon.c (mon.2) 2023-09-23T22:32:28.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:28 smithi052 conmon[100224]: 101 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:29.434 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[151068]: 2023-09-23T22:32:29.094+0000 7f9ee5e25200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:32:29.434 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[151068]: 2023-09-23T22:32:29.238+0000 7f9ee5e25200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:32:29.434 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[151068]: 2023-09-23T22:32:29.331+0000 7f9ee5e25200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:32:29.434 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[151068]: 2023-09-23T22:32:29.432+0000 7f9ee5e25200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:32:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[100454]: audit 2023-09-23T22:32:29.198696+0000 mon.c (mon.2 2023-09-23T22:32:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[100454]: ) 102 : audit [DBG] from='client.? 172.21.15.52:0/3250524711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:29.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:29 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:29.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:29 smithi052 conmon[110082]: 32:29.198696+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.52:0/3250524711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:29.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:29 smithi052 conmon[100224]: audit 2023-09-23T22:32:29.198696+0000 mon.c (mon 2023-09-23T22:32:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:29 smithi052 conmon[100224]: .2) 102 : audit [DBG] from='client.? 172.21.15.52:0/3250524711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:30.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:29 smithi151 conmon[151068]: 2023-09-23T22:32:29.925+0000 7f9ee5e25200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:32:30.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:30 smithi151 conmon[100454]: cluster 2023-09-23T22:32:29.273020+0000 mgr.y (mgr.15174) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:30.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:30 smithi151 conmon[151068]: 2023-09-23T22:32:30.641+0000 7f9ee5e25200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:32:30.949 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:30 smithi151 conmon[151068]: 2023-09-23T22:32:30.740+0000 7f9ee5e25200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:32:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:30 smithi052 conmon[110082]: cluster 2023-09-23T22:32: 2023-09-23T22:32:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:30 smithi052 conmon[110082]: 29.273020+0000 mgr.y (mgr.15174) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:30.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:30 smithi052 conmon[100224]: cluster 2023-09-23T22:32:29.273020+0000 mgr.y (mgr.15174) 28 2023-09-23T22:32:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:30 smithi052 conmon[100224]: : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:31.897 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:31 smithi151 conmon[151068]: 2023-09-23T22:32:31.597+0000 7f9ee5e25200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:31.897 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:31 smithi151 conmon[151068]: 2023-09-23T22:32:31.695+0000 7f9ee5e25200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:32:32.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:31 smithi151 conmon[151068]: 2023-09-23T22:32:31.895+0000 7f9ee5e25200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: audit 2023-09-23T22:32:31.090420+0000 mon.a (mon.0) 926 : audit 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: audit 2023-09-23T22:32:31.096471+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: -09-23T22:32:31.273427+0000 mgr.y (mgr.15174) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: 23T22:32:31.604441+0000 2023-09-23T22:32:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: mon.c (mon.2) 103 : audit 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: [DBG] from='client.? 172.21.15.52:0/889914656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: audit 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: 2023-09-23T22:32: 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: 31.786345+0000 mon.a (mon.0) 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: 928 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: 09-23T22:32:31.791658 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: +0000 mon.a (mon.0) 929 2023-09-23T22:32:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[100454]: : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: audit 2023-09-23T22:32:31.090420 2023-09-23T22:32:32.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: +0000 mon.a (mon.0) 926 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: -23T22:32:31.096471+0000 mon.a (mon.0) 927 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: :32:31.273427+0000 mgr.y (mgr. 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: 15174) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: 23T22:32:31.604441+0000 mon.c 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: (mon.2) 103 : audit [DBG] from='client.? 172.21.15.52:0/889914656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:32.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: 09-23T22:32:31.786345+0000 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: mon.a (mon.0) 928 : audit [INF] 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: audit 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: 2023-09-23T22:32:31.791658+0000 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: mon.a (mon.0) 929 : audit [INF] 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[110082]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: audit 2023-09-23T22: 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: 32:31.090420+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: audit 2023-09-23T22:32: 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: 31.096471+0000 mon.a (mon.0) 927 : audit 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: cluster 2023-09-23 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: T22:32:31.273427+0000 mgr.y (mgr. 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: 15174) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: 23T22:32:31.604441+0000 mon.c (mon.2) 103 : audit 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: [DBG] from='client.? 172.21.15.52:0/889914656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: :32:31.786345+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: audit 2023-09-23T22:32:31. 2023-09-23T22:32:32.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:32 smithi052 conmon[100224]: 791658+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:32.551 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[151068]: 2023-09-23T22:32:32.218+0000 7f9ee5e25200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:32:32.551 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[151068]: 2023-09-23T22:32:32.458+0000 7f9ee5e25200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:32:32.903 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[151068]: 2023-09-23T22:32:32.622+0000 7f9ee5e25200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:32:33.185 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:32 smithi151 conmon[151068]: 2023-09-23T22:32:32.901+0000 7f9ee5e25200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:32:33.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:32:33] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.33.4" 2023-09-23T22:32:33.793 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:33 smithi151 conmon[151068]: 2023-09-23T22:32:33.650+0000 7f9ee5e25200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:32:34.094 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:33 smithi151 conmon[151068]: 2023-09-23T22:32:33.791+0000 7f9ee5e25200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:32:34.094 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:33 smithi151 conmon[151068]: 2023-09-23T22:32:33.996+0000 7f9ee5e25200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:32:34.357 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:34 smithi151 conmon[151068]: 2023-09-23T22:32:34.119+0000 7f9ee5e25200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:34.357 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:34 smithi151 conmon[151068]: 2023-09-23T22:32:34.355+0000 7f9ee5e25200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:32:34.357 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:34 smithi151 conmon[100454]: cluster 2023-09-23T22:32:33.273979+0000 mgr.y (mgr.15174) 30 : cluster 2023-09-23T22:32:34.357 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:34 smithi151 conmon[100454]: [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:34.358 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:34 smithi151 conmon[100454]: audit 2023-09-23T22:32:34.014789+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.52:0/3048669992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[100224]: cluster 2023-09-23T22:32:33.273979+0000 mgr.y (mgr.15174) 30 : cluster 2023-09-23T22:32:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[100224]: [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:34.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.014789+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.52:0/3048669992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[110082]: cluster 2023-09-23T22:32:33.273979 2023-09-23T22:32:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[110082]: +0000 mgr.y (mgr.15174) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[110082]: 2023-09-23T22:32:34.014789+0000 mon.a (mon.0 2023-09-23T22:32:34.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:34 smithi052 conmon[110082]: ) 930 : audit [DBG] from='client.? 172.21.15.52:0/3048669992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: 2023-09-23T22:32:35.251+0000 7f9ee5e25200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: [23/Sep/2023:22:32:35] ENGINE Bus STARTING 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: CherryPy Checker: 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: The Application mounted at '' has an empty config. 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: [23/Sep/2023:22:32:35] ENGINE Serving on http://:::9283 2023-09-23T22:32:35.669 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[151068]: [23/Sep/2023:22:32:35] ENGINE Bus STARTED 2023-09-23T22:32:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:34.669792+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:34.676874+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 22:32:34.678743+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.679990+0000 mon.c (mon.2) 105 : 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:34.686631+0000 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: mon.a (mon.0) 933 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.825926+0000 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: mon.c (mon.2) 106 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.829203+0000 mon.c (mon.2) 107 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:34.829941+0000 mgr.y (mgr.15174) 2023-09-23T22:32:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 31 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.834455+0000 mon.a (mon.0) 934 : audit [INF] 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:34. 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 835585+0000 mon.c (mon.2) 108 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.835903+0000 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: mon.a (mon.0) 935 : audit [INF] 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.839174+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: -23T22:32:34.840178+0000 mon.c (mon.2) 109 : audit [INF] 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.840470+0000 mon.a 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: (mon.0) 937 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:34.844342+0000 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: mon.a (mon.0) 938 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:34.845928+0000 mon.c (mon. 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2) 110 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:34.846749+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: cephadm 2023-09-23T22:32 2023-09-23T22:32:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: :34.847355+0000 mgr.y (mgr.15174) 32 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: cluster 2023-09-23T22:32:35 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: .265963+0000 mon.a (mon.0) 939 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: cluster 2023-09-23T22:32: 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 35.266103+0000 mon.a (mon.0) 940 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.267726 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: +0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.268195+0000 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.270092+0000 mon.b (mon.1 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: ) 107 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: audit 2023-09-23T22:32:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: 2023-09-23T22:32:35.271032+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:35.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: cluster 2023-09-23T22:32:35 2023-09-23T22:32:35.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:35 smithi151 conmon[100454]: .274329+0000 mgr.y (mgr.15174) 33 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:35.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.669792+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.676874+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.678743+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.679990+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.686631+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.825926+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.829203+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: cephadm 2023-09- 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: 23T22:32:34.829941+0000 mgr.y (mgr.15174) 31 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.834455+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.835585+0000 mon.c (mon.2) 108 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:32:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.835903+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.839174+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.840178+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.840470+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.844342+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:34.845928+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: 2023-09-23T22:32:34.846749+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:34.847355+0000 mgr.y (mgr.15174) 32 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: cluster 2023-09-23T22:32:35.265963+0000 mon.a (mon 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: .0) 939 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:32:35.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: cluster 2023-09-23T22:32:35 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.669792+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.676874+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.678743+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 09-23T22:32:34.679990+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.686631+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.825926+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.829203+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:34.829941+0000 mgr.y (mgr.15174) 31 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-09-23T22:32:35.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.834455+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:34.835585+0000 mon.c (mon.2) 108 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09-23T22:32:34.835903+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: -09-23T22:32:34 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: .839174+0000 mon.a (mon. 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 0) 936 : audit [INF] 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: from='mgr.15174 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: -09-23T22:32: 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 34.840178+0000 mon.c ( 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: mon.2) 109 : audit 2023-09-23T22:32:35.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09- 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 23T22:32:34. 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 840470+0000 mon.a (mon.0 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: ) 937 : audit [INF] from='mgr.15174 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023- 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 09-23T22:32:34 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: .844342+0000 mon.a (mon 2023-09-23T22:32:35.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: .0) 938 : audit [INF] 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: from='mgr.15174 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: -23T22:32:34 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: .266103+0000 mon.a (mon.0) 940 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: 22:32:35.267726+0000 mon.b (mon.1) 105 : audit 2023-09-23T22:32:35.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: -09-23T22:32:35.268195+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09- 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: 23T22:32:35.270092+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: audit 2023-09-23T22 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: :32:35.271032+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: cluster 2023-09- 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: 23T22:32:35.274329+0000 mgr.y (mgr.15174) 33 : cluster 2023-09-23T22:32:36.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[100224]: [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: .845928+0000 mon.c (mon 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: .2) 110 : audit [DBG] 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023- 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 09-23T22:32: 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 34.846749+0000 mon.c ( 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: mon.2) 111 : audit 2023-09-23T22:32:36.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 32:34.847355+0000 mgr.y (mgr.15174) 32 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: cluster 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09-23T22:32:35 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: .265963+0000 mon.a (mon.0) 939 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: cluster 2023-09-23T 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 22:32:35.266103+0000 mon.a (mon.0) 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 940 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: :35.267726+0000 mon.b (mon.1) 105 : audit 2023-09-23T22:32:36.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 32:35.268195+0000 mon.b (mon.1 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: ) 106 : audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 2023-09-23T22:32:35.270092 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: +0000 mon.b (mon.1) 107 : audit [DBG] 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 32:35.271032+0000 mon.b (mon.1) 108 : 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: audit [DBG] from='mgr.? 172.21.15.151:0/190323256' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:36.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:32:36.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: 32:35.274329+0000 mgr.y (mgr.15174) 33 : cluster 2023-09-23T22:32:36.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:35 smithi052 conmon[110082]: [DBG] pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: cluster 2023-09-23T22:32:36.202308+0000 mon.a 2023-09-23T22:32:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: (mon.0) 941 : cluster [DBG] mgrmap e31: y(active, since 24s), standbys: x 2023-09-23T22:32:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:36.313631+0000 mgr.y (mgr.15174) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T22:32:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: audit 2023-09-23T22:32:36.319295+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: audit 2023-09-23T22:32:36.320299+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:32:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: audit 2023-09-23T22:32:36.320968+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:32:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: audit 2023-09-23T22:32:36.321613+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: cephadm 2023-09-23T22:32:36.322610+0000 mgr.y (mgr.15174) 35 : cephadm [INF] Deploying daemon mon.a on smithi052 2023-09-23T22:32:37.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: audit 2023-09-23T22:32:36.418206+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.52:0/815090877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: cluster 2023-09-23T22:32 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: :36.202308+0000 mon.a (mon.0) 941 : cluster [DBG] mgrmap e31: y(active, since 24s), standbys: x 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:36.313631+0000 mgr.y (mgr.15174) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.319295+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.320299+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.320968+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:32:37.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: audit 2023-09-23T22:32:36.321613+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:36.322610+0000 mgr.y ( 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: mgr.15174) 35 : cephadm [INF] 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: Deploying daemon mon.a on smithi052 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: audit 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: 2023-09-23 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: T22:32:36.418206+0000 mon.c (mon.2) 115 : 2023-09-23T22:32:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[110082]: audit [DBG] from='client.? 172.21.15.52:0/815090877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: cluster 2023-09-23T22:32:36.202308+0000 mon.a (mon.0) 941 : cluster [DBG] mgrmap e31: y(active, since 24s), standbys: x 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:36.313631+0000 mgr.y (mgr.15174) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: audit 2023-09-23T22:32:36.319295+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.15174 ' entity='mgr.y' 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: audit 2023-09-23T22:32:36.320299+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: audit 2023-09-23T22:32: 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: 36.320968+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: audit 2023-09-23T22:32:36.321613+0000 mon.c (mon.2) 114 : audit 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: [DBG] from='mgr.15174 172.21.15.52:0/2647665496' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: cephadm 2023-09-23 2023-09-23T22:32:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: T22:32:36.322610+0000 mgr.y (mgr.15174) 35 : cephadm [INF] Deploying daemon mon.a on smithi052 2023-09-23T22:32:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: audit 2023-09-23T22:32:36.418206+0000 mon.c (mon. 2023-09-23T22:32:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:37 smithi151 conmon[100454]: 2) 115 : audit [DBG] from='client.? 172.21.15.52:0/815090877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:37.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 systemd[1]: Stopping Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:38.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: debug 2023-09-23T22:32:37.949+0000 7f1a5d36f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T22:32:38.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:37 smithi052 conmon[100224]: debug 2023-09-23T22:32:37.949+0000 7f1a5d36f700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T22:32:38.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:38 smithi151 conmon[150039]: ts=2023-09-23T22:32:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:32:38.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:38 smithi052 bash[180527]: da0d22fd61dec586ac9139c0da8f0a6ab794d365934165843b368ff4eb3a51a6 2023-09-23T22:32:38.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:38 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T22:32:38.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:38 smithi052 systemd[1]: Stopped Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:39.467 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 systemd[1]: Starting Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:32:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 bash[180913]: bb846ce0af7315753c8e17de98294cd12716fed3bf4743e2bf2753f14ac7f118 2023-09-23T22:32:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: pidfile_write: ignore empty --pid-file 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 systemd[1]: Started Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: load: jerasure load: lrc 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Git sha 0 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: DB SUMMARY 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: DB Session ID: S67OFYME9OBPXQCZ425F 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:32:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: MANIFEST file: MANIFEST-000008 size: 780 Bytes 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000020.sst 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000018.log size: 327771 ; 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.env: 0x563b2add3720 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:32:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.info_log: 0x563b2d231420 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.statistics: (nil) 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.use_fsync: 0 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:32:39.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:32:39.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:32:39.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:32:39.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.db_log_dir: 2023-09-23T22:32:39.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.wal_dir: 2023-09-23T22:32:39.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:32:39.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:32:39.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:32:39.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.write_buffer_manager: 0x563b2c4c0c80 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:32:39.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.unordered_write: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.row_cache: None 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.wal_filter: None 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:32:40.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.wal_compression: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:32:40.001 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:32:40.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_open_files: -1 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Compression algorithms supported: 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kZSTD supported: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:32:40.003 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.merge_operator: 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_filter: None 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:32:40.004 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563b2d231520) 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: index_type: 0 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: data_block_index_type: 0 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: index_shortening: 1 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: checksum: 4 2023-09-23T22:32:40.005 INFO:journalctl@ceph.mon.a.smithi052.stdout: no_block_cache: 0 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_cache: 0x563b2c543350 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_cache_options: 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: capacity : 536870912 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: num_shard_bits : 4 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: strict_capacity_limit : 0 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_cache_compressed: (nil) 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: persistent_cache: (nil) 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_size: 4096 2023-09-23T22:32:40.006 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_size_deviation: 10 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_restart_interval: 16 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: index_block_restart_interval: 1 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: metadata_block_size: 4096 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: partition_filters: 0 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: use_delta_encoding: 1 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: filter_policy: bloomfilter 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: whole_key_filtering: 1 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: verify_compression: 0 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: format_version: 5 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: enable_index_compression: 1 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: block_align: 0 2023-09-23T22:32:40.007 INFO:journalctl@ceph.mon.a.smithi052.stdout: max_auto_readahead_size: 262144 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout: prepopulate_block_cache: 0 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression: NoCompression 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.num_levels: 7 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:32:40.008 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:32:40.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:32:40.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:32:40.011 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:32:40.012 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:32:40.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.ttl: 2592000 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:32:40.014 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 22, last_sequence is 10413, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 34bed5a4-5a12-42e1-a967-b2a78ff070f8 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508359560637, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508359562387, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 328581, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10405, "largest_seqno": 10456, "table_properties": {"data_size": 327222, "index_size": 219, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 860, "raw_average_key_size": 24, "raw_value_size": 326271, "raw_average_value_size": 9322, "num_data_blocks": 10, "num_entries": 35, "num_filter_entries": 35, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695508359, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "34bed5a4-5a12-42e1-a967-b2a78ff070f8", "db_session_id": "S67OFYME9OBPXQCZ425F", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508359562466, "job": 1, "event": "recovery_finished"} 2023-09-23T22:32:40.015 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563b2c5de000 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: DB pointer 0x563b2c5cc000 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout: ** DB Stats ** 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:32:40.016 INFO:journalctl@ceph.mon.a.smithi052.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: ** Compaction Stats [default] ** 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: L0 1/0 320.88 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 284.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: L6 1/0 24.95 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: Sum 2/0 25.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 284.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 284.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.017 INFO:journalctl@ceph.mon.a.smithi052.stdout: ** Compaction Stats [default] ** 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 284.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:32:40.018 INFO:journalctl@ceph.mon.a.smithi052.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: Cumulative compaction: 0.00 GB write, 76.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: Interval compaction: 0.00 GB write, 76.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: Block cache BinnedLRUCache@0x563b2c543350#7 capacity: 512.00 MB usage: 0.47 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.30 KB,5.66244e-05%) Misc(1,0.00 KB,0%) 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: starting mon.a rank 0 at public addrs [v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] at bind addrs [v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???) e3 preinit fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).mds e7 new map 2023-09-23T22:32:40.019 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).mds e7 print_map 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: e7 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: legacy client fscid: 1 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: Filesystem 'foo' (1) 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: fs_name foo 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: epoch 5 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: created 2023-09-23T22:28:42.663514+0000 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: modified 2023-09-23T22:28:50.691835+0000 2023-09-23T22:32:40.020 INFO:journalctl@ceph.mon.a.smithi052.stdout: tableserver 0 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: root 0 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: session_timeout 60 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: session_autoclose 300 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: max_file_size 1099511627776 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: required_client_features {} 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: last_failure 0 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: last_failure_osd_epoch 0 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: max_mds 1 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: in 0 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: up {0=24518} 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: failed 2023-09-23T22:32:40.021 INFO:journalctl@ceph.mon.a.smithi052.stdout: damaged 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: stopped 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: data_pools [4] 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: metadata_pool 3 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: inline_data disabled 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: balancer 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: bal_rank_mask -1 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: standby_count_wanted 1 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: [mds.foo.smithi052.thpzxu{0:24518} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.022 INFO:journalctl@ceph.mon.a.smithi052.stdout: Standby daemons: 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout: 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout: [mds.foo.smithi151.rbxgii{-1:24529} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.151:6832/507517307,v1:172.21.15.151:6833/507517307] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expand map: {default=false} 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta from 'false' to 'false' 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expanded map: {default=false} 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expand map: {default=info} 2023-09-23T22:32:40.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta from 'info' to 'info' 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expanded map: {default=info} 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expand map: {default=daemon} 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expand map: {default=debug} 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T22:32:40.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:32:39 smithi052 ceph-mon[180979]: expand_channel_meta expanded map: {default=debug} 2023-09-23T22:32:41.270 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T22:32:41.308 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: ignoring --setuser ceph since I am not root 2023-09-23T22:32:41.308 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: ignoring --setgroup ceph since I am not root 2023-09-23T22:32:41.308 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 ceph-mgr[173992]: -- 172.21.15.52:0/1596685955 <== mon.0 v2:172.21.15.52:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56146be391e0 con 0x56146be10c00 2023-09-23T22:32:41.308 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: 2023-09-23T22:32:41.209+0000 7f39f98fd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:32:41.308 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: 2023-09-23T22:32:41.307+0000 7f39f98fd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 32:38.837640+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.52:0/821675625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09-23T22:32:39.275545+0000 mgr.y (mgr.15174) 37 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09-23T22:32:39.775118+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: -23T22:32:40.990097+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09-23 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: T22:32:40.994038+0000 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: mon.a (mon.0) 3 2023-09-23T22:32:41.309 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 2023-09-23T22: 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 32:40.994573+0000 mon.a ( 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: mon.0) 4 : cluster 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 09-23T22:32: 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 40.994608+0000 mon.a (mon. 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 0) 5 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09 2023-09-23T22:32:41.310 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: -23T22:32:40. 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 995528+0000 mon.a (mon.0 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: ) 6 : cluster [DBG] mgrmap e31: y(active, since 29s), standbys: x 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023-09- 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 23T22:32:41. 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 011586+0000 mon.a (mon.0 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: ) 7 : cluster [DBG] mgrmap e32: y(active, since 29s), standbys: x 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: cluster 2023- 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: 09-23T22:32:41 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: .011627+0000 mon.a (mon.0 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: ) 8 : cluster [INF] overall HEALTH_OK 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: audit 2023-09-23T22:32:41.011730+0000 2023-09-23T22:32:41.311 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[110082]: mon.a (mon.0) 9 : audit [INF] from='mgr.15174 ' entity='' 2023-09-23T22:32:41.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: ignoring --setuser ceph since I am not root 2023-09-23T22:32:41.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: ignoring --setgroup ceph since I am not root 2023-09-23T22:32:41.347 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 ceph-mgr[151092]: -- 172.21.15.151:0/3339768901 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562c923131e0 con 0x562c922eac00 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: 2023-09-23T22:32:41.177+0000 7fb1a125a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: 2023-09-23T22:32:41.256+0000 7fb1a125a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: 2023-09-23T22:32:41.346+0000 7fb1a125a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: audit 2023-09-23T22:32:38.837640+0000 mon.c (mon.2) 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 116 : audit [DBG] from='client.? 172.21.15.52:0/821675625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023-09-23T22:32:39.275545+0000 mgr.y (mgr.15174) 37 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: -09-23T22:32:39.775118+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023-09-23T22:32:41.348 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 2023-09-23T22:32:40.990097+0000 mon.a (mon.0 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: ) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023-09-23T22:32:40. 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 994038+0000 mon.a (mon.0) 3 : cluster [DBG] 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023-09- 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 23T22:32:40.994573+0000 mon.a (mon.0) 4 : cluster [DBG] 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023- 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 09-23T22:32:40.994608+0000 mon.a (mon.0) 5 : cluster 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:32:41.349 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: -09-23T22:32:40.995528+0000 mon.a (mon.0) 6 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: : cluster [DBG] mgrmap e31: y(active, since 29s), standbys: x 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 2023-09-23T22:32:41.011586+0000 mon.a (mon. 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 0) 7 : cluster [DBG] mgrmap e32: y(active, since 29s), standbys: x 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: cluster 2023-09-23T22:32:41 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: .011627+0000 mon.a (mon.0) 8 : cluster 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: [INF] overall HEALTH_OK 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: 22:32:41.011730+0000 mon.a (mon.0 2023-09-23T22:32:41.350 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[100454]: ) 9 : audit [INF] from='mgr.15174 ' entity='' 2023-09-23T22:32:41.610 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: 2023-09-23T22:32:41.438+0000 7fb1a125a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:32:41.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: 2023-09-23T22:32:41.426+0000 7f39f98fd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:32:41.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: 2023-09-23T22:32:41.543+0000 7f39f98fd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:32:42.026 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:41 smithi052 conmon[173965]: 2023-09-23T22:32:41.752+0000 7f39f98fd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:32:42.045 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:32:42.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:42 smithi151 conmon[100454]: audit 2023-09-23T22:32:41.255699+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.52:0/3437008378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:42.286 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: 2023-09-23T22:32:41.609+0000 7fb1a125a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:32:42.286 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:41 smithi151 conmon[151068]: 2023-09-23T22:32:41.972+0000 7fb1a125a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:32:42.286 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:42 smithi151 conmon[151068]: 2023-09-23T22:32:42.102+0000 7fb1a125a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:32:42.287 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:42 smithi151 conmon[151068]: 2023-09-23T22:32:42.190+0000 7fb1a125a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:32:42.427 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:42 smithi052 conmon[173965]: 2023-09-23T22:32:42.242+0000 7f39f98fd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:32:42.427 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:42 smithi052 conmon[110082]: audit 2023-09-23T22:32:41.255699+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.52:0/3437008378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:32:42.695 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:42 smithi052 conmon[173965]: 2023-09-23T22:32:42.426+0000 7f39f98fd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:32:42.695 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:42 smithi052 conmon[173965]: 2023-09-23T22:32:42.539+0000 7f39f98fd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:32:42.695 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:42 smithi052 conmon[173965]: 2023-09-23T22:32:42.657+0000 7f39f98fd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:32:43.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:42 smithi151 conmon[151068]: 2023-09-23T22:32:42.284+0000 7fb1a125a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:32:43.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:42 smithi151 conmon[151068]: 2023-09-23T22:32:42.710+0000 7fb1a125a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:32:43.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:43 smithi052 conmon[173965]: 2023-09-23T22:32:43.187+0000 7f39f98fd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:32:43.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:43 smithi151 conmon[151068]: 2023-09-23T22:32:43.371+0000 7fb1a125a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:32:43.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:43 smithi151 conmon[151068]: 2023-09-23T22:32:43.454+0000 7fb1a125a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:32:44.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:44 smithi052 conmon[173965]: 2023-09-23T22:32:44.027+0000 7f39f98fd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:32:44.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:44 smithi052 conmon[173965]: 2023-09-23T22:32:44.130+0000 7f39f98fd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:32:44.504 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:44 smithi151 conmon[151068]: 2023-09-23T22:32:44.236+0000 7fb1a125a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:44.504 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:44 smithi151 conmon[151068]: 2023-09-23T22:32:44.328+0000 7fb1a125a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:32:44.788 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:44 smithi151 conmon[151068]: 2023-09-23T22:32:44.503+0000 7fb1a125a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:45.166 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:44 smithi151 conmon[151068]: 2023-09-23T22:32:44.787+0000 7fb1a125a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:32:45.166 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:45 smithi151 conmon[151068]: 2023-09-23T22:32:45.016+0000 7fb1a125a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:32:45.433 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:45 smithi151 conmon[151068]: 2023-09-23T22:32:45.164+0000 7fb1a125a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:32:45.455 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:45 smithi052 conmon[173965]: 2023-09-23T22:32:45.115+0000 7f39f98fd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:45.455 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:45 smithi052 conmon[173965]: 2023-09-23T22:32:45.228+0000 7f39f98fd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:32:45.455 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:45 smithi052 conmon[173965]: 2023-09-23T22:32:45.454+0000 7f39f98fd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:32:45.698 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:45 smithi151 conmon[151068]: 2023-09-23T22:32:45.432+0000 7fb1a125a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:32:46.070 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:45 smithi052 conmon[173965]: 2023-09-23T22:32:45.812+0000 7f39f98fd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:32:46.448 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:46 smithi151 conmon[151068]: 2023-09-23T22:32:46.143+0000 7fb1a125a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:32:46.449 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:46 smithi151 conmon[151068]: 2023-09-23T22:32:46.277+0000 7fb1a125a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:32:46.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:46 smithi052 conmon[173965]: 2023-09-23T22:32:46.069+0000 7f39f98fd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:32:46.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:46 smithi052 conmon[173965]: 2023-09-23T22:32:46.254+0000 7f39f98fd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:32:46.813 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:46 smithi151 conmon[151068]: 2023-09-23T22:32:46.475+0000 7fb1a125a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:32:46.813 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:46 smithi151 conmon[151068]: 2023-09-23T22:32:46.591+0000 7fb1a125a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:46.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:46 smithi052 conmon[173965]: 2023-09-23T22:32:46.591+0000 7f39f98fd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:32:47.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:46 smithi151 conmon[151068]: 2023-09-23T22:32:46.811+0000 7fb1a125a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:32:47.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[173965]: 2023-09-23T22:32:47.431+0000 7f39f98fd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:32:47.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[173965]: 2023-09-23T22:32:47.594+0000 7f39f98fd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:32:48.198 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: 2023-09-23T22:32:47.698+0000 7fb1a125a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: [23/Sep/2023:22:32:47] ENGINE Bus STARTING 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: CherryPy Checker: 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: The Application mounted at '' has an empty config. 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: [23/Sep/2023:22:32:47] ENGINE Serving on http://:::9283 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[151068]: [23/Sep/2023:22:32:47] ENGINE Bus STARTED 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: cluster 2023-09-23T22:32:47.702224+0000 mon.a (mon.0) 10 : 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:32:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: cluster 2023-09-23T22:32:47.702325+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: audit 2023-09-23T22:32 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: :47.704816+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: -23T22:32:47.705167+0000 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: mon.b (mon.1) 110 : audit [DBG] 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: -09-23T22:32:47. 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: 707678+0000 mon.b (mon.1) 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: 111 : audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: -23T22:32:47.708007 2023-09-23T22:32:48.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: +0000 mon.b (mon.1) 112 2023-09-23T22:32:48.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:47 smithi151 conmon[100454]: : audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:48.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[173965]: 2023-09-23T22:32:47.851+0000 7f39f98fd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:32:48.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[173965]: 2023-09-23T22:32:47.993+0000 7f39f98fd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:32:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: cluster 2023-09-23T22:32:47.702224+0000 mon.a (mon.0 2023-09-23T22:32:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: ) 10 : cluster [DBG] Standby manager daemon x restarted 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: cluster 2023-09-23T22:32:47.702325+0000 mon.a (mon.0) 11 : cluster [DBG] 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: Standby manager daemon x started 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: audit 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: 2023-09-23T22:32:47.704816+0000 mon.b (mon. 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: 1) 109 : audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: -23T22:32:47.705167+0000 mon.b (mon 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: .1) 110 : audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: 09-23T22:32:47.707678+0000 mon.b ( 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: mon.1) 111 : audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:32:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: -09-23T22:32:47.708007+0000 mon.b (mon.1) 112 : 2023-09-23T22:32:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:47 smithi052 conmon[110082]: audit [DBG] from='mgr.? 172.21.15.151:0/4272070661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:32:48.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:48 smithi052 conmon[173965]: 2023-09-23T22:32:48.275+0000 7f39f98fd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:32:49.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:48 smithi151 conmon[100454]: cluster 2023-09-23T22:32:47.757852+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e33: y(active, since 36s), standbys: x 2023-09-23T22:32:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:48 smithi151 conmon[100454]: 2023-09-23T22:32:49.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:48 smithi052 conmon[110082]: cluster 2023-09-23T22:32:47.757852+0000 mon.a (mon. 2023-09-23T22:32:49.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:48 smithi052 conmon[110082]: 0) 12 : cluster [DBG] mgrmap e33: y(active, since 36s), standbys: x 2023-09-23T22:32:49.596 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: debug 2023-09-23T22:32:49.361+0000 7f7672b62700 -1 mon.c@2(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.344+0000 7f39f98fd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: [23/Sep/2023:22:32:49] ENGINE Bus STARTING 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f395a476700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f395a476700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f395a476700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f395a476700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f395a476700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f3953468700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.598 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f3953468700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.598 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f3953468700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.598 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f3953468700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.598 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.513+0000 7f3953468700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:32:49.598 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: CherryPy Checker: 2023-09-23T22:32:49.598 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: The Application mounted at '' has an empty config. 2023-09-23T22:32:49.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: debug 2023-09-23T22:32:49.361+0000 7fdd18485700 -1 mon.b@1(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:32:49.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: 2023-09-23T22:32:49.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: [23/Sep/2023:22:32:49] ENGINE Serving on http://:::9283 2023-09-23T22:32:49.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[173965]: [23/Sep/2023:22:32:49] ENGINE Bus STARTED 2023-09-23T22:32:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023-09-23T22:32:49 2023-09-23T22:32:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .359668+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-09-23T22:32:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023-09-23T22:32:49.359991+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-09-23T22:32:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023-09-23T22:32:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023- 2023-09-23T22:32:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 09-23T22:32:49. 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 364923+0000 mon.a (mon.0) 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 15 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32:49. 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 365200+0000 mon.a (mon.0 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 16 : cluster [DBG] mgrmap e34: y(active, starting, since 0.00528947s), standbys: x 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32:49 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .368200+0000 osd.4 (osd. 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 4) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 22:32:49.372783+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32: 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 49.372930+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 32:49.373069+0000 mon.a (mon.0) 19 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: :49.373343+0000 mon.a (mon.0) 20 : audit [DBG] 2023-09-23T22:32:49.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:49. 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 373724+0000 mon.a (mon.0) 21 : audit [DBG] 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:49. 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 373780+0000 osd.4 (osd.4 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32:49 2023-09-23T22:32:49.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .374555+0000 mon.a (mon.0 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 22 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32: 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 49.374852+0000 mon.a (mon.0 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 23 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 22:32:49.375168+0000 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: mon.a (mon.0) 24 : 2023-09-23T22:32:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -23T22:32:49. 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 375463+0000 mon.a (mon.0 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 25 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 09-23T22:32:49 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .375643+0000 mon.a (mon. 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 0) 26 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:32:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: :32:49.375809+0000 mon.a 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: (mon.0) 27 : audit 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32:49. 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 375973+0000 mon.a (mon.0) 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 28 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: :49.376164+0000 mon.a (mon 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .0) 29 : audit [DBG] 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 23T22:32:49.376444+0000 mon.a (mon 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .0) 30 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -23T22:32:49. 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 376677+0000 mon.a (mon.0 2023-09-23T22:32:50.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 31 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32:49 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .376896+0000 mon.a (mon. 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 0) 32 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: T22:32:49.377130 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: +0000 mon.a (mon.0) 33 : 2023-09-23T22:32:50.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: T22:32:49.377526 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: +0000 mon.a (mon.0) 34 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32: 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 49.396111+0000 mon.a (mon. 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 0) 35 : cluster [INF] 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: Manager daemon y is now available 2023-09-23T22:32:50.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09- 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 23T22:32:49.422327 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: +0000 mon.a (mon.0) 36 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 09-23T22:32:49.424961+0000 mon.a ( 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: mon.0) 37 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T22:32 2023-09-23T22:32:50.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: :49.448104+0000 mon.a ( 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: mon.0) 38 : audit [DBG] 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 09-23T22:32:49 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: .459833+0000 mon.a (mon.0 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: ) 39 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 2023-09-23T 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 22:32:49.481600+0000 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: osd.2 (osd.2) 3 : 2023-09-23T22:32:50.005 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.006 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: audit 2023 2023-09-23T22:32:50.006 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: -09-23T22:32: 2023-09-23T22:32:50.006 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 49.487342+0000 mon.a (mon. 2023-09-23T22:32:50.006 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: 0) 40 : audit [INF] 2023-09-23T22:32:50.006 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:49 smithi052 conmon[110082]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.359668+0000 mon.a (mon.0) 13 : cluster [INF] 2023-09-23T22:32:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: Active manager daemon y restarted 2023-09-23T22:32:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.359991+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-09-23T22:32:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.364923+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:32:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.365200+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e34: y(active, starting, since 0.00528947s), standbys: x 2023-09-23T22:32:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.368200+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.372783+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.372930+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.373069+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.373343+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi052.thpzxu"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.373724+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.rbxgii"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.373780+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.374555+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.374852+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.375168+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.375463+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.375643+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:32:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.375809+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.375973+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.376164+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.376444+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.376677+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.376896+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.377130+0000 mon.a (mon.0) 33 : audit [DBG] 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.377526+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.396111+0000 mon.a (mon.0) 35 : cluster [INF] Manager daemon y is now available 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.422327+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.424961+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: 32:49.448104+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:49.459833+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.481600+0000 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: audit 2023-09-23T22:32:50.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:49 smithi151 conmon[100454]: 2023-09-23T22:32:49.487342+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:32:50.390 INFO:teuthology.orchestra.run.smithi052.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:32:50.390 INFO:teuthology.orchestra.run.smithi052.stdout:alertmanager.a smithi052 *:9093,9094 running (86s) - 4m 17.9M - 0.23.0 ba2b418f427c a82e31979683 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:grafana.a smithi151 *:3000 running (52s) - 4m 46.2M - 8.3.5 dad864ee21e9 8803df2494ff 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi052.thpzxu smithi052 running (4m) - 4m 19.3M - 16.2.14-37-g2e1f1015 1e6a756174a8 d8befa844e73 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi151.rbxgii smithi151 running (3m) - 3m 15.0M - 16.2.14-37-g2e1f1015 1e6a756174a8 61e8f44cb3b6 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.x smithi151 *:8443,9283,8765 running (22s) - 8m 46.3M - 18.2.0-460-gbb0daeff e32c2a64c126 648eee1c1677 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.y smithi052 *:8443,9283,8765 running (48s) - 10m 479M - 18.2.0-460-gbb0daeff e32c2a64c126 d2a437c50b3c 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a smithi052 running (10m) - 10m 112M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 da0d22fd61de 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b smithi151 running (8m) - 8m 97.4M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 4a3cec32371a 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c smithi052 running (8m) - 8m 89.0M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 58c863a9b169 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.a smithi052 *:9100 running (94s) - 5m 21.7M - 1.3.1 1dbe0e931976 3bd65f3ab9c9 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.b smithi151 *:9100 running (72s) - 4m 17.4M - 1.3.1 1dbe0e931976 7e4657a95489 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 smithi052 running (8m) - 8m 65.2M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 667e33a7d512 2023-09-23T22:32:50.391 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 smithi052 running (7m) - 7m 66.9M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 cb0928f7abc9 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 smithi052 running (7m) - 7m 141M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 6d0289679661 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 smithi052 running (7m) - 7m 62.0M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 1744ec2d869b 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 smithi151 running (6m) - 6m 69.8M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 b2126e632b7b 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 smithi151 running (6m) - 6m 90.3M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 46585800d682 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 smithi151 running (6m) - 6m 62.0M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 17e250c1eb43 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 smithi151 running (5m) - 5m 101M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 73447cd68609 2023-09-23T22:32:50.392 INFO:teuthology.orchestra.run.smithi052.stdout:prometheus.a smithi151 *:9095 running (27s) - 5m 43.3M - 2.33.4 514e6a882f6e 88e6208a51e9 2023-09-23T22:32:50.660 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[173965]: 2023-09-23T22:32:50.373+0000 7f39bb05d700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T22:32:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:50 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.366648+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:50 smithi151 conmon[100454]: cluster 2023-09-23T22:32:49.471740+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:50 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:49.873299+0000 mgr.y (mgr.25214) 1 : cephadm [INF] [23/Sep/2023:22:32:49] ENGINE Bus STARTING 2023-09-23T22:32:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:50 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:49.994932+0000 mgr.y (mgr.25214) 2 : cephadm [INF] [23/Sep/2023:22:32:49] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:32:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:50 smithi151 conmon[100454]: cluster 2023-09-23T22:32:50.369961+0000 mon.a (mon.0) 41 : cluster [DBG] mgrmap e35: y(active, since 1.01005s), standbys: x 2023-09-23T22:32:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: cluster 2023-09-23T22:32:49.366648+0000 osd.1 (osd 2023-09-23T22:32:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: .1) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: cluster 2023-09-23T22:32:49.471740+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-09-23T22:32:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:49.873299+0000 mgr.y (mgr.25214) 1 : cephadm [INF] [23/Sep/2023:22:32:49] ENGINE Bus STARTING 2023-09-23T22:32:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:49.994932+0000 mgr.y (mgr.25214) 2 : cephadm [INF] [23/Sep/2023:22:32:49] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:32:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: cluster 2023-09-23T22:32:50. 2023-09-23T22:32:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:50 smithi052 conmon[110082]: 369961+0000 mon.a (mon.0) 41 : cluster [DBG] mgrmap e35: y(active, since 1.01005s), standbys: x 2023-09-23T22:32:51.337 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:32:52.078 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:51 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:50.096441+0000 mgr.y (mgr.25214) 3 : cephadm [INF] [23/Sep/2023:22:32:50] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:32:52.078 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:51 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:50.096603+0000 mgr.y (mgr.25214) 4 : cephadm [INF] [23/Sep/2023:22:32:50] ENGINE Bus STARTED 2023-09-23T22:32:52.078 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:51 smithi052 conmon[110082]: audit 2023-09-23T22:32:50.376596+0000 mgr.y (mgr.25214) 5 : audit [DBG] from='client.25220 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:32:52.078 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:51 smithi052 conmon[110082]: cluster 2023-09-23T22:32:50.378236+0000 mgr.y (mgr.25214) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:52.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:50.096441+0000 mgr.y (mgr.25214) 3 : cephadm [INF] [23/Sep/2023:22:32:50] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:50.096603+0000 mgr.y (mgr.25214) 4 : cephadm [INF] [23/Sep/2023:22:32:50] ENGINE Bus STARTED 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: audit 2023-09-23T22: 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: 32:50.376596+0000 mgr.y (mgr.25214) 5 : audit [DBG] from='client.25220 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: cluster 2023-09 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: -23T22:32:50.378236+0000 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: mgr.y (mgr.25214) 6 : cluster [DBG] 2023-09-23T22:32:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:51 smithi151 conmon[100454]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:53.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:52 smithi052 conmon[110082]: cluster 2023-09-23T22:32:51.376463+0000 2023-09-23T22:32:53.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:52 smithi052 conmon[110082]: mgr.y (mgr.25214) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:53.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:52 smithi052 conmon[110082]: cluster 2023-09-23T22:32:52.375107+0000 mon.a (mon.0) 42 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2023-09-23T22:32:53.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: cluster 2023-09-23T22:32:51.376463 2023-09-23T22:32:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: +0000 mgr.y (mgr.25214) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: cluster 2023-09-23T22:32:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: 2023-09-23T22:32: 2023-09-23T22:32:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: 52.375107+0000 mon.a (mon. 2023-09-23T22:32:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: 0) 42 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2023-09-23T22:32:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:52 smithi151 conmon[100454]: 2023-09-23T22:32:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:32:53] "GET /metrics HTTP/1.1" 200 31227 "" "Prometheus/2.33.4" 2023-09-23T22:32:53.591 INFO:teuthology.orchestra.run.smithi052.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:32:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: audit 2023-09-23T22:32:52.831013+0000 mon.a 2023-09-23T22:32:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: (mon.0) 43 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: audit 2023-09-23T22:32:52.836973+0000 mon.a (mon.0) 44 : audit 2023-09-23T22:32:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: audit 2023- 2023-09-23T22:32:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: 09-23T22:32:53.522881+0000 mon.a (mon.0) 45 : audit 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: audit 2023-09-23T22:32 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: :53.527500+0000 mon.a (mon.0) 46 : 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: :32:53.590527+0000 mon.a (mon.0) 2023-09-23T22:32:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:53 smithi052 conmon[110082]: 47 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:54.096 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:53 smithi151 conmon[100454]: audit 2023-09-23T22:32:52.831013+0000 mon.a 2023-09-23T22:32:54.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:53 smithi151 conmon[100454]: (mon.0) 43 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:54.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:53 smithi151 conmon[100454]: audit 2023-09-23T22:32:52.836973+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:54.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:53 smithi151 conmon[100454]: audit 2023-09-23T22:32:53.522881+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:54.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:53 smithi151 conmon[100454]: audit 2023-09-23T22:32:53.527500+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:54.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:53 smithi151 conmon[100454]: audit 2023-09-23T22:32:53.590527+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:54.369 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:32:55.150 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:54 smithi151 conmon[100454]: cluster 2023-09-23T22:32:53.376838+0000 mgr.y (mgr.25214) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:55.150 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:54 smithi151 conmon[100454]: audit 2023-09-23T22:32:53.586844+0000 mgr.y (mgr.25214) 9 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:32:55.150 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:54 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:53.587253+0000 mgr.y (mgr.25214) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:32:55.151 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:54 smithi151 conmon[100454]: audit 2023-09-23T22:32:54.392443+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:55.151 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:54 smithi151 conmon[100454]: audit 2023-09-23T22:32:54.397445+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:55.218 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:54 smithi052 conmon[110082]: cluster 2023-09-23T22 2023-09-23T22:32:55.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:54 smithi052 conmon[110082]: :32:53.376838+0000 mgr.y (mgr.25214) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:55.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:54 smithi052 conmon[110082]: audit 2023-09-23T22:32:53.586844+0000 mgr.y (mgr.25214) 9 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:32:55.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:54 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:53.587253+0000 mgr.y (mgr.25214) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:32:55.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:54 smithi052 conmon[110082]: audit 2023-09-23T22:32:54.392443+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:55.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:54 smithi052 conmon[110082]: audit 2023-09-23T22:32:54.397445+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:56 smithi052 conmon[110082]: audit 2023-09-23T22:32:55.233803+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:56 smithi052 conmon[110082]: audit 2023-09-23T22:32:55.238340+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:56 smithi052 conmon[110082]: cluster 2023-09-23T22:32:55.377390+0000 mgr.y (mgr.25214) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:56.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:56 smithi151 conmon[100454]: audit 2023-09-23T22:32:55.233803+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:56.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:56 smithi151 conmon[100454]: audit 2023-09-23T22:32:55.238340+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:56.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:56 smithi151 conmon[100454]: cluster 2023-09-23T22:32:55.377390+0000 mgr.y (mgr.25214) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: audit 2023-09-23T22:32:56.304696+0000 mon.a 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: (mon.0) 52 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: audit 2023-09-23T22:32:56.309574+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: audit 2023-09-23T22:32:56.311135+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: audit 2023-09-23T22:32:56.311991+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:56.313136 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: +0000 mgr.y (mgr.25214) 12 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: cephadm 2023-09-23 2023-09-23T22:32:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: T22:32:56.313361+0000 mgr.y (mgr.25214) 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: 13 : cephadm [INF] Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: cephadm 2023-09-23 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: T22:32:56.577626+0000 mgr.y 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: (mgr.25214) 14 : cephadm [INF] 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: 2023-09-23T22:32: 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: 56.578140+0000 mgr.y (mgr. 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: 25214) 15 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: cephadm 2023-09-23T22:32: 2023-09-23T22:32:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: 56.842355+0000 mgr.y (mgr.25214) 16 : cephadm 2023-09-23T22:32:57.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:57.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:56 2023-09-23T22:32:57.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: .842623+0000 mgr.y (mgr.25214) 17 : cephadm [INF] 2023-09-23T22:32:57.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:57 smithi151 conmon[100454]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: audit 2023-09-23T22:32:56.304696 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: +0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: audit 2023-09-23T22:32:56.309574+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: audit 2023-09-23T22:32:56. 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: 311135+0000 mon.a (mon.0) 54 : audit 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: audit 2023-09-23 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: T22:32:56.311991+0000 mon.a (mon.0) 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: 55 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm 2023-09 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: -23T22:32:56.313136+0000 mgr.y 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: (mgr.25214) 12 : cephadm [INF] Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm 2023 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: -09-23T22:32:56.313361 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: +0000 mgr.y (mgr.25214) 13 : cephadm [INF] 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: 2023-09-23T22:32:56.577626+0000 mgr.y (mgr.25214) 14 : 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm 2023-09-23 2023-09-23T22:32:57.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: T22:32:56.578140+0000 mgr.y (mgr.25214) 15 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:32:57.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:56.842355+0000 mgr.y (mgr.25214) 16 : cephadm 2023-09-23T22:32:57.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: [INF] Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:57.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: cephadm 2023-09-23T22: 2023-09-23T22:32:57.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: 32:56.842623+0000 mgr.y (mgr.25214) 17 : cephadm [INF] 2023-09-23T22:32:57.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:57 smithi052 conmon[110082]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:32:58.659 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:57.107138+0000 mgr.y (mgr.25214) 18 : cephadm [INF] 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: cephadm 2023-09-23T22:32:57.107307+0000 mgr.y (mgr.25214) 19 : cephadm [INF] Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023-09-23T22:32:57.376114+0000 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: mon.a (mon.0) 56 : audit [INF] 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: cluster 2023 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -09-23T22:32:57 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: .377713+0000 mgr.y (mgr.25214) 20 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:32:58.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -23T22:32:57.378810+0000 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: mon.a (mon.0) 57 : audit [INF] 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: 2023-09-23T22:32 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: :57.382978+0000 mon.a (mon. 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: 0) 58 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -09-23T22:32:57.387585 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: +0000 mon.a (mon.0) 59 : 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:58.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -09-23T22:32:57. 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: 399185+0000 mon.a (mon.0) 60 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -09-23T22:32:57.536809 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: +0000 mon.a (mon.0) 61 : audit 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -09-23T22:32:57.538185 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: +0000 mon.a (mon.0) 62 : 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023 2023-09-23T22:32:58.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: -09-23T22:32:57. 2023-09-23T22:32:58.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: 538777+0000 mon.a (mon.0) 63 2023-09-23T22:32:58.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:58.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:32:58.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: 09-23T22:32:57.543192+0000 2023-09-23T22:32:58.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: mon.a (mon.0) 64 : audit 2023-09-23T22:32:58.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[100454]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:57.107138+0000 mgr.y (mgr. 2023-09-23T22:32:58.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 25214) 18 : cephadm [INF] Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: cephadm 2023-09-23T22:32:57.107307+0000 mgr.y (mgr.25214) 19 : cephadm [INF] 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09-23T22 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: :32:57.376114+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: cluster 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 2023-09-23T22:32:57.377713+0000 mgr.y (mgr. 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 25214) 20 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: -23T22:32:57.378810+0000 mon.a (mon.0) 57 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 23T22:32:57.382978+0000 mon.a (mon.0) 58 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09- 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 23T22:32:57.387585+0000 mon.a (mon.0 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: ) 59 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: -23T22:32:57.399185+0000 mon.a (mon. 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 0) 60 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 2023-09-23T22 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: :32:57.536809+0000 mon.a (mon.0) 61 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:32:58.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 2023-09-23T22:32:57.538185 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: +0000 mon.a (mon.0) 62 : audit [DBG] 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 32:57.538777+0000 mon.a (mon.0) 63 : audit 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: audit 2023-09-23T22: 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: 32:57.543192+0000 mon.a (mon.0) 64 : audit 2023-09-23T22:32:58.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:32:58 smithi052 conmon[110082]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:32:58.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:32:58 smithi151 conmon[150039]: ts=2023-09-23T22:32:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:00 smithi151 conmon[100454]: cluster 2023-09-23T22:32:59.378337+0000 mgr.y (mgr.25214) 21 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:00 smithi151 conmon[100454]: 2023-09-23T22:33:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:00 smithi052 conmon[110082]: cluster 2023-09-23T22:32:59.378337+0000 mgr.y (mgr.25214 2023-09-23T22:33:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:00 smithi052 conmon[110082]: ) 21 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:02 smithi151 conmon[100454]: cluster 2023-09-23T22:33:01.379214+0000 mgr.y (mgr.25214) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:02 smithi052 conmon[110082]: cluster 2023-09-23T22:33:01.379214 2023-09-23T22:33:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:02 smithi052 conmon[110082]: +0000 mgr.y (mgr.25214) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:33:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:33:03] "GET /metrics HTTP/1.1" 200 31227 "" "Prometheus/2.33.4" 2023-09-23T22:33:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:04 smithi151 conmon[100454]: cluster 2023-09-23T22:33:03.379712+0000 mgr.y (mgr.25214) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:04 smithi151 conmon[100454]: 2023-09-23T22:33:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:04 smithi151 conmon[100454]: audit 2023-09-23T22:33:04.448396+0000 mon.a (mon.0) 65 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:04 smithi052 conmon[110082]: cluster 2023-09-23T22:33:03.379712+0000 mgr.y (mgr.25214) 23 2023-09-23T22:33:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:04 smithi052 conmon[110082]: : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:04 smithi052 conmon[110082]: audit 2023-09-23T22:33:04.448396+0000 mon.a (mon.0) 65 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:06.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:06 smithi151 conmon[100454]: cluster 2023-09-23T22:33:05.380274+0000 mgr.y (mgr.25214) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:06 smithi052 conmon[110082]: cluster 2023-09-23T22:33:05.380274+0000 mgr.y ( 2023-09-23T22:33:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:06 smithi052 conmon[110082]: mgr.25214) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:08 smithi151 conmon[100454]: cluster 2023-09-23T22:33:07.380703+0000 mgr.y (mgr.25214) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:08.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:33:08 smithi151 conmon[150039]: ts=2023-09-23T22:33:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:08 smithi052 conmon[110082]: cluster 2023-09-23T22:33:07.380703+0000 mgr.y ( 2023-09-23T22:33:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:08 smithi052 conmon[110082]: mgr.25214) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:10.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:10 smithi151 conmon[100454]: cluster 2023-09-23T22:33:09.381334+0000 mgr.y (mgr.25214) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:10 smithi052 conmon[110082]: cluster 2023-09-23T22:33:09.381334+0000 mgr.y 2023-09-23T22:33:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:10 smithi052 conmon[110082]: (mgr.25214) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:12 smithi151 conmon[100454]: cluster 2023-09-23T22:33:11.382086+0000 mgr.y (mgr.25214) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:12 smithi052 conmon[110082]: cluster 2023-09-23T22:33:11.382086+0000 mgr.y ( 2023-09-23T22:33:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:12 smithi052 conmon[110082]: mgr.25214) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:33:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:33:13] "GET /metrics HTTP/1.1" 200 33856 "" "Prometheus/2.33.4" 2023-09-23T22:33:14.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:14 smithi151 conmon[100454]: cluster 2023-09-23T22:33:13.382542+0000 mgr.y (mgr.25214) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:14 smithi052 conmon[110082]: cluster 2023-09-23T22:33:13.382542+0000 mgr.y (mgr.25214) 28 : cluster [DBG] 2023-09-23T22:33:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:14 smithi052 conmon[110082]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:16.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:16 smithi151 conmon[100454]: cluster 2023-09-23T22:33:15.383261+0000 mgr.y (mgr.25214) 29 : cluster [DBG] 2023-09-23T22:33:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:16 smithi151 conmon[100454]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:16 smithi052 conmon[110082]: cluster 2023-09-23T22:33:15.383261+0000 mgr.y (mgr.25214 2023-09-23T22:33:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:16 smithi052 conmon[110082]: ) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:18 smithi151 conmon[100454]: cluster 2023-09-23T22:33:17.383860+0000 mgr.y (mgr.25214) 30 : cluster [DBG] 2023-09-23T22:33:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:18 smithi151 conmon[100454]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:18.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:33:18 smithi151 conmon[150039]: ts=2023-09-23T22:33:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:18 smithi052 conmon[110082]: cluster 2023-09-23T22:33:17.383860+0000 mgr.y (mgr.25214 2023-09-23T22:33:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:18 smithi052 conmon[110082]: ) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:19 smithi151 conmon[100454]: audit 2023-09-23T22:33:19.448869+0000 mon.a (mon.0) 66 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:19 smithi052 conmon[110082]: audit 2023-09-23T22:33:19.448869+0000 mon.a (mon.0 2023-09-23T22:33:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:19 smithi052 conmon[110082]: ) 66 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:20 smithi151 conmon[100454]: cluster 2023-09-23T22:33:19.384401+0000 mgr.y (mgr.25214) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:20 smithi052 conmon[110082]: cluster 2023-09-23T22:33:19.384401+0000 mgr.y (mgr.25214 2023-09-23T22:33:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:20 smithi052 conmon[110082]: ) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:22 smithi151 conmon[100454]: cluster 2023-09-23T22:33:21.385014+0000 mgr.y (mgr.25214) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:22 smithi052 conmon[110082]: cluster 2023-09-23T22:33:21.385014+0000 mgr.y (mgr.25214 2023-09-23T22:33:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:22 smithi052 conmon[110082]: ) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:33:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:33:23] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-09-23T22:33:24.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:24 smithi151 conmon[100454]: cluster 2023-09-23T22:33:23.385486+0000 mgr.y (mgr.25214) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:24 smithi052 conmon[110082]: cluster 2023-09-23T22:33:23.385486+0000 mgr.y (mgr.25214) 33 : 2023-09-23T22:33:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:24 smithi052 conmon[110082]: cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:26 smithi151 conmon[100454]: cluster 2023-09-23T22:33:25.386265+0000 mgr.y (mgr.25214) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:26 smithi151 conmon[100454]: 2023-09-23T22:33:26.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:26 smithi052 conmon[110082]: cluster 2023-09-23T22:33:25.386265+0000 mgr.y 2023-09-23T22:33:26.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:26 smithi052 conmon[110082]: (mgr.25214) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:28 smithi151 conmon[100454]: cluster 2023-09-23T22:33:27.386809+0000 mgr.y (mgr.25214) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:28.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:33:28 smithi151 conmon[150039]: ts=2023-09-23T22:33:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:28 smithi052 conmon[110082]: cluster 2023-09-23T22:33:27.386809+0000 mgr.y (mgr.25214) 35 2023-09-23T22:33:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:28 smithi052 conmon[110082]: : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:30 smithi151 conmon[100454]: cluster 2023-09-23T22:33:29.387231+0000 mgr.y (mgr.25214) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:30 smithi052 conmon[110082]: cluster 2023-09-23T22: 2023-09-23T22:33:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:30 smithi052 conmon[110082]: 33:29.387231+0000 mgr.y (mgr.25214) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:32.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:32 smithi151 conmon[100454]: cluster 2023-09-23T22:33:31.387793+0000 mgr.y (mgr.25214) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:32 smithi052 conmon[110082]: cluster 2023-09-23T22:33:31.387793 2023-09-23T22:33:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:32 smithi052 conmon[110082]: +0000 mgr.y (mgr.25214) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:33.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:33:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:33:33] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-09-23T22:33:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:34 smithi151 conmon[100454]: cluster 2023-09-23T22:33:33.388209+0000 mgr.y (mgr.25214) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:34 smithi151 conmon[100454]: audit 2023-09-23T22:33:34.449149+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:34 smithi052 conmon[110082]: cluster 2023-09-23T22:33:33.388209+0000 mgr.y (mgr 2023-09-23T22:33:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:34 smithi052 conmon[110082]: .25214) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:34 smithi052 conmon[110082]: audit 2023-09-23T22:33:34.449149+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:36.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:36 smithi151 conmon[100454]: cluster 2023-09-23T22:33:35.388909+0000 mgr.y (mgr.25214) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:36 smithi052 conmon[110082]: cluster 2023-09-23T22:33 2023-09-23T22:33:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:36 smithi052 conmon[110082]: :35.388909+0000 mgr.y (mgr.25214) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:38.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:38 smithi151 conmon[100454]: cluster 2023-09-23T22:33:37.389426+0000 mgr.y (mgr.25214) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:38.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:33:38 smithi151 conmon[150039]: ts=2023-09-23T22:33:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:38.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:38 smithi052 conmon[110082]: cluster 2023-09-23T22:33: 2023-09-23T22:33:38.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:38 smithi052 conmon[110082]: 37.389426+0000 mgr.y (mgr.25214) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:40.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:40 smithi151 conmon[100454]: cluster 2023-09-23T22:33:39.389945+0000 mgr.y (mgr.25214) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:40.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:40 smithi052 conmon[110082]: cluster 2023-09-23T22:33:39.389945+0000 mgr.y 2023-09-23T22:33:40.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:40 smithi052 conmon[110082]: (mgr.25214) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:42.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:42 smithi151 conmon[100454]: cluster 2023-09-23T22:33:41.390703+0000 mgr.y (mgr.25214) 42 : cluster [DBG] 2023-09-23T22:33:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:42 smithi151 conmon[100454]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:42 smithi052 conmon[110082]: cluster 2023-09-23T22:33:41.390703+0000 mgr.y (mgr.25214 2023-09-23T22:33:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:42 smithi052 conmon[110082]: ) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:33:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:33:43] "GET /metrics HTTP/1.1" 200 33849 "" "Prometheus/2.33.4" 2023-09-23T22:33:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:44 smithi151 conmon[100454]: cluster 2023-09-23T22:33:43.391344 2023-09-23T22:33:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:44 smithi151 conmon[100454]: +0000 mgr.y (mgr.25214) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:44 smithi052 conmon[110082]: cluster 2023-09-23T22:33:43.391344+0000 mgr.y (mgr.25214) 43 : cluster 2023-09-23T22:33:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:44 smithi052 conmon[110082]: [DBG] pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:46 smithi151 conmon[100454]: cluster 2023-09-23T22:33:45.391881+0000 mgr.y (mgr.25214) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:46 smithi052 conmon[110082]: cluster 2023-09-23T22:33:45.391881+0000 mgr.y (mgr. 2023-09-23T22:33:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:46 smithi052 conmon[110082]: 25214) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:48 smithi151 conmon[100454]: cluster 2023-09-23T22:33:47.392303+0000 mgr.y (mgr. 2023-09-23T22:33:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:48 smithi151 conmon[100454]: 25214) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:48.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:33:48 smithi151 conmon[150039]: ts=2023-09-23T22:33:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:48 smithi052 conmon[110082]: cluster 2023-09-23T22:33:47.392303+0000 2023-09-23T22:33:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:48 smithi052 conmon[110082]: mgr.y (mgr.25214) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:49.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:49 smithi151 conmon[100454]: audit 2023-09-23T22:33:49.450476+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:49.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:49 smithi052 conmon[110082]: audit 2023-09-23T22:33:49.450476+0000 mon.a (mon.0 2023-09-23T22:33:49.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:49 smithi052 conmon[110082]: ) 68 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:50.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:50 smithi151 conmon[100454]: cluster 2023-09-23T22:33:49.392725+0000 mgr.y (mgr.25214) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:50 smithi052 conmon[110082]: cluster 2023-09-23T22:33:49.392725+0000 mgr.y (mgr.25214) 46 : cluster [DBG] 2023-09-23T22:33:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:50 smithi052 conmon[110082]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:52.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:52 smithi151 conmon[100454]: cluster 2023-09-23T22:33:51.393357+0000 mgr.y (mgr.25214) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:52 smithi052 conmon[110082]: cluster 2023-09-23T22:33:51.393357+0000 mgr.y ( 2023-09-23T22:33:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:52 smithi052 conmon[110082]: mgr.25214) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:53.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:33:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:33:53] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-09-23T22:33:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:54 smithi151 conmon[100454]: cluster 2023-09-23T22:33:53.393757+0000 mgr.y (mgr.25214) 2023-09-23T22:33:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:54 smithi151 conmon[100454]: 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:54.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:54 smithi052 conmon[110082]: cluster 2023-09-23T22:33:53.393757+0000 mgr.y (mgr.25214) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:56.512 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:33:56.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:56 smithi151 conmon[100454]: cluster 2023-09-23T22:33:55.394318+0000 mgr.y (mgr.25214) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:56.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:56 smithi052 conmon[110082]: cluster 2023-09-23T22:33:55.394318+0000 mgr.y (mgr.25214) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:57.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:57 smithi151 conmon[100454]: audit 2023-09-23T22:33:57.650257+0000 mon.a (mon.0) 69 : audit 2023-09-23T22:33:57.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:57 smithi151 conmon[100454]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:33:57.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:57 smithi052 conmon[110082]: audit 2023-09-23T22:33:57 2023-09-23T22:33:57.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:57 smithi052 conmon[110082]: .650257+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:33:58.430 INFO:teuthology.orchestra.run.smithi052.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:33:58.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[150039]: ts=2023-09-23T22:33:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:33:59.215 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T22:33:59.242 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: cluster 2023-09-23T22:33:57.394647+0000 mgr.y (mgr.25214) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: audit 2023-09-23T22:33:57.971902+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: audit 2023-09-23T22:33:57.976456+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: audit 2023-09-23T22:33:57.981104+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: audit 2023-09-23T22:33:57.984960+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: audit 2023-09-23T22:33:58.430130+0000 mon.a (mon.0) 74 : audit 2023-09-23T22:33:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:58 smithi052 conmon[110082]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: cluster 2023-09-23T22:33:57.394647+0000 mgr.y (mgr.25214) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:33:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: audit 2023-09-23T22:33:57.971902+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: audit 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: 2023-09-23T22:33:57.976456+0000 mon.a (mon.0) 71 : audit [INF] 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: 09-23T22:33:57.981104+0000 mon.a (mon.0) 72 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: audit 2023-09-23T22:33:57.984960+0000 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: mon.a (mon.0) 73 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: audit 2023-09- 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: 23T22:33:58.430130+0000 mon.a (mon.0) 74 : audit [INF] 2023-09-23T22:33:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:58 smithi151 conmon[100454]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:00.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:59 smithi052 conmon[110082]: audit 2023-09-23T22:33:58.426107+0000 mgr.y (mgr 2023-09-23T22:34:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:59 smithi052 conmon[110082]: .25214) 51 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:33:59 smithi052 conmon[110082]: cephadm 2023-09-23T22:33:58.426479+0000 mgr.y (mgr.25214) 52 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:00.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:59 smithi151 conmon[100454]: audit 2023-09-23T22:33:58.426107+0000 mgr.y (mgr.25214) 2023-09-23T22:34:00.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:59 smithi151 conmon[100454]: 51 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:00.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:33:59 smithi151 conmon[100454]: cephadm 2023-09-23T22:33:58.426479+0000 mgr.y (mgr.25214) 52 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:01.066 INFO:teuthology.orchestra.run.smithi052.stdout:wait for majority of mons to upgrade 2023-09-23T22:34:01.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:00 smithi052 conmon[110082]: cluster 2023-09-23T22:33:59.394967+0000 mgr.y (mgr.25214) 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:01.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:00 smithi052 conmon[110082]: audit 2023-09-23T22:34:00.805309+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:01.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:00 smithi052 conmon[110082]: audit 2023-09-23T22:34:00.809816+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:01.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:00 smithi151 conmon[100454]: cluster 2023-09-23T22:33:59.394967+0000 mgr.y (mgr.25214) 2023-09-23T22:34:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:00 smithi151 conmon[100454]: 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:00 smithi151 conmon[100454]: audit 2023-09-23T22:34:00.805309+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:01.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:00 smithi151 conmon[100454]: audit 2023-09-23T22:34:00.809816+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:01.599 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T22:34:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:02 smithi052 conmon[110082]: cluster 2023-09-23T22:34:01.395534+0000 mgr.y (mgr.25214) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:02 smithi151 conmon[100454]: cluster 2023-09-23T22:34:01.395534+0000 mgr.y (mgr.25214) 54 : cluster [DBG] 2023-09-23T22:34:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:02 smithi151 conmon[100454]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:34:03] "GET /metrics HTTP/1.1" 200 33855 "" "Prometheus/2.33.4" 2023-09-23T22:34:04.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.219760+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:04.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.225299+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:04.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: cluster 2023-09-23T22:34:03.395850+0000 mgr.y (mgr.25214) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:04.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.796311+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:04.481 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.797079+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.801740+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.904872+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.52:0/2432123542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.940063+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.942522+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.943671+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: audit 2023-09-23T22:34:03.944281+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:34:04.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:04 smithi052 conmon[110082]: cephadm 2023-09-23T22:34:03.944728+0000 mgr.y (mgr.25214) 56 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09-23T22:34:03.219760+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09-23T22:34:03.225299+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: cluster 2023-09-23T22:34:03.395850+0000 mgr.y (mgr.25214) 55 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09-23T22:34:03.796311+0000 mon.a (mon.0) 79 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09-23T22:34:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 2023-09-23T22:34:03.797079 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: +0000 mon.a (mon.0) 80 : audit 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 09-23T22:34:03.801740+0000 mon.a 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: (mon.0) 81 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: -23T22:34:03.904872+0000 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: mon.c (mon.2) 118 : audit [DBG] 2023-09-23T22:34:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/2432123542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 2023-09-23T22:34:03 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: .940063+0000 mon.a (mon.0) 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 82 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 09-23T22:34:03.942522+0000 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: mon.a (mon.0) 83 : audit 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:34:04.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: T22:34:03.943671+0000 mon.a (mon 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: .0) 84 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: audit 2023- 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 09-23T22:34:03.944281+0000 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: mon.a (mon.0) 85 : audit [DBG] 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: cephadm 2023- 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: 09-23T22:34:03.944728+0000 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: mgr.y (mgr.25214) 56 : cephadm [INF] 2023-09-23T22:34:04.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:04 smithi151 conmon[100454]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:34:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:05 smithi052 conmon[110082]: audit 2023-09-23T22:34: 2023-09-23T22:34:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:05 smithi052 conmon[110082]: 04.450663+0000 mon.a (mon.0) 86 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:05.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:05 smithi151 conmon[100454]: audit 2023-09-23T22:34:04.450663+0000 mon.a (mon.0) 86 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:06.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: cluster 2023-09-23T22:34:05.396455+0000 mgr.y 2023-09-23T22:34:06.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: (mgr.25214) 57 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:06.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: cephadm 2023-09-23T22:34:05.411929+0000 mgr.y (mgr.25214) 58 : cephadm [INF] Upgrade: Updating mon.c 2023-09-23T22:34:06.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: audit 2023-09-23T22:34:05.416073+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:06.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: audit 2023-09-23T22:34:05.416920+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:06.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: audit 2023-09-23T22:34:05.417538+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:06.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: audit 2023-09-23T22:34:05.418138+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:06.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: cephadm 2023-09-23T22:34:05.418999+0000 mgr.y (mgr.25214) 59 : cephadm [INF] Deploying daemon mon.c on smithi052 2023-09-23T22:34:06.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 conmon[110082]: audit 2023-09-23T22:34:06.310082+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.52:0/2379170911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: cluster 2023-09-23T22:34:05.396455+0000 mgr.y (mgr.25214) 57 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: cephadm 2023-09-23T22:34:05.411929+0000 mgr.y (mgr.25214) 58 : cephadm [INF] Upgrade: Updating mon.c 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: audit 2023-09-23T22:34:05.416073+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: audit 2023-09-23T22:34:05.416920+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: audit 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 2023-09-23T22:34:05.417538+0000 mon.a (mon. 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 0) 89 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: audit 2023-09-23T 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 22:34:05.418138+0000 mon.a (mon.0) 90 : audit 2023-09-23T22:34:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: cephadm 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 2023-09-23T22: 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 34:05.418999+0000 mgr.y (mgr.25214) 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: 59 : cephadm [INF] Deploying daemon mon.c on smithi052 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: audit 2023-09-23 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: T22:34:06.310082+0000 mon.c (mon.2) 119 : 2023-09-23T22:34:06.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:06 smithi151 conmon[100454]: audit [DBG] from='client.? 172.21.15.52:0/2379170911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:06 smithi052 systemd[1]: Stopping Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:34:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:07 smithi052 conmon[110082]: debug 2023-09-23T22:34:07.121+0000 7f767936f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T22:34:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:07 smithi052 conmon[110082]: debug 2023-09-23T22:34:07.121+0000 7f767936f700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T22:34:07.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:07 smithi052 bash[187298]: 58c863a9b1690898cce0a821af07f1b4f2349db2dc27f0c816900d59674b48b0 2023-09-23T22:34:07.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:07 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T22:34:07.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:07 smithi052 systemd[1]: Stopped Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:34:08.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:07 smithi052 systemd[1]: Starting Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:34:08.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:34:08 smithi151 conmon[150039]: ts=2023-09-23T22:34:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:34:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 bash[187654]: 0c16f625a72c8c5971c4027c92b5533bddb5a04c1cd40d11d92fd61d9ad54ea6 2023-09-23T22:34:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:34:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: pidfile_write: ignore empty --pid-file 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 systemd[1]: Started Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: load: jerasure load: lrc 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Git sha 0 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: DB SUMMARY 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: DB Session ID: JTP7BJ64IBJPZHAB84UH 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: MANIFEST file: MANIFEST-000005 size: 891 Bytes 2023-09-23T22:34:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000020.sst 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000018.log size: 2683925 ; 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.env: 0x55d0ffeea720 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.info_log: 0x55d101e7d420 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:34:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.statistics: (nil) 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.use_fsync: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:34:08.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.db_log_dir: 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.wal_dir: 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:34:08.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.write_buffer_manager: 0x55d10110cc80 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.unordered_write: 0 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:34:08.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.row_cache: None 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.wal_filter: None 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.wal_compression: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:34:09.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:34:09.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_open_files: -1 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:34:09.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Compression algorithms supported: 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kZSTD supported: 0 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:34:09.003 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.merge_operator: 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_filter: None 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d101e7d520) 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout: index_type: 0 2023-09-23T22:34:09.004 INFO:journalctl@ceph.mon.c.smithi052.stdout: data_block_index_type: 0 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: index_shortening: 1 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: checksum: 4 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: no_block_cache: 0 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_cache: 0x55d10118f1f0 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_cache_options: 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: capacity : 536870912 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: num_shard_bits : 4 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: strict_capacity_limit : 0 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_cache_compressed: (nil) 2023-09-23T22:34:09.005 INFO:journalctl@ceph.mon.c.smithi052.stdout: persistent_cache: (nil) 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_size: 4096 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_size_deviation: 10 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_restart_interval: 16 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: index_block_restart_interval: 1 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: metadata_block_size: 4096 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: partition_filters: 0 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: use_delta_encoding: 1 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: filter_policy: bloomfilter 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: whole_key_filtering: 1 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: verify_compression: 0 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: format_version: 5 2023-09-23T22:34:09.006 INFO:journalctl@ceph.mon.c.smithi052.stdout: enable_index_compression: 1 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout: block_align: 0 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout: max_auto_readahead_size: 262144 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout: prepopulate_block_cache: 0 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression: NoCompression 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.num_levels: 7 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:34:09.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:34:09.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:34:09.009 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:34:09.010 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:34:09.011 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.ttl: 2592000 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:34:09.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:34:09.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12015, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 24a0f6ef-d672-4b57-876e-1074044f09cc 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508448508999, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508448515672, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 1759809, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12020, "largest_seqno": 12535, "table_properties": {"data_size": 1756081, "index_size": 2266, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 3258, "raw_average_key_size": 20, "raw_value_size": 1751337, "raw_average_value_size": 11084, "num_data_blocks": 100, "num_entries": 158, "num_filter_entries": 158, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695508448, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "24a0f6ef-d672-4b57-876e-1074044f09cc", "db_session_id": "JTP7BJ64IBJPZHAB84UH", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508448515768, "job": 1, "event": "recovery_finished"} 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:34:09.014 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d10122a000 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: DB pointer 0x55d101218000 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: ** DB Stats ** 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.015 INFO:journalctl@ceph.mon.c.smithi052.stdout: ** Compaction Stats [default] ** 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: L0 1/0 1.68 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 509.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: L6 1/0 24.57 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: Sum 2/0 26.25 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 509.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 509.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: ** Compaction Stats [default] ** 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 509.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.016 INFO:journalctl@ceph.mon.c.smithi052.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Cumulative compaction: 0.00 GB write, 166.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Interval compaction: 0.00 GB write, 166.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Block cache BinnedLRUCache@0x55d10118f1f0#7 capacity: 512.00 MB usage: 2.83 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.017 INFO:journalctl@ceph.mon.c.smithi052.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: starting mon.c rank 2 at public addrs [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] at bind addrs [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???) e3 preinit fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).mds e7 new map 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).mds e7 print_map 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: e7 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: legacy client fscid: 1 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: Filesystem 'foo' (1) 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: fs_name foo 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: epoch 5 2023-09-23T22:34:09.018 INFO:journalctl@ceph.mon.c.smithi052.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: created 2023-09-23T22:28:42.663514+0000 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: modified 2023-09-23T22:28:50.691835+0000 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: tableserver 0 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: root 0 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: session_timeout 60 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: session_autoclose 300 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: max_file_size 1099511627776 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: required_client_features {} 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: last_failure 0 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: last_failure_osd_epoch 0 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: max_mds 1 2023-09-23T22:34:09.019 INFO:journalctl@ceph.mon.c.smithi052.stdout: in 0 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: up {0=24518} 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: failed 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: damaged 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: stopped 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: data_pools [4] 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: metadata_pool 3 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: inline_data disabled 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: balancer 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: bal_rank_mask -1 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: standby_count_wanted 1 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: [mds.foo.smithi052.thpzxu{0:24518} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.020 INFO:journalctl@ceph.mon.c.smithi052.stdout: Standby daemons: 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout: 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout: [mds.foo.smithi151.rbxgii{-1:24529} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.151:6832/507517307,v1:172.21.15.151:6833/507517307] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expand map: {default=false} 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta from 'false' to 'false' 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expanded map: {default=false} 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expand map: {default=info} 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta from 'info' to 'info' 2023-09-23T22:34:09.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expanded map: {default=info} 2023-09-23T22:34:09.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expand map: {default=daemon} 2023-09-23T22:34:09.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T22:34:09.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T22:34:09.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expand map: {default=debug} 2023-09-23T22:34:09.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T22:34:09.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:08 smithi052 ceph-mon[187743]: expand_channel_meta expanded map: {default=debug} 2023-09-23T22:34:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:34:13] "GET /metrics HTTP/1.1" 200 33853 "" "Prometheus/2.33.4" 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: audit 2023-09-23T22:34:08.716438+0000 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: mon.a (mon.0) 91 : audit [DBG] from='client.? 172.21.15.52:0/2332178255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:08.729875+0000 mon.b (mon.1) 113 : cluster [INF] mon.b calling monitor election 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:08.730717+0000 mon.a (mon.0) 92 : cluster [INF] mon.a calling monitor election 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 22:34:09.397313+0000 mgr.y (mgr.25214) 61 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 2023-09-23T22:34:11.398088+0000 mgr.y (mgr.25214) 62 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:13.398535+0000 mgr.y (mgr.25214) 63 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34: 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 13.732127+0000 mon.a (mon.0) 93 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 2023-09-23T22:34:13.736768+0000 mon.a (mon.0) 94 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:13.748667+0000 mon.a (mon.0) 95 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:13.748702+0000 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: mon.a (mon.0) 96 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22: 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 34:13.749091+0000 mon.a (mon.0) 97 : cluster [DBG] mgrmap e36: y(active, since 84s), standbys: x 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: 2023-09-23T22:34:13.749280+0000 mon.a (mon.0) 98 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:13.754780+0000 mon.a ( 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: mon.0) 99 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23T22:34:13.754807+0000 mon.a (mon.0) 100 : cluster 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: cluster 2023-09-23 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: T22:34:13.754816+0000 mon.a (mon.0) 101 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] is down (out of quorum) 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: audit 2023-09-23T22:34:13.757340+0000 mon.a (mon.0) 102 : audit 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:15.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: audit 2023-09 2023-09-23T22:34:15.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: -23T22:34:13.763379+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:15.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: audit 2023-09-23T22:34:13.801290+0000 mon.a (mon 2023-09-23T22:34:15.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: .0) 104 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:15.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: audit 2023-09-23T22:34:14.138367+0000 mon.a (mon.0) 105 : audit [DBG] 2023-09-23T22:34:15.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:14 smithi151 conmon[100454]: from='client.? 172.21.15.52:0/393704791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:08.729158+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.744901+0000 mon.a (mon.0) 106 : cluster [INF] mon.a calling monitor election 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.746075+0000 mon.b (mon.1) 114 : cluster 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: [INF] mon.b calling monitor election 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.758803+0000 mon.a (mon.0) 107 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.762931+0000 mon.a (mon.0) 108 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14. 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: 763629+0000 mon.a (mon.0) 109 : cluster [DBG] fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023- 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: 09-23T22:34:14.763675+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.764201+0000 mon.a (mon.0) 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: 111 : cluster [DBG] mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: 2023-09-23T22:34:14.764399+0000 mon.a (mon.0) 112 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.764425+0000 2023-09-23T22:34:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: mon.a (mon.0) 113 : cluster [INF] Cluster is now healthy 2023-09-23T22:34:16.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: cluster 2023-09-23T22:34:14.775770+0000 mon.a (mon.0 2023-09-23T22:34:16.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:15 smithi151 conmon[100454]: ) 114 : cluster [INF] overall HEALTH_OK 2023-09-23T22:34:16.584 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T22:34:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:16 smithi151 conmon[100454]: cluster 2023-09-23T22:34:15.399021+0000 mgr.y (mgr.25214) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:16 smithi151 conmon[100454]: audit 2023-09-23T22:34:16.476330+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:16 smithi151 conmon[100454]: audit 2023-09-23T22:34:16.481239+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:16 smithi151 conmon[100454]: audit 2023-09-23T22:34:16.571491+0000 mon.a (mon.0) 117 : audit [DBG] from='client.? 172.21.15.52:0/964861373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:17.260 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:34:18.798 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:18 smithi052 conmon[173965]: 2023-09-23T22:34:18.526+0000 7f39bb05d700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T22:34:18.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:34:18 smithi151 conmon[150039]: ts=2023-09-23T22:34:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:34:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:18 smithi151 conmon[100454]: cluster 2023-09-23T22:34:17.399366+0000 mgr.y (mgr.25214) 65 : cluster 2023-09-23T22:34:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:18 smithi151 conmon[100454]: [DBG] pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:18 smithi151 conmon[100454]: audit 2023-09-23T22:34:17.517528+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:18 smithi151 conmon[100454]: audit 2023-09-23T22:34:17.522702+0000 mon.a (mon.0 2023-09-23T22:34:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:18 smithi151 conmon[100454]: ) 119 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:alertmanager.a smithi052 *:9093,9094 running (2m) - 6m 25.5M - 0.23.0 ba2b418f427c a82e31979683 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:grafana.a smithi151 *:3000 running (2m) - 5m 53.6M - 8.3.5 dad864ee21e9 8803df2494ff 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi052.thpzxu smithi052 running (5m) - 5m 20.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 d8befa844e73 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi151.rbxgii smithi151 running (5m) - 5m 15.5M - 16.2.14-37-g2e1f1015 1e6a756174a8 61e8f44cb3b6 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.x smithi151 *:8443,9283,8765 running (111s) - 10m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 648eee1c1677 2023-09-23T22:34:19.689 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.y smithi052 *:8443,9283,8765 running (2m) - 11m 503M - 18.2.0-460-gbb0daeff e32c2a64c126 d2a437c50b3c 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a smithi052 running (100s) - 11m 75.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 bb846ce0af73 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b smithi151 running (10m) - 10m 97.2M 2048M 16.2.14-37-g2e1f1015 1e6a756174a8 4a3cec32371a 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c smithi052 running (11s) - 10m 20.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0c16f625a72c 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.a smithi052 *:9100 running (3m) - 6m 22.0M - 1.3.1 1dbe0e931976 3bd65f3ab9c9 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.b smithi151 *:9100 running (2m) - 6m 23.1M - 1.3.1 1dbe0e931976 7e4657a95489 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 smithi052 running (9m) - 9m 69.4M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 667e33a7d512 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 smithi052 running (9m) - 9m 72.0M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 cb0928f7abc9 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 smithi052 running (9m) - 9m 144M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 6d0289679661 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 smithi052 running (8m) - 8m 66.9M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 1744ec2d869b 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 smithi151 running (8m) - 8m 73.8M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 b2126e632b7b 2023-09-23T22:34:19.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 smithi151 running (7m) - 7m 93.3M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 46585800d682 2023-09-23T22:34:19.691 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 smithi151 running (7m) - 7m 65.0M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 17e250c1eb43 2023-09-23T22:34:19.691 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 smithi151 running (7m) - 7m 106M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 73447cd68609 2023-09-23T22:34:19.691 INFO:teuthology.orchestra.run.smithi052.stdout:prometheus.a smithi151 *:9095 running (116s) - 6m 59.4M - 2.33.4 514e6a882f6e 88e6208a51e9 2023-09-23T22:34:19.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:19 smithi151 conmon[100454]: audit 2023-09-23T22:34:19.451029+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:20.364 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:34:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:20 smithi151 conmon[100454]: cluster 2023-09-23T22:34:19.399699+0000 mgr.y (mgr.25214) 66 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:20 smithi151 conmon[100454]: audit 2023-09-23T22:34:19.676561+0000 mgr.y (mgr.25214) 67 : audit [DBG] from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:21.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:21 smithi151 conmon[100454]: cluster 2023-09-23T22:34:20.598540+0000 mon.a (mon.0) 121 : cluster 2023-09-23T22:34:21.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:21 smithi151 conmon[100454]: [DBG] mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T22:34:22.712 INFO:teuthology.orchestra.run.smithi052.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: cluster 2023-09-23T22:34:21.400201+0000 mgr.y (mgr.25214) 68 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:21.949381+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:21.953762+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:21.954730+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:21.955473+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:21.959846+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:22.099005+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:22.101657+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:22.102856+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:22.103527+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:34:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:22 smithi151 conmon[100454]: audit 2023-09-23T22:34:22.712315+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:23.408 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:34:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:34:23] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.33.4" 2023-09-23T22:34:24.302 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: cephadm 2023-09-23T22:34:22.104042+0000 mgr.y (mgr.25214 2023-09-23T22:34:24.302 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: ) 69 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-09-23T22:34:24.302 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: audit 2023-09-23T22:34:22.708610+0000 mgr.y (mgr.25214) 70 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:24.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: cephadm 2023-09-23T22:34:22.708933+0000 mgr.y (mgr.25214) 71 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:24.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: audit 2023-09-23T22:34:23.474806+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:24.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: audit 2023-09-23T22:34:23.475670+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:24.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: audit 2023-09-23T22:34:23.476277+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:24.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: audit 2023-09-23T22:34:23.476872+0000 mon.a (mon.0) 135 : audit 2023-09-23T22:34:24.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:23 smithi151 conmon[100454]: [DBG] from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:25.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:24 smithi151 systemd[1]: Stopping Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:34:25.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:24 smithi151 conmon[100454]: cluster 2023-09-23T22:34:23.400539+0000 mgr.y (mgr.25214) 72 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:25.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:24 smithi151 conmon[100454]: cephadm 2023-09-23T22:34:23.470478+0000 mgr.y (mgr.25214) 73 : cephadm [INF] Upgrade: Updating mon.b 2023-09-23T22:34:25.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:24 smithi151 conmon[100454]: cephadm 2023-09-23T22:34:23.477825+0000 mgr.y (mgr.25214) 74 : cephadm [INF] Deploying daemon mon.b on smithi151 2023-09-23T22:34:25.150 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:25 smithi151 conmon[100454]: debug 2023-09-23T22:34:25.148+0000 7fdd1ec92700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-09-23T22:34:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:25 smithi151 conmon[100454]: debug 2023-09-23T22:34:25.148+0000 7fdd1ec92700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T22:34:25.995 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:25 smithi151 bash[158784]: 4a3cec32371a1a87e3092bbbd4727088a62baa73c648ed17d686cc0c15ff2bf3 2023-09-23T22:34:25.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:25 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T22:34:25.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:25 smithi151 systemd[1]: Stopped Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:34:26.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:25 smithi151 systemd[1]: Starting Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:34:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:34:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: pidfile_write: ignore empty --pid-file 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 bash[159145]: 0f16dac83db3ce897a0379c31ee2672482d60140ef9dbbc281c54c08de5c8345 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: load: jerasure load: lrc 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Git sha 0 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: DB SUMMARY 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: DB Session ID: 9L8YPPAIBUR99RCGKW7P 2023-09-23T22:34:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: MANIFEST file: MANIFEST-000005 size: 1078 Bytes 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000023.sst 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000021.log size: 90451 ; 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.env: 0x55b318f92720 2023-09-23T22:34:26.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.info_log: 0x55b31beed420 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.statistics: (nil) 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.use_fsync: 0 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:34:26.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.db_log_dir: 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.wal_dir: 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:34:26.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.write_buffer_manager: 0x55b31b17cc80 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:34:26.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.unordered_write: 0 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.row_cache: None 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.wal_filter: None 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.wal_compression: 0 2023-09-23T22:34:26.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:34:26.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_open_files: -1 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:34:26.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Compression algorithms supported: 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kZSTD supported: 0 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:34:26.958 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.merge_operator: 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_filter: None 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b31beed520) 2023-09-23T22:34:26.959 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_type: 0 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_index_type: 0 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_shortening: 1 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: checksum: 4 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: no_block_cache: 0 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache: 0x55b31b1ff350 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_options: 2023-09-23T22:34:26.960 INFO:journalctl@ceph.mon.b.smithi151.stdout: capacity : 536870912 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_shard_bits : 4 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: strict_capacity_limit : 0 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_compressed: (nil) 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: persistent_cache: (nil) 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size: 4096 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size_deviation: 10 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_restart_interval: 16 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_block_restart_interval: 1 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: metadata_block_size: 4096 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: partition_filters: 0 2023-09-23T22:34:26.961 INFO:journalctl@ceph.mon.b.smithi151.stdout: use_delta_encoding: 1 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: filter_policy: bloomfilter 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: whole_key_filtering: 1 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: verify_compression: 0 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: format_version: 5 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_index_compression: 1 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_align: 0 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_auto_readahead_size: 262144 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: prepopulate_block_cache: 0 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:34:26.962 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression: NoCompression 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.num_levels: 7 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:34:26.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:34:26.964 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:34:26.965 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:34:26.966 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:34:26.967 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:34:26.968 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:34:26.969 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.ttl: 2592000 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:34:26.970 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 25, last_sequence is 13211, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: eecba52e-bd22-455f-be64-ca0d792424dd 2023-09-23T22:34:26.971 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508466536913, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508466537568, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 61561, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13212, "largest_seqno": 13221, "table_properties": {"data_size": 60414, "index_size": 74, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 216, "raw_average_key_size": 24, "raw_value_size": 60156, "raw_average_value_size": 6684, "num_data_blocks": 3, "num_entries": 9, "num_filter_entries": 9, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695508466, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "eecba52e-bd22-455f-be64-ca0d792424dd", "db_session_id": "9L8YPPAIBUR99RCGKW7P", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695508466537652, "job": 1, "event": "recovery_finished"} 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b31b29a000 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: DB pointer 0x55b31b288000 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** DB Stats ** 2023-09-23T22:34:26.972 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:34:26.973 INFO:journalctl@ceph.mon.b.smithi151.stdout: L0 1/0 60.12 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 131.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: L6 1/0 26.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: Sum 2/0 26.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 131.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 131.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 131.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.974 INFO:journalctl@ceph.mon.b.smithi151.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative compaction: 0.00 GB write, 21.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval compaction: 0.00 GB write, 21.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Block cache BinnedLRUCache@0x55b31b1ff350#7 capacity: 512.00 MB usage: 0.27 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2023-09-23T22:34:26.975 INFO:journalctl@ceph.mon.b.smithi151.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.16 KB,2.98023e-05%) Misc(1,0.00 KB,0%) 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: starting mon.b rank 1 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???) e3 preinit fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).mds e7 new map 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 systemd[1]: Started Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).mds e7 print_map 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: e7 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: legacy client fscid: 1 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.976 INFO:journalctl@ceph.mon.b.smithi151.stdout: Filesystem 'foo' (1) 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: fs_name foo 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: epoch 5 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: created 2023-09-23T22:28:42.663514+0000 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: modified 2023-09-23T22:28:50.691835+0000 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: tableserver 0 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: root 0 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: session_timeout 60 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: session_autoclose 300 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_file_size 1099511627776 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: required_client_features {} 2023-09-23T22:34:26.977 INFO:journalctl@ceph.mon.b.smithi151.stdout: last_failure 0 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: last_failure_osd_epoch 0 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_mds 1 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: in 0 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: up {0=24518} 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: failed 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: damaged 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: stopped 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_pools [4] 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: metadata_pool 3 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: inline_data disabled 2023-09-23T22:34:26.978 INFO:journalctl@ceph.mon.b.smithi151.stdout: balancer 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: bal_rank_mask -1 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: standby_count_wanted 1 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: [mds.foo.smithi052.thpzxu{0:24518} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.52:6834/200598653,v1:172.21.15.52:6835/200598653] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: Standby daemons: 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout: [mds.foo.smithi151.rbxgii{-1:24529} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.151:6832/507517307,v1:172.21.15.151:6833/507517307] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:34:26.979 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expand map: {default=false} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta from 'false' to 'false' 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expanded map: {default=false} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expand map: {default=info} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta from 'info' to 'info' 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expanded map: {default=info} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expand map: {default=daemon} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expand map: {default=debug} 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T22:34:26.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:26 smithi151 ceph-mon[159210]: expand_channel_meta expanded map: {default=debug} 2023-09-23T22:34:28.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mkfs 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:28.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:34:28.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:34:28.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring daemon node-exporter.b on smithi151 2023-09-23T22:34:28.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring daemon grafana.a on smithi151 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Updating mgr.y 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Deploying daemon mgr.y on smithi052 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:11] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:12] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:12] ENGINE Bus STARTED 2023-09-23T22:34:28.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:34:28.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Updating mgr.x 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Deploying daemon mgr.x on smithi151 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Setting container_image for all mgr 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Updating mon.a 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Deploying daemon mon.a on smithi052 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:49] ENGINE Bus STARTING 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:49] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:50] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [23/Sep/2023:22:32:50] ENGINE Bus STARTED 2023-09-23T22:34:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mkfs 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring daemon node-exporter.b on smithi151 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring daemon grafana.a on smithi151 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Updating mgr.y 2023-09-23T22:34:28.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Deploying daemon mgr.y on smithi052 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:11] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:12] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:12] ENGINE Bus STARTED 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Updating mgr.x 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Deploying daemon mgr.x on smithi151 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Setting container_image for all mgr 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Updating mon.a 2023-09-23T22:34:28.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Deploying daemon mon.a on smithi052 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:49] ENGINE Bus STARTING 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:49] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:50] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [23/Sep/2023:22:32:50] ENGINE Bus STARTED 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.506 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.506 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.506 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.506 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:28.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.506 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.507 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:28.509 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2432123542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.510 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Updating mon.c 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Deploying daemon mon.c on smithi052 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2379170911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2332178255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.c calling monitor election 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.b calling monitor election 2023-09-23T22:34:28.511 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.a calling monitor election 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mgrmap e36: y(active, since 84s), standbys: x 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:34:28.512 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.c (rank 2) addr [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] is down (out of quorum) 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2432123542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:28.513 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Updating mon.c 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:28.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Deploying daemon mon.c on smithi052 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2379170911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2332178255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.c calling monitor election 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.b calling monitor election 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.a calling monitor election 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:34:28.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mgrmap e36: y(active, since 84s), standbys: x 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.c (rank 2) addr [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] is down (out of quorum) 2023-09-23T22:34:28.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/393704791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.a calling monitor election 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.b calling monitor election 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: overall HEALTH_OK 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/964861373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:28.519 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.520 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.520 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:28.520 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:28.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.520 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/393704791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.a calling monitor election 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.b calling monitor election 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:34:28.521 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: overall HEALTH_OK 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/964861373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.523 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: It appears safe to stop mon.b 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Upgrade: Updating mon.b 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: Deploying daemon mon.b on smithi151 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: overall HEALTH_OK 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.b calling monitor election 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.c calling monitor election 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.a calling monitor election 2023-09-23T22:34:28.525 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: monmap e4: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: mgrmap e37: y(active, since 97s), standbys: x 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: overall HEALTH_OK 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.526 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[187743]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: It appears safe to stop mon.b 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.527 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Upgrade: Updating mon.b 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: Deploying daemon mon.b on smithi151 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: overall HEALTH_OK 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.b calling monitor election 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.c calling monitor election 2023-09-23T22:34:28.528 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.a calling monitor election 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: monmap e4: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: mgrmap e37: y(active, since 97s), standbys: x 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: overall HEALTH_OK 2023-09-23T22:34:28.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.530 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:28 smithi052 ceph-mon[180979]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mkfs 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:34:28.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:34:28.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring daemon node-exporter.b on smithi151 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring daemon grafana.a on smithi151 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Updating mgr.y 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Deploying daemon mgr.y on smithi052 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:11] ENGINE Bus STARTING 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:11] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:12] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:34:28.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:12] ENGINE Bus STARTED 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Reconfiguring daemon prometheus.a on smithi151 2023-09-23T22:34:28.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Updating mgr.x 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Deploying daemon mgr.x on smithi151 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Setting container_image for all mgr 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Updating mon.a 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Deploying daemon mon.a on smithi052 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:49] ENGINE Bus STARTING 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:49] ENGINE Serving on https://172.21.15.52:7150 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:50] ENGINE Serving on http://172.21.15.52:8765 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [23/Sep/2023:22:32:50] ENGINE Bus STARTED 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/etc/ceph/ceph.conf 2023-09-23T22:34:28.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/etc/ceph/ceph.conf 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Updating smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.665 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2432123542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Updating mon.c 2023-09-23T22:34:28.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Deploying daemon mon.c on smithi052 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2379170911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2332178255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.c calling monitor election 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.b calling monitor election 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.a calling monitor election 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mgrmap e36: y(active, since 84s), standbys: x 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.c (rank 2) addr [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0] is down (out of quorum) 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.669 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/393704791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.a calling monitor election 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.b calling monitor election 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: monmap e3: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: overall HEALTH_OK 2023-09-23T22:34:28.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/964861373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: It appears safe to stop mon.b 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Upgrade: Updating mon.b 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: Deploying daemon mon.b on smithi151 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: overall HEALTH_OK 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.b calling monitor election 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.c calling monitor election 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.a calling monitor election 2023-09-23T22:34:28.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: monmap e4: 3 mons at {a=[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]} removed_ranks: {} 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: fsmap foo:1 {0=foo.smithi052.thpzxu=up:active} 1 up:standby 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: mgrmap e37: y(active, since 97s), standbys: x 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: overall HEALTH_OK 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:28.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:28 smithi151 ceph-mon[159210]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:28.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:34:28 smithi151 conmon[150039]: ts=2023-09-23T22:34:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:34:30.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:30 smithi151 ceph-mon[159210]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:30 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:30 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:30 smithi052 ceph-mon[187743]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:30 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:30 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:30 smithi052 ceph-mon[180979]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:30 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:30 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:31.995 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:31 smithi052 conmon[173965]: 2023-09-23T22:34:31.547+0000 7f39bb05d700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T22:34:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:32.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:32.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:32.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:32.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:32.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:33.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:33 smithi052 ceph-mon[187743]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:33.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:33 smithi052 ceph-mon[180979]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:33 smithi151 ceph-mon[159210]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:34:33.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:34:33] "GET /metrics HTTP/1.1" 200 33848 "" "Prometheus/2.33.4" 2023-09-23T22:34:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:35.487 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:35.488 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:35.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:35.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: Reconfiguring mon.a (monmap changed)... 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: Reconfiguring daemon mon.a on smithi052 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:34:36.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: Reconfiguring mon.a (monmap changed)... 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: Reconfiguring daemon mon.a on smithi052 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:34:36.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:36 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: Reconfiguring mon.a (monmap changed)... 2023-09-23T22:34:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: Reconfiguring daemon mon.a on smithi052 2023-09-23T22:34:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:34:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:34:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:36 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:37.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:37.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: Reconfiguring daemon mgr.y on smithi052 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:37.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: Reconfiguring daemon mgr.y on smithi052 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: Reconfiguring daemon mgr.y on smithi052 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:38 smithi151 ceph-mon[159210]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:34:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:38 smithi151 ceph-mon[159210]: Reconfiguring daemon mon.c on smithi052 2023-09-23T22:34:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:38 smithi052 ceph-mon[187743]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:34:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:38 smithi052 ceph-mon[187743]: Reconfiguring daemon mon.c on smithi052 2023-09-23T22:34:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:38 smithi052 ceph-mon[180979]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:34:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:38 smithi052 ceph-mon[180979]: Reconfiguring daemon mon.c on smithi052 2023-09-23T22:34:38.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:34:38 smithi151 conmon[150039]: ts=2023-09-23T22:34:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:34:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[187743]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[180979]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:34:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:39 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:39.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:39 smithi151 ceph-mon[159210]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:39.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:39 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:39 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:39 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:34:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:39 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:40.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[187743]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T22:34:40.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.0 on smithi052 2023-09-23T22:34:40.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:40.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[180979]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.0 on smithi052 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:34:40.239 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:40 smithi151 ceph-mon[159210]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T22:34:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:40 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.0 on smithi052 2023-09-23T22:34:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:34:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.1 on smithi052 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:34:41.279 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.1 on smithi052 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:34:41.280 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T22:34:41.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.1 on smithi052 2023-09-23T22:34:41.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:41.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:41.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:34:41.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:42.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:42 smithi151 ceph-mon[159210]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T22:34:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:42 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.2 on smithi052 2023-09-23T22:34:42.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:42 smithi052 ceph-mon[187743]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T22:34:42.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:42 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.2 on smithi052 2023-09-23T22:34:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:42 smithi052 ceph-mon[180979]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T22:34:42.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:42 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.2 on smithi052 2023-09-23T22:34:43.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[180979]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[187743]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:34:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:43 smithi151 ceph-mon[159210]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:34:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:34:43] "GET /metrics HTTP/1.1" 200 33847 "" "Prometheus/2.33.4" 2023-09-23T22:34:44.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[187743]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T22:34:44.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.3 on smithi052 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[180979]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.3 on smithi052 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:44.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:34:44.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:44.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:44 smithi151 ceph-mon[159210]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T22:34:44.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:44 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.3 on smithi052 2023-09-23T22:34:44.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:44.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:44.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi052.thpzxu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:34:44.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:45.307 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:45.307 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: Reconfiguring mds.foo.smithi052.thpzxu (monmap changed)... 2023-09-23T22:34:45.307 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: Reconfiguring daemon mds.foo.smithi052.thpzxu on smithi052 2023-09-23T22:34:45.307 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:45.307 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:45.308 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:45.308 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:45.308 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:45.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:45.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: Reconfiguring mds.foo.smithi052.thpzxu (monmap changed)... 2023-09-23T22:34:45.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: Reconfiguring daemon mds.foo.smithi052.thpzxu on smithi052 2023-09-23T22:34:45.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: Reconfiguring mds.foo.smithi052.thpzxu (monmap changed)... 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: Reconfiguring daemon mds.foo.smithi052.thpzxu on smithi052 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:45.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:34:45.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:34:45.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:46 smithi151 ceph-mon[159210]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:34:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:46 smithi151 ceph-mon[159210]: Reconfiguring daemon mon.b on smithi151 2023-09-23T22:34:46.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:46 smithi052 ceph-mon[180979]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:34:46.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:46 smithi052 ceph-mon[180979]: Reconfiguring daemon mon.b on smithi151 2023-09-23T22:34:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:46 smithi052 ceph-mon[187743]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:34:46.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:46 smithi052 ceph-mon[187743]: Reconfiguring daemon mon.b on smithi151 2023-09-23T22:34:47.390 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:47 smithi151 ceph-mon[159210]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:47.390 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:47.390 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:47.390 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:34:47.390 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:34:47.391 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[187743]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:34:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:34:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[180979]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:34:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:34:47.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:48.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:48 smithi151 ceph-mon[159210]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T22:34:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:48 smithi151 ceph-mon[159210]: Reconfiguring daemon mgr.x on smithi151 2023-09-23T22:34:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:34:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[187743]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T22:34:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[187743]: Reconfiguring daemon mgr.x on smithi151 2023-09-23T22:34:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[180979]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[180979]: Reconfiguring daemon mgr.x on smithi151 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:34:48.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:48.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:34:48 smithi151 conmon[150039]: ts=2023-09-23T22:34:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.4 on smithi151 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:34:49.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T22:34:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.4 on smithi151 2023-09-23T22:34:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:49.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.4 on smithi151 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:34:49.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:50.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T22:34:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.5 on smithi151 2023-09-23T22:34:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:34:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T22:34:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.5 on smithi151 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.5 on smithi151 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:50.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:50.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:34:50.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:51.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:51 smithi151 ceph-mon[159210]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:51.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:51 smithi151 ceph-mon[159210]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T22:34:51.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:51 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.6 on smithi151 2023-09-23T22:34:51.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:51 smithi052 ceph-mon[187743]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:51 smithi052 ceph-mon[187743]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T22:34:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:51 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.6 on smithi151 2023-09-23T22:34:51.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:51 smithi052 ceph-mon[180979]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:51.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:51 smithi052 ceph-mon[180979]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T22:34:51.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:51 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.6 on smithi151 2023-09-23T22:34:52.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:52.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:52.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T22:34:52.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:34:52.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:52.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: Reconfiguring daemon osd.7 on smithi151 2023-09-23T22:34:52.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:52 smithi151 ceph-mon[159210]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: Reconfiguring daemon osd.7 on smithi151 2023-09-23T22:34:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[187743]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: Reconfiguring daemon osd.7 on smithi151 2023-09-23T22:34:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:52 smithi052 ceph-mon[180979]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:53.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:34:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:34:53] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.33.4" 2023-09-23T22:34:53.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:53.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:53.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:53 smithi151 ceph-mon[159210]: Reconfiguring mds.foo.smithi151.rbxgii (monmap changed)... 2023-09-23T22:34:53.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:34:53.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:53.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:53 smithi151 ceph-mon[159210]: Reconfiguring daemon mds.foo.smithi151.rbxgii on smithi151 2023-09-23T22:34:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[187743]: Reconfiguring mds.foo.smithi151.rbxgii (monmap changed)... 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[187743]: Reconfiguring daemon mds.foo.smithi151.rbxgii on smithi151 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[180979]: Reconfiguring mds.foo.smithi151.rbxgii (monmap changed)... 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.rbxgii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:34:54.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:54.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:53 smithi052 ceph-mon[180979]: Reconfiguring daemon mds.foo.smithi151.rbxgii on smithi151 2023-09-23T22:34:55.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:55.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:55.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:55.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:55.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:34:57.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:56 smithi052 ceph-mon[187743]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:57.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:56 smithi052 ceph-mon[180979]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:57.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:56 smithi151 ceph-mon[159210]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:58.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:34:58 smithi151 conmon[150039]: ts=2023-09-23T22:34:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:34:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:34:58 smithi052 ceph-mon[187743]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:34:58 smithi052 ceph-mon[180979]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:34:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:34:58 smithi151 ceph-mon[159210]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:01.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:00 smithi052 ceph-mon[187743]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:00 smithi052 ceph-mon[180979]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:01.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:00 smithi151 ceph-mon[159210]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:02 smithi052 ceph-mon[187743]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:03.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:02 smithi052 ceph-mon[180979]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:02 smithi151 ceph-mon[159210]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:35:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:35:03] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.33.4" 2023-09-23T22:35:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:04 smithi052 ceph-mon[187743]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:04 smithi052 ceph-mon[180979]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:05.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:04 smithi151 ceph-mon[159210]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:06 smithi052 ceph-mon[187743]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:07.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:06 smithi052 ceph-mon[180979]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:07.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:06 smithi151 ceph-mon[159210]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:08.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:08 smithi151 conmon[150039]: ts=2023-09-23T22:35:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:35:09.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:08 smithi052 ceph-mon[180979]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:09.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:08 smithi052 ceph-mon[187743]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:09.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:08 smithi151 ceph-mon[159210]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:11.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:10 smithi052 ceph-mon[180979]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:11.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:10 smithi052 ceph-mon[187743]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:11.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:10 smithi151 ceph-mon[159210]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:12 smithi052 ceph-mon[187743]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:13.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:12 smithi052 ceph-mon[180979]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:13.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:12 smithi151 ceph-mon[159210]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:35:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:35:13] "GET /metrics HTTP/1.1" 200 33852 "" "Prometheus/2.33.4" 2023-09-23T22:35:15.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:14 smithi052 ceph-mon[180979]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:15.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:14 smithi052 ceph-mon[187743]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:15.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:14 smithi151 ceph-mon[159210]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:17.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:16 smithi052 ceph-mon[180979]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:17.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:16 smithi052 ceph-mon[187743]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:17.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:16 smithi151 ceph-mon[159210]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:18.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:18 smithi151 conmon[150039]: ts=2023-09-23T22:35:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:35:19.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:18 smithi052 ceph-mon[180979]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:19.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:18 smithi052 ceph-mon[187743]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:18 smithi151 ceph-mon[159210]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:20.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:20.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:21.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:20 smithi052 ceph-mon[187743]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:21.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:20 smithi052 ceph-mon[180979]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:20 smithi151 ceph-mon[159210]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:23.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:22 smithi052 ceph-mon[187743]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:23.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:22 smithi052 ceph-mon[180979]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:22 smithi151 ceph-mon[159210]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:35:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:35:23] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.33.4" 2023-09-23T22:35:25.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:24 smithi151 ceph-mon[159210]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:25.476 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:24 smithi052 ceph-mon[180979]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:25.476 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:24 smithi052 ceph-mon[187743]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:25.493 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:35:27.143 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:26 smithi052 ceph-mon[187743]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:27.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:26 smithi052 ceph-mon[180979]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:26 smithi151 ceph-mon[159210]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:27.549 INFO:teuthology.orchestra.run.smithi052.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:28.295 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T22:35:28.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[180979]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:28.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[180979]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:28.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[180979]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:28.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:28.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:28.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[187743]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:28.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[187743]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:28.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[187743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:28.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:28.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:28.948 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:28 smithi151 conmon[150039]: ts=2023-09-23T22:35:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:35:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:28 smithi151 ceph-mon[159210]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:28 smithi151 ceph-mon[159210]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:28 smithi151 ceph-mon[159210]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:30.430 INFO:teuthology.orchestra.run.smithi052.stdout:wait for all mons to upgrade 2023-09-23T22:35:30.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:30 smithi151 ceph-mon[159210]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:30.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:30 smithi052 ceph-mon[180979]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:30 smithi052 ceph-mon[187743]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:31.113 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T22:35:32.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:32 smithi151 ceph-mon[159210]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:32.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:32 smithi052 ceph-mon[187743]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:32.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:32.952 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:32.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:32 smithi052 ceph-mon[180979]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:32.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:32.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:35:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:35:33] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.33.4" 2023-09-23T22:35:33.723 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: Upgrade: Setting container_image for all mon 2023-09-23T22:35:33.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: Upgrade: Setting container_image for all crash 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:35:33.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[180979]: Upgrade: osd.0 is safe to restart 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: Upgrade: Setting container_image for all mon 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.884 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: Upgrade: Setting container_image for all crash 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:35:33.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:33 smithi052 ceph-mon[187743]: Upgrade: osd.0 is safe to restart 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: Upgrade: Setting container_image for all mon 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: Upgrade: Setting container_image for all crash 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:35:33.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:33 smithi151 ceph-mon[159210]: Upgrade: osd.0 is safe to restart 2023-09-23T22:35:34.691 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:35:34.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:34 smithi052 ceph-mon[187743]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:34.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1717079876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:35:34.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:34.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:34 smithi052 ceph-mon[180979]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:34.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1717079876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:35:34.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:34.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:34 smithi151 ceph-mon[159210]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1717079876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:35:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:35.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[187743]: Upgrade: Updating osd.0 2023-09-23T22:35:35.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:35.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[187743]: Deploying daemon osd.0 on smithi052 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[180979]: Upgrade: Updating osd.0 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:35 smithi052 ceph-mon[180979]: Deploying daemon osd.0 on smithi052 2023-09-23T22:35:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:35 smithi151 ceph-mon[159210]: Upgrade: Updating osd.0 2023-09-23T22:35:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:35:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:35 smithi151 ceph-mon[159210]: Deploying daemon osd.0 on smithi052 2023-09-23T22:35:36.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:36 smithi052 ceph-mon[187743]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:36.941 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:36 smithi052 ceph-mon[180979]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:37.131 INFO:teuthology.orchestra.run.smithi052.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:35:37.131 INFO:teuthology.orchestra.run.smithi052.stdout:alertmanager.a smithi052 *:9093,9094 running (4m) - 7m 25.5M - 0.23.0 ba2b418f427c a82e31979683 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:grafana.a smithi151 *:3000 running (3m) - 7m 56.5M - 8.3.5 dad864ee21e9 8803df2494ff 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi052.thpzxu smithi052 running (6m) - 6m 20.5M - 16.2.14-37-g2e1f1015 1e6a756174a8 d8befa844e73 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi151.rbxgii smithi151 running (6m) - 6m 15.8M - 16.2.14-37-g2e1f1015 1e6a756174a8 61e8f44cb3b6 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.x smithi151 *:8443,9283,8765 running (3m) - 11m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 648eee1c1677 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.y smithi052 *:8443,9283,8765 running (3m) - 12m 507M - 18.2.0-460-gbb0daeff e32c2a64c126 d2a437c50b3c 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a smithi052 running (2m) - 13m 85.4M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 bb846ce0af73 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b smithi151 running (70s) - 11m 20.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0f16dac83db3 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c smithi052 running (88s) - 11m 69.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0c16f625a72c 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.a smithi052 *:9100 running (4m) - 7m 22.2M - 1.3.1 1dbe0e931976 3bd65f3ab9c9 2023-09-23T22:35:37.132 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.b smithi151 *:9100 running (3m) - 7m 20.2M - 1.3.1 1dbe0e931976 7e4657a95489 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 smithi052 running (11m) - 11m 72.1M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 667e33a7d512 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 smithi052 running (10m) - 10m 74.5M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 cb0928f7abc9 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 smithi052 running (10m) - 10m 146M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 6d0289679661 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 smithi052 running (9m) - 9m 69.5M 2501M 16.2.14-37-g2e1f1015 1e6a756174a8 1744ec2d869b 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 smithi151 running (9m) - 9m 75.2M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 b2126e632b7b 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 smithi151 running (9m) - 9m 94.1M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 46585800d682 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 smithi151 running (8m) - 8m 66.0M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 17e250c1eb43 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 smithi151 running (8m) - 8m 107M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 73447cd68609 2023-09-23T22:35:37.133 INFO:teuthology.orchestra.run.smithi052.stdout:prometheus.a smithi151 *:9095 running (3m) - 7m 59.6M - 2.33.4 514e6a882f6e 88e6208a51e9 2023-09-23T22:35:37.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:36 smithi151 ceph-mon[159210]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:37.871 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:35:38.864 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:38 smithi052 systemd[1]: Stopping Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:35:38.942 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:38 smithi151 conmon[150039]: ts=2023-09-23T22:35:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:35:39.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:38 smithi151 ceph-mon[159210]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:39.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:38 smithi151 ceph-mon[159210]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:38 smithi052 ceph-mon[180979]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:38 smithi052 ceph-mon[180979]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:39.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:38 smithi052 ceph-mon[187743]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:39.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:38 smithi052 ceph-mon[187743]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:39.649 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:39 smithi052 conmon[115173]: debug 2023-09-23T22:35:39.443+0000 7f1aebde7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T22:35:39.649 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:39 smithi052 conmon[115173]: debug 2023-09-23T22:35:39.443+0000 7f1aebde7700 -1 osd.0 75 *** Got signal Terminated *** 2023-09-23T22:35:39.649 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:39 smithi052 conmon[115173]: debug 2023-09-23T22:35:39.443+0000 7f1aebde7700 -1 osd.0 75 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:35:39.944 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:39 smithi052 ceph-mon[180979]: osd.0 marked itself down and dead 2023-09-23T22:35:40.101 INFO:teuthology.orchestra.run.smithi052.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:40.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:39 smithi151 ceph-mon[159210]: osd.0 marked itself down and dead 2023-09-23T22:35:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:39 smithi052 ceph-mon[187743]: osd.0 marked itself down and dead 2023-09-23T22:35:40.946 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:40 smithi052 bash[196451]: 667e33a7d51219a7e9befe95d9d74f5db14f5e241ab6656db3f2166b7bac17c5 2023-09-23T22:35:40.984 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: failed to encode map e76 with expected crc 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: failed to encode map e76 with expected crc 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: osdmap e76: 8 total, 7 up, 8 in 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:35:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: failed to encode map e76 with expected crc 2023-09-23T22:35:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: failed to encode map e76 with expected crc 2023-09-23T22:35:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: osdmap e76: 8 total, 7 up, 8 in 2023-09-23T22:35:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: failed to encode map e76 with expected crc 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: failed to encode map e76 with expected crc 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: osdmap e76: 8 total, 7 up, 8 in 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:35:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:41.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: osdmap e77: 8 total, 7 up, 8 in 2023-09-23T22:35:41.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[187743]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: osdmap e77: 8 total, 7 up, 8 in 2023-09-23T22:35:41.961 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:41.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:41 smithi052 ceph-mon[180979]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: osdmap e77: 8 total, 7 up, 8 in 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:41 smithi151 ceph-mon[159210]: failed to encode map e77 with expected crc 2023-09-23T22:35:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:42 smithi052 ceph-mon[187743]: pgmap v91: 73 pgs: 26 peering, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:42 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T22:35:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:42 smithi052 ceph-mon[180979]: pgmap v91: 73 pgs: 26 peering, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:42 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T22:35:43.244 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:42 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T22:35:43.245 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:42 smithi052 systemd[1]: Stopped Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:35:43.245 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:43 smithi052 systemd[1]: Starting Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:35:43.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:42 smithi151 ceph-mon[159210]: pgmap v91: 73 pgs: 26 peering, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:42 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T22:35:43.698 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:35:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:35:43] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.33.4" 2023-09-23T22:35:44.493 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:35:44.493 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:35:44.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T22:35:44.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:35:44.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:35:44.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T22:35:44.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:35:44.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:44 smithi052 bash[197500]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T22:35:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:44 smithi052 ceph-mon[187743]: pgmap v92: 73 pgs: 26 peering, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:45.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:44 smithi052 ceph-mon[180979]: pgmap v92: 73 pgs: 26 peering, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:45.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:44 smithi151 ceph-mon[159210]: pgmap v92: 73 pgs: 26 peering, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:45.743 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:45 smithi052 ceph-osd[197965]: -- 172.21.15.52:0/4093436029 <== mon.0 v2:172.21.15.52:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56327bb2a4e0 con 0x56327bafcc00 2023-09-23T22:35:45.743 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:45 smithi052 bash[197900]: b70c0aa9b5abc19f9172fd4758c71d234777969ca7072c6e2d9f9e1ee6bfca6a 2023-09-23T22:35:45.744 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:45 smithi052 systemd[1]: Started Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:35:46.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:45 smithi052 conmon[197941]: 2023-09-23T22:35:45.895+0000 7feac763d640 -1 Falling back to public interface 2023-09-23T22:35:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:46 smithi151 ceph-mon[159210]: pgmap v93: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:46 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:46 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T22:35:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[187743]: pgmap v93: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T22:35:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[180979]: pgmap v93: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:46.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:46 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T22:35:48.699 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:48 smithi151 conmon[150039]: ts=2023-09-23T22:35:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:35:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:48 smithi151 ceph-mon[159210]: pgmap v94: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:48 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[180979]: pgmap v94: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[187743]: pgmap v94: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:48 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:49.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:48 smithi052 conmon[197941]: 2023-09-23T22:35:48.841+0000 7feac763d640 -1 osd.0 75 log_to_monitors true 2023-09-23T22:35:49.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:49 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:35:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:49.743 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:35:49 smithi052 conmon[197941]: 2023-09-23T22:35:49.447+0000 7feab72a7700 -1 osd.0 75 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:35:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:49 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:35:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:49 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:35:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: pgmap v95: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: failed to encode map e78 with expected crc 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: failed to encode map e78 with expected crc 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: osdmap e78: 8 total, 7 up, 8 in 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: pgmap v95: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: failed to encode map e78 with expected crc 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: failed to encode map e78 with expected crc 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: osdmap e78: 8 total, 7 up, 8 in 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:35:50.722 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: pgmap v95: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: failed to encode map e78 with expected crc 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: failed to encode map e78 with expected crc 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: osdmap e78: 8 total, 7 up, 8 in 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:50.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: failed to encode map e78 with expected crc 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: failed to encode map e79 with expected crc 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: failed to encode map e79 with expected crc 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800] boot 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:35:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:51 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:35:51.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: failed to encode map e78 with expected crc 2023-09-23T22:35:51.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: failed to encode map e79 with expected crc 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: failed to encode map e79 with expected crc 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800] boot 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: failed to encode map e78 with expected crc 2023-09-23T22:35:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:35:51.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: failed to encode map e79 with expected crc 2023-09-23T22:35:51.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: failed to encode map e79 with expected crc 2023-09-23T22:35:51.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: osd.0 [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800] boot 2023-09-23T22:35:51.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:35:51.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:51 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:35:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: pgmap v98: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: pgmap v98: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:52.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: pgmap v98: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: failed to encode map e80 with expected crc 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:52.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:35:53.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:35:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:35:53] "GET /metrics HTTP/1.1" 200 33840 "" "Prometheus/2.33.4" 2023-09-23T22:35:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:54 smithi052 ceph-mon[187743]: pgmap v100: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:54.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:54 smithi052 ceph-mon[180979]: pgmap v100: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:54 smithi151 ceph-mon[159210]: pgmap v100: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:35:55.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:55 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded) 2023-09-23T22:35:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:55 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:35:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:55 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded) 2023-09-23T22:35:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:55 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:35:55.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:55 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded) 2023-09-23T22:35:55.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:55 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:35:56.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:56 smithi151 ceph-mon[159210]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:56 smithi052 ceph-mon[180979]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:56.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:56 smithi052 ceph-mon[187743]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:58.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:35:58 smithi151 ceph-mon[159210]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:58.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:58 smithi151 conmon[150039]: ts=2023-09-23T22:35:58.659Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-09-23T22:35:58.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:35:58 smithi151 conmon[150039]: ts=2023-09-23T22:35:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:35:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:35:58 smithi052 ceph-mon[187743]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:35:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:35:58 smithi052 ceph-mon[180979]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:00 smithi151 ceph-mon[159210]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:00.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:00 smithi052 ceph-mon[187743]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:00.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:00 smithi052 ceph-mon[180979]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:02 smithi151 ceph-mon[159210]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:02 smithi052 ceph-mon[187743]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:02.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:02 smithi052 ceph-mon[180979]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:36:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:36:03] "GET /metrics HTTP/1.1" 200 33840 "" "Prometheus/2.33.4" 2023-09-23T22:36:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:04 smithi151 ceph-mon[159210]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[187743]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[180979]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:06.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:06 smithi151 ceph-mon[159210]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:06.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:06 smithi052 ceph-mon[180979]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:06 smithi052 ceph-mon[187743]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:08 smithi151 ceph-mon[159210]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:08.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:36:08 smithi151 conmon[150039]: ts=2023-09-23T22:36:08.659Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-09-23T22:36:08.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:36:08 smithi151 conmon[150039]: ts=2023-09-23T22:36:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:36:08.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:08 smithi052 ceph-mon[187743]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:08.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:08 smithi052 ceph-mon[180979]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:10 smithi151 ceph-mon[159210]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:10 smithi052 ceph-mon[187743]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:10 smithi052 ceph-mon[180979]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:12.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:12 smithi151 ceph-mon[159210]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:12 smithi052 ceph-mon[187743]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:12.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:12 smithi052 ceph-mon[180979]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:13.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:36:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:36:13] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.33.4" 2023-09-23T22:36:14.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:14 smithi151 ceph-mon[159210]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:14.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:14 smithi052 ceph-mon[180979]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:14 smithi052 ceph-mon[187743]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:16.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:16 smithi151 ceph-mon[159210]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:16 smithi052 ceph-mon[187743]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:16.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:16 smithi052 ceph-mon[180979]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:18 smithi151 ceph-mon[159210]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:18.949 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 22:36:18 smithi151 conmon[150039]: ts=2023-09-23T22:36:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi052:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)\", cluster_addr=\"172.21.15.52\", device_class=\"ssd\", hostname=\"smithi052\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.52\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:36:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:18 smithi052 ceph-mon[180979]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:18 smithi052 ceph-mon[187743]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:19.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:20.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:20.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:20 smithi052 ceph-mon[180979]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:20 smithi052 ceph-mon[187743]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:21.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:20 smithi151 ceph-mon[159210]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:22.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:22 smithi052 ceph-mon[180979]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:22.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:22 smithi052 ceph-mon[187743]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:23.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:22 smithi151 ceph-mon[159210]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:36:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:36:23] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.33.4" 2023-09-23T22:36:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:24 smithi052 ceph-mon[180979]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:24.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:24 smithi052 ceph-mon[187743]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:25.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:24 smithi151 ceph-mon[159210]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:26.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:26 smithi052 ceph-mon[180979]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:26.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:26 smithi052 ceph-mon[187743]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:27.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:26 smithi151 ceph-mon[159210]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:28.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:28 smithi052 ceph-mon[180979]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:28.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:28 smithi052 ceph-mon[187743]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:29.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:28 smithi151 ceph-mon[159210]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:30.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:30 smithi052 ceph-mon[187743]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:30.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:30 smithi052 ceph-mon[180979]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:31.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:30 smithi151 ceph-mon[159210]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:32 smithi052 ceph-mon[187743]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:32.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:32 smithi052 ceph-mon[180979]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:33.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:32 smithi151 ceph-mon[159210]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:33.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:36:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:36:33] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.33.4" 2023-09-23T22:36:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:34 smithi052 ceph-mon[187743]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:34 smithi052 ceph-mon[180979]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:34 smithi151 ceph-mon[159210]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:36 smithi052 ceph-mon[187743]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:36.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:36 smithi052 ceph-mon[180979]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:37.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:36 smithi151 ceph-mon[159210]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:38.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:38 smithi052 ceph-mon[187743]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:38.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:38 smithi052 ceph-mon[180979]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:39.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:38 smithi151 ceph-mon[159210]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:40.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:40 smithi052 ceph-mon[187743]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:40.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:40 smithi052 ceph-mon[180979]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:41.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:40 smithi151 ceph-mon[159210]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:42 smithi052 ceph-mon[187743]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:42.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:42 smithi052 ceph-mon[180979]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:43.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:42 smithi151 ceph-mon[159210]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:43.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:36:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:36:43] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.33.4" 2023-09-23T22:36:43.909 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:36:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:44 smithi052 ceph-mon[187743]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:44 smithi052 ceph-mon[180979]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:45.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:44 smithi151 ceph-mon[159210]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:45.772 INFO:teuthology.orchestra.run.smithi052.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:36:46.478 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T22:36:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:36:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:36:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:36:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:36:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:36:46.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[187743]: Upgrade: osd.1 is safe to restart 2023-09-23T22:36:46.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:46.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:36:46.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:36:46.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:46.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:46.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:36:46.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:36:46.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:46 smithi052 ceph-mon[180979]: Upgrade: osd.1 is safe to restart 2023-09-23T22:36:47.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:36:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:36:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:36:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:46 smithi151 ceph-mon[159210]: Upgrade: osd.1 is safe to restart 2023-09-23T22:36:48.600 INFO:teuthology.orchestra.run.smithi052.stdout:wait for half of osds to upgrade 2023-09-23T22:36:49.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[187743]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:49.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[187743]: Upgrade: Updating osd.1 2023-09-23T22:36:49.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:49.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:36:49.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:36:49.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[187743]: Deploying daemon osd.1 on smithi052 2023-09-23T22:36:49.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[180979]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:49.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[180979]: Upgrade: Updating osd.1 2023-09-23T22:36:49.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:49.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:36:49.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:36:49.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:49 smithi052 ceph-mon[180979]: Deploying daemon osd.1 on smithi052 2023-09-23T22:36:49.361 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T22:36:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:49 smithi151 ceph-mon[159210]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:49 smithi151 ceph-mon[159210]: Upgrade: Updating osd.1 2023-09-23T22:36:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:36:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:36:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:49 smithi151 ceph-mon[159210]: Deploying daemon osd.1 on smithi052 2023-09-23T22:36:50.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:50.130 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:50.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:51.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:51 smithi052 ceph-mon[187743]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:51 smithi052 ceph-mon[180979]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:51.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:51 smithi151 ceph-mon[159210]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:52.077 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:51 smithi052 systemd[1]: Stopping Ceph osd.1 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:36:52.493 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:52 smithi052 conmon[121096]: debug 2023-09-23T22:36:52.075+0000 7fbe50ac2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T22:36:52.493 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:52 smithi052 conmon[121096]: debug 2023-09-23T22:36:52.075+0000 7fbe50ac2700 -1 osd.1 80 *** Got signal Terminated *** 2023-09-23T22:36:52.493 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:52 smithi052 conmon[121096]: debug 2023-09-23T22:36:52.075+0000 7fbe50ac2700 -1 osd.1 80 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:36:53.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:53 smithi052 ceph-mon[187743]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:53.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:53 smithi052 ceph-mon[187743]: osd.1 marked itself down and dead 2023-09-23T22:36:53.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2215981034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:53.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:53 smithi052 ceph-mon[180979]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:53.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:53 smithi052 ceph-mon[180979]: osd.1 marked itself down and dead 2023-09-23T22:36:53.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2215981034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:53 smithi151 ceph-mon[159210]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:53 smithi151 ceph-mon[159210]: osd.1 marked itself down and dead 2023-09-23T22:36:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2215981034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:53.690 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:36:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:36:53] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.33.4" 2023-09-23T22:36:53.993 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:53 smithi052 bash[202446]: cb0928f7abc90f22823fe4f7d9e74ba380360715dc213d854827748152cec6c3 2023-09-23T22:36:54.314 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:36:54.314 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[187743]: failed to encode map e81 with expected crc 2023-09-23T22:36:54.314 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[187743]: failed to encode map e81 with expected crc 2023-09-23T22:36:54.314 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[187743]: osdmap e81: 8 total, 7 up, 8 in 2023-09-23T22:36:54.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:36:54.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[180979]: failed to encode map e81 with expected crc 2023-09-23T22:36:54.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[180979]: failed to encode map e81 with expected crc 2023-09-23T22:36:54.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:54 smithi052 ceph-mon[180979]: osdmap e81: 8 total, 7 up, 8 in 2023-09-23T22:36:54.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:54 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:36:54.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:54 smithi151 ceph-mon[159210]: failed to encode map e81 with expected crc 2023-09-23T22:36:54.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:54 smithi151 ceph-mon[159210]: failed to encode map e81 with expected crc 2023-09-23T22:36:54.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:54 smithi151 ceph-mon[159210]: osdmap e81: 8 total, 7 up, 8 in 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: pgmap v131: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: osdmap e82: 8 total, 7 up, 8 in 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2842474356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:55.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: pgmap v131: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: osdmap e82: 8 total, 7 up, 8 in 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.382 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.383 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.383 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.383 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2842474356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: pgmap v131: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: osdmap e82: 8 total, 7 up, 8 in 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: failed to encode map e82 with expected crc 2023-09-23T22:36:55.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2842474356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:55.744 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:55 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T22:36:55.744 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:55 smithi052 systemd[1]: Stopped Ceph osd.1 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:36:55.744 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:55 smithi052 systemd[1]: Starting Ceph osd.1 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:36:56.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:56 smithi052 ceph-mon[187743]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:36:56.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:56 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:56.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:56 smithi052 ceph-mon[180979]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:36:56.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:56 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:56.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:56 smithi151 ceph-mon[159210]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:36:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:56 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:56.743 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:36:56.743 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T22:36:56.743 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T22:36:56.743 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T22:36:56.743 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T22:36:56.744 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T22:36:56.744 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T22:36:56.744 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:56 smithi052 bash[203118]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T22:36:57.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1807388606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:57.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1807388606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:57.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1807388606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:36:57.993 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:57 smithi052 ceph-osd[203614]: -- 172.21.15.52:0/3818340029 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d3527b04e0 con 0x55d352783000 2023-09-23T22:36:57.993 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:57 smithi052 bash[203549]: 4ff251b65413b4a0f154dc2ebbf538b71a9cb49edb4cf7c8db9e5aae830ef33a 2023-09-23T22:36:57.993 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:57 smithi052 systemd[1]: Started Ceph osd.1 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:36:58.493 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:36:58 smithi052 conmon[203590]: 2023-09-23T22:36:58.227+0000 7fbb69e4a640 -1 Falling back to public interface 2023-09-23T22:36:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:58 smithi151 ceph-mon[159210]: pgmap v134: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:36:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:58 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:58 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:58 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[187743]: pgmap v134: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:36:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[180979]: pgmap v134: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:36:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:36:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:58 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:00.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:36:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2033442735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:00.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:36:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2033442735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:00.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:36:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2033442735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:00 smithi151 ceph-mon[159210]: pgmap v135: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.243 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:37:00 smithi052 conmon[203590]: 2023-09-23T22:37:00.878+0000 7fbb69e4a640 -1 osd.1 80 log_to_monitors true 2023-09-23T22:37:01.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[187743]: pgmap v135: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:01.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[180979]: pgmap v135: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.133 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[187743]: from='osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:02.133 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[187743]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:02.133 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.133 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.133 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2218154178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:02.134 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 22:37:01 smithi052 conmon[203590]: 2023-09-23T22:37:01.806+0000 7fbb592b3700 -1 osd.1 80 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:37:02.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[180979]: from='osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:02.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[180979]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:02.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2218154178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:02.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:01 smithi151 ceph-mon[159210]: from='osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:02.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:01 smithi151 ceph-mon[159210]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:01 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:01 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2218154178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:03.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: pgmap v136: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:03.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: osdmap e83: 8 total, 7 up, 8 in 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: from='osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[180979]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: pgmap v136: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: osdmap e83: 8 total, 7 up, 8 in 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: from='osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:03.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:03.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:02 smithi052 ceph-mon[187743]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: pgmap v136: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: osdmap e83: 8 total, 7 up, 8 in 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: from='osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:02 smithi151 ceph-mon[159210]: failed to encode map e83 with expected crc 2023-09-23T22:37:03.626 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:37:03] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.33.4" 2023-09-23T22:37:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: failed to encode map e84 with expected crc 2023-09-23T22:37:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443] boot 2023-09-23T22:37:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T22:37:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:37:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1151253392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[187743]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: failed to encode map e84 with expected crc 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443] boot 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1151253392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:03 smithi052 ceph-mon[180979]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: failed to encode map e84 with expected crc 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: osd.1 [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443] boot 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1151253392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:03 smithi151 ceph-mon[159210]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T22:37:04.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:04 smithi052 conmon[173965]: 2023-09-23T22:37:04.444+0000 7f39bb05d700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: pgmap v139: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:05.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: pgmap v139: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:05.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: pgmap v139: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: failed to encode map e85 with expected crc 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:05.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:05.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:06.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:05 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:05 smithi151 ceph-mon[159210]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:37:06.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:05 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:06.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:05 smithi052 ceph-mon[180979]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:37:06.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:05 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:06.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:05 smithi052 ceph-mon[187743]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:37:07.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:06 smithi151 ceph-mon[159210]: pgmap v141: 73 pgs: 8 active+undersized, 65 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:06 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2023-09-23T22:37:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:06 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:37:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1369302671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[187743]: pgmap v141: 73 pgs: 8 active+undersized, 65 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2023-09-23T22:37:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:37:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1369302671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[180979]: pgmap v141: 73 pgs: 8 active+undersized, 65 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2023-09-23T22:37:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:37:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1369302671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:08 smithi151 ceph-mon[159210]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2985405704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:09.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:08 smithi052 ceph-mon[187743]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:09.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2985405704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:09.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:08 smithi052 ceph-mon[180979]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2985405704' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:10 smithi151 ceph-mon[159210]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3067870326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:11.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:10 smithi052 ceph-mon[187743]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:11.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3067870326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:11.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:10 smithi052 ceph-mon[180979]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:11.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3067870326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:13.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:12 smithi151 ceph-mon[159210]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:13.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:12 smithi052 ceph-mon[180979]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:12 smithi052 ceph-mon[187743]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:37:13] "GET /metrics HTTP/1.1" 200 33831 "" "Prometheus/2.33.4" 2023-09-23T22:37:14.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/140253022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:14.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/140253022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:14.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/140253022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:14 smithi151 ceph-mon[159210]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:14 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:15.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:14 smithi052 ceph-mon[187743]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:15.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:14 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:15.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:14 smithi052 ceph-mon[180979]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:15.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:14 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:16.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/548000743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:16.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/548000743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:16.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/548000743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:17.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:16 smithi151 ceph-mon[159210]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:17.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:16 smithi052 ceph-mon[187743]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:17.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:16 smithi052 ceph-mon[180979]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:18.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2487430727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:18.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2487430727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:18.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2487430727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:19.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:18 smithi151 ceph-mon[159210]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:19.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:18 smithi052 ceph-mon[187743]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:19.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:18 smithi052 ceph-mon[180979]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: Upgrade: osd.2 is safe to restart 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4156079539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:20.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:20.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:20.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: Upgrade: osd.2 is safe to restart 2023-09-23T22:37:20.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4156079539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:37:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: Upgrade: osd.2 is safe to restart 2023-09-23T22:37:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4156079539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:22.181 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[187743]: Upgrade: Updating osd.2 2023-09-23T22:37:22.181 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:22.181 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:37:22.181 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:22.181 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[187743]: Deploying daemon osd.2 on smithi052 2023-09-23T22:37:22.181 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[180979]: Upgrade: Updating osd.2 2023-09-23T22:37:22.182 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:22.182 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:37:22.182 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:22.182 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:21 smithi052 ceph-mon[180979]: Deploying daemon osd.2 on smithi052 2023-09-23T22:37:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:21 smithi151 ceph-mon[159210]: Upgrade: Updating osd.2 2023-09-23T22:37:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:37:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:22.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:21 smithi151 ceph-mon[159210]: Deploying daemon osd.2 on smithi052 2023-09-23T22:37:23.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:22 smithi052 ceph-mon[187743]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:23.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1127086140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:23.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:22 smithi052 ceph-mon[180979]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:23.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1127086140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:23.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:22 smithi151 ceph-mon[159210]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1127086140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:23.621 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:37:23] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.33.4" 2023-09-23T22:37:23.965 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:23 smithi052 systemd[1]: Stopping Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:37:24.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:23 smithi052 ceph-mon[180979]: osd.2 marked itself down and dead 2023-09-23T22:37:24.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:23 smithi052 ceph-mon[187743]: osd.2 marked itself down and dead 2023-09-23T22:37:24.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:23 smithi052 conmon[126863]: debug 2023-09-23T22:37:23.963+0000 7f698b14d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T22:37:24.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:23 smithi052 conmon[126863]: debug 2023-09-23T22:37:23.963+0000 7f698b14d700 -1 osd.2 85 *** Got signal Terminated *** 2023-09-23T22:37:24.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:23 smithi052 conmon[126863]: debug 2023-09-23T22:37:23.963+0000 7f698b14d700 -1 osd.2 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:37:24.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:24 smithi052 bash[207338]: 6d0289679661817b00c797e986a8dbc7cff667e22b745555a0ea055897bd6324 2023-09-23T22:37:24.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:23 smithi151 ceph-mon[159210]: osd.2 marked itself down and dead 2023-09-23T22:37:25.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: osdmap e86: 8 total, 7 up, 8 in 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4246484634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:25.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: osdmap e86: 8 total, 7 up, 8 in 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4246484634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: osdmap e86: 8 total, 7 up, 8 in 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: failed to encode map e86 with expected crc 2023-09-23T22:37:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4246484634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:26.338 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.338 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.338 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: osdmap e87: 8 total, 7 up, 8 in 2023-09-23T22:37:26.338 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.338 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.338 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.339 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.339 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[187743]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.339 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:26 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T22:37:26.339 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:26 smithi052 systemd[1]: Stopped Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:37:26.339 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:26 smithi052 systemd[1]: Starting Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:37:26.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.339 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: osdmap e87: 8 total, 7 up, 8 in 2023-09-23T22:37:26.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.340 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:26 smithi052 ceph-mon[180979]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: osdmap e87: 8 total, 7 up, 8 in 2023-09-23T22:37:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:26 smithi151 ceph-mon[159210]: failed to encode map e87 with expected crc 2023-09-23T22:37:27.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:27 smithi052 ceph-mon[180979]: pgmap v153: 73 pgs: 1 active+undersized+degraded, 2 peering, 1 stale+active+clean, 12 active+undersized, 57 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:27.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:27 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:37:27.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:27 smithi052 ceph-mon[187743]: pgmap v153: 73 pgs: 1 active+undersized+degraded, 2 peering, 1 stale+active+clean, 12 active+undersized, 57 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:27.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:27 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:37:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:27 smithi151 ceph-mon[159210]: pgmap v153: 73 pgs: 1 active+undersized+degraded, 2 peering, 1 stale+active+clean, 12 active+undersized, 57 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:27 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:37:27.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:37:27.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:37:27.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T22:37:27.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:37:27.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:37:27.744 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T22:37:27.744 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:37:27.744 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:27 smithi052 bash[207990]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T22:37:28.385 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/293500621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:28.385 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/293500621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:28.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/293500621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:28.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:28 smithi052 ceph-osd[208481]: -- 172.21.15.52:0/2511475717 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f9b5bc84e0 con 0x55f9b5b9b000 2023-09-23T22:37:28.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:28 smithi052 bash[208416]: 58cbbf66d491acc74e901bcfbdf626aff8956a6073e1a44f0937efa927caaf50 2023-09-23T22:37:28.743 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:28 smithi052 systemd[1]: Started Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:37:29.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[187743]: pgmap v154: 73 pgs: 1 active+undersized+degraded, 2 peering, 16 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:29.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:28 smithi052 conmon[208457]: 2023-09-23T22:37:28.984+0000 7ffb753e6640 -1 Falling back to public interface 2023-09-23T22:37:29.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[180979]: pgmap v154: 73 pgs: 1 active+undersized+degraded, 2 peering, 16 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:29.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:29.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:29.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:29.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:29 smithi151 ceph-mon[159210]: pgmap v154: 73 pgs: 1 active+undersized+degraded, 2 peering, 16 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:29.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:29.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:29.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:30.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2099880075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:30.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2099880075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:30.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2099880075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:31 smithi151 ceph-mon[159210]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 2 peering, 16 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:31.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:31 smithi052 ceph-mon[187743]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 2 peering, 16 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:31.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:31 smithi052 ceph-mon[180979]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 2 peering, 16 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:37:32.591 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.591 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.591 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[187743]: pgmap v156: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:37:32.591 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2750154111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:32.591 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[180979]: pgmap v156: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2750154111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:32 smithi151 ceph-mon[159210]: pgmap v156: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:37:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2750154111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:33.478 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:37:33] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.33.4" 2023-09-23T22:37:33.479 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:33 smithi052 conmon[208457]: 2023-09-23T22:37:33.213+0000 7ffb753e6640 -1 osd.2 85 log_to_monitors true 2023-09-23T22:37:33.479 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:37:33 smithi052 conmon[208457]: 2023-09-23T22:37:33.274+0000 7ffb65851700 -1 osd.2 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:37:33.479 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:33 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:33.479 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:33 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:37:33.480 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:33 smithi052 ceph-mon[180979]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:37:33.480 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:33 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:33.480 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:33 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:37:33.480 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:33 smithi052 ceph-mon[187743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:37:33.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:33 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:33 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:37:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:33 smithi151 ceph-mon[159210]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:37:34.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: failed to encode map e88 with expected crc 2023-09-23T22:37:34.515 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: osdmap e88: 8 total, 7 up, 8 in 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: pgmap v158: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/782198652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: failed to encode map e88 with expected crc 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: osdmap e88: 8 total, 7 up, 8 in 2023-09-23T22:37:34.516 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:34.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:34.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: pgmap v158: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:37:34.517 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/782198652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:34.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: failed to encode map e88 with expected crc 2023-09-23T22:37:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:37:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: osdmap e88: 8 total, 7 up, 8 in 2023-09-23T22:37:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:37:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: pgmap v158: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:37:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/782198652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:35.243 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:35 smithi052 conmon[173965]: 2023-09-23T22:37:35.169+0000 7f39bb05d700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: failed to encode map e88 with expected crc 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: failed to encode map e88 with expected crc 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046] boot 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:37:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: failed to encode map e88 with expected crc 2023-09-23T22:37:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: failed to encode map e88 with expected crc 2023-09-23T22:37:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046] boot 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: failed to encode map e88 with expected crc 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: failed to encode map e88 with expected crc 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: osd.2 [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046] boot 2023-09-23T22:37:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:37:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:37:35.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:36.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:36 smithi151 ceph-mon[159210]: pgmap v161: 73 pgs: 2 peering, 6 active+undersized, 65 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:36.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[180979]: pgmap v161: 73 pgs: 2 peering, 6 active+undersized, 65 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: failed to encode map e90 with expected crc 2023-09-23T22:37:36.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:36 smithi052 ceph-mon[187743]: pgmap v161: 73 pgs: 2 peering, 6 active+undersized, 65 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:37 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2023-09-23T22:37:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:37 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:37:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3393750403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:37 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2023-09-23T22:37:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:37 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:37:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3393750403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:37 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2023-09-23T22:37:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:37 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:37:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3393750403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:38 smithi151 ceph-mon[159210]: pgmap v162: 73 pgs: 2 peering, 3 active+undersized, 68 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:38.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:38 smithi052 ceph-mon[187743]: pgmap v162: 73 pgs: 2 peering, 3 active+undersized, 68 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:38.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:38 smithi052 ceph-mon[180979]: pgmap v162: 73 pgs: 2 peering, 3 active+undersized, 68 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:39.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1307723339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:39.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1307723339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1307723339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:40 smithi151 ceph-mon[159210]: pgmap v163: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:40.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:40 smithi052 ceph-mon[187743]: pgmap v163: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:40.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:40 smithi052 ceph-mon[180979]: pgmap v163: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:41.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/866274922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:41.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/866274922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:41.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/866274922' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:42 smithi151 ceph-mon[159210]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:42 smithi052 ceph-mon[187743]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:42.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:42 smithi052 ceph-mon[180979]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:43.647 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:37:43] "GET /metrics HTTP/1.1" 200 33829 "" "Prometheus/2.33.4" 2023-09-23T22:37:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2371829973' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:43.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2371829973' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:43.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2371829973' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:44 smithi151 ceph-mon[159210]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:44.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:44 smithi052 ceph-mon[180979]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:44 smithi052 ceph-mon[187743]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:46 smithi151 ceph-mon[159210]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/524860499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:46.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:46 smithi052 ceph-mon[180979]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:46.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/524860499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:46 smithi052 ceph-mon[187743]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:46.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/524860499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:48 smithi151 ceph-mon[159210]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2205310235' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:48 smithi052 ceph-mon[187743]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2205310235' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:48 smithi052 ceph-mon[180979]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2205310235' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:50.729 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[187743]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/532029693' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[180979]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:50.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:50.731 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:50.731 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/532029693' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:50.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:50 smithi151 ceph-mon[159210]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/532029693' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:51.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:51 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:51.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:51 smithi052 ceph-mon[187743]: Upgrade: osd.3 is safe to restart 2023-09-23T22:37:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:51 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:51 smithi052 ceph-mon[180979]: Upgrade: osd.3 is safe to restart 2023-09-23T22:37:51.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:51 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:37:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:51 smithi151 ceph-mon[159210]: Upgrade: osd.3 is safe to restart 2023-09-23T22:37:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:52 smithi151 ceph-mon[159210]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:52 smithi151 ceph-mon[159210]: Upgrade: Updating osd.3 2023-09-23T22:37:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:37:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:52 smithi151 ceph-mon[159210]: Deploying daemon osd.3 on smithi052 2023-09-23T22:37:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[187743]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[187743]: Upgrade: Updating osd.3 2023-09-23T22:37:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[187743]: Deploying daemon osd.3 on smithi052 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[180979]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[180979]: Upgrade: Updating osd.3 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:52 smithi052 ceph-mon[180979]: Deploying daemon osd.3 on smithi052 2023-09-23T22:37:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:37:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:37:53] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.33.4" 2023-09-23T22:37:53.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2317455168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:53.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2317455168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2317455168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:54.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:54 smithi052 ceph-mon[180979]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:54.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:54 smithi052 ceph-mon[187743]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:54.744 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:54 smithi052 systemd[1]: Stopping Ceph osd.3 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:37:54.744 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:54 smithi052 conmon[132705]: debug 2023-09-23T22:37:54.664+0000 7f72561b4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T22:37:54.745 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:54 smithi052 conmon[132705]: debug 2023-09-23T22:37:54.664+0000 7f72561b4700 -1 osd.3 90 *** Got signal Terminated *** 2023-09-23T22:37:54.745 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:54 smithi052 conmon[132705]: debug 2023-09-23T22:37:54.664+0000 7f72561b4700 -1 osd.3 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:37:54.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:54 smithi151 ceph-mon[159210]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:55.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:55 smithi151 ceph-mon[159210]: osd.3 marked itself down and dead 2023-09-23T22:37:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3145926735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:55.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:55 smithi052 ceph-mon[180979]: osd.3 marked itself down and dead 2023-09-23T22:37:55.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3145926735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:55.990 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:55 smithi052 ceph-mon[187743]: osd.3 marked itself down and dead 2023-09-23T22:37:55.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3145926735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:56.243 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:55 smithi052 bash[212180]: 1744ec2d869b840d5fc6a99078bd39acb5615617d211f9b7f9e6707fc4174fe7 2023-09-23T22:37:56.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:56 smithi151 ceph-mon[159210]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[187743]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: failed to encode map e91 with expected crc 2023-09-23T22:37:56.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:56 smithi052 ceph-mon[180979]: failed to encode map e91 with expected crc 2023-09-23T22:37:57.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3102114470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3102114470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: failed to encode map e92 with expected crc 2023-09-23T22:37:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3102114470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:58.493 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:58 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T22:37:58.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:58 smithi052 systemd[1]: Stopped Ceph osd.3 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:37:58.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:58 smithi052 systemd[1]: Starting Ceph osd.3 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:37:58.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:58 smithi151 ceph-mon[159210]: pgmap v174: 73 pgs: 10 stale+active+clean, 4 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:58 smithi052 ceph-mon[187743]: pgmap v174: 73 pgs: 10 stale+active+clean, 4 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:58 smithi052 ceph-mon[180979]: pgmap v174: 73 pgs: 10 stale+active+clean, 4 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T22:37:59.494 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:37:59 smithi052 bash[212870]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T22:37:59.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:37:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4024263601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:59.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:37:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4024263601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:37:59.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:37:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4024263601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:00.689 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-osd[213378]: -- 172.21.15.52:0/2384254840 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a3e923e4e0 con 0x55a3e9211400 2023-09-23T22:38:00.689 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:38:00 smithi052 bash[213313]: 8408de2fa10661c3962ba56929b7242881d1aec053d012164e6f06b9ba066fdf 2023-09-23T22:38:00.689 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:38:00 smithi052 systemd[1]: Started Ceph osd.3 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:38:00.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[180979]: pgmap v175: 73 pgs: 1 active+undersized+degraded, 6 stale+active+clean, 15 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-09-23T22:38:00.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:38:00.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:00.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:00.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:00 smithi151 ceph-mon[159210]: pgmap v175: 73 pgs: 1 active+undersized+degraded, 6 stale+active+clean, 15 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-09-23T22:38:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:00 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:38:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:00.993 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:38:00 smithi052 conmon[213354]: 2023-09-23T22:38:00.850+0000 7f0160af2640 -1 Falling back to public interface 2023-09-23T22:38:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[187743]: pgmap v175: 73 pgs: 1 active+undersized+degraded, 6 stale+active+clean, 15 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-09-23T22:38:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:38:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:02 smithi151 ceph-mon[159210]: pgmap v176: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:38:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3819813528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:02 smithi052 ceph-mon[187743]: pgmap v176: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:38:02.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3819813528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:02 smithi052 ceph-mon[180979]: pgmap v176: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:38:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3819813528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:38:03] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.33.4" 2023-09-23T22:38:04.116 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:38:03 smithi052 conmon[213354]: 2023-09-23T22:38:03.759+0000 7f0160af2640 -1 osd.3 90 log_to_monitors true 2023-09-23T22:38:04.117 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.117 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.117 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[180979]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:38:04.117 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.118 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.377 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 22:38:04 smithi052 conmon[213354]: 2023-09-23T22:38:04.148+0000 7f014f75a700 -1 osd.3 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:38:04.378 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.378 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.378 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[187743]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:38:04.378 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.378 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:04 smithi151 ceph-mon[159210]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:38:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:05.448 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:38:05.448 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:38:05.448 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3245046055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: failed to encode map e93 with expected crc 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[180979]: failed to encode map e93 with expected crc 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T22:38:05.449 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3245046055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: failed to encode map e93 with expected crc 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:05 smithi052 ceph-mon[187743]: failed to encode map e93 with expected crc 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:38:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T22:38:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: from='osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:38:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3245046055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: failed to encode map e93 with expected crc 2023-09-23T22:38:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:05 smithi151 ceph-mon[159210]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: failed to encode map e94 with expected crc 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872] boot 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:38:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:06 smithi151 ceph-mon[159210]: pgmap v180: 73 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:38:06.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: failed to encode map e94 with expected crc 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872] boot 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[180979]: pgmap v180: 73 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: failed to encode map e93 with expected crc 2023-09-23T22:38:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:38:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: failed to encode map e94 with expected crc 2023-09-23T22:38:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: osd.3 [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872] boot 2023-09-23T22:38:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T22:38:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:38:06.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:06 smithi052 ceph-mon[187743]: pgmap v180: 73 pgs: 9 peering, 2 active+undersized+degraded, 20 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:38:06.693 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T22:38:06.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:06 smithi052 conmon[173965]: 2023-09-23T22:38:06.939+0000 7f39bb05d700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:38:07.141 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2420590409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:38:07.143 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[180979]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2420590409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:07.145 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:38:07.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:38:07.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:07 smithi052 ceph-mon[187743]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:38:07.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: failed to encode map e95 with expected crc 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2420590409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:07.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:38:07.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:38:07.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:07 smithi151 ceph-mon[159210]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:38:08.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:08 smithi151 ceph-mon[159210]: pgmap v182: 73 pgs: 9 peering, 2 active+undersized+degraded, 19 active+undersized, 43 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:38:08.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:08 smithi052 ceph-mon[187743]: pgmap v182: 73 pgs: 9 peering, 2 active+undersized+degraded, 19 active+undersized, 43 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:38:08.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:08 smithi052 ceph-mon[180979]: pgmap v182: 73 pgs: 9 peering, 2 active+undersized+degraded, 19 active+undersized, 43 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T22:38:09.344 INFO:teuthology.orchestra.run.smithi052.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:38:09.978 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:38:10.610 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[187743]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:10.610 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[187743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:38:10.610 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:10.610 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[187743]: pgmap v183: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:10.610 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:10.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[180979]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:10.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[180979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:38:10.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:10.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[180979]: pgmap v183: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:10.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:10 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:10.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:10 smithi151 ceph-mon[159210]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:10 smithi151 ceph-mon[159210]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:38:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:10 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:10 smithi151 ceph-mon[159210]: pgmap v183: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:10 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:11.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:11 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T22:38:11.523 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:11 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:38:11.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:11 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T22:38:11.524 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:11 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:38:11.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:11 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T22:38:11.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:11 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:38:11.898 INFO:teuthology.orchestra.run.smithi052.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:38:11.898 INFO:teuthology.orchestra.run.smithi052.stdout:alertmanager.a smithi052 *:9093,9094 running (6m) - 10m 25.6M - 0.23.0 ba2b418f427c a82e31979683 2023-09-23T22:38:11.898 INFO:teuthology.orchestra.run.smithi052.stdout:grafana.a smithi151 *:3000 running (6m) - 9m 56.2M - 8.3.5 dad864ee21e9 8803df2494ff 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi052.thpzxu smithi052 running (9m) - 9m 21.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 d8befa844e73 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi151.rbxgii smithi151 running (9m) - 9m 16.3M - 16.2.14-37-g2e1f1015 1e6a756174a8 61e8f44cb3b6 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.x smithi151 *:8443,9283,8765 running (5m) - 13m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 648eee1c1677 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.y smithi052 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-460-gbb0daeff e32c2a64c126 d2a437c50b3c 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a smithi052 running (5m) - 15m 97.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 bb846ce0af73 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b smithi151 running (3m) - 14m 43.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0f16dac83db3 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c smithi052 running (4m) - 14m 71.8M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0c16f625a72c 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.a smithi052 *:9100 running (6m) - 10m 22.5M - 1.3.1 1dbe0e931976 3bd65f3ab9c9 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.b smithi151 *:9100 running (6m) - 10m 20.4M - 1.3.1 1dbe0e931976 7e4657a95489 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 smithi052 running (2m) - 13m 58.3M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 b70c0aa9b5ab 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 smithi052 running (74s) - 13m 53.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 4ff251b65413 2023-09-23T22:38:11.899 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 smithi052 running (43s) - 12m 52.3M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 58cbbf66d491 2023-09-23T22:38:11.900 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 smithi052 running (11s) - 12m 12.4M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 8408de2fa106 2023-09-23T22:38:11.900 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 smithi151 running (12m) - 12m 85.6M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 b2126e632b7b 2023-09-23T22:38:11.900 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 smithi151 running (11m) - 11m 101M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 46585800d682 2023-09-23T22:38:11.900 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 smithi151 running (11m) - 11m 73.5M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 17e250c1eb43 2023-09-23T22:38:11.900 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 smithi151 running (11m) - 11m 116M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 73447cd68609 2023-09-23T22:38:11.900 INFO:teuthology.orchestra.run.smithi052.stdout:prometheus.a smithi151 *:9095 running (5m) - 10m 70.9M - 2.33.4 514e6a882f6e 88e6208a51e9 2023-09-23T22:38:12.494 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:38:12.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:12 smithi151 ceph-mon[159210]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:12 smithi151 ceph-mon[159210]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:12 smithi052 ceph-mon[187743]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:12.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:12 smithi052 ceph-mon[187743]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:12 smithi052 ceph-mon[180979]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:12.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:12 smithi052 ceph-mon[180979]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:13.727 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:38:13] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.33.4" 2023-09-23T22:38:14.430 INFO:teuthology.orchestra.run.smithi052.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:38:14.430 INFO:teuthology.orchestra.run.smithi052.stdout:alertmanager.a smithi052 *:9093,9094 running (6m) - 10m 25.6M - 0.23.0 ba2b418f427c a82e31979683 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:grafana.a smithi151 *:3000 running (6m) - 9m 56.2M - 8.3.5 dad864ee21e9 8803df2494ff 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi052.thpzxu smithi052 running (9m) - 9m 21.2M - 16.2.14-37-g2e1f1015 1e6a756174a8 d8befa844e73 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mds.foo.smithi151.rbxgii smithi151 running (9m) - 9m 16.3M - 16.2.14-37-g2e1f1015 1e6a756174a8 61e8f44cb3b6 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.x smithi151 *:8443,9283,8765 running (5m) - 13m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 648eee1c1677 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mgr.y smithi052 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-460-gbb0daeff e32c2a64c126 d2a437c50b3c 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a smithi052 running (5m) - 15m 97.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 bb846ce0af73 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b smithi151 running (3m) - 14m 43.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0f16dac83db3 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c smithi052 running (4m) - 14m 71.8M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0c16f625a72c 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.a smithi052 *:9100 running (6m) - 10m 22.5M - 1.3.1 1dbe0e931976 3bd65f3ab9c9 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:node-exporter.b smithi151 *:9100 running (6m) - 10m 20.4M - 1.3.1 1dbe0e931976 7e4657a95489 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 smithi052 running (2m) - 13m 58.3M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 b70c0aa9b5ab 2023-09-23T22:38:14.431 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 smithi052 running (76s) - 13m 53.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 4ff251b65413 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 smithi052 running (46s) - 12m 52.3M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 58cbbf66d491 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 smithi052 running (14s) - 12m 12.4M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 8408de2fa106 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 smithi151 running (12m) - 12m 85.6M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 b2126e632b7b 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 smithi151 running (11m) - 11m 101M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 46585800d682 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 smithi151 running (11m) - 11m 73.5M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 17e250c1eb43 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 smithi151 running (11m) - 11m 116M 2500M 16.2.14-37-g2e1f1015 1e6a756174a8 73447cd68609 2023-09-23T22:38:14.432 INFO:teuthology.orchestra.run.smithi052.stdout:prometheus.a smithi151 *:9095 running (5m) - 10m 70.9M - 2.33.4 514e6a882f6e 88e6208a51e9 2023-09-23T22:38:14.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:14 smithi052 ceph-mon[187743]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:14.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:14 smithi052 ceph-mon[180979]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:14.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:14 smithi151 ceph-mon[159210]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:15.102 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T22:38:15.909 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:15 smithi052 ceph-mon[187743]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:15.910 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:15 smithi052 ceph-mon[180979]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:15.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:15 smithi151 ceph-mon[159210]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:38:16.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:16 smithi151 ceph-mon[159210]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:16.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:16 smithi052 ceph-mon[180979]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:16 smithi052 ceph-mon[187743]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:17.051 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:38:17.051 INFO:teuthology.orchestra.run.smithi052.stdout: "mon": { 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr": { 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "osd": { 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 4, 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "mds": { 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 2 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:38:17.052 INFO:teuthology.orchestra.run.smithi052.stdout: "overall": { 2023-09-23T22:38:17.053 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 16.2.14-37-g2e1f1015 (2e1f10153ef2437dea8ce9168df827af9bb64fd7) pacific (stable)": 6, 2023-09-23T22:38:17.053 INFO:teuthology.orchestra.run.smithi052.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T22:38:17.053 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:38:17.053 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:38:17.617 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:17.617 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:38:17.627 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T22:38:17.627 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T22:38:17.635 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:38:17.635 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:38:17.636 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:38:17.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1058583209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1058583209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:17.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1058583209' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:38:18.000 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:18.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:18 smithi151 ceph-mon[159210]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:18 smithi052 ceph-mon[187743]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:18 smithi052 ceph-mon[180979]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:19.687 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:38:20.310 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":37,"active_gid":25214,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6800","nonce":3137988186},{"type":"v1","addr":"172.21.15.52:6801","nonce":3137988186}]},"active_addr":"172.21.15.52:6801/3137988186","active_change":"2023-09-23T22:32:49.359898+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25111,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":75,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":2751724358}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":551565457}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":3448800119}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.52:0","nonce":2922970471}]}]} 2023-09-23T22:38:20.314 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:38:20.314 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:38:20.314 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:38:20.682 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[180979]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/746110141' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:38:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[187743]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/746110141' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:38:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:20 smithi151 ceph-mon[159210]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/746110141' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:38:22.320 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:38:22.320 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":95,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:38:06.133584+0000","last_up_change":"2023-09-23T22:38:05.125918+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:28:41.273306+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:28:42.097486+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":75,"up_from":79,"up_thru":94,"down_at":76,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6803","nonce":570465800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6805","nonce":570465800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6809","nonce":570465800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6807","nonce":570465800}]},"public_addr":"172.21.15.52:6803/570465800","cluster_addr":"172.21.15.52:6805/570465800","heartbeat_back_addr":"172.21.15.52:6809/570465800","heartbeat_front_addr":"172.21.15.52:6807/570465800","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":94,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6811","nonce":1231582443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6813","nonce":1231582443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6817","nonce":1231582443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6815","nonce":1231582443}]},"public_addr":"172.21.15.52:6811/1231582443","cluster_addr":"172.21.15.52:6813/1231582443","heartbeat_back_addr":"172.21.15.52:6817/1231582443","heartbeat_front_addr":"172.21.15.52:6815/1231582443","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":85,"up_from":89,"up_thru":94,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6819","nonce":1755732046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6821","nonce":1755732046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6825","nonce":1755732046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6823","nonce":1755732046}]},"public_addr":"172.21.15.52:6819/1755732046","cluster_addr":"172.21.15.52:6821/1755732046","heartbeat_back_addr":"172.21.15.52:6825/1755732046","heartbeat_front_addr":"172.21.15.52:6823/1755732046","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":90,"up_from":94,"up_thru":94,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6827","nonce":133265872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6829","nonce":133265872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6833","nonce":133265872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6831","nonce":133265872}]},"public_addr":"172.21.15.52:6827/133265872","cluster_addr":"172.21.15.52:6829/133265872","heartbeat_back_addr":"172.21.15.52:6833/133265872","heartbeat_front_addr":"172.21.15.52:6831/133265872","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":75},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":85},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":90},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:6801/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3645713772":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3758668631":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/1038804029":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3912833337":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/3773142740":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000","172.21.15.151:0/1914484619":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/1262640861":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.151:0/3484573808":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.151:6834/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/1078027830":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:6835/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:0/483361858":"2023-09-24T22:32:09.537934+0000","172.21.15.151:0/4181179150":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/3070502744":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:0/3637583595":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/4148041136":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:38:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:22 smithi052 ceph-mon[187743]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:22 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:22.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2297982917' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:38:22.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:22 smithi052 ceph-mon[180979]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:22.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:22 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:22.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2297982917' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:38:22.909 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:38:22.909 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:38:22.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:22 smithi151 ceph-mon[159210]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:22 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2297982917' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:38:23.273 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:23.643 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:38:23] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.33.4" 2023-09-23T22:38:24.897 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:38:24.897 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":95,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T22:38:06.133584+0000","last_up_change":"2023-09-23T22:38:05.125918+0000","last_in_change":"2023-09-23T22:26:55.709232+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:28:41.273306+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:28:42.097486+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":75,"up_from":79,"up_thru":94,"down_at":76,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6803","nonce":570465800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6805","nonce":570465800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6809","nonce":570465800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":570465800},{"type":"v1","addr":"172.21.15.52:6807","nonce":570465800}]},"public_addr":"172.21.15.52:6803/570465800","cluster_addr":"172.21.15.52:6805/570465800","heartbeat_back_addr":"172.21.15.52:6809/570465800","heartbeat_front_addr":"172.21.15.52:6807/570465800","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":94,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6811","nonce":1231582443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6813","nonce":1231582443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6817","nonce":1231582443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6815","nonce":1231582443}]},"public_addr":"172.21.15.52:6811/1231582443","cluster_addr":"172.21.15.52:6813/1231582443","heartbeat_back_addr":"172.21.15.52:6817/1231582443","heartbeat_front_addr":"172.21.15.52:6815/1231582443","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":85,"up_from":89,"up_thru":94,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6819","nonce":1755732046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6821","nonce":1755732046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6825","nonce":1755732046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":1755732046},{"type":"v1","addr":"172.21.15.52:6823","nonce":1755732046}]},"public_addr":"172.21.15.52:6819/1755732046","cluster_addr":"172.21.15.52:6821/1755732046","heartbeat_back_addr":"172.21.15.52:6825/1755732046","heartbeat_front_addr":"172.21.15.52:6823/1755732046","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":90,"up_from":94,"up_thru":94,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6827","nonce":133265872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6829","nonce":133265872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6833","nonce":133265872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6831","nonce":133265872}]},"public_addr":"172.21.15.52:6827/133265872","cluster_addr":"172.21.15.52:6829/133265872","heartbeat_back_addr":"172.21.15.52:6833/133265872","heartbeat_front_addr":"172.21.15.52:6831/133265872","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":94,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":75},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":85},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":90},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:6801/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3645713772":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3758668631":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/1038804029":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3912833337":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/3773142740":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000","172.21.15.151:0/1914484619":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/1262640861":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.151:0/3484573808":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.151:6834/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/1078027830":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:6835/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:0/483361858":"2023-09-24T22:32:09.537934+0000","172.21.15.151:0/4181179150":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/3070502744":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:0/3637583595":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/4148041136":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:38:24.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:24 smithi052 ceph-mon[187743]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:24.921 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:24 smithi052 ceph-mon[180979]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:24.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:24 smithi151 ceph-mon[159210]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:25.557 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:38:25.558 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:38:25.558 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:38:25.559 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:38:25.559 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:38:25.559 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:38:25.560 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:38:25.560 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:38:25.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:25.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:25.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1507059183' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:38:25.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:25.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:25.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1507059183' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:25.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1507059183' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:26.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:26.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:26.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:38:26.623 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:26.748 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:26.775 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:26.917 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:26.943 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:27.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:26 smithi151 ceph-mon[159210]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:27.209 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:27.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:26 smithi052 ceph-mon[187743]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:27.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:26 smithi052 ceph-mon[180979]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:27.247 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:27.891 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:29.002 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:28 smithi052 ceph-mon[187743]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:29.002 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:28 smithi052 ceph-mon[180979]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:29.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:28 smithi151 ceph-mon[159210]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:30.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:30 smithi052 ceph-mon[187743]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:30.961 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:30 smithi052 ceph-mon[180979]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:31.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:30 smithi151 ceph-mon[159210]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:33.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:32 smithi151 ceph-mon[159210]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:33.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:32 smithi052 ceph-mon[187743]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:33.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:32 smithi052 ceph-mon[180979]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:33.623 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:38:33] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.33.4" 2023-09-23T22:38:33.781 INFO:teuthology.orchestra.run.smithi052.stdout:128849019030 2023-09-23T22:38:33.781 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:38:34.415 INFO:teuthology.orchestra.run.smithi052.stdout:360777252884 2023-09-23T22:38:34.416 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:38:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:34 smithi052 ceph-mon[187743]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:34 smithi052 ceph-mon[180979]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:34 smithi151 ceph-mon[159210]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:35.513 INFO:teuthology.orchestra.run.smithi052.stdout:403726925831 2023-09-23T22:38:35.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:38:37.101 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:36 smithi052 ceph-mon[187743]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:37.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:36 smithi052 ceph-mon[180979]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:37.163 INFO:teuthology.orchestra.run.smithi052.stdout:382252089358 2023-09-23T22:38:37.164 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:38:37.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:36 smithi151 ceph-mon[159210]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:37.488 INFO:teuthology.orchestra.run.smithi052.stdout:180388626575 2023-09-23T22:38:37.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi052.stdout:339302416419 2023-09-23T22:38:38.073 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:38:38.414 INFO:teuthology.orchestra.run.smithi052.stdout:154618822803 2023-09-23T22:38:38.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:38:38.954 INFO:teuthology.orchestra.run.smithi052.stdout:206158430347 2023-09-23T22:38:38.954 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:38:39.038 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:39.179 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:39.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:38 smithi151 ceph-mon[159210]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:39.206 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:39.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:38 smithi052 ceph-mon[180979]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:38 smithi052 ceph-mon[187743]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:39.442 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:39.541 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:39.553 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:41.157 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:40 smithi052 ceph-mon[187743]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:41.157 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:40 smithi052 ceph-mon[180979]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:41.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:40 smithi151 ceph-mon[159210]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:41.855 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:42.956 INFO:teuthology.orchestra.run.smithi052.stdout:403726925833 2023-09-23T22:38:43.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:42 smithi151 ceph-mon[159210]: pgmap v199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:43.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:42 smithi052 ceph-mon[187743]: pgmap v199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:43.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:42 smithi052 ceph-mon[180979]: pgmap v199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:43.615 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:38:43] "GET /metrics HTTP/1.1" 200 33969 "" "Prometheus/2.33.4" 2023-09-23T22:38:43.884 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1755756013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:38:44.173 INFO:teuthology.orchestra.run.smithi052.stdout:360777252886 2023-09-23T22:38:44.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1755756013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:38:44.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1755756013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:38:44.430 INFO:teuthology.orchestra.run.smithi052.stdout:382252089360 2023-09-23T22:38:44.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:44 smithi052 ceph-mon[180979]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:44.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/749126263' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:38:44.887 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2278168914' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:38:44.964 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925831 got 403726925833 for osd.3 2023-09-23T22:38:44.964 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:45.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:44 smithi151 ceph-mon[159210]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:45.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/749126263' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:38:45.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2278168914' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:38:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:44 smithi052 ceph-mon[187743]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:45.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/749126263' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:38:45.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2278168914' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:38:46.308 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777252884 got 360777252886 for osd.1 2023-09-23T22:38:46.308 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:46.753 INFO:teuthology.orchestra.run.smithi052.stdout:128849019033 2023-09-23T22:38:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:46 smithi052 ceph-mon[187743]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1352841761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:38:46.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:46 smithi052 ceph-mon[180979]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:46.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1352841761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:38:47.177 INFO:teuthology.orchestra.run.smithi052.stdout:180388626577 2023-09-23T22:38:47.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:46 smithi151 ceph-mon[159210]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1352841761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:38:47.471 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089358 got 382252089360 for osd.2 2023-09-23T22:38:47.472 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:47.957 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019030 got 128849019033 for osd.4 2023-09-23T22:38:47.997 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:48.118 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:48.175 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4015955588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:38:48.175 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4015955588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:38:48.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4015955588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:38:48.239 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626575 got 180388626577 for osd.6 2023-09-23T22:38:48.239 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:48.583 INFO:teuthology.orchestra.run.smithi052.stdout:339302416421 2023-09-23T22:38:48.894 INFO:teuthology.orchestra.run.smithi052.stdout:154618822805 2023-09-23T22:38:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:48 smithi052 ceph-mon[187743]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1825983271' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:38:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:48 smithi052 ceph-mon[180979]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1825983271' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:38:49.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:48 smithi151 ceph-mon[159210]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1825983271' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:38:50.074 INFO:tasks.cephadm.ceph_manager.ceph:need seq 339302416419 got 339302416421 for osd.0 2023-09-23T22:38:50.074 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:50.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1606511433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:38:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:50.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1606511433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:38:50.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1606511433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:38:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:50.338 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822803 got 154618822805 for osd.5 2023-09-23T22:38:50.338 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:50.973 INFO:teuthology.orchestra.run.smithi052.stdout:206158430349 2023-09-23T22:38:51.172 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:50 smithi052 ceph-mon[187743]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:51.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:50 smithi052 ceph-mon[180979]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:51.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:50 smithi151 ceph-mon[159210]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:51.639 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430347 got 206158430349 for osd.7 2023-09-23T22:38:51.639 DEBUG:teuthology.parallel:result is None 2023-09-23T22:38:51.639 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:38:51.639 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:38:52.012 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:52.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2776246184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:38:52.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2776246184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:38:52.222 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2776246184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:38:53.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:52 smithi151 ceph-mon[159210]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:52 smithi052 ceph-mon[187743]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:53.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:52 smithi052 ceph-mon[180979]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:53.715 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:38:53.735 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:38:53.737 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:38:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:38:53] "GET /metrics HTTP/1.1" 200 33968 "" "Prometheus/2.33.4" 2023-09-23T22:38:54.321 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":205,"stamp":"2023-09-23T22:38:53.474320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348296,"kb_used_data":9416,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612920,"statfs":{"total":767960285184,"available":766579630080,"internally_reserved":0,"allocated":9641984,"data_stored":5951558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003376"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140027+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:38:05.140027+0000","last_peered":"2023-09-23T22:38:05.140027+0000","last_clean":"2023-09-23T22:38:05.140027+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:38:05.140027+0000","last_undegraded":"2023-09-23T22:38:05.140027+0000","last_fullsized":"2023-09-23T22:38:05.140027+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147480+0000","last_change":"2023-09-23T22:37:35.270139+0000","last_active":"2023-09-23T22:38:05.147480+0000","last_peered":"2023-09-23T22:38:05.147480+0000","last_clean":"2023-09-23T22:38:05.147480+0000","last_became_active":"2023-09-23T22:37:35.269761+0000","last_became_peered":"2023-09-23T22:37:35.269761+0000","last_unstale":"2023-09-23T22:38:05.147480+0000","last_undegraded":"2023-09-23T22:38:05.147480+0000","last_fullsized":"2023-09-23T22:38:05.147480+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:08.396803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.147696+0000","last_change":"2023-09-23T22:38:06.147696+0000","last_active":"2023-09-23T22:38:06.147696+0000","last_peered":"2023-09-23T22:38:06.147696+0000","last_clean":"2023-09-23T22:38:06.147696+0000","last_became_active":"2023-09-23T22:38:06.147382+0000","last_became_peered":"2023-09-23T22:38:06.147382+0000","last_unstale":"2023-09-23T22:38:06.147696+0000","last_undegraded":"2023-09-23T22:38:06.147696+0000","last_fullsized":"2023-09-23T22:38:06.147696+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:04.570885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.149889+0000","last_change":"2023-09-23T22:38:06.149889+0000","last_active":"2023-09-23T22:38:06.149889+0000","last_peered":"2023-09-23T22:38:06.149889+0000","last_clean":"2023-09-23T22:38:06.149889+0000","last_became_active":"2023-09-23T22:38:06.149711+0000","last_became_peered":"2023-09-23T22:38:06.149711+0000","last_unstale":"2023-09-23T22:38:06.149889+0000","last_undegraded":"2023-09-23T22:38:06.149889+0000","last_fullsized":"2023-09-23T22:38:06.149889+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:06.335320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151879+0000","last_change":"2023-09-23T22:38:06.151879+0000","last_active":"2023-09-23T22:38:06.151879+0000","last_peered":"2023-09-23T22:38:06.151879+0000","last_clean":"2023-09-23T22:38:06.151879+0000","last_became_active":"2023-09-23T22:38:06.151519+0000","last_became_peered":"2023-09-23T22:38:06.151519+0000","last_unstale":"2023-09-23T22:38:06.151879+0000","last_undegraded":"2023-09-23T22:38:06.151879+0000","last_fullsized":"2023-09-23T22:38:06.151879+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.135034+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:38:05.135034+0000","last_peered":"2023-09-23T22:38:05.135034+0000","last_clean":"2023-09-23T22:38:05.135034+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:38:05.135034+0000","last_undegraded":"2023-09-23T22:38:05.135034+0000","last_fullsized":"2023-09-23T22:38:05.135034+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153986+0000","last_change":"2023-09-23T22:38:06.153986+0000","last_active":"2023-09-23T22:38:06.153986+0000","last_peered":"2023-09-23T22:38:06.153986+0000","last_clean":"2023-09-23T22:38:06.153986+0000","last_became_active":"2023-09-23T22:38:06.153604+0000","last_became_peered":"2023-09-23T22:38:06.153604+0000","last_unstale":"2023-09-23T22:38:06.153986+0000","last_undegraded":"2023-09-23T22:38:06.153986+0000","last_fullsized":"2023-09-23T22:38:06.153986+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134780+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:38:05.134780+0000","last_peered":"2023-09-23T22:38:05.134780+0000","last_clean":"2023-09-23T22:38:05.134780+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:38:05.134780+0000","last_undegraded":"2023-09-23T22:38:05.134780+0000","last_fullsized":"2023-09-23T22:38:05.134780+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152423+0000","last_change":"2023-09-23T22:38:06.152423+0000","last_active":"2023-09-23T22:38:06.152423+0000","last_peered":"2023-09-23T22:38:06.152423+0000","last_clean":"2023-09-23T22:38:06.152423+0000","last_became_active":"2023-09-23T22:38:06.152083+0000","last_became_peered":"2023-09-23T22:38:06.152083+0000","last_unstale":"2023-09-23T22:38:06.152423+0000","last_undegraded":"2023-09-23T22:38:06.152423+0000","last_fullsized":"2023-09-23T22:38:06.152423+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:49.106880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.144944+0000","last_change":"2023-09-23T22:35:51.475988+0000","last_active":"2023-09-23T22:38:06.144944+0000","last_peered":"2023-09-23T22:38:06.144944+0000","last_clean":"2023-09-23T22:38:06.144944+0000","last_became_active":"2023-09-23T22:35:51.475664+0000","last_became_peered":"2023-09-23T22:35:51.475664+0000","last_unstale":"2023-09-23T22:38:06.144944+0000","last_undegraded":"2023-09-23T22:38:06.144944+0000","last_fullsized":"2023-09-23T22:38:06.144944+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:22.107425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140271+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:38:05.140271+0000","last_peered":"2023-09-23T22:38:05.140271+0000","last_clean":"2023-09-23T22:38:05.140271+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:38:05.140271+0000","last_undegraded":"2023-09-23T22:38:05.140271+0000","last_fullsized":"2023-09-23T22:38:05.140271+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.146941+0000","last_change":"2023-09-23T22:37:03.801120+0000","last_active":"2023-09-23T22:38:05.146941+0000","last_peered":"2023-09-23T22:38:05.146941+0000","last_clean":"2023-09-23T22:38:05.146941+0000","last_became_active":"2023-09-23T22:37:03.800596+0000","last_became_peered":"2023-09-23T22:37:03.800596+0000","last_unstale":"2023-09-23T22:38:05.146941+0000","last_undegraded":"2023-09-23T22:38:05.146941+0000","last_fullsized":"2023-09-23T22:38:05.146941+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:03.424170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136253+0000","last_change":"2023-09-23T22:37:35.276863+0000","last_active":"2023-09-23T22:38:05.136253+0000","last_peered":"2023-09-23T22:38:05.136253+0000","last_clean":"2023-09-23T22:38:05.136253+0000","last_became_active":"2023-09-23T22:37:35.276428+0000","last_became_peered":"2023-09-23T22:37:35.276428+0000","last_unstale":"2023-09-23T22:38:05.136253+0000","last_undegraded":"2023-09-23T22:38:05.136253+0000","last_fullsized":"2023-09-23T22:38:05.136253+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155276+0000","last_change":"2023-09-23T22:38:06.155276+0000","last_active":"2023-09-23T22:38:06.155276+0000","last_peered":"2023-09-23T22:38:06.155276+0000","last_clean":"2023-09-23T22:38:06.155276+0000","last_became_active":"2023-09-23T22:38:06.154550+0000","last_became_peered":"2023-09-23T22:38:06.154550+0000","last_unstale":"2023-09-23T22:38:06.155276+0000","last_undegraded":"2023-09-23T22:38:06.155276+0000","last_fullsized":"2023-09-23T22:38:06.155276+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:06.092371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140029+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:38:05.140029+0000","last_peered":"2023-09-23T22:38:05.140029+0000","last_clean":"2023-09-23T22:38:05.140029+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:38:05.140029+0000","last_undegraded":"2023-09-23T22:38:05.140029+0000","last_fullsized":"2023-09-23T22:38:05.140029+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155104+0000","last_change":"2023-09-23T22:38:06.155104+0000","last_active":"2023-09-23T22:38:06.155104+0000","last_peered":"2023-09-23T22:38:06.155104+0000","last_clean":"2023-09-23T22:38:06.155104+0000","last_became_active":"2023-09-23T22:38:06.154630+0000","last_became_peered":"2023-09-23T22:38:06.154630+0000","last_unstale":"2023-09-23T22:38:06.155104+0000","last_undegraded":"2023-09-23T22:38:06.155104+0000","last_fullsized":"2023-09-23T22:38:06.155104+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.232885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144233+0000","last_change":"2023-09-23T22:35:51.477103+0000","last_active":"2023-09-23T22:38:05.144233+0000","last_peered":"2023-09-23T22:38:05.144233+0000","last_clean":"2023-09-23T22:38:05.144233+0000","last_became_active":"2023-09-23T22:35:51.476467+0000","last_became_peered":"2023-09-23T22:35:51.476467+0000","last_unstale":"2023-09-23T22:38:05.144233+0000","last_undegraded":"2023-09-23T22:38:05.144233+0000","last_fullsized":"2023-09-23T22:38:05.144233+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:24.234310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142041+0000","last_change":"2023-09-23T22:37:35.271043+0000","last_active":"2023-09-23T22:38:05.142041+0000","last_peered":"2023-09-23T22:38:05.142041+0000","last_clean":"2023-09-23T22:38:05.142041+0000","last_became_active":"2023-09-23T22:37:35.270808+0000","last_became_peered":"2023-09-23T22:37:35.270808+0000","last_unstale":"2023-09-23T22:38:05.142041+0000","last_undegraded":"2023-09-23T22:38:05.142041+0000","last_fullsized":"2023-09-23T22:38:05.142041+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:00.218174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141189+0000","last_change":"2023-09-23T22:35:51.476177+0000","last_active":"2023-09-23T22:38:05.141189+0000","last_peered":"2023-09-23T22:38:05.141189+0000","last_clean":"2023-09-23T22:38:05.141189+0000","last_became_active":"2023-09-23T22:35:51.475644+0000","last_became_peered":"2023-09-23T22:35:51.475644+0000","last_unstale":"2023-09-23T22:38:05.141189+0000","last_undegraded":"2023-09-23T22:38:05.141189+0000","last_fullsized":"2023-09-23T22:38:05.141189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147016+0000","last_change":"2023-09-23T22:37:35.266620+0000","last_active":"2023-09-23T22:38:05.147016+0000","last_peered":"2023-09-23T22:38:05.147016+0000","last_clean":"2023-09-23T22:38:05.147016+0000","last_became_active":"2023-09-23T22:37:35.266391+0000","last_became_peered":"2023-09-23T22:37:35.266391+0000","last_unstale":"2023-09-23T22:38:05.147016+0000","last_undegraded":"2023-09-23T22:38:05.147016+0000","last_fullsized":"2023-09-23T22:38:05.147016+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:15.615607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150984+0000","last_change":"2023-09-23T22:38:06.150984+0000","last_active":"2023-09-23T22:38:06.150984+0000","last_peered":"2023-09-23T22:38:06.150984+0000","last_clean":"2023-09-23T22:38:06.150984+0000","last_became_active":"2023-09-23T22:38:06.150668+0000","last_became_peered":"2023-09-23T22:38:06.150668+0000","last_unstale":"2023-09-23T22:38:06.150984+0000","last_undegraded":"2023-09-23T22:38:06.150984+0000","last_fullsized":"2023-09-23T22:38:06.150984+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:25.323709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139228+0000","last_change":"2023-09-23T22:37:03.808191+0000","last_active":"2023-09-23T22:38:05.139228+0000","last_peered":"2023-09-23T22:38:05.139228+0000","last_clean":"2023-09-23T22:38:05.139228+0000","last_became_active":"2023-09-23T22:37:03.807779+0000","last_became_peered":"2023-09-23T22:37:03.807779+0000","last_unstale":"2023-09-23T22:38:05.139228+0000","last_undegraded":"2023-09-23T22:38:05.139228+0000","last_fullsized":"2023-09-23T22:38:05.139228+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136454+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:38:05.136454+0000","last_peered":"2023-09-23T22:38:05.136454+0000","last_clean":"2023-09-23T22:38:05.136454+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:38:05.136454+0000","last_undegraded":"2023-09-23T22:38:05.136454+0000","last_fullsized":"2023-09-23T22:38:05.136454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154384+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:38:06.154384+0000","last_peered":"2023-09-23T22:38:06.154384+0000","last_clean":"2023-09-23T22:38:06.154384+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:38:06.154384+0000","last_undegraded":"2023-09-23T22:38:06.154384+0000","last_fullsized":"2023-09-23T22:38:06.154384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":297,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139145+0000","last_change":"2023-09-23T22:35:51.477611+0000","last_active":"2023-09-23T22:38:05.139145+0000","last_peered":"2023-09-23T22:38:05.139145+0000","last_clean":"2023-09-23T22:38:05.139145+0000","last_became_active":"2023-09-23T22:35:51.477061+0000","last_became_peered":"2023-09-23T22:35:51.477061+0000","last_unstale":"2023-09-23T22:38:05.139145+0000","last_undegraded":"2023-09-23T22:38:05.139145+0000","last_fullsized":"2023-09-23T22:38:05.139145+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133688+0000","last_change":"2023-09-23T22:37:35.270450+0000","last_active":"2023-09-23T22:38:05.133688+0000","last_peered":"2023-09-23T22:38:05.133688+0000","last_clean":"2023-09-23T22:38:05.133688+0000","last_became_active":"2023-09-23T22:37:35.270097+0000","last_became_peered":"2023-09-23T22:37:35.270097+0000","last_unstale":"2023-09-23T22:38:05.133688+0000","last_undegraded":"2023-09-23T22:38:05.133688+0000","last_fullsized":"2023-09-23T22:38:05.133688+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140469+0000","last_change":"2023-09-23T22:37:03.801645+0000","last_active":"2023-09-23T22:38:05.140469+0000","last_peered":"2023-09-23T22:38:05.140469+0000","last_clean":"2023-09-23T22:38:05.140469+0000","last_became_active":"2023-09-23T22:37:03.799407+0000","last_became_peered":"2023-09-23T22:37:03.799407+0000","last_unstale":"2023-09-23T22:38:05.140469+0000","last_undegraded":"2023-09-23T22:38:05.140469+0000","last_fullsized":"2023-09-23T22:38:05.140469+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.396991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141322+0000","last_change":"2023-09-23T22:37:35.271109+0000","last_active":"2023-09-23T22:38:05.141322+0000","last_peered":"2023-09-23T22:38:05.141322+0000","last_clean":"2023-09-23T22:38:05.141322+0000","last_became_active":"2023-09-23T22:37:35.270850+0000","last_became_peered":"2023-09-23T22:37:35.270850+0000","last_unstale":"2023-09-23T22:38:05.141322+0000","last_undegraded":"2023-09-23T22:38:05.141322+0000","last_fullsized":"2023-09-23T22:38:05.141322+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:24.045725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148923+0000","last_change":"2023-09-23T22:38:06.148923+0000","last_active":"2023-09-23T22:38:06.148923+0000","last_peered":"2023-09-23T22:38:06.148923+0000","last_clean":"2023-09-23T22:38:06.148923+0000","last_became_active":"2023-09-23T22:38:06.148636+0000","last_became_peered":"2023-09-23T22:38:06.148636+0000","last_unstale":"2023-09-23T22:38:06.148923+0000","last_undegraded":"2023-09-23T22:38:06.148923+0000","last_fullsized":"2023-09-23T22:38:06.148923+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:01.486987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156455+0000","last_change":"2023-09-23T22:38:06.156455+0000","last_active":"2023-09-23T22:38:06.156455+0000","last_peered":"2023-09-23T22:38:06.156455+0000","last_clean":"2023-09-23T22:38:06.156455+0000","last_became_active":"2023-09-23T22:38:06.156203+0000","last_became_peered":"2023-09-23T22:38:06.156203+0000","last_unstale":"2023-09-23T22:38:06.156455+0000","last_undegraded":"2023-09-23T22:38:06.156455+0000","last_fullsized":"2023-09-23T22:38:06.156455+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142051+0000","last_change":"2023-09-23T22:35:51.474457+0000","last_active":"2023-09-23T22:38:05.142051+0000","last_peered":"2023-09-23T22:38:05.142051+0000","last_clean":"2023-09-23T22:38:05.142051+0000","last_became_active":"2023-09-23T22:35:51.474166+0000","last_became_peered":"2023-09-23T22:35:51.474166+0000","last_unstale":"2023-09-23T22:38:05.142051+0000","last_undegraded":"2023-09-23T22:38:05.142051+0000","last_fullsized":"2023-09-23T22:38:05.142051+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133761+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:38:05.133761+0000","last_peered":"2023-09-23T22:38:05.133761+0000","last_clean":"2023-09-23T22:38:05.133761+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:38:05.133761+0000","last_undegraded":"2023-09-23T22:38:05.133761+0000","last_fullsized":"2023-09-23T22:38:05.133761+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139384+0000","last_change":"2023-09-23T22:37:03.797729+0000","last_active":"2023-09-23T22:38:05.139384+0000","last_peered":"2023-09-23T22:38:05.139384+0000","last_clean":"2023-09-23T22:38:05.139384+0000","last_became_active":"2023-09-23T22:37:03.797377+0000","last_became_peered":"2023-09-23T22:37:03.797377+0000","last_unstale":"2023-09-23T22:38:05.139384+0000","last_undegraded":"2023-09-23T22:38:05.139384+0000","last_fullsized":"2023-09-23T22:38:05.139384+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:18.065155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157484+0000","last_change":"2023-09-23T22:38:06.157484+0000","last_active":"2023-09-23T22:38:06.157484+0000","last_peered":"2023-09-23T22:38:06.157484+0000","last_clean":"2023-09-23T22:38:06.157484+0000","last_became_active":"2023-09-23T22:38:06.157175+0000","last_became_peered":"2023-09-23T22:38:06.157175+0000","last_unstale":"2023-09-23T22:38:06.157484+0000","last_undegraded":"2023-09-23T22:38:06.157484+0000","last_fullsized":"2023-09-23T22:38:06.157484+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142777+0000","last_change":"2023-09-23T22:37:03.805141+0000","last_active":"2023-09-23T22:38:05.142777+0000","last_peered":"2023-09-23T22:38:05.142777+0000","last_clean":"2023-09-23T22:38:05.142777+0000","last_became_active":"2023-09-23T22:37:03.804606+0000","last_became_peered":"2023-09-23T22:37:03.804606+0000","last_unstale":"2023-09-23T22:38:05.142777+0000","last_undegraded":"2023-09-23T22:38:05.142777+0000","last_fullsized":"2023-09-23T22:38:05.142777+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:28.826826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153406+0000","last_change":"2023-09-23T22:38:06.153406+0000","last_active":"2023-09-23T22:38:06.153406+0000","last_peered":"2023-09-23T22:38:06.153406+0000","last_clean":"2023-09-23T22:38:06.153406+0000","last_became_active":"2023-09-23T22:38:06.153092+0000","last_became_peered":"2023-09-23T22:38:06.153092+0000","last_unstale":"2023-09-23T22:38:06.153406+0000","last_undegraded":"2023-09-23T22:38:06.153406+0000","last_fullsized":"2023-09-23T22:38:06.153406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:53.462379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155404+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:38:06.155404+0000","last_peered":"2023-09-23T22:38:06.155404+0000","last_clean":"2023-09-23T22:38:06.155404+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:38:06.155404+0000","last_undegraded":"2023-09-23T22:38:06.155404+0000","last_fullsized":"2023-09-23T22:38:06.155404+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142766+0000","last_change":"2023-09-23T22:37:03.800899+0000","last_active":"2023-09-23T22:38:05.142766+0000","last_peered":"2023-09-23T22:38:05.142766+0000","last_clean":"2023-09-23T22:38:05.142766+0000","last_became_active":"2023-09-23T22:37:03.800586+0000","last_became_peered":"2023-09-23T22:37:03.800586+0000","last_unstale":"2023-09-23T22:38:05.142766+0000","last_undegraded":"2023-09-23T22:38:05.142766+0000","last_fullsized":"2023-09-23T22:38:05.142766+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140171+0000","last_change":"2023-09-23T22:37:35.274945+0000","last_active":"2023-09-23T22:38:05.140171+0000","last_peered":"2023-09-23T22:38:05.140171+0000","last_clean":"2023-09-23T22:38:05.140171+0000","last_became_active":"2023-09-23T22:37:35.274580+0000","last_became_peered":"2023-09-23T22:37:35.274580+0000","last_unstale":"2023-09-23T22:38:05.140171+0000","last_undegraded":"2023-09-23T22:38:05.140171+0000","last_fullsized":"2023-09-23T22:38:05.140171+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141735+0000","last_change":"2023-09-23T22:37:35.276836+0000","last_active":"2023-09-23T22:38:05.141735+0000","last_peered":"2023-09-23T22:38:05.141735+0000","last_clean":"2023-09-23T22:38:05.141735+0000","last_became_active":"2023-09-23T22:37:35.276422+0000","last_became_peered":"2023-09-23T22:37:35.276422+0000","last_unstale":"2023-09-23T22:38:05.141735+0000","last_undegraded":"2023-09-23T22:38:05.141735+0000","last_fullsized":"2023-09-23T22:38:05.141735+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137068+0000","last_change":"2023-09-23T22:37:03.804557+0000","last_active":"2023-09-23T22:38:05.137068+0000","last_peered":"2023-09-23T22:38:05.137068+0000","last_clean":"2023-09-23T22:38:05.137068+0000","last_became_active":"2023-09-23T22:37:03.804229+0000","last_became_peered":"2023-09-23T22:37:03.804229+0000","last_unstale":"2023-09-23T22:38:05.137068+0000","last_undegraded":"2023-09-23T22:38:05.137068+0000","last_fullsized":"2023-09-23T22:38:05.137068+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152590+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:38:06.152590+0000","last_peered":"2023-09-23T22:38:06.152590+0000","last_clean":"2023-09-23T22:38:06.152590+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:38:06.152590+0000","last_undegraded":"2023-09-23T22:38:06.152590+0000","last_fullsized":"2023-09-23T22:38:06.152590+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151577+0000","last_change":"2023-09-23T22:38:06.151577+0000","last_active":"2023-09-23T22:38:06.151577+0000","last_peered":"2023-09-23T22:38:06.151577+0000","last_clean":"2023-09-23T22:38:06.151577+0000","last_became_active":"2023-09-23T22:38:06.151229+0000","last_became_peered":"2023-09-23T22:38:06.151229+0000","last_unstale":"2023-09-23T22:38:06.151577+0000","last_undegraded":"2023-09-23T22:38:06.151577+0000","last_fullsized":"2023-09-23T22:38:06.151577+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:25.864388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144329+0000","last_change":"2023-09-23T22:37:03.800764+0000","last_active":"2023-09-23T22:38:05.144329+0000","last_peered":"2023-09-23T22:38:05.144329+0000","last_clean":"2023-09-23T22:38:05.144329+0000","last_became_active":"2023-09-23T22:37:03.800407+0000","last_became_peered":"2023-09-23T22:37:03.800407+0000","last_unstale":"2023-09-23T22:38:05.144329+0000","last_undegraded":"2023-09-23T22:38:05.144329+0000","last_fullsized":"2023-09-23T22:38:05.144329+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:10.143602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155895+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:38:06.155895+0000","last_peered":"2023-09-23T22:38:06.155895+0000","last_clean":"2023-09-23T22:38:06.155895+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:38:06.155895+0000","last_undegraded":"2023-09-23T22:38:06.155895+0000","last_fullsized":"2023-09-23T22:38:06.155895+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136911+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:38:05.136911+0000","last_peered":"2023-09-23T22:38:05.136911+0000","last_clean":"2023-09-23T22:38:05.136911+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:38:05.136911+0000","last_undegraded":"2023-09-23T22:38:05.136911+0000","last_fullsized":"2023-09-23T22:38:05.136911+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141209+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:38:05.141209+0000","last_peered":"2023-09-23T22:38:05.141209+0000","last_clean":"2023-09-23T22:38:05.141209+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:38:05.141209+0000","last_undegraded":"2023-09-23T22:38:05.141209+0000","last_fullsized":"2023-09-23T22:38:05.141209+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154190+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:38:06.154190+0000","last_peered":"2023-09-23T22:38:06.154190+0000","last_clean":"2023-09-23T22:38:06.154190+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:38:06.154190+0000","last_undegraded":"2023-09-23T22:38:06.154190+0000","last_fullsized":"2023-09-23T22:38:06.154190+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.138291+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:38:05.138291+0000","last_peered":"2023-09-23T22:38:05.138291+0000","last_clean":"2023-09-23T22:38:05.138291+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:38:05.138291+0000","last_undegraded":"2023-09-23T22:38:05.138291+0000","last_fullsized":"2023-09-23T22:38:05.138291+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134242+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:38:05.134242+0000","last_peered":"2023-09-23T22:38:05.134242+0000","last_clean":"2023-09-23T22:38:05.134242+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:38:05.134242+0000","last_undegraded":"2023-09-23T22:38:05.134242+0000","last_fullsized":"2023-09-23T22:38:05.134242+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139635+0000","last_change":"2023-09-23T22:35:51.479873+0000","last_active":"2023-09-23T22:38:05.139635+0000","last_peered":"2023-09-23T22:38:05.139635+0000","last_clean":"2023-09-23T22:38:05.139635+0000","last_became_active":"2023-09-23T22:35:51.479506+0000","last_became_peered":"2023-09-23T22:35:51.479506+0000","last_unstale":"2023-09-23T22:38:05.139635+0000","last_undegraded":"2023-09-23T22:38:05.139635+0000","last_fullsized":"2023-09-23T22:38:05.139635+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154965+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:38:06.154965+0000","last_peered":"2023-09-23T22:38:06.154965+0000","last_clean":"2023-09-23T22:38:06.154965+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:38:06.154965+0000","last_undegraded":"2023-09-23T22:38:06.154965+0000","last_fullsized":"2023-09-23T22:38:06.154965+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":70,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136898+0000","last_change":"2023-09-23T22:35:51.475694+0000","last_active":"2023-09-23T22:38:05.136898+0000","last_peered":"2023-09-23T22:38:05.136898+0000","last_clean":"2023-09-23T22:38:05.136898+0000","last_became_active":"2023-09-23T22:35:51.475326+0000","last_became_peered":"2023-09-23T22:35:51.475326+0000","last_unstale":"2023-09-23T22:38:05.136898+0000","last_undegraded":"2023-09-23T22:38:05.136898+0000","last_fullsized":"2023-09-23T22:38:05.136898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156874+0000","last_change":"2023-09-23T22:38:06.156215+0000","last_active":"2023-09-23T22:38:06.156874+0000","last_peered":"2023-09-23T22:38:06.156874+0000","last_clean":"2023-09-23T22:38:06.156874+0000","last_became_active":"2023-09-23T22:38:06.155344+0000","last_became_peered":"2023-09-23T22:38:06.155344+0000","last_unstale":"2023-09-23T22:38:06.156874+0000","last_undegraded":"2023-09-23T22:38:06.156874+0000","last_fullsized":"2023-09-23T22:38:06.156874+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.252155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137434+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:38:05.137434+0000","last_peered":"2023-09-23T22:38:05.137434+0000","last_clean":"2023-09-23T22:38:05.137434+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:38:05.137434+0000","last_undegraded":"2023-09-23T22:38:05.137434+0000","last_fullsized":"2023-09-23T22:38:05.137434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157553+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:38:06.157553+0000","last_peered":"2023-09-23T22:38:06.157553+0000","last_clean":"2023-09-23T22:38:06.157553+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:38:06.157553+0000","last_undegraded":"2023-09-23T22:38:06.157553+0000","last_fullsized":"2023-09-23T22:38:06.157553+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152547+0000","last_change":"2023-09-23T22:38:06.151401+0000","last_active":"2023-09-23T22:38:06.152547+0000","last_peered":"2023-09-23T22:38:06.152547+0000","last_clean":"2023-09-23T22:38:06.152547+0000","last_became_active":"2023-09-23T22:38:06.151071+0000","last_became_peered":"2023-09-23T22:38:06.151071+0000","last_unstale":"2023-09-23T22:38:06.152547+0000","last_undegraded":"2023-09-23T22:38:06.152547+0000","last_fullsized":"2023-09-23T22:38:06.152547+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:14.260144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154184+0000","last_change":"2023-09-23T22:38:06.154184+0000","last_active":"2023-09-23T22:38:06.154184+0000","last_peered":"2023-09-23T22:38:06.154184+0000","last_clean":"2023-09-23T22:38:06.154184+0000","last_became_active":"2023-09-23T22:38:06.153914+0000","last_became_peered":"2023-09-23T22:38:06.153914+0000","last_unstale":"2023-09-23T22:38:06.154184+0000","last_undegraded":"2023-09-23T22:38:06.154184+0000","last_fullsized":"2023-09-23T22:38:06.154184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:53.068518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137833+0000","last_change":"2023-09-23T22:37:35.268294+0000","last_active":"2023-09-23T22:38:05.137833+0000","last_peered":"2023-09-23T22:38:05.137833+0000","last_clean":"2023-09-23T22:38:05.137833+0000","last_became_active":"2023-09-23T22:37:35.267886+0000","last_became_peered":"2023-09-23T22:37:35.267886+0000","last_unstale":"2023-09-23T22:38:05.137833+0000","last_undegraded":"2023-09-23T22:38:05.137833+0000","last_fullsized":"2023-09-23T22:38:05.137833+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141916+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:38:05.141916+0000","last_peered":"2023-09-23T22:38:05.141916+0000","last_clean":"2023-09-23T22:38:05.141916+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:38:05.141916+0000","last_undegraded":"2023-09-23T22:38:05.141916+0000","last_fullsized":"2023-09-23T22:38:05.141916+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141885+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:38:05.141885+0000","last_peered":"2023-09-23T22:38:05.141885+0000","last_clean":"2023-09-23T22:38:05.141885+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:38:05.141885+0000","last_undegraded":"2023-09-23T22:38:05.141885+0000","last_fullsized":"2023-09-23T22:38:05.141885+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139285+0000","last_change":"2023-09-23T22:37:35.272454+0000","last_active":"2023-09-23T22:38:05.139285+0000","last_peered":"2023-09-23T22:38:05.139285+0000","last_clean":"2023-09-23T22:38:05.139285+0000","last_became_active":"2023-09-23T22:37:35.271917+0000","last_became_peered":"2023-09-23T22:37:35.271917+0000","last_unstale":"2023-09-23T22:38:05.139285+0000","last_undegraded":"2023-09-23T22:38:05.139285+0000","last_fullsized":"2023-09-23T22:38:05.139285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:42.398917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157765+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:38:06.157765+0000","last_peered":"2023-09-23T22:38:06.157765+0000","last_clean":"2023-09-23T22:38:06.157765+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:38:06.157765+0000","last_undegraded":"2023-09-23T22:38:06.157765+0000","last_fullsized":"2023-09-23T22:38:06.157765+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151474+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:38:06.151474+0000","last_peered":"2023-09-23T22:38:06.151474+0000","last_clean":"2023-09-23T22:38:06.151474+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:38:06.151474+0000","last_undegraded":"2023-09-23T22:38:06.151474+0000","last_fullsized":"2023-09-23T22:38:06.151474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134905+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:38:05.134905+0000","last_peered":"2023-09-23T22:38:05.134905+0000","last_clean":"2023-09-23T22:38:05.134905+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:38:05.134905+0000","last_undegraded":"2023-09-23T22:38:05.134905+0000","last_fullsized":"2023-09-23T22:38:05.134905+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148885+0000","last_change":"2023-09-23T22:38:06.148885+0000","last_active":"2023-09-23T22:38:06.148885+0000","last_peered":"2023-09-23T22:38:06.148885+0000","last_clean":"2023-09-23T22:38:06.148885+0000","last_became_active":"2023-09-23T22:38:06.148470+0000","last_became_peered":"2023-09-23T22:38:06.148470+0000","last_unstale":"2023-09-23T22:38:06.148885+0000","last_undegraded":"2023-09-23T22:38:06.148885+0000","last_fullsized":"2023-09-23T22:38:06.148885+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:02.793510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153759+0000","last_change":"2023-09-23T22:38:06.153487+0000","last_active":"2023-09-23T22:38:06.153759+0000","last_peered":"2023-09-23T22:38:06.153759+0000","last_clean":"2023-09-23T22:38:06.153759+0000","last_became_active":"2023-09-23T22:38:06.153156+0000","last_became_peered":"2023-09-23T22:38:06.153156+0000","last_unstale":"2023-09-23T22:38:06.153759+0000","last_undegraded":"2023-09-23T22:38:06.153759+0000","last_fullsized":"2023-09-23T22:38:06.153759+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:49.053379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136355+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:38:05.136355+0000","last_peered":"2023-09-23T22:38:05.136355+0000","last_clean":"2023-09-23T22:38:05.136355+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:38:05.136355+0000","last_undegraded":"2023-09-23T22:38:05.136355+0000","last_fullsized":"2023-09-23T22:38:05.136355+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148692+0000","last_change":"2023-09-23T22:38:06.148692+0000","last_active":"2023-09-23T22:38:06.148692+0000","last_peered":"2023-09-23T22:38:06.148692+0000","last_clean":"2023-09-23T22:38:06.148692+0000","last_became_active":"2023-09-23T22:38:06.148096+0000","last_became_peered":"2023-09-23T22:38:06.148096+0000","last_unstale":"2023-09-23T22:38:06.148692+0000","last_undegraded":"2023-09-23T22:38:06.148692+0000","last_fullsized":"2023-09-23T22:38:06.148692+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:15.042597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156081+0000","last_change":"2023-09-23T22:38:06.156081+0000","last_active":"2023-09-23T22:38:06.156081+0000","last_peered":"2023-09-23T22:38:06.156081+0000","last_clean":"2023-09-23T22:38:06.156081+0000","last_became_active":"2023-09-23T22:38:06.155768+0000","last_became_peered":"2023-09-23T22:38:06.155768+0000","last_unstale":"2023-09-23T22:38:06.156081+0000","last_undegraded":"2023-09-23T22:38:06.156081+0000","last_fullsized":"2023-09-23T22:38:06.156081+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:21.557265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157097+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:38:06.157097+0000","last_peered":"2023-09-23T22:38:06.157097+0000","last_clean":"2023-09-23T22:38:06.157097+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:38:06.157097+0000","last_undegraded":"2023-09-23T22:38:06.157097+0000","last_fullsized":"2023-09-23T22:38:06.157097+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150667+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:38:06.150667+0000","last_peered":"2023-09-23T22:38:06.150667+0000","last_clean":"2023-09-23T22:38:06.150667+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:38:06.150667+0000","last_undegraded":"2023-09-23T22:38:06.150667+0000","last_fullsized":"2023-09-23T22:38:06.150667+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141602+0000","last_change":"2023-09-23T22:37:35.275167+0000","last_active":"2023-09-23T22:38:05.141602+0000","last_peered":"2023-09-23T22:38:05.141602+0000","last_clean":"2023-09-23T22:38:05.141602+0000","last_became_active":"2023-09-23T22:37:35.274809+0000","last_became_peered":"2023-09-23T22:37:35.274809+0000","last_unstale":"2023-09-23T22:38:05.141602+0000","last_undegraded":"2023-09-23T22:38:05.141602+0000","last_fullsized":"2023-09-23T22:38:05.141602+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430350,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.72899999999999998,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.61399999999999999,"15min":0.62},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.78300000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.56499999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.48899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60499999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:37:28 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58399999999999996,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.32400000000000001},"max":{"1min":0.79600000000000004,"5min":0.85399999999999998,"15min":1.4970000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.48499999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81200000000000006,"5min":0.81299999999999994,"15min":1.087},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47899999999999998,"15min":0.498},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.26300000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":5.3140000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.51800000000000002,"15min":0.502},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.93799999999999994},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.80600000000000005,"5min":0.85799999999999998,"15min":1.0669999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57099999999999995,"15min":0.58299999999999996},"min":{"1min":0.42399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":5.3490000000000002},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56699999999999995,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.95799999999999996},"last":0.628},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61699999999999999,"15min":0.59199999999999997},"min":{"1min":0.504,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":1.038},"last":0.64900000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626578,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1560576,"data_stored":1111799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.59199999999999997,"15min":0.60499999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72999999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.60199999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.627}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.68799999999999994}]},{"osd":2,"last update":"Sat Sep 23 22:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:37:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":0.81100000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.34300000000000003},"max":{"1min":0.79800000000000004,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.66400000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51300000000000001,"15min":0.46700000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53600000000000003,"15min":0.47899999999999998},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":0.748,"5min":0.80600000000000005,"15min":0.81299999999999994},"last":0.40000000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54100000000000004,"15min":0.497},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":0.755,"5min":0.77300000000000002,"15min":0.84899999999999998},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57799999999999996,"15min":0.52500000000000002},"min":{"1min":0.38200000000000001,"5min":0.377,"15min":0.32400000000000001},"max":{"1min":0.83899999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.59099999999999997,"15min":0.55000000000000004},"min":{"1min":0.46000000000000002,"5min":0.41799999999999998,"15min":0.34799999999999998},"max":{"1min":0.76000000000000001,"5min":0.81999999999999995,"15min":0.83199999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.29399999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":1.357},"last":0.433}]}]},{"osd":1,"up_from":84,"seq":360777252888,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32412,"kb_used_data":980,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712740,"statfs":{"total":95995035648,"available":95961845760,"internally_reserved":0,"allocated":1003520,"data_stored":527042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.27100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.38400000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.629}]},{"osd":5,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.65000000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416422,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33068,"kb_used_data":1572,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712084,"statfs":{"total":95995035648,"available":95961174016,"internally_reserved":0,"allocated":1609728,"data_stored":1117239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.317},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.379},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.46300000000000002,"15min":0.47099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.46500000000000002,"15min":0.47399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.625,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.67300000000000004,"15min":0.68100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.66800000000000004,"15min":0.67400000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.624}]},{"osd":5,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.628,"15min":0.63600000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.66000000000000003,"15min":0.66800000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.59899999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.66300000000000003,"15min":0.67300000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.66300000000000003,"15min":0.67500000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.74199999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.66800000000000004,"15min":0.67500000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.70199999999999996,"15min":0.71699999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.66100000000000003}]}]},{"osd":2,"up_from":89,"seq":382252089362,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36628,"kb_used_data":972,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708524,"statfs":{"total":95995035648,"available":95957528576,"internally_reserved":0,"allocated":995328,"data_stored":524245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.438},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.41199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.69799999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.73599999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925835,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":521160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28599999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41799999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019034,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.79200000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.66500000000000004,"15min":0.67100000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80600000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.58199999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.63600000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.36299999999999999},"max":{"1min":0.81699999999999995,"5min":0.90500000000000003,"15min":1.482},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":0.85999999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:37:56 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.505,"15min":0.46500000000000002},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":0.69499999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54300000000000004,"15min":0.48399999999999999},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":0.72099999999999997,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57299999999999995,"15min":0.5},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76400000000000001,"5min":0.875,"15min":0.875},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.52100000000000002},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.79200000000000004,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.60299999999999998,"15min":0.55200000000000005},"min":{"1min":0.41999999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.754,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59999999999999998,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.71699999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822806,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306216,"kb_used_data":928,"kb_used_omap":0,"kb_used_meta":305280,"kb_avail":93438936,"statfs":{"total":95995035648,"available":95681470464,"internally_reserved":0,"allocated":950272,"data_stored":516621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312606720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.58399999999999996,"15min":0.59599999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.66100000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.625,"15min":0.63600000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.501},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59299999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.34999999999999998},"max":{"1min":0.68400000000000005,"5min":0.79000000000000004,"15min":1.278},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.64000000000000001,"15min":0.61899999999999999},"min":{"1min":0.51700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.3109999999999999},"last":0.61599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.48799999999999999,"15min":0.45800000000000002},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.50800000000000001,"5min":0.81299999999999994,"15min":0.89000000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.50700000000000001,"15min":0.47399999999999998},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.27600000000000002},"max":{"1min":0.72099999999999997,"5min":0.73099999999999998,"15min":0.88300000000000001},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.48599999999999999},"min":{"1min":0.35899999999999999,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":0.69499999999999995,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56599999999999995,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":0.79200000000000004,"5min":0.84599999999999997,"15min":0.90300000000000002},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.59099999999999997,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.89500000000000002},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59399999999999997,"15min":0.55400000000000005},"min":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":0.77000000000000002,"5min":0.78400000000000003,"15min":0.85199999999999998},"last":0.63100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:38:54.327 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:38:54.728 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:55.171 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:54 smithi052 ceph-mon[187743]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:55.172 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:54 smithi052 ceph-mon[187743]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:38:55.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:54 smithi052 ceph-mon[180979]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:55.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:54 smithi052 ceph-mon[180979]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:38:55.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:54 smithi151 ceph-mon[159210]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:55.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:54 smithi151 ceph-mon[159210]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:38:56.438 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:38:56.439 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:38:57.055 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":206,"stamp":"2023-09-23T22:38:55.474718+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348296,"kb_used_data":9416,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612920,"statfs":{"total":767960285184,"available":766579630080,"internally_reserved":0,"allocated":9641984,"data_stored":5951558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002650"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140027+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:38:05.140027+0000","last_peered":"2023-09-23T22:38:05.140027+0000","last_clean":"2023-09-23T22:38:05.140027+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:38:05.140027+0000","last_undegraded":"2023-09-23T22:38:05.140027+0000","last_fullsized":"2023-09-23T22:38:05.140027+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147480+0000","last_change":"2023-09-23T22:37:35.270139+0000","last_active":"2023-09-23T22:38:05.147480+0000","last_peered":"2023-09-23T22:38:05.147480+0000","last_clean":"2023-09-23T22:38:05.147480+0000","last_became_active":"2023-09-23T22:37:35.269761+0000","last_became_peered":"2023-09-23T22:37:35.269761+0000","last_unstale":"2023-09-23T22:38:05.147480+0000","last_undegraded":"2023-09-23T22:38:05.147480+0000","last_fullsized":"2023-09-23T22:38:05.147480+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:08.396803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.147696+0000","last_change":"2023-09-23T22:38:06.147696+0000","last_active":"2023-09-23T22:38:06.147696+0000","last_peered":"2023-09-23T22:38:06.147696+0000","last_clean":"2023-09-23T22:38:06.147696+0000","last_became_active":"2023-09-23T22:38:06.147382+0000","last_became_peered":"2023-09-23T22:38:06.147382+0000","last_unstale":"2023-09-23T22:38:06.147696+0000","last_undegraded":"2023-09-23T22:38:06.147696+0000","last_fullsized":"2023-09-23T22:38:06.147696+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:04.570885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.149889+0000","last_change":"2023-09-23T22:38:06.149889+0000","last_active":"2023-09-23T22:38:06.149889+0000","last_peered":"2023-09-23T22:38:06.149889+0000","last_clean":"2023-09-23T22:38:06.149889+0000","last_became_active":"2023-09-23T22:38:06.149711+0000","last_became_peered":"2023-09-23T22:38:06.149711+0000","last_unstale":"2023-09-23T22:38:06.149889+0000","last_undegraded":"2023-09-23T22:38:06.149889+0000","last_fullsized":"2023-09-23T22:38:06.149889+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:06.335320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151879+0000","last_change":"2023-09-23T22:38:06.151879+0000","last_active":"2023-09-23T22:38:06.151879+0000","last_peered":"2023-09-23T22:38:06.151879+0000","last_clean":"2023-09-23T22:38:06.151879+0000","last_became_active":"2023-09-23T22:38:06.151519+0000","last_became_peered":"2023-09-23T22:38:06.151519+0000","last_unstale":"2023-09-23T22:38:06.151879+0000","last_undegraded":"2023-09-23T22:38:06.151879+0000","last_fullsized":"2023-09-23T22:38:06.151879+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.135034+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:38:05.135034+0000","last_peered":"2023-09-23T22:38:05.135034+0000","last_clean":"2023-09-23T22:38:05.135034+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:38:05.135034+0000","last_undegraded":"2023-09-23T22:38:05.135034+0000","last_fullsized":"2023-09-23T22:38:05.135034+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153986+0000","last_change":"2023-09-23T22:38:06.153986+0000","last_active":"2023-09-23T22:38:06.153986+0000","last_peered":"2023-09-23T22:38:06.153986+0000","last_clean":"2023-09-23T22:38:06.153986+0000","last_became_active":"2023-09-23T22:38:06.153604+0000","last_became_peered":"2023-09-23T22:38:06.153604+0000","last_unstale":"2023-09-23T22:38:06.153986+0000","last_undegraded":"2023-09-23T22:38:06.153986+0000","last_fullsized":"2023-09-23T22:38:06.153986+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134780+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:38:05.134780+0000","last_peered":"2023-09-23T22:38:05.134780+0000","last_clean":"2023-09-23T22:38:05.134780+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:38:05.134780+0000","last_undegraded":"2023-09-23T22:38:05.134780+0000","last_fullsized":"2023-09-23T22:38:05.134780+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152423+0000","last_change":"2023-09-23T22:38:06.152423+0000","last_active":"2023-09-23T22:38:06.152423+0000","last_peered":"2023-09-23T22:38:06.152423+0000","last_clean":"2023-09-23T22:38:06.152423+0000","last_became_active":"2023-09-23T22:38:06.152083+0000","last_became_peered":"2023-09-23T22:38:06.152083+0000","last_unstale":"2023-09-23T22:38:06.152423+0000","last_undegraded":"2023-09-23T22:38:06.152423+0000","last_fullsized":"2023-09-23T22:38:06.152423+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:49.106880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.144944+0000","last_change":"2023-09-23T22:35:51.475988+0000","last_active":"2023-09-23T22:38:06.144944+0000","last_peered":"2023-09-23T22:38:06.144944+0000","last_clean":"2023-09-23T22:38:06.144944+0000","last_became_active":"2023-09-23T22:35:51.475664+0000","last_became_peered":"2023-09-23T22:35:51.475664+0000","last_unstale":"2023-09-23T22:38:06.144944+0000","last_undegraded":"2023-09-23T22:38:06.144944+0000","last_fullsized":"2023-09-23T22:38:06.144944+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:22.107425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140271+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:38:05.140271+0000","last_peered":"2023-09-23T22:38:05.140271+0000","last_clean":"2023-09-23T22:38:05.140271+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:38:05.140271+0000","last_undegraded":"2023-09-23T22:38:05.140271+0000","last_fullsized":"2023-09-23T22:38:05.140271+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.146941+0000","last_change":"2023-09-23T22:37:03.801120+0000","last_active":"2023-09-23T22:38:05.146941+0000","last_peered":"2023-09-23T22:38:05.146941+0000","last_clean":"2023-09-23T22:38:05.146941+0000","last_became_active":"2023-09-23T22:37:03.800596+0000","last_became_peered":"2023-09-23T22:37:03.800596+0000","last_unstale":"2023-09-23T22:38:05.146941+0000","last_undegraded":"2023-09-23T22:38:05.146941+0000","last_fullsized":"2023-09-23T22:38:05.146941+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:03.424170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136253+0000","last_change":"2023-09-23T22:37:35.276863+0000","last_active":"2023-09-23T22:38:05.136253+0000","last_peered":"2023-09-23T22:38:05.136253+0000","last_clean":"2023-09-23T22:38:05.136253+0000","last_became_active":"2023-09-23T22:37:35.276428+0000","last_became_peered":"2023-09-23T22:37:35.276428+0000","last_unstale":"2023-09-23T22:38:05.136253+0000","last_undegraded":"2023-09-23T22:38:05.136253+0000","last_fullsized":"2023-09-23T22:38:05.136253+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155276+0000","last_change":"2023-09-23T22:38:06.155276+0000","last_active":"2023-09-23T22:38:06.155276+0000","last_peered":"2023-09-23T22:38:06.155276+0000","last_clean":"2023-09-23T22:38:06.155276+0000","last_became_active":"2023-09-23T22:38:06.154550+0000","last_became_peered":"2023-09-23T22:38:06.154550+0000","last_unstale":"2023-09-23T22:38:06.155276+0000","last_undegraded":"2023-09-23T22:38:06.155276+0000","last_fullsized":"2023-09-23T22:38:06.155276+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:06.092371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140029+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:38:05.140029+0000","last_peered":"2023-09-23T22:38:05.140029+0000","last_clean":"2023-09-23T22:38:05.140029+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:38:05.140029+0000","last_undegraded":"2023-09-23T22:38:05.140029+0000","last_fullsized":"2023-09-23T22:38:05.140029+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155104+0000","last_change":"2023-09-23T22:38:06.155104+0000","last_active":"2023-09-23T22:38:06.155104+0000","last_peered":"2023-09-23T22:38:06.155104+0000","last_clean":"2023-09-23T22:38:06.155104+0000","last_became_active":"2023-09-23T22:38:06.154630+0000","last_became_peered":"2023-09-23T22:38:06.154630+0000","last_unstale":"2023-09-23T22:38:06.155104+0000","last_undegraded":"2023-09-23T22:38:06.155104+0000","last_fullsized":"2023-09-23T22:38:06.155104+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.232885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144233+0000","last_change":"2023-09-23T22:35:51.477103+0000","last_active":"2023-09-23T22:38:05.144233+0000","last_peered":"2023-09-23T22:38:05.144233+0000","last_clean":"2023-09-23T22:38:05.144233+0000","last_became_active":"2023-09-23T22:35:51.476467+0000","last_became_peered":"2023-09-23T22:35:51.476467+0000","last_unstale":"2023-09-23T22:38:05.144233+0000","last_undegraded":"2023-09-23T22:38:05.144233+0000","last_fullsized":"2023-09-23T22:38:05.144233+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:24.234310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142041+0000","last_change":"2023-09-23T22:37:35.271043+0000","last_active":"2023-09-23T22:38:05.142041+0000","last_peered":"2023-09-23T22:38:05.142041+0000","last_clean":"2023-09-23T22:38:05.142041+0000","last_became_active":"2023-09-23T22:37:35.270808+0000","last_became_peered":"2023-09-23T22:37:35.270808+0000","last_unstale":"2023-09-23T22:38:05.142041+0000","last_undegraded":"2023-09-23T22:38:05.142041+0000","last_fullsized":"2023-09-23T22:38:05.142041+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:00.218174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141189+0000","last_change":"2023-09-23T22:35:51.476177+0000","last_active":"2023-09-23T22:38:05.141189+0000","last_peered":"2023-09-23T22:38:05.141189+0000","last_clean":"2023-09-23T22:38:05.141189+0000","last_became_active":"2023-09-23T22:35:51.475644+0000","last_became_peered":"2023-09-23T22:35:51.475644+0000","last_unstale":"2023-09-23T22:38:05.141189+0000","last_undegraded":"2023-09-23T22:38:05.141189+0000","last_fullsized":"2023-09-23T22:38:05.141189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147016+0000","last_change":"2023-09-23T22:37:35.266620+0000","last_active":"2023-09-23T22:38:05.147016+0000","last_peered":"2023-09-23T22:38:05.147016+0000","last_clean":"2023-09-23T22:38:05.147016+0000","last_became_active":"2023-09-23T22:37:35.266391+0000","last_became_peered":"2023-09-23T22:37:35.266391+0000","last_unstale":"2023-09-23T22:38:05.147016+0000","last_undegraded":"2023-09-23T22:38:05.147016+0000","last_fullsized":"2023-09-23T22:38:05.147016+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:15.615607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150984+0000","last_change":"2023-09-23T22:38:06.150984+0000","last_active":"2023-09-23T22:38:06.150984+0000","last_peered":"2023-09-23T22:38:06.150984+0000","last_clean":"2023-09-23T22:38:06.150984+0000","last_became_active":"2023-09-23T22:38:06.150668+0000","last_became_peered":"2023-09-23T22:38:06.150668+0000","last_unstale":"2023-09-23T22:38:06.150984+0000","last_undegraded":"2023-09-23T22:38:06.150984+0000","last_fullsized":"2023-09-23T22:38:06.150984+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:25.323709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139228+0000","last_change":"2023-09-23T22:37:03.808191+0000","last_active":"2023-09-23T22:38:05.139228+0000","last_peered":"2023-09-23T22:38:05.139228+0000","last_clean":"2023-09-23T22:38:05.139228+0000","last_became_active":"2023-09-23T22:37:03.807779+0000","last_became_peered":"2023-09-23T22:37:03.807779+0000","last_unstale":"2023-09-23T22:38:05.139228+0000","last_undegraded":"2023-09-23T22:38:05.139228+0000","last_fullsized":"2023-09-23T22:38:05.139228+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136454+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:38:05.136454+0000","last_peered":"2023-09-23T22:38:05.136454+0000","last_clean":"2023-09-23T22:38:05.136454+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:38:05.136454+0000","last_undegraded":"2023-09-23T22:38:05.136454+0000","last_fullsized":"2023-09-23T22:38:05.136454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154384+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:38:06.154384+0000","last_peered":"2023-09-23T22:38:06.154384+0000","last_clean":"2023-09-23T22:38:06.154384+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:38:06.154384+0000","last_undegraded":"2023-09-23T22:38:06.154384+0000","last_fullsized":"2023-09-23T22:38:06.154384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":297,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139145+0000","last_change":"2023-09-23T22:35:51.477611+0000","last_active":"2023-09-23T22:38:05.139145+0000","last_peered":"2023-09-23T22:38:05.139145+0000","last_clean":"2023-09-23T22:38:05.139145+0000","last_became_active":"2023-09-23T22:35:51.477061+0000","last_became_peered":"2023-09-23T22:35:51.477061+0000","last_unstale":"2023-09-23T22:38:05.139145+0000","last_undegraded":"2023-09-23T22:38:05.139145+0000","last_fullsized":"2023-09-23T22:38:05.139145+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133688+0000","last_change":"2023-09-23T22:37:35.270450+0000","last_active":"2023-09-23T22:38:05.133688+0000","last_peered":"2023-09-23T22:38:05.133688+0000","last_clean":"2023-09-23T22:38:05.133688+0000","last_became_active":"2023-09-23T22:37:35.270097+0000","last_became_peered":"2023-09-23T22:37:35.270097+0000","last_unstale":"2023-09-23T22:38:05.133688+0000","last_undegraded":"2023-09-23T22:38:05.133688+0000","last_fullsized":"2023-09-23T22:38:05.133688+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140469+0000","last_change":"2023-09-23T22:37:03.801645+0000","last_active":"2023-09-23T22:38:05.140469+0000","last_peered":"2023-09-23T22:38:05.140469+0000","last_clean":"2023-09-23T22:38:05.140469+0000","last_became_active":"2023-09-23T22:37:03.799407+0000","last_became_peered":"2023-09-23T22:37:03.799407+0000","last_unstale":"2023-09-23T22:38:05.140469+0000","last_undegraded":"2023-09-23T22:38:05.140469+0000","last_fullsized":"2023-09-23T22:38:05.140469+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.396991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141322+0000","last_change":"2023-09-23T22:37:35.271109+0000","last_active":"2023-09-23T22:38:05.141322+0000","last_peered":"2023-09-23T22:38:05.141322+0000","last_clean":"2023-09-23T22:38:05.141322+0000","last_became_active":"2023-09-23T22:37:35.270850+0000","last_became_peered":"2023-09-23T22:37:35.270850+0000","last_unstale":"2023-09-23T22:38:05.141322+0000","last_undegraded":"2023-09-23T22:38:05.141322+0000","last_fullsized":"2023-09-23T22:38:05.141322+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:24.045725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148923+0000","last_change":"2023-09-23T22:38:06.148923+0000","last_active":"2023-09-23T22:38:06.148923+0000","last_peered":"2023-09-23T22:38:06.148923+0000","last_clean":"2023-09-23T22:38:06.148923+0000","last_became_active":"2023-09-23T22:38:06.148636+0000","last_became_peered":"2023-09-23T22:38:06.148636+0000","last_unstale":"2023-09-23T22:38:06.148923+0000","last_undegraded":"2023-09-23T22:38:06.148923+0000","last_fullsized":"2023-09-23T22:38:06.148923+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:01.486987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156455+0000","last_change":"2023-09-23T22:38:06.156455+0000","last_active":"2023-09-23T22:38:06.156455+0000","last_peered":"2023-09-23T22:38:06.156455+0000","last_clean":"2023-09-23T22:38:06.156455+0000","last_became_active":"2023-09-23T22:38:06.156203+0000","last_became_peered":"2023-09-23T22:38:06.156203+0000","last_unstale":"2023-09-23T22:38:06.156455+0000","last_undegraded":"2023-09-23T22:38:06.156455+0000","last_fullsized":"2023-09-23T22:38:06.156455+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142051+0000","last_change":"2023-09-23T22:35:51.474457+0000","last_active":"2023-09-23T22:38:05.142051+0000","last_peered":"2023-09-23T22:38:05.142051+0000","last_clean":"2023-09-23T22:38:05.142051+0000","last_became_active":"2023-09-23T22:35:51.474166+0000","last_became_peered":"2023-09-23T22:35:51.474166+0000","last_unstale":"2023-09-23T22:38:05.142051+0000","last_undegraded":"2023-09-23T22:38:05.142051+0000","last_fullsized":"2023-09-23T22:38:05.142051+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133761+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:38:05.133761+0000","last_peered":"2023-09-23T22:38:05.133761+0000","last_clean":"2023-09-23T22:38:05.133761+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:38:05.133761+0000","last_undegraded":"2023-09-23T22:38:05.133761+0000","last_fullsized":"2023-09-23T22:38:05.133761+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139384+0000","last_change":"2023-09-23T22:37:03.797729+0000","last_active":"2023-09-23T22:38:05.139384+0000","last_peered":"2023-09-23T22:38:05.139384+0000","last_clean":"2023-09-23T22:38:05.139384+0000","last_became_active":"2023-09-23T22:37:03.797377+0000","last_became_peered":"2023-09-23T22:37:03.797377+0000","last_unstale":"2023-09-23T22:38:05.139384+0000","last_undegraded":"2023-09-23T22:38:05.139384+0000","last_fullsized":"2023-09-23T22:38:05.139384+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:18.065155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157484+0000","last_change":"2023-09-23T22:38:06.157484+0000","last_active":"2023-09-23T22:38:06.157484+0000","last_peered":"2023-09-23T22:38:06.157484+0000","last_clean":"2023-09-23T22:38:06.157484+0000","last_became_active":"2023-09-23T22:38:06.157175+0000","last_became_peered":"2023-09-23T22:38:06.157175+0000","last_unstale":"2023-09-23T22:38:06.157484+0000","last_undegraded":"2023-09-23T22:38:06.157484+0000","last_fullsized":"2023-09-23T22:38:06.157484+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142777+0000","last_change":"2023-09-23T22:37:03.805141+0000","last_active":"2023-09-23T22:38:05.142777+0000","last_peered":"2023-09-23T22:38:05.142777+0000","last_clean":"2023-09-23T22:38:05.142777+0000","last_became_active":"2023-09-23T22:37:03.804606+0000","last_became_peered":"2023-09-23T22:37:03.804606+0000","last_unstale":"2023-09-23T22:38:05.142777+0000","last_undegraded":"2023-09-23T22:38:05.142777+0000","last_fullsized":"2023-09-23T22:38:05.142777+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:28.826826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153406+0000","last_change":"2023-09-23T22:38:06.153406+0000","last_active":"2023-09-23T22:38:06.153406+0000","last_peered":"2023-09-23T22:38:06.153406+0000","last_clean":"2023-09-23T22:38:06.153406+0000","last_became_active":"2023-09-23T22:38:06.153092+0000","last_became_peered":"2023-09-23T22:38:06.153092+0000","last_unstale":"2023-09-23T22:38:06.153406+0000","last_undegraded":"2023-09-23T22:38:06.153406+0000","last_fullsized":"2023-09-23T22:38:06.153406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:53.462379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155404+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:38:06.155404+0000","last_peered":"2023-09-23T22:38:06.155404+0000","last_clean":"2023-09-23T22:38:06.155404+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:38:06.155404+0000","last_undegraded":"2023-09-23T22:38:06.155404+0000","last_fullsized":"2023-09-23T22:38:06.155404+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142766+0000","last_change":"2023-09-23T22:37:03.800899+0000","last_active":"2023-09-23T22:38:05.142766+0000","last_peered":"2023-09-23T22:38:05.142766+0000","last_clean":"2023-09-23T22:38:05.142766+0000","last_became_active":"2023-09-23T22:37:03.800586+0000","last_became_peered":"2023-09-23T22:37:03.800586+0000","last_unstale":"2023-09-23T22:38:05.142766+0000","last_undegraded":"2023-09-23T22:38:05.142766+0000","last_fullsized":"2023-09-23T22:38:05.142766+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140171+0000","last_change":"2023-09-23T22:37:35.274945+0000","last_active":"2023-09-23T22:38:05.140171+0000","last_peered":"2023-09-23T22:38:05.140171+0000","last_clean":"2023-09-23T22:38:05.140171+0000","last_became_active":"2023-09-23T22:37:35.274580+0000","last_became_peered":"2023-09-23T22:37:35.274580+0000","last_unstale":"2023-09-23T22:38:05.140171+0000","last_undegraded":"2023-09-23T22:38:05.140171+0000","last_fullsized":"2023-09-23T22:38:05.140171+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141735+0000","last_change":"2023-09-23T22:37:35.276836+0000","last_active":"2023-09-23T22:38:05.141735+0000","last_peered":"2023-09-23T22:38:05.141735+0000","last_clean":"2023-09-23T22:38:05.141735+0000","last_became_active":"2023-09-23T22:37:35.276422+0000","last_became_peered":"2023-09-23T22:37:35.276422+0000","last_unstale":"2023-09-23T22:38:05.141735+0000","last_undegraded":"2023-09-23T22:38:05.141735+0000","last_fullsized":"2023-09-23T22:38:05.141735+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137068+0000","last_change":"2023-09-23T22:37:03.804557+0000","last_active":"2023-09-23T22:38:05.137068+0000","last_peered":"2023-09-23T22:38:05.137068+0000","last_clean":"2023-09-23T22:38:05.137068+0000","last_became_active":"2023-09-23T22:37:03.804229+0000","last_became_peered":"2023-09-23T22:37:03.804229+0000","last_unstale":"2023-09-23T22:38:05.137068+0000","last_undegraded":"2023-09-23T22:38:05.137068+0000","last_fullsized":"2023-09-23T22:38:05.137068+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152590+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:38:06.152590+0000","last_peered":"2023-09-23T22:38:06.152590+0000","last_clean":"2023-09-23T22:38:06.152590+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:38:06.152590+0000","last_undegraded":"2023-09-23T22:38:06.152590+0000","last_fullsized":"2023-09-23T22:38:06.152590+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151577+0000","last_change":"2023-09-23T22:38:06.151577+0000","last_active":"2023-09-23T22:38:06.151577+0000","last_peered":"2023-09-23T22:38:06.151577+0000","last_clean":"2023-09-23T22:38:06.151577+0000","last_became_active":"2023-09-23T22:38:06.151229+0000","last_became_peered":"2023-09-23T22:38:06.151229+0000","last_unstale":"2023-09-23T22:38:06.151577+0000","last_undegraded":"2023-09-23T22:38:06.151577+0000","last_fullsized":"2023-09-23T22:38:06.151577+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:25.864388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144329+0000","last_change":"2023-09-23T22:37:03.800764+0000","last_active":"2023-09-23T22:38:05.144329+0000","last_peered":"2023-09-23T22:38:05.144329+0000","last_clean":"2023-09-23T22:38:05.144329+0000","last_became_active":"2023-09-23T22:37:03.800407+0000","last_became_peered":"2023-09-23T22:37:03.800407+0000","last_unstale":"2023-09-23T22:38:05.144329+0000","last_undegraded":"2023-09-23T22:38:05.144329+0000","last_fullsized":"2023-09-23T22:38:05.144329+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:10.143602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155895+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:38:06.155895+0000","last_peered":"2023-09-23T22:38:06.155895+0000","last_clean":"2023-09-23T22:38:06.155895+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:38:06.155895+0000","last_undegraded":"2023-09-23T22:38:06.155895+0000","last_fullsized":"2023-09-23T22:38:06.155895+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136911+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:38:05.136911+0000","last_peered":"2023-09-23T22:38:05.136911+0000","last_clean":"2023-09-23T22:38:05.136911+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:38:05.136911+0000","last_undegraded":"2023-09-23T22:38:05.136911+0000","last_fullsized":"2023-09-23T22:38:05.136911+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141209+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:38:05.141209+0000","last_peered":"2023-09-23T22:38:05.141209+0000","last_clean":"2023-09-23T22:38:05.141209+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:38:05.141209+0000","last_undegraded":"2023-09-23T22:38:05.141209+0000","last_fullsized":"2023-09-23T22:38:05.141209+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154190+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:38:06.154190+0000","last_peered":"2023-09-23T22:38:06.154190+0000","last_clean":"2023-09-23T22:38:06.154190+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:38:06.154190+0000","last_undegraded":"2023-09-23T22:38:06.154190+0000","last_fullsized":"2023-09-23T22:38:06.154190+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.138291+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:38:05.138291+0000","last_peered":"2023-09-23T22:38:05.138291+0000","last_clean":"2023-09-23T22:38:05.138291+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:38:05.138291+0000","last_undegraded":"2023-09-23T22:38:05.138291+0000","last_fullsized":"2023-09-23T22:38:05.138291+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134242+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:38:05.134242+0000","last_peered":"2023-09-23T22:38:05.134242+0000","last_clean":"2023-09-23T22:38:05.134242+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:38:05.134242+0000","last_undegraded":"2023-09-23T22:38:05.134242+0000","last_fullsized":"2023-09-23T22:38:05.134242+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139635+0000","last_change":"2023-09-23T22:35:51.479873+0000","last_active":"2023-09-23T22:38:05.139635+0000","last_peered":"2023-09-23T22:38:05.139635+0000","last_clean":"2023-09-23T22:38:05.139635+0000","last_became_active":"2023-09-23T22:35:51.479506+0000","last_became_peered":"2023-09-23T22:35:51.479506+0000","last_unstale":"2023-09-23T22:38:05.139635+0000","last_undegraded":"2023-09-23T22:38:05.139635+0000","last_fullsized":"2023-09-23T22:38:05.139635+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154965+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:38:06.154965+0000","last_peered":"2023-09-23T22:38:06.154965+0000","last_clean":"2023-09-23T22:38:06.154965+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:38:06.154965+0000","last_undegraded":"2023-09-23T22:38:06.154965+0000","last_fullsized":"2023-09-23T22:38:06.154965+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":70,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136898+0000","last_change":"2023-09-23T22:35:51.475694+0000","last_active":"2023-09-23T22:38:05.136898+0000","last_peered":"2023-09-23T22:38:05.136898+0000","last_clean":"2023-09-23T22:38:05.136898+0000","last_became_active":"2023-09-23T22:35:51.475326+0000","last_became_peered":"2023-09-23T22:35:51.475326+0000","last_unstale":"2023-09-23T22:38:05.136898+0000","last_undegraded":"2023-09-23T22:38:05.136898+0000","last_fullsized":"2023-09-23T22:38:05.136898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156874+0000","last_change":"2023-09-23T22:38:06.156215+0000","last_active":"2023-09-23T22:38:06.156874+0000","last_peered":"2023-09-23T22:38:06.156874+0000","last_clean":"2023-09-23T22:38:06.156874+0000","last_became_active":"2023-09-23T22:38:06.155344+0000","last_became_peered":"2023-09-23T22:38:06.155344+0000","last_unstale":"2023-09-23T22:38:06.156874+0000","last_undegraded":"2023-09-23T22:38:06.156874+0000","last_fullsized":"2023-09-23T22:38:06.156874+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.252155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137434+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:38:05.137434+0000","last_peered":"2023-09-23T22:38:05.137434+0000","last_clean":"2023-09-23T22:38:05.137434+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:38:05.137434+0000","last_undegraded":"2023-09-23T22:38:05.137434+0000","last_fullsized":"2023-09-23T22:38:05.137434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157553+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:38:06.157553+0000","last_peered":"2023-09-23T22:38:06.157553+0000","last_clean":"2023-09-23T22:38:06.157553+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:38:06.157553+0000","last_undegraded":"2023-09-23T22:38:06.157553+0000","last_fullsized":"2023-09-23T22:38:06.157553+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152547+0000","last_change":"2023-09-23T22:38:06.151401+0000","last_active":"2023-09-23T22:38:06.152547+0000","last_peered":"2023-09-23T22:38:06.152547+0000","last_clean":"2023-09-23T22:38:06.152547+0000","last_became_active":"2023-09-23T22:38:06.151071+0000","last_became_peered":"2023-09-23T22:38:06.151071+0000","last_unstale":"2023-09-23T22:38:06.152547+0000","last_undegraded":"2023-09-23T22:38:06.152547+0000","last_fullsized":"2023-09-23T22:38:06.152547+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:14.260144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154184+0000","last_change":"2023-09-23T22:38:06.154184+0000","last_active":"2023-09-23T22:38:06.154184+0000","last_peered":"2023-09-23T22:38:06.154184+0000","last_clean":"2023-09-23T22:38:06.154184+0000","last_became_active":"2023-09-23T22:38:06.153914+0000","last_became_peered":"2023-09-23T22:38:06.153914+0000","last_unstale":"2023-09-23T22:38:06.154184+0000","last_undegraded":"2023-09-23T22:38:06.154184+0000","last_fullsized":"2023-09-23T22:38:06.154184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:53.068518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137833+0000","last_change":"2023-09-23T22:37:35.268294+0000","last_active":"2023-09-23T22:38:05.137833+0000","last_peered":"2023-09-23T22:38:05.137833+0000","last_clean":"2023-09-23T22:38:05.137833+0000","last_became_active":"2023-09-23T22:37:35.267886+0000","last_became_peered":"2023-09-23T22:37:35.267886+0000","last_unstale":"2023-09-23T22:38:05.137833+0000","last_undegraded":"2023-09-23T22:38:05.137833+0000","last_fullsized":"2023-09-23T22:38:05.137833+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141916+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:38:05.141916+0000","last_peered":"2023-09-23T22:38:05.141916+0000","last_clean":"2023-09-23T22:38:05.141916+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:38:05.141916+0000","last_undegraded":"2023-09-23T22:38:05.141916+0000","last_fullsized":"2023-09-23T22:38:05.141916+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141885+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:38:05.141885+0000","last_peered":"2023-09-23T22:38:05.141885+0000","last_clean":"2023-09-23T22:38:05.141885+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:38:05.141885+0000","last_undegraded":"2023-09-23T22:38:05.141885+0000","last_fullsized":"2023-09-23T22:38:05.141885+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139285+0000","last_change":"2023-09-23T22:37:35.272454+0000","last_active":"2023-09-23T22:38:05.139285+0000","last_peered":"2023-09-23T22:38:05.139285+0000","last_clean":"2023-09-23T22:38:05.139285+0000","last_became_active":"2023-09-23T22:37:35.271917+0000","last_became_peered":"2023-09-23T22:37:35.271917+0000","last_unstale":"2023-09-23T22:38:05.139285+0000","last_undegraded":"2023-09-23T22:38:05.139285+0000","last_fullsized":"2023-09-23T22:38:05.139285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:42.398917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157765+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:38:06.157765+0000","last_peered":"2023-09-23T22:38:06.157765+0000","last_clean":"2023-09-23T22:38:06.157765+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:38:06.157765+0000","last_undegraded":"2023-09-23T22:38:06.157765+0000","last_fullsized":"2023-09-23T22:38:06.157765+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151474+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:38:06.151474+0000","last_peered":"2023-09-23T22:38:06.151474+0000","last_clean":"2023-09-23T22:38:06.151474+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:38:06.151474+0000","last_undegraded":"2023-09-23T22:38:06.151474+0000","last_fullsized":"2023-09-23T22:38:06.151474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134905+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:38:05.134905+0000","last_peered":"2023-09-23T22:38:05.134905+0000","last_clean":"2023-09-23T22:38:05.134905+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:38:05.134905+0000","last_undegraded":"2023-09-23T22:38:05.134905+0000","last_fullsized":"2023-09-23T22:38:05.134905+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148885+0000","last_change":"2023-09-23T22:38:06.148885+0000","last_active":"2023-09-23T22:38:06.148885+0000","last_peered":"2023-09-23T22:38:06.148885+0000","last_clean":"2023-09-23T22:38:06.148885+0000","last_became_active":"2023-09-23T22:38:06.148470+0000","last_became_peered":"2023-09-23T22:38:06.148470+0000","last_unstale":"2023-09-23T22:38:06.148885+0000","last_undegraded":"2023-09-23T22:38:06.148885+0000","last_fullsized":"2023-09-23T22:38:06.148885+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:02.793510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153759+0000","last_change":"2023-09-23T22:38:06.153487+0000","last_active":"2023-09-23T22:38:06.153759+0000","last_peered":"2023-09-23T22:38:06.153759+0000","last_clean":"2023-09-23T22:38:06.153759+0000","last_became_active":"2023-09-23T22:38:06.153156+0000","last_became_peered":"2023-09-23T22:38:06.153156+0000","last_unstale":"2023-09-23T22:38:06.153759+0000","last_undegraded":"2023-09-23T22:38:06.153759+0000","last_fullsized":"2023-09-23T22:38:06.153759+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:49.053379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136355+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:38:05.136355+0000","last_peered":"2023-09-23T22:38:05.136355+0000","last_clean":"2023-09-23T22:38:05.136355+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:38:05.136355+0000","last_undegraded":"2023-09-23T22:38:05.136355+0000","last_fullsized":"2023-09-23T22:38:05.136355+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148692+0000","last_change":"2023-09-23T22:38:06.148692+0000","last_active":"2023-09-23T22:38:06.148692+0000","last_peered":"2023-09-23T22:38:06.148692+0000","last_clean":"2023-09-23T22:38:06.148692+0000","last_became_active":"2023-09-23T22:38:06.148096+0000","last_became_peered":"2023-09-23T22:38:06.148096+0000","last_unstale":"2023-09-23T22:38:06.148692+0000","last_undegraded":"2023-09-23T22:38:06.148692+0000","last_fullsized":"2023-09-23T22:38:06.148692+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:15.042597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156081+0000","last_change":"2023-09-23T22:38:06.156081+0000","last_active":"2023-09-23T22:38:06.156081+0000","last_peered":"2023-09-23T22:38:06.156081+0000","last_clean":"2023-09-23T22:38:06.156081+0000","last_became_active":"2023-09-23T22:38:06.155768+0000","last_became_peered":"2023-09-23T22:38:06.155768+0000","last_unstale":"2023-09-23T22:38:06.156081+0000","last_undegraded":"2023-09-23T22:38:06.156081+0000","last_fullsized":"2023-09-23T22:38:06.156081+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:21.557265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157097+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:38:06.157097+0000","last_peered":"2023-09-23T22:38:06.157097+0000","last_clean":"2023-09-23T22:38:06.157097+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:38:06.157097+0000","last_undegraded":"2023-09-23T22:38:06.157097+0000","last_fullsized":"2023-09-23T22:38:06.157097+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150667+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:38:06.150667+0000","last_peered":"2023-09-23T22:38:06.150667+0000","last_clean":"2023-09-23T22:38:06.150667+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:38:06.150667+0000","last_undegraded":"2023-09-23T22:38:06.150667+0000","last_fullsized":"2023-09-23T22:38:06.150667+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141602+0000","last_change":"2023-09-23T22:37:35.275167+0000","last_active":"2023-09-23T22:38:05.141602+0000","last_peered":"2023-09-23T22:38:05.141602+0000","last_clean":"2023-09-23T22:38:05.141602+0000","last_became_active":"2023-09-23T22:37:35.274809+0000","last_became_peered":"2023-09-23T22:37:35.274809+0000","last_unstale":"2023-09-23T22:38:05.141602+0000","last_undegraded":"2023-09-23T22:38:05.141602+0000","last_fullsized":"2023-09-23T22:38:05.141602+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430351,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57299999999999995,"15min":0.59899999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.60099999999999998,"15min":0.61599999999999999},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.755,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.63300000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.505},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:37:28 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58399999999999996,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.32400000000000001},"max":{"1min":0.79600000000000004,"5min":0.85399999999999998,"15min":1.4970000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.48499999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81200000000000006,"5min":0.81299999999999994,"15min":1.087},"last":0.64700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47899999999999998,"15min":0.498},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.26300000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":5.3140000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.51800000000000002,"15min":0.502},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.93799999999999994},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.80600000000000005,"5min":0.85799999999999998,"15min":1.0669999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57099999999999995,"15min":0.58299999999999996},"min":{"1min":0.42399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":5.3490000000000002},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56699999999999995,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.95799999999999996},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61699999999999999,"15min":0.59199999999999997},"min":{"1min":0.504,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":1.038},"last":0.51800000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626579,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1560576,"data_stored":1111799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.59199999999999997,"15min":0.60499999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72999999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.60199999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.501}]},{"osd":2,"last update":"Sat Sep 23 22:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.63},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.67800000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:37:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":0.81100000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.34300000000000003},"max":{"1min":0.79800000000000004,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.74399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51300000000000001,"15min":0.46700000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53600000000000003,"15min":0.47899999999999998},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":0.748,"5min":0.80600000000000005,"15min":0.81299999999999994},"last":0.77300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54100000000000004,"15min":0.497},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":0.755,"5min":0.77300000000000002,"15min":0.84899999999999998},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57799999999999996,"15min":0.52500000000000002},"min":{"1min":0.38200000000000001,"5min":0.377,"15min":0.32400000000000001},"max":{"1min":0.83899999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.73099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.59099999999999997,"15min":0.55000000000000004},"min":{"1min":0.46000000000000002,"5min":0.41799999999999998,"15min":0.34799999999999998},"max":{"1min":0.76000000000000001,"5min":0.81999999999999995,"15min":0.83199999999999996},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.29399999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":1.357},"last":0.69799999999999995}]}]},{"osd":1,"up_from":84,"seq":360777252888,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32412,"kb_used_data":980,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712740,"statfs":{"total":95995035648,"available":95961845760,"internally_reserved":0,"allocated":1003520,"data_stored":527042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.27100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.38400000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.629}]},{"osd":5,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.65000000000000002}]}]},{"osd":0,"up_from":79,"seq":339302416423,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33068,"kb_used_data":1572,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712084,"statfs":{"total":95995035648,"available":95961174016,"internally_reserved":0,"allocated":1609728,"data_stored":1117239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.46300000000000002,"15min":0.47099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.374},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.46500000000000002,"15min":0.47399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.625,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66900000000000004,"15min":0.67900000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.66300000000000003,"15min":0.67200000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.98099999999999998,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.64700000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.63200000000000001},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.64900000000000002,"15min":0.66500000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91100000000000003,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.67600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.64800000000000002,"15min":0.66800000000000004},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.879,"5min":1.036,"15min":1.036},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.65500000000000003,"15min":0.67300000000000004},"min":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.66100000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.67200000000000004},"min":{"1min":0.41199999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.98899999999999999,"5min":1.085,"15min":1.085},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67900000000000005,"15min":0.70899999999999996},"min":{"1min":0.44,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89900000000000002,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.69899999999999995}]}]},{"osd":2,"up_from":89,"seq":382252089362,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36628,"kb_used_data":972,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708524,"statfs":{"total":95995035648,"available":95957528576,"internally_reserved":0,"allocated":995328,"data_stored":524245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.438},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.41199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.69799999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.73599999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925836,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":521160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56599999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61799999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019035,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.79200000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.66500000000000004,"15min":0.67100000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80600000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.64900000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.53700000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.36299999999999999},"max":{"1min":0.81699999999999995,"5min":0.90500000000000003,"15min":1.482},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":0.85999999999999999},"last":0.73299999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:37:56 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.505,"15min":0.46500000000000002},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":0.69499999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54300000000000004,"15min":0.48399999999999999},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":0.72099999999999997,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57299999999999995,"15min":0.5},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76400000000000001,"5min":0.875,"15min":0.875},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.52100000000000002},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.79200000000000004,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.60299999999999998,"15min":0.55200000000000005},"min":{"1min":0.41999999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.754,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59999999999999998,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.58999999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822807,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306216,"kb_used_data":928,"kb_used_omap":0,"kb_used_meta":305280,"kb_avail":93438936,"statfs":{"total":95995035648,"available":95681470464,"internally_reserved":0,"allocated":950272,"data_stored":516621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312606720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:37:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.58399999999999996,"15min":0.59599999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.66100000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.432},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.625,"15min":0.63600000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.34999999999999998},"max":{"1min":0.68400000000000005,"5min":0.79000000000000004,"15min":1.278},"last":0.498},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.64000000000000001,"15min":0.61899999999999999},"min":{"1min":0.51700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.3109999999999999},"last":0.59999999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:51 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28799999999999998},"max":{"1min":0.72199999999999998,"5min":0.81299999999999994,"15min":0.89000000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.47899999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":0.70099999999999996,"5min":0.73099999999999998,"15min":0.88300000000000001},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.48599999999999999},"min":{"1min":0.35899999999999999,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":0.69499999999999995,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56599999999999995,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":0.79200000000000004,"5min":0.84599999999999997,"15min":0.90300000000000002},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.59099999999999997,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.89500000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59399999999999997,"15min":0.55400000000000005},"min":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":0.77000000000000002,"5min":0.78400000000000003,"15min":0.85199999999999998},"last":0.63600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:38:57.060 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:38:57.060 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:38:57.060 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:38:57.060 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:38:57.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:56 smithi151 ceph-mon[159210]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:57.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:56 smithi052 ceph-mon[187743]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:57.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:56 smithi052 ceph-mon[180979]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:57.428 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:57.915 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:57 smithi052 ceph-mon[180979]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:38:58.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:57 smithi151 ceph-mon[159210]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:38:58.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:57 smithi052 ceph-mon[187743]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:38:59.066 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:38:59.067 INFO:teuthology.orchestra.run.smithi052.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:38:59.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:58 smithi151 ceph-mon[159210]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:59.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:58 smithi052 ceph-mon[187743]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:58 smithi052 ceph-mon[180979]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:38:59.687 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:38:59.687 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:38:59.696 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T22:38:59.697 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T22:38:59.704 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T22:38:59.705 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:38:59.705 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T22:38:59.706 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T22:38:59.706 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T22:38:59.707 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T22:38:59.707 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T22:38:59.707 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:38:59.707 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:38:59.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:38:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2901513618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:38:59.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:38:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2901513618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:39:00.072 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:00.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:38:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2901513618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:39:01.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:00 smithi151 ceph-mon[159210]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:01.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:00 smithi052 ceph-mon[187743]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:01.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:00 smithi052 ceph-mon[180979]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:01.652 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:39:01.654 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:39:02.219 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":209,"stamp":"2023-09-23T22:39:01.476316+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348296,"kb_used_data":9416,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612920,"statfs":{"total":767960285184,"available":766579630080,"internally_reserved":0,"allocated":9641984,"data_stored":5951558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002867"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140027+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:38:05.140027+0000","last_peered":"2023-09-23T22:38:05.140027+0000","last_clean":"2023-09-23T22:38:05.140027+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:38:05.140027+0000","last_undegraded":"2023-09-23T22:38:05.140027+0000","last_fullsized":"2023-09-23T22:38:05.140027+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147480+0000","last_change":"2023-09-23T22:37:35.270139+0000","last_active":"2023-09-23T22:38:05.147480+0000","last_peered":"2023-09-23T22:38:05.147480+0000","last_clean":"2023-09-23T22:38:05.147480+0000","last_became_active":"2023-09-23T22:37:35.269761+0000","last_became_peered":"2023-09-23T22:37:35.269761+0000","last_unstale":"2023-09-23T22:38:05.147480+0000","last_undegraded":"2023-09-23T22:38:05.147480+0000","last_fullsized":"2023-09-23T22:38:05.147480+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:08.396803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.147696+0000","last_change":"2023-09-23T22:38:06.147696+0000","last_active":"2023-09-23T22:38:06.147696+0000","last_peered":"2023-09-23T22:38:06.147696+0000","last_clean":"2023-09-23T22:38:06.147696+0000","last_became_active":"2023-09-23T22:38:06.147382+0000","last_became_peered":"2023-09-23T22:38:06.147382+0000","last_unstale":"2023-09-23T22:38:06.147696+0000","last_undegraded":"2023-09-23T22:38:06.147696+0000","last_fullsized":"2023-09-23T22:38:06.147696+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:04.570885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.149889+0000","last_change":"2023-09-23T22:38:06.149889+0000","last_active":"2023-09-23T22:38:06.149889+0000","last_peered":"2023-09-23T22:38:06.149889+0000","last_clean":"2023-09-23T22:38:06.149889+0000","last_became_active":"2023-09-23T22:38:06.149711+0000","last_became_peered":"2023-09-23T22:38:06.149711+0000","last_unstale":"2023-09-23T22:38:06.149889+0000","last_undegraded":"2023-09-23T22:38:06.149889+0000","last_fullsized":"2023-09-23T22:38:06.149889+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:06.335320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151879+0000","last_change":"2023-09-23T22:38:06.151879+0000","last_active":"2023-09-23T22:38:06.151879+0000","last_peered":"2023-09-23T22:38:06.151879+0000","last_clean":"2023-09-23T22:38:06.151879+0000","last_became_active":"2023-09-23T22:38:06.151519+0000","last_became_peered":"2023-09-23T22:38:06.151519+0000","last_unstale":"2023-09-23T22:38:06.151879+0000","last_undegraded":"2023-09-23T22:38:06.151879+0000","last_fullsized":"2023-09-23T22:38:06.151879+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.135034+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:38:05.135034+0000","last_peered":"2023-09-23T22:38:05.135034+0000","last_clean":"2023-09-23T22:38:05.135034+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:38:05.135034+0000","last_undegraded":"2023-09-23T22:38:05.135034+0000","last_fullsized":"2023-09-23T22:38:05.135034+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153986+0000","last_change":"2023-09-23T22:38:06.153986+0000","last_active":"2023-09-23T22:38:06.153986+0000","last_peered":"2023-09-23T22:38:06.153986+0000","last_clean":"2023-09-23T22:38:06.153986+0000","last_became_active":"2023-09-23T22:38:06.153604+0000","last_became_peered":"2023-09-23T22:38:06.153604+0000","last_unstale":"2023-09-23T22:38:06.153986+0000","last_undegraded":"2023-09-23T22:38:06.153986+0000","last_fullsized":"2023-09-23T22:38:06.153986+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134780+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:38:05.134780+0000","last_peered":"2023-09-23T22:38:05.134780+0000","last_clean":"2023-09-23T22:38:05.134780+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:38:05.134780+0000","last_undegraded":"2023-09-23T22:38:05.134780+0000","last_fullsized":"2023-09-23T22:38:05.134780+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152423+0000","last_change":"2023-09-23T22:38:06.152423+0000","last_active":"2023-09-23T22:38:06.152423+0000","last_peered":"2023-09-23T22:38:06.152423+0000","last_clean":"2023-09-23T22:38:06.152423+0000","last_became_active":"2023-09-23T22:38:06.152083+0000","last_became_peered":"2023-09-23T22:38:06.152083+0000","last_unstale":"2023-09-23T22:38:06.152423+0000","last_undegraded":"2023-09-23T22:38:06.152423+0000","last_fullsized":"2023-09-23T22:38:06.152423+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:49.106880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.144944+0000","last_change":"2023-09-23T22:35:51.475988+0000","last_active":"2023-09-23T22:38:06.144944+0000","last_peered":"2023-09-23T22:38:06.144944+0000","last_clean":"2023-09-23T22:38:06.144944+0000","last_became_active":"2023-09-23T22:35:51.475664+0000","last_became_peered":"2023-09-23T22:35:51.475664+0000","last_unstale":"2023-09-23T22:38:06.144944+0000","last_undegraded":"2023-09-23T22:38:06.144944+0000","last_fullsized":"2023-09-23T22:38:06.144944+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:22.107425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140271+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:38:05.140271+0000","last_peered":"2023-09-23T22:38:05.140271+0000","last_clean":"2023-09-23T22:38:05.140271+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:38:05.140271+0000","last_undegraded":"2023-09-23T22:38:05.140271+0000","last_fullsized":"2023-09-23T22:38:05.140271+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.146941+0000","last_change":"2023-09-23T22:37:03.801120+0000","last_active":"2023-09-23T22:38:05.146941+0000","last_peered":"2023-09-23T22:38:05.146941+0000","last_clean":"2023-09-23T22:38:05.146941+0000","last_became_active":"2023-09-23T22:37:03.800596+0000","last_became_peered":"2023-09-23T22:37:03.800596+0000","last_unstale":"2023-09-23T22:38:05.146941+0000","last_undegraded":"2023-09-23T22:38:05.146941+0000","last_fullsized":"2023-09-23T22:38:05.146941+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:03.424170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136253+0000","last_change":"2023-09-23T22:37:35.276863+0000","last_active":"2023-09-23T22:38:05.136253+0000","last_peered":"2023-09-23T22:38:05.136253+0000","last_clean":"2023-09-23T22:38:05.136253+0000","last_became_active":"2023-09-23T22:37:35.276428+0000","last_became_peered":"2023-09-23T22:37:35.276428+0000","last_unstale":"2023-09-23T22:38:05.136253+0000","last_undegraded":"2023-09-23T22:38:05.136253+0000","last_fullsized":"2023-09-23T22:38:05.136253+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155276+0000","last_change":"2023-09-23T22:38:06.155276+0000","last_active":"2023-09-23T22:38:06.155276+0000","last_peered":"2023-09-23T22:38:06.155276+0000","last_clean":"2023-09-23T22:38:06.155276+0000","last_became_active":"2023-09-23T22:38:06.154550+0000","last_became_peered":"2023-09-23T22:38:06.154550+0000","last_unstale":"2023-09-23T22:38:06.155276+0000","last_undegraded":"2023-09-23T22:38:06.155276+0000","last_fullsized":"2023-09-23T22:38:06.155276+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:06.092371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140029+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:38:05.140029+0000","last_peered":"2023-09-23T22:38:05.140029+0000","last_clean":"2023-09-23T22:38:05.140029+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:38:05.140029+0000","last_undegraded":"2023-09-23T22:38:05.140029+0000","last_fullsized":"2023-09-23T22:38:05.140029+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155104+0000","last_change":"2023-09-23T22:38:06.155104+0000","last_active":"2023-09-23T22:38:06.155104+0000","last_peered":"2023-09-23T22:38:06.155104+0000","last_clean":"2023-09-23T22:38:06.155104+0000","last_became_active":"2023-09-23T22:38:06.154630+0000","last_became_peered":"2023-09-23T22:38:06.154630+0000","last_unstale":"2023-09-23T22:38:06.155104+0000","last_undegraded":"2023-09-23T22:38:06.155104+0000","last_fullsized":"2023-09-23T22:38:06.155104+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.232885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144233+0000","last_change":"2023-09-23T22:35:51.477103+0000","last_active":"2023-09-23T22:38:05.144233+0000","last_peered":"2023-09-23T22:38:05.144233+0000","last_clean":"2023-09-23T22:38:05.144233+0000","last_became_active":"2023-09-23T22:35:51.476467+0000","last_became_peered":"2023-09-23T22:35:51.476467+0000","last_unstale":"2023-09-23T22:38:05.144233+0000","last_undegraded":"2023-09-23T22:38:05.144233+0000","last_fullsized":"2023-09-23T22:38:05.144233+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:24.234310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142041+0000","last_change":"2023-09-23T22:37:35.271043+0000","last_active":"2023-09-23T22:38:05.142041+0000","last_peered":"2023-09-23T22:38:05.142041+0000","last_clean":"2023-09-23T22:38:05.142041+0000","last_became_active":"2023-09-23T22:37:35.270808+0000","last_became_peered":"2023-09-23T22:37:35.270808+0000","last_unstale":"2023-09-23T22:38:05.142041+0000","last_undegraded":"2023-09-23T22:38:05.142041+0000","last_fullsized":"2023-09-23T22:38:05.142041+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:00.218174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141189+0000","last_change":"2023-09-23T22:35:51.476177+0000","last_active":"2023-09-23T22:38:05.141189+0000","last_peered":"2023-09-23T22:38:05.141189+0000","last_clean":"2023-09-23T22:38:05.141189+0000","last_became_active":"2023-09-23T22:35:51.475644+0000","last_became_peered":"2023-09-23T22:35:51.475644+0000","last_unstale":"2023-09-23T22:38:05.141189+0000","last_undegraded":"2023-09-23T22:38:05.141189+0000","last_fullsized":"2023-09-23T22:38:05.141189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147016+0000","last_change":"2023-09-23T22:37:35.266620+0000","last_active":"2023-09-23T22:38:05.147016+0000","last_peered":"2023-09-23T22:38:05.147016+0000","last_clean":"2023-09-23T22:38:05.147016+0000","last_became_active":"2023-09-23T22:37:35.266391+0000","last_became_peered":"2023-09-23T22:37:35.266391+0000","last_unstale":"2023-09-23T22:38:05.147016+0000","last_undegraded":"2023-09-23T22:38:05.147016+0000","last_fullsized":"2023-09-23T22:38:05.147016+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:15.615607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150984+0000","last_change":"2023-09-23T22:38:06.150984+0000","last_active":"2023-09-23T22:38:06.150984+0000","last_peered":"2023-09-23T22:38:06.150984+0000","last_clean":"2023-09-23T22:38:06.150984+0000","last_became_active":"2023-09-23T22:38:06.150668+0000","last_became_peered":"2023-09-23T22:38:06.150668+0000","last_unstale":"2023-09-23T22:38:06.150984+0000","last_undegraded":"2023-09-23T22:38:06.150984+0000","last_fullsized":"2023-09-23T22:38:06.150984+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:25.323709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139228+0000","last_change":"2023-09-23T22:37:03.808191+0000","last_active":"2023-09-23T22:38:05.139228+0000","last_peered":"2023-09-23T22:38:05.139228+0000","last_clean":"2023-09-23T22:38:05.139228+0000","last_became_active":"2023-09-23T22:37:03.807779+0000","last_became_peered":"2023-09-23T22:37:03.807779+0000","last_unstale":"2023-09-23T22:38:05.139228+0000","last_undegraded":"2023-09-23T22:38:05.139228+0000","last_fullsized":"2023-09-23T22:38:05.139228+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136454+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:38:05.136454+0000","last_peered":"2023-09-23T22:38:05.136454+0000","last_clean":"2023-09-23T22:38:05.136454+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:38:05.136454+0000","last_undegraded":"2023-09-23T22:38:05.136454+0000","last_fullsized":"2023-09-23T22:38:05.136454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154384+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:38:06.154384+0000","last_peered":"2023-09-23T22:38:06.154384+0000","last_clean":"2023-09-23T22:38:06.154384+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:38:06.154384+0000","last_undegraded":"2023-09-23T22:38:06.154384+0000","last_fullsized":"2023-09-23T22:38:06.154384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":297,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139145+0000","last_change":"2023-09-23T22:35:51.477611+0000","last_active":"2023-09-23T22:38:05.139145+0000","last_peered":"2023-09-23T22:38:05.139145+0000","last_clean":"2023-09-23T22:38:05.139145+0000","last_became_active":"2023-09-23T22:35:51.477061+0000","last_became_peered":"2023-09-23T22:35:51.477061+0000","last_unstale":"2023-09-23T22:38:05.139145+0000","last_undegraded":"2023-09-23T22:38:05.139145+0000","last_fullsized":"2023-09-23T22:38:05.139145+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133688+0000","last_change":"2023-09-23T22:37:35.270450+0000","last_active":"2023-09-23T22:38:05.133688+0000","last_peered":"2023-09-23T22:38:05.133688+0000","last_clean":"2023-09-23T22:38:05.133688+0000","last_became_active":"2023-09-23T22:37:35.270097+0000","last_became_peered":"2023-09-23T22:37:35.270097+0000","last_unstale":"2023-09-23T22:38:05.133688+0000","last_undegraded":"2023-09-23T22:38:05.133688+0000","last_fullsized":"2023-09-23T22:38:05.133688+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140469+0000","last_change":"2023-09-23T22:37:03.801645+0000","last_active":"2023-09-23T22:38:05.140469+0000","last_peered":"2023-09-23T22:38:05.140469+0000","last_clean":"2023-09-23T22:38:05.140469+0000","last_became_active":"2023-09-23T22:37:03.799407+0000","last_became_peered":"2023-09-23T22:37:03.799407+0000","last_unstale":"2023-09-23T22:38:05.140469+0000","last_undegraded":"2023-09-23T22:38:05.140469+0000","last_fullsized":"2023-09-23T22:38:05.140469+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.396991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141322+0000","last_change":"2023-09-23T22:37:35.271109+0000","last_active":"2023-09-23T22:38:05.141322+0000","last_peered":"2023-09-23T22:38:05.141322+0000","last_clean":"2023-09-23T22:38:05.141322+0000","last_became_active":"2023-09-23T22:37:35.270850+0000","last_became_peered":"2023-09-23T22:37:35.270850+0000","last_unstale":"2023-09-23T22:38:05.141322+0000","last_undegraded":"2023-09-23T22:38:05.141322+0000","last_fullsized":"2023-09-23T22:38:05.141322+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:24.045725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148923+0000","last_change":"2023-09-23T22:38:06.148923+0000","last_active":"2023-09-23T22:38:06.148923+0000","last_peered":"2023-09-23T22:38:06.148923+0000","last_clean":"2023-09-23T22:38:06.148923+0000","last_became_active":"2023-09-23T22:38:06.148636+0000","last_became_peered":"2023-09-23T22:38:06.148636+0000","last_unstale":"2023-09-23T22:38:06.148923+0000","last_undegraded":"2023-09-23T22:38:06.148923+0000","last_fullsized":"2023-09-23T22:38:06.148923+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:01.486987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156455+0000","last_change":"2023-09-23T22:38:06.156455+0000","last_active":"2023-09-23T22:38:06.156455+0000","last_peered":"2023-09-23T22:38:06.156455+0000","last_clean":"2023-09-23T22:38:06.156455+0000","last_became_active":"2023-09-23T22:38:06.156203+0000","last_became_peered":"2023-09-23T22:38:06.156203+0000","last_unstale":"2023-09-23T22:38:06.156455+0000","last_undegraded":"2023-09-23T22:38:06.156455+0000","last_fullsized":"2023-09-23T22:38:06.156455+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142051+0000","last_change":"2023-09-23T22:35:51.474457+0000","last_active":"2023-09-23T22:38:05.142051+0000","last_peered":"2023-09-23T22:38:05.142051+0000","last_clean":"2023-09-23T22:38:05.142051+0000","last_became_active":"2023-09-23T22:35:51.474166+0000","last_became_peered":"2023-09-23T22:35:51.474166+0000","last_unstale":"2023-09-23T22:38:05.142051+0000","last_undegraded":"2023-09-23T22:38:05.142051+0000","last_fullsized":"2023-09-23T22:38:05.142051+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133761+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:38:05.133761+0000","last_peered":"2023-09-23T22:38:05.133761+0000","last_clean":"2023-09-23T22:38:05.133761+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:38:05.133761+0000","last_undegraded":"2023-09-23T22:38:05.133761+0000","last_fullsized":"2023-09-23T22:38:05.133761+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139384+0000","last_change":"2023-09-23T22:37:03.797729+0000","last_active":"2023-09-23T22:38:05.139384+0000","last_peered":"2023-09-23T22:38:05.139384+0000","last_clean":"2023-09-23T22:38:05.139384+0000","last_became_active":"2023-09-23T22:37:03.797377+0000","last_became_peered":"2023-09-23T22:37:03.797377+0000","last_unstale":"2023-09-23T22:38:05.139384+0000","last_undegraded":"2023-09-23T22:38:05.139384+0000","last_fullsized":"2023-09-23T22:38:05.139384+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:18.065155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157484+0000","last_change":"2023-09-23T22:38:06.157484+0000","last_active":"2023-09-23T22:38:06.157484+0000","last_peered":"2023-09-23T22:38:06.157484+0000","last_clean":"2023-09-23T22:38:06.157484+0000","last_became_active":"2023-09-23T22:38:06.157175+0000","last_became_peered":"2023-09-23T22:38:06.157175+0000","last_unstale":"2023-09-23T22:38:06.157484+0000","last_undegraded":"2023-09-23T22:38:06.157484+0000","last_fullsized":"2023-09-23T22:38:06.157484+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142777+0000","last_change":"2023-09-23T22:37:03.805141+0000","last_active":"2023-09-23T22:38:05.142777+0000","last_peered":"2023-09-23T22:38:05.142777+0000","last_clean":"2023-09-23T22:38:05.142777+0000","last_became_active":"2023-09-23T22:37:03.804606+0000","last_became_peered":"2023-09-23T22:37:03.804606+0000","last_unstale":"2023-09-23T22:38:05.142777+0000","last_undegraded":"2023-09-23T22:38:05.142777+0000","last_fullsized":"2023-09-23T22:38:05.142777+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:28.826826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153406+0000","last_change":"2023-09-23T22:38:06.153406+0000","last_active":"2023-09-23T22:38:06.153406+0000","last_peered":"2023-09-23T22:38:06.153406+0000","last_clean":"2023-09-23T22:38:06.153406+0000","last_became_active":"2023-09-23T22:38:06.153092+0000","last_became_peered":"2023-09-23T22:38:06.153092+0000","last_unstale":"2023-09-23T22:38:06.153406+0000","last_undegraded":"2023-09-23T22:38:06.153406+0000","last_fullsized":"2023-09-23T22:38:06.153406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:53.462379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155404+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:38:06.155404+0000","last_peered":"2023-09-23T22:38:06.155404+0000","last_clean":"2023-09-23T22:38:06.155404+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:38:06.155404+0000","last_undegraded":"2023-09-23T22:38:06.155404+0000","last_fullsized":"2023-09-23T22:38:06.155404+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142766+0000","last_change":"2023-09-23T22:37:03.800899+0000","last_active":"2023-09-23T22:38:05.142766+0000","last_peered":"2023-09-23T22:38:05.142766+0000","last_clean":"2023-09-23T22:38:05.142766+0000","last_became_active":"2023-09-23T22:37:03.800586+0000","last_became_peered":"2023-09-23T22:37:03.800586+0000","last_unstale":"2023-09-23T22:38:05.142766+0000","last_undegraded":"2023-09-23T22:38:05.142766+0000","last_fullsized":"2023-09-23T22:38:05.142766+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140171+0000","last_change":"2023-09-23T22:37:35.274945+0000","last_active":"2023-09-23T22:38:05.140171+0000","last_peered":"2023-09-23T22:38:05.140171+0000","last_clean":"2023-09-23T22:38:05.140171+0000","last_became_active":"2023-09-23T22:37:35.274580+0000","last_became_peered":"2023-09-23T22:37:35.274580+0000","last_unstale":"2023-09-23T22:38:05.140171+0000","last_undegraded":"2023-09-23T22:38:05.140171+0000","last_fullsized":"2023-09-23T22:38:05.140171+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141735+0000","last_change":"2023-09-23T22:37:35.276836+0000","last_active":"2023-09-23T22:38:05.141735+0000","last_peered":"2023-09-23T22:38:05.141735+0000","last_clean":"2023-09-23T22:38:05.141735+0000","last_became_active":"2023-09-23T22:37:35.276422+0000","last_became_peered":"2023-09-23T22:37:35.276422+0000","last_unstale":"2023-09-23T22:38:05.141735+0000","last_undegraded":"2023-09-23T22:38:05.141735+0000","last_fullsized":"2023-09-23T22:38:05.141735+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137068+0000","last_change":"2023-09-23T22:37:03.804557+0000","last_active":"2023-09-23T22:38:05.137068+0000","last_peered":"2023-09-23T22:38:05.137068+0000","last_clean":"2023-09-23T22:38:05.137068+0000","last_became_active":"2023-09-23T22:37:03.804229+0000","last_became_peered":"2023-09-23T22:37:03.804229+0000","last_unstale":"2023-09-23T22:38:05.137068+0000","last_undegraded":"2023-09-23T22:38:05.137068+0000","last_fullsized":"2023-09-23T22:38:05.137068+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152590+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:38:06.152590+0000","last_peered":"2023-09-23T22:38:06.152590+0000","last_clean":"2023-09-23T22:38:06.152590+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:38:06.152590+0000","last_undegraded":"2023-09-23T22:38:06.152590+0000","last_fullsized":"2023-09-23T22:38:06.152590+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151577+0000","last_change":"2023-09-23T22:38:06.151577+0000","last_active":"2023-09-23T22:38:06.151577+0000","last_peered":"2023-09-23T22:38:06.151577+0000","last_clean":"2023-09-23T22:38:06.151577+0000","last_became_active":"2023-09-23T22:38:06.151229+0000","last_became_peered":"2023-09-23T22:38:06.151229+0000","last_unstale":"2023-09-23T22:38:06.151577+0000","last_undegraded":"2023-09-23T22:38:06.151577+0000","last_fullsized":"2023-09-23T22:38:06.151577+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:25.864388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144329+0000","last_change":"2023-09-23T22:37:03.800764+0000","last_active":"2023-09-23T22:38:05.144329+0000","last_peered":"2023-09-23T22:38:05.144329+0000","last_clean":"2023-09-23T22:38:05.144329+0000","last_became_active":"2023-09-23T22:37:03.800407+0000","last_became_peered":"2023-09-23T22:37:03.800407+0000","last_unstale":"2023-09-23T22:38:05.144329+0000","last_undegraded":"2023-09-23T22:38:05.144329+0000","last_fullsized":"2023-09-23T22:38:05.144329+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:10.143602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155895+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:38:06.155895+0000","last_peered":"2023-09-23T22:38:06.155895+0000","last_clean":"2023-09-23T22:38:06.155895+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:38:06.155895+0000","last_undegraded":"2023-09-23T22:38:06.155895+0000","last_fullsized":"2023-09-23T22:38:06.155895+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136911+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:38:05.136911+0000","last_peered":"2023-09-23T22:38:05.136911+0000","last_clean":"2023-09-23T22:38:05.136911+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:38:05.136911+0000","last_undegraded":"2023-09-23T22:38:05.136911+0000","last_fullsized":"2023-09-23T22:38:05.136911+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141209+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:38:05.141209+0000","last_peered":"2023-09-23T22:38:05.141209+0000","last_clean":"2023-09-23T22:38:05.141209+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:38:05.141209+0000","last_undegraded":"2023-09-23T22:38:05.141209+0000","last_fullsized":"2023-09-23T22:38:05.141209+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154190+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:38:06.154190+0000","last_peered":"2023-09-23T22:38:06.154190+0000","last_clean":"2023-09-23T22:38:06.154190+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:38:06.154190+0000","last_undegraded":"2023-09-23T22:38:06.154190+0000","last_fullsized":"2023-09-23T22:38:06.154190+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.138291+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:38:05.138291+0000","last_peered":"2023-09-23T22:38:05.138291+0000","last_clean":"2023-09-23T22:38:05.138291+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:38:05.138291+0000","last_undegraded":"2023-09-23T22:38:05.138291+0000","last_fullsized":"2023-09-23T22:38:05.138291+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134242+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:38:05.134242+0000","last_peered":"2023-09-23T22:38:05.134242+0000","last_clean":"2023-09-23T22:38:05.134242+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:38:05.134242+0000","last_undegraded":"2023-09-23T22:38:05.134242+0000","last_fullsized":"2023-09-23T22:38:05.134242+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139635+0000","last_change":"2023-09-23T22:35:51.479873+0000","last_active":"2023-09-23T22:38:05.139635+0000","last_peered":"2023-09-23T22:38:05.139635+0000","last_clean":"2023-09-23T22:38:05.139635+0000","last_became_active":"2023-09-23T22:35:51.479506+0000","last_became_peered":"2023-09-23T22:35:51.479506+0000","last_unstale":"2023-09-23T22:38:05.139635+0000","last_undegraded":"2023-09-23T22:38:05.139635+0000","last_fullsized":"2023-09-23T22:38:05.139635+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154965+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:38:06.154965+0000","last_peered":"2023-09-23T22:38:06.154965+0000","last_clean":"2023-09-23T22:38:06.154965+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:38:06.154965+0000","last_undegraded":"2023-09-23T22:38:06.154965+0000","last_fullsized":"2023-09-23T22:38:06.154965+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":70,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136898+0000","last_change":"2023-09-23T22:35:51.475694+0000","last_active":"2023-09-23T22:38:05.136898+0000","last_peered":"2023-09-23T22:38:05.136898+0000","last_clean":"2023-09-23T22:38:05.136898+0000","last_became_active":"2023-09-23T22:35:51.475326+0000","last_became_peered":"2023-09-23T22:35:51.475326+0000","last_unstale":"2023-09-23T22:38:05.136898+0000","last_undegraded":"2023-09-23T22:38:05.136898+0000","last_fullsized":"2023-09-23T22:38:05.136898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156874+0000","last_change":"2023-09-23T22:38:06.156215+0000","last_active":"2023-09-23T22:38:06.156874+0000","last_peered":"2023-09-23T22:38:06.156874+0000","last_clean":"2023-09-23T22:38:06.156874+0000","last_became_active":"2023-09-23T22:38:06.155344+0000","last_became_peered":"2023-09-23T22:38:06.155344+0000","last_unstale":"2023-09-23T22:38:06.156874+0000","last_undegraded":"2023-09-23T22:38:06.156874+0000","last_fullsized":"2023-09-23T22:38:06.156874+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.252155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137434+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:38:05.137434+0000","last_peered":"2023-09-23T22:38:05.137434+0000","last_clean":"2023-09-23T22:38:05.137434+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:38:05.137434+0000","last_undegraded":"2023-09-23T22:38:05.137434+0000","last_fullsized":"2023-09-23T22:38:05.137434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157553+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:38:06.157553+0000","last_peered":"2023-09-23T22:38:06.157553+0000","last_clean":"2023-09-23T22:38:06.157553+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:38:06.157553+0000","last_undegraded":"2023-09-23T22:38:06.157553+0000","last_fullsized":"2023-09-23T22:38:06.157553+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152547+0000","last_change":"2023-09-23T22:38:06.151401+0000","last_active":"2023-09-23T22:38:06.152547+0000","last_peered":"2023-09-23T22:38:06.152547+0000","last_clean":"2023-09-23T22:38:06.152547+0000","last_became_active":"2023-09-23T22:38:06.151071+0000","last_became_peered":"2023-09-23T22:38:06.151071+0000","last_unstale":"2023-09-23T22:38:06.152547+0000","last_undegraded":"2023-09-23T22:38:06.152547+0000","last_fullsized":"2023-09-23T22:38:06.152547+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:14.260144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154184+0000","last_change":"2023-09-23T22:38:06.154184+0000","last_active":"2023-09-23T22:38:06.154184+0000","last_peered":"2023-09-23T22:38:06.154184+0000","last_clean":"2023-09-23T22:38:06.154184+0000","last_became_active":"2023-09-23T22:38:06.153914+0000","last_became_peered":"2023-09-23T22:38:06.153914+0000","last_unstale":"2023-09-23T22:38:06.154184+0000","last_undegraded":"2023-09-23T22:38:06.154184+0000","last_fullsized":"2023-09-23T22:38:06.154184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:53.068518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137833+0000","last_change":"2023-09-23T22:37:35.268294+0000","last_active":"2023-09-23T22:38:05.137833+0000","last_peered":"2023-09-23T22:38:05.137833+0000","last_clean":"2023-09-23T22:38:05.137833+0000","last_became_active":"2023-09-23T22:37:35.267886+0000","last_became_peered":"2023-09-23T22:37:35.267886+0000","last_unstale":"2023-09-23T22:38:05.137833+0000","last_undegraded":"2023-09-23T22:38:05.137833+0000","last_fullsized":"2023-09-23T22:38:05.137833+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141916+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:38:05.141916+0000","last_peered":"2023-09-23T22:38:05.141916+0000","last_clean":"2023-09-23T22:38:05.141916+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:38:05.141916+0000","last_undegraded":"2023-09-23T22:38:05.141916+0000","last_fullsized":"2023-09-23T22:38:05.141916+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141885+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:38:05.141885+0000","last_peered":"2023-09-23T22:38:05.141885+0000","last_clean":"2023-09-23T22:38:05.141885+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:38:05.141885+0000","last_undegraded":"2023-09-23T22:38:05.141885+0000","last_fullsized":"2023-09-23T22:38:05.141885+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139285+0000","last_change":"2023-09-23T22:37:35.272454+0000","last_active":"2023-09-23T22:38:05.139285+0000","last_peered":"2023-09-23T22:38:05.139285+0000","last_clean":"2023-09-23T22:38:05.139285+0000","last_became_active":"2023-09-23T22:37:35.271917+0000","last_became_peered":"2023-09-23T22:37:35.271917+0000","last_unstale":"2023-09-23T22:38:05.139285+0000","last_undegraded":"2023-09-23T22:38:05.139285+0000","last_fullsized":"2023-09-23T22:38:05.139285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:42.398917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157765+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:38:06.157765+0000","last_peered":"2023-09-23T22:38:06.157765+0000","last_clean":"2023-09-23T22:38:06.157765+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:38:06.157765+0000","last_undegraded":"2023-09-23T22:38:06.157765+0000","last_fullsized":"2023-09-23T22:38:06.157765+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151474+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:38:06.151474+0000","last_peered":"2023-09-23T22:38:06.151474+0000","last_clean":"2023-09-23T22:38:06.151474+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:38:06.151474+0000","last_undegraded":"2023-09-23T22:38:06.151474+0000","last_fullsized":"2023-09-23T22:38:06.151474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134905+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:38:05.134905+0000","last_peered":"2023-09-23T22:38:05.134905+0000","last_clean":"2023-09-23T22:38:05.134905+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:38:05.134905+0000","last_undegraded":"2023-09-23T22:38:05.134905+0000","last_fullsized":"2023-09-23T22:38:05.134905+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148885+0000","last_change":"2023-09-23T22:38:06.148885+0000","last_active":"2023-09-23T22:38:06.148885+0000","last_peered":"2023-09-23T22:38:06.148885+0000","last_clean":"2023-09-23T22:38:06.148885+0000","last_became_active":"2023-09-23T22:38:06.148470+0000","last_became_peered":"2023-09-23T22:38:06.148470+0000","last_unstale":"2023-09-23T22:38:06.148885+0000","last_undegraded":"2023-09-23T22:38:06.148885+0000","last_fullsized":"2023-09-23T22:38:06.148885+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:02.793510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153759+0000","last_change":"2023-09-23T22:38:06.153487+0000","last_active":"2023-09-23T22:38:06.153759+0000","last_peered":"2023-09-23T22:38:06.153759+0000","last_clean":"2023-09-23T22:38:06.153759+0000","last_became_active":"2023-09-23T22:38:06.153156+0000","last_became_peered":"2023-09-23T22:38:06.153156+0000","last_unstale":"2023-09-23T22:38:06.153759+0000","last_undegraded":"2023-09-23T22:38:06.153759+0000","last_fullsized":"2023-09-23T22:38:06.153759+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:49.053379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136355+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:38:05.136355+0000","last_peered":"2023-09-23T22:38:05.136355+0000","last_clean":"2023-09-23T22:38:05.136355+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:38:05.136355+0000","last_undegraded":"2023-09-23T22:38:05.136355+0000","last_fullsized":"2023-09-23T22:38:05.136355+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148692+0000","last_change":"2023-09-23T22:38:06.148692+0000","last_active":"2023-09-23T22:38:06.148692+0000","last_peered":"2023-09-23T22:38:06.148692+0000","last_clean":"2023-09-23T22:38:06.148692+0000","last_became_active":"2023-09-23T22:38:06.148096+0000","last_became_peered":"2023-09-23T22:38:06.148096+0000","last_unstale":"2023-09-23T22:38:06.148692+0000","last_undegraded":"2023-09-23T22:38:06.148692+0000","last_fullsized":"2023-09-23T22:38:06.148692+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:15.042597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156081+0000","last_change":"2023-09-23T22:38:06.156081+0000","last_active":"2023-09-23T22:38:06.156081+0000","last_peered":"2023-09-23T22:38:06.156081+0000","last_clean":"2023-09-23T22:38:06.156081+0000","last_became_active":"2023-09-23T22:38:06.155768+0000","last_became_peered":"2023-09-23T22:38:06.155768+0000","last_unstale":"2023-09-23T22:38:06.156081+0000","last_undegraded":"2023-09-23T22:38:06.156081+0000","last_fullsized":"2023-09-23T22:38:06.156081+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:21.557265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157097+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:38:06.157097+0000","last_peered":"2023-09-23T22:38:06.157097+0000","last_clean":"2023-09-23T22:38:06.157097+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:38:06.157097+0000","last_undegraded":"2023-09-23T22:38:06.157097+0000","last_fullsized":"2023-09-23T22:38:06.157097+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150667+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:38:06.150667+0000","last_peered":"2023-09-23T22:38:06.150667+0000","last_clean":"2023-09-23T22:38:06.150667+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:38:06.150667+0000","last_undegraded":"2023-09-23T22:38:06.150667+0000","last_fullsized":"2023-09-23T22:38:06.150667+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141602+0000","last_change":"2023-09-23T22:37:35.275167+0000","last_active":"2023-09-23T22:38:05.141602+0000","last_peered":"2023-09-23T22:38:05.141602+0000","last_clean":"2023-09-23T22:38:05.141602+0000","last_became_active":"2023-09-23T22:37:35.274809+0000","last_became_peered":"2023-09-23T22:37:35.274809+0000","last_unstale":"2023-09-23T22:38:05.141602+0000","last_undegraded":"2023-09-23T22:38:05.141602+0000","last_fullsized":"2023-09-23T22:38:05.141602+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430352,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57299999999999995,"15min":0.59899999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.60099999999999998,"15min":0.61599999999999999},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.755,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.499}]},{"osd":1,"last update":"Sat Sep 23 22:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:37:28 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58399999999999996,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.32400000000000001},"max":{"1min":0.79600000000000004,"5min":0.85399999999999998,"15min":1.4970000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.48499999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81200000000000006,"5min":0.81299999999999994,"15min":1.087},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47899999999999998,"15min":0.498},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.26300000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":5.3140000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.51800000000000002,"15min":0.502},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.93799999999999994},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.80600000000000005,"5min":0.85799999999999998,"15min":1.0669999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57099999999999995,"15min":0.58299999999999996},"min":{"1min":0.42399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":5.3490000000000002},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56699999999999995,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.95799999999999996},"last":0.438},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61699999999999999,"15min":0.59199999999999997},"min":{"1min":0.504,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":1.038},"last":0.58299999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626580,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1560576,"data_stored":1111799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.57099999999999995,"15min":0.59799999999999998},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.68400000000000005,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.44},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57999999999999996,"15min":0.59799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.47399999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 22:37:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":0.81100000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.34300000000000003},"max":{"1min":0.79800000000000004,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.61699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51300000000000001,"15min":0.46700000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53600000000000003,"15min":0.47899999999999998},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":0.748,"5min":0.80600000000000005,"15min":0.81299999999999994},"last":0.65100000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54100000000000004,"15min":0.497},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":0.755,"5min":0.77300000000000002,"15min":0.84899999999999998},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57799999999999996,"15min":0.52500000000000002},"min":{"1min":0.38200000000000001,"5min":0.377,"15min":0.32400000000000001},"max":{"1min":0.83899999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.60899999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.59099999999999997,"15min":0.55000000000000004},"min":{"1min":0.46000000000000002,"5min":0.41799999999999998,"15min":0.34799999999999998},"max":{"1min":0.76000000000000001,"5min":0.81999999999999995,"15min":0.83199999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.29399999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":1.357},"last":0.58299999999999996}]}]},{"osd":1,"up_from":84,"seq":360777252889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32412,"kb_used_data":980,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712740,"statfs":{"total":95995035648,"available":95961845760,"internally_reserved":0,"allocated":1003520,"data_stored":527042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.33600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.77100000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.68700000000000006}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.83799999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.80300000000000005}]}]},{"osd":0,"up_from":79,"seq":339302416424,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33068,"kb_used_data":1572,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712084,"statfs":{"total":95995035648,"available":95961174016,"internally_reserved":0,"allocated":1609728,"data_stored":1117239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.312},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.27700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.46300000000000002,"15min":0.47099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.46500000000000002,"15min":0.47399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.625,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66900000000000004,"15min":0.67900000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.66300000000000003,"15min":0.67200000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.98099999999999998,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.63200000000000001},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.64900000000000002,"15min":0.66500000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91100000000000003,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.64800000000000002,"15min":0.66800000000000004},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.879,"5min":1.036,"15min":1.036},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.65500000000000003,"15min":0.67300000000000004},"min":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.67200000000000004},"min":{"1min":0.41199999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.98899999999999999,"5min":1.085,"15min":1.085},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67900000000000005,"15min":0.70899999999999996},"min":{"1min":0.44,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89900000000000002,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.52900000000000003}]}]},{"osd":2,"up_from":89,"seq":382252089363,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36628,"kb_used_data":972,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708524,"statfs":{"total":95995035648,"available":95957528576,"internally_reserved":0,"allocated":995328,"data_stored":524245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.435}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.77200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.69499999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.83099999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.97399999999999998}]}]},{"osd":3,"up_from":94,"seq":403726925837,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":521160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25600000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28999999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33300000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019036,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.58899999999999997,"15min":0.60899999999999999},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.64100000000000001,"15min":0.66300000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69699999999999995,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 22:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.36299999999999999},"max":{"1min":0.81699999999999995,"5min":0.90500000000000003,"15min":1.482},"last":0.434},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":0.85999999999999999},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51100000000000001,"15min":0.46999999999999997},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":0.65900000000000003,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.623},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54000000000000004,"15min":0.48799999999999999},"min":{"1min":0.34899999999999998,"5min":0.32800000000000001,"15min":0.29399999999999998},"max":{"1min":0.65600000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57299999999999995,"15min":0.5},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76400000000000001,"5min":0.875,"15min":0.875},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.52100000000000002},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.79200000000000004,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.60299999999999998,"15min":0.55200000000000005},"min":{"1min":0.41999999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.754,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59999999999999998,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.53600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822808,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306216,"kb_used_data":928,"kb_used_omap":0,"kb_used_meta":305280,"kb_avail":93438936,"statfs":{"total":95995035648,"available":95681470464,"internally_reserved":0,"allocated":950272,"data_stored":516621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312606720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.56299999999999994,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80200000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.63},"min":{"1min":0.34100000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.79000000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.79000000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.77900000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.751}]},{"osd":3,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.34999999999999998},"max":{"1min":0.68400000000000005,"5min":0.79000000000000004,"15min":1.278},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.64000000000000001,"15min":0.61899999999999999},"min":{"1min":0.51700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.3109999999999999},"last":0.75800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:51 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28799999999999998},"max":{"1min":0.72199999999999998,"5min":0.81299999999999994,"15min":0.89000000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.47899999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":0.70099999999999996,"5min":0.73099999999999998,"15min":0.88300000000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.48599999999999999},"min":{"1min":0.35899999999999999,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":0.69499999999999995,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56599999999999995,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":0.79200000000000004,"5min":0.84599999999999997,"15min":0.90300000000000002},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.59099999999999997,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.89500000000000002},"last":0.497},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59399999999999997,"15min":0.55400000000000005},"min":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":0.77000000000000002,"5min":0.78400000000000003,"15min":0.85199999999999998},"last":0.46400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:39:02.226 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:39:02.590 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:03.120 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:02 smithi052 ceph-mon[187743]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:03.120 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:02 smithi052 ceph-mon[187743]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:03.121 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:02 smithi052 ceph-mon[180979]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:03.121 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:02 smithi052 ceph-mon[180979]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:03.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:02 smithi151 ceph-mon[159210]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:02 smithi151 ceph-mon[159210]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:39:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:39:03] "GET /metrics HTTP/1.1" 200 33968 "" "Prometheus/2.33.4" 2023-09-23T22:39:04.201 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:39:04.202 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:39:04.854 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":210,"stamp":"2023-09-23T22:39:03.476754+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1348296,"kb_used_data":9416,"kb_used_omap":126,"kb_used_meta":1338689,"kb_avail":748612920,"statfs":{"total":767960285184,"available":766579630080,"internally_reserved":0,"allocated":9641984,"data_stored":5951558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1370818171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002897"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140027+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:38:05.140027+0000","last_peered":"2023-09-23T22:38:05.140027+0000","last_clean":"2023-09-23T22:38:05.140027+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:38:05.140027+0000","last_undegraded":"2023-09-23T22:38:05.140027+0000","last_fullsized":"2023-09-23T22:38:05.140027+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147480+0000","last_change":"2023-09-23T22:37:35.270139+0000","last_active":"2023-09-23T22:38:05.147480+0000","last_peered":"2023-09-23T22:38:05.147480+0000","last_clean":"2023-09-23T22:38:05.147480+0000","last_became_active":"2023-09-23T22:37:35.269761+0000","last_became_peered":"2023-09-23T22:37:35.269761+0000","last_unstale":"2023-09-23T22:38:05.147480+0000","last_undegraded":"2023-09-23T22:38:05.147480+0000","last_fullsized":"2023-09-23T22:38:05.147480+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:08.396803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.147696+0000","last_change":"2023-09-23T22:38:06.147696+0000","last_active":"2023-09-23T22:38:06.147696+0000","last_peered":"2023-09-23T22:38:06.147696+0000","last_clean":"2023-09-23T22:38:06.147696+0000","last_became_active":"2023-09-23T22:38:06.147382+0000","last_became_peered":"2023-09-23T22:38:06.147382+0000","last_unstale":"2023-09-23T22:38:06.147696+0000","last_undegraded":"2023-09-23T22:38:06.147696+0000","last_fullsized":"2023-09-23T22:38:06.147696+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:04.570885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.149889+0000","last_change":"2023-09-23T22:38:06.149889+0000","last_active":"2023-09-23T22:38:06.149889+0000","last_peered":"2023-09-23T22:38:06.149889+0000","last_clean":"2023-09-23T22:38:06.149889+0000","last_became_active":"2023-09-23T22:38:06.149711+0000","last_became_peered":"2023-09-23T22:38:06.149711+0000","last_unstale":"2023-09-23T22:38:06.149889+0000","last_undegraded":"2023-09-23T22:38:06.149889+0000","last_fullsized":"2023-09-23T22:38:06.149889+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:06.335320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151879+0000","last_change":"2023-09-23T22:38:06.151879+0000","last_active":"2023-09-23T22:38:06.151879+0000","last_peered":"2023-09-23T22:38:06.151879+0000","last_clean":"2023-09-23T22:38:06.151879+0000","last_became_active":"2023-09-23T22:38:06.151519+0000","last_became_peered":"2023-09-23T22:38:06.151519+0000","last_unstale":"2023-09-23T22:38:06.151879+0000","last_undegraded":"2023-09-23T22:38:06.151879+0000","last_fullsized":"2023-09-23T22:38:06.151879+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.135034+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:38:05.135034+0000","last_peered":"2023-09-23T22:38:05.135034+0000","last_clean":"2023-09-23T22:38:05.135034+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:38:05.135034+0000","last_undegraded":"2023-09-23T22:38:05.135034+0000","last_fullsized":"2023-09-23T22:38:05.135034+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153986+0000","last_change":"2023-09-23T22:38:06.153986+0000","last_active":"2023-09-23T22:38:06.153986+0000","last_peered":"2023-09-23T22:38:06.153986+0000","last_clean":"2023-09-23T22:38:06.153986+0000","last_became_active":"2023-09-23T22:38:06.153604+0000","last_became_peered":"2023-09-23T22:38:06.153604+0000","last_unstale":"2023-09-23T22:38:06.153986+0000","last_undegraded":"2023-09-23T22:38:06.153986+0000","last_fullsized":"2023-09-23T22:38:06.153986+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134780+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:38:05.134780+0000","last_peered":"2023-09-23T22:38:05.134780+0000","last_clean":"2023-09-23T22:38:05.134780+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:38:05.134780+0000","last_undegraded":"2023-09-23T22:38:05.134780+0000","last_fullsized":"2023-09-23T22:38:05.134780+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152423+0000","last_change":"2023-09-23T22:38:06.152423+0000","last_active":"2023-09-23T22:38:06.152423+0000","last_peered":"2023-09-23T22:38:06.152423+0000","last_clean":"2023-09-23T22:38:06.152423+0000","last_became_active":"2023-09-23T22:38:06.152083+0000","last_became_peered":"2023-09-23T22:38:06.152083+0000","last_unstale":"2023-09-23T22:38:06.152423+0000","last_undegraded":"2023-09-23T22:38:06.152423+0000","last_fullsized":"2023-09-23T22:38:06.152423+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:49.106880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.144944+0000","last_change":"2023-09-23T22:35:51.475988+0000","last_active":"2023-09-23T22:38:06.144944+0000","last_peered":"2023-09-23T22:38:06.144944+0000","last_clean":"2023-09-23T22:38:06.144944+0000","last_became_active":"2023-09-23T22:35:51.475664+0000","last_became_peered":"2023-09-23T22:35:51.475664+0000","last_unstale":"2023-09-23T22:38:06.144944+0000","last_undegraded":"2023-09-23T22:38:06.144944+0000","last_fullsized":"2023-09-23T22:38:06.144944+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:22.107425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140271+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:38:05.140271+0000","last_peered":"2023-09-23T22:38:05.140271+0000","last_clean":"2023-09-23T22:38:05.140271+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:38:05.140271+0000","last_undegraded":"2023-09-23T22:38:05.140271+0000","last_fullsized":"2023-09-23T22:38:05.140271+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.146941+0000","last_change":"2023-09-23T22:37:03.801120+0000","last_active":"2023-09-23T22:38:05.146941+0000","last_peered":"2023-09-23T22:38:05.146941+0000","last_clean":"2023-09-23T22:38:05.146941+0000","last_became_active":"2023-09-23T22:37:03.800596+0000","last_became_peered":"2023-09-23T22:37:03.800596+0000","last_unstale":"2023-09-23T22:38:05.146941+0000","last_undegraded":"2023-09-23T22:38:05.146941+0000","last_fullsized":"2023-09-23T22:38:05.146941+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:03.424170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136253+0000","last_change":"2023-09-23T22:37:35.276863+0000","last_active":"2023-09-23T22:38:05.136253+0000","last_peered":"2023-09-23T22:38:05.136253+0000","last_clean":"2023-09-23T22:38:05.136253+0000","last_became_active":"2023-09-23T22:37:35.276428+0000","last_became_peered":"2023-09-23T22:37:35.276428+0000","last_unstale":"2023-09-23T22:38:05.136253+0000","last_undegraded":"2023-09-23T22:38:05.136253+0000","last_fullsized":"2023-09-23T22:38:05.136253+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155276+0000","last_change":"2023-09-23T22:38:06.155276+0000","last_active":"2023-09-23T22:38:06.155276+0000","last_peered":"2023-09-23T22:38:06.155276+0000","last_clean":"2023-09-23T22:38:06.155276+0000","last_became_active":"2023-09-23T22:38:06.154550+0000","last_became_peered":"2023-09-23T22:38:06.154550+0000","last_unstale":"2023-09-23T22:38:06.155276+0000","last_undegraded":"2023-09-23T22:38:06.155276+0000","last_fullsized":"2023-09-23T22:38:06.155276+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:06.092371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140029+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:38:05.140029+0000","last_peered":"2023-09-23T22:38:05.140029+0000","last_clean":"2023-09-23T22:38:05.140029+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:38:05.140029+0000","last_undegraded":"2023-09-23T22:38:05.140029+0000","last_fullsized":"2023-09-23T22:38:05.140029+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155104+0000","last_change":"2023-09-23T22:38:06.155104+0000","last_active":"2023-09-23T22:38:06.155104+0000","last_peered":"2023-09-23T22:38:06.155104+0000","last_clean":"2023-09-23T22:38:06.155104+0000","last_became_active":"2023-09-23T22:38:06.154630+0000","last_became_peered":"2023-09-23T22:38:06.154630+0000","last_unstale":"2023-09-23T22:38:06.155104+0000","last_undegraded":"2023-09-23T22:38:06.155104+0000","last_fullsized":"2023-09-23T22:38:06.155104+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:24:53.232885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144233+0000","last_change":"2023-09-23T22:35:51.477103+0000","last_active":"2023-09-23T22:38:05.144233+0000","last_peered":"2023-09-23T22:38:05.144233+0000","last_clean":"2023-09-23T22:38:05.144233+0000","last_became_active":"2023-09-23T22:35:51.476467+0000","last_became_peered":"2023-09-23T22:35:51.476467+0000","last_unstale":"2023-09-23T22:38:05.144233+0000","last_undegraded":"2023-09-23T22:38:05.144233+0000","last_fullsized":"2023-09-23T22:38:05.144233+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:24.234310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142041+0000","last_change":"2023-09-23T22:37:35.271043+0000","last_active":"2023-09-23T22:38:05.142041+0000","last_peered":"2023-09-23T22:38:05.142041+0000","last_clean":"2023-09-23T22:38:05.142041+0000","last_became_active":"2023-09-23T22:37:35.270808+0000","last_became_peered":"2023-09-23T22:37:35.270808+0000","last_unstale":"2023-09-23T22:38:05.142041+0000","last_undegraded":"2023-09-23T22:38:05.142041+0000","last_fullsized":"2023-09-23T22:38:05.142041+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:00.218174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141189+0000","last_change":"2023-09-23T22:35:51.476177+0000","last_active":"2023-09-23T22:38:05.141189+0000","last_peered":"2023-09-23T22:38:05.141189+0000","last_clean":"2023-09-23T22:38:05.141189+0000","last_became_active":"2023-09-23T22:35:51.475644+0000","last_became_peered":"2023-09-23T22:35:51.475644+0000","last_unstale":"2023-09-23T22:38:05.141189+0000","last_undegraded":"2023-09-23T22:38:05.141189+0000","last_fullsized":"2023-09-23T22:38:05.141189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.147016+0000","last_change":"2023-09-23T22:37:35.266620+0000","last_active":"2023-09-23T22:38:05.147016+0000","last_peered":"2023-09-23T22:38:05.147016+0000","last_clean":"2023-09-23T22:38:05.147016+0000","last_became_active":"2023-09-23T22:37:35.266391+0000","last_became_peered":"2023-09-23T22:37:35.266391+0000","last_unstale":"2023-09-23T22:38:05.147016+0000","last_undegraded":"2023-09-23T22:38:05.147016+0000","last_fullsized":"2023-09-23T22:38:05.147016+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:15.615607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150984+0000","last_change":"2023-09-23T22:38:06.150984+0000","last_active":"2023-09-23T22:38:06.150984+0000","last_peered":"2023-09-23T22:38:06.150984+0000","last_clean":"2023-09-23T22:38:06.150984+0000","last_became_active":"2023-09-23T22:38:06.150668+0000","last_became_peered":"2023-09-23T22:38:06.150668+0000","last_unstale":"2023-09-23T22:38:06.150984+0000","last_undegraded":"2023-09-23T22:38:06.150984+0000","last_fullsized":"2023-09-23T22:38:06.150984+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:25.323709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139228+0000","last_change":"2023-09-23T22:37:03.808191+0000","last_active":"2023-09-23T22:38:05.139228+0000","last_peered":"2023-09-23T22:38:05.139228+0000","last_clean":"2023-09-23T22:38:05.139228+0000","last_became_active":"2023-09-23T22:37:03.807779+0000","last_became_peered":"2023-09-23T22:37:03.807779+0000","last_unstale":"2023-09-23T22:38:05.139228+0000","last_undegraded":"2023-09-23T22:38:05.139228+0000","last_fullsized":"2023-09-23T22:38:05.139228+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136454+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:38:05.136454+0000","last_peered":"2023-09-23T22:38:05.136454+0000","last_clean":"2023-09-23T22:38:05.136454+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:38:05.136454+0000","last_undegraded":"2023-09-23T22:38:05.136454+0000","last_fullsized":"2023-09-23T22:38:05.136454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154384+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:38:06.154384+0000","last_peered":"2023-09-23T22:38:06.154384+0000","last_clean":"2023-09-23T22:38:06.154384+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:38:06.154384+0000","last_undegraded":"2023-09-23T22:38:06.154384+0000","last_fullsized":"2023-09-23T22:38:06.154384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":297,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139145+0000","last_change":"2023-09-23T22:35:51.477611+0000","last_active":"2023-09-23T22:38:05.139145+0000","last_peered":"2023-09-23T22:38:05.139145+0000","last_clean":"2023-09-23T22:38:05.139145+0000","last_became_active":"2023-09-23T22:35:51.477061+0000","last_became_peered":"2023-09-23T22:35:51.477061+0000","last_unstale":"2023-09-23T22:38:05.139145+0000","last_undegraded":"2023-09-23T22:38:05.139145+0000","last_fullsized":"2023-09-23T22:38:05.139145+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133688+0000","last_change":"2023-09-23T22:37:35.270450+0000","last_active":"2023-09-23T22:38:05.133688+0000","last_peered":"2023-09-23T22:38:05.133688+0000","last_clean":"2023-09-23T22:38:05.133688+0000","last_became_active":"2023-09-23T22:37:35.270097+0000","last_became_peered":"2023-09-23T22:37:35.270097+0000","last_unstale":"2023-09-23T22:38:05.133688+0000","last_undegraded":"2023-09-23T22:38:05.133688+0000","last_fullsized":"2023-09-23T22:38:05.133688+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140469+0000","last_change":"2023-09-23T22:37:03.801645+0000","last_active":"2023-09-23T22:38:05.140469+0000","last_peered":"2023-09-23T22:38:05.140469+0000","last_clean":"2023-09-23T22:38:05.140469+0000","last_became_active":"2023-09-23T22:37:03.799407+0000","last_became_peered":"2023-09-23T22:37:03.799407+0000","last_unstale":"2023-09-23T22:38:05.140469+0000","last_undegraded":"2023-09-23T22:38:05.140469+0000","last_fullsized":"2023-09-23T22:38:05.140469+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.396991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":49,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.141323+0000","last_change":"2023-09-23T22:37:35.271109+0000","last_active":"2023-09-23T22:38:06.141323+0000","last_peered":"2023-09-23T22:38:06.141323+0000","last_clean":"2023-09-23T22:38:06.141323+0000","last_became_active":"2023-09-23T22:37:35.270850+0000","last_became_peered":"2023-09-23T22:37:35.270850+0000","last_unstale":"2023-09-23T22:38:06.141323+0000","last_undegraded":"2023-09-23T22:38:06.141323+0000","last_fullsized":"2023-09-23T22:38:06.141323+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:24.045725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148923+0000","last_change":"2023-09-23T22:38:06.148923+0000","last_active":"2023-09-23T22:38:06.148923+0000","last_peered":"2023-09-23T22:38:06.148923+0000","last_clean":"2023-09-23T22:38:06.148923+0000","last_became_active":"2023-09-23T22:38:06.148636+0000","last_became_peered":"2023-09-23T22:38:06.148636+0000","last_unstale":"2023-09-23T22:38:06.148923+0000","last_undegraded":"2023-09-23T22:38:06.148923+0000","last_fullsized":"2023-09-23T22:38:06.148923+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:01.486987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156455+0000","last_change":"2023-09-23T22:38:06.156455+0000","last_active":"2023-09-23T22:38:06.156455+0000","last_peered":"2023-09-23T22:38:06.156455+0000","last_clean":"2023-09-23T22:38:06.156455+0000","last_became_active":"2023-09-23T22:38:06.156203+0000","last_became_peered":"2023-09-23T22:38:06.156203+0000","last_unstale":"2023-09-23T22:38:06.156455+0000","last_undegraded":"2023-09-23T22:38:06.156455+0000","last_fullsized":"2023-09-23T22:38:06.156455+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142051+0000","last_change":"2023-09-23T22:35:51.474457+0000","last_active":"2023-09-23T22:38:05.142051+0000","last_peered":"2023-09-23T22:38:05.142051+0000","last_clean":"2023-09-23T22:38:05.142051+0000","last_became_active":"2023-09-23T22:35:51.474166+0000","last_became_peered":"2023-09-23T22:35:51.474166+0000","last_unstale":"2023-09-23T22:38:05.142051+0000","last_undegraded":"2023-09-23T22:38:05.142051+0000","last_fullsized":"2023-09-23T22:38:05.142051+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.133761+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:38:05.133761+0000","last_peered":"2023-09-23T22:38:05.133761+0000","last_clean":"2023-09-23T22:38:05.133761+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:38:05.133761+0000","last_undegraded":"2023-09-23T22:38:05.133761+0000","last_fullsized":"2023-09-23T22:38:05.133761+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139384+0000","last_change":"2023-09-23T22:37:03.797729+0000","last_active":"2023-09-23T22:38:05.139384+0000","last_peered":"2023-09-23T22:38:05.139384+0000","last_clean":"2023-09-23T22:38:05.139384+0000","last_became_active":"2023-09-23T22:37:03.797377+0000","last_became_peered":"2023-09-23T22:37:03.797377+0000","last_unstale":"2023-09-23T22:38:05.139384+0000","last_undegraded":"2023-09-23T22:38:05.139384+0000","last_fullsized":"2023-09-23T22:38:05.139384+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:18.065155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157484+0000","last_change":"2023-09-23T22:38:06.157484+0000","last_active":"2023-09-23T22:38:06.157484+0000","last_peered":"2023-09-23T22:38:06.157484+0000","last_clean":"2023-09-23T22:38:06.157484+0000","last_became_active":"2023-09-23T22:38:06.157175+0000","last_became_peered":"2023-09-23T22:38:06.157175+0000","last_unstale":"2023-09-23T22:38:06.157484+0000","last_undegraded":"2023-09-23T22:38:06.157484+0000","last_fullsized":"2023-09-23T22:38:06.157484+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142777+0000","last_change":"2023-09-23T22:37:03.805141+0000","last_active":"2023-09-23T22:38:05.142777+0000","last_peered":"2023-09-23T22:38:05.142777+0000","last_clean":"2023-09-23T22:38:05.142777+0000","last_became_active":"2023-09-23T22:37:03.804606+0000","last_became_peered":"2023-09-23T22:37:03.804606+0000","last_unstale":"2023-09-23T22:38:05.142777+0000","last_undegraded":"2023-09-23T22:38:05.142777+0000","last_fullsized":"2023-09-23T22:38:05.142777+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:28.826826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153406+0000","last_change":"2023-09-23T22:38:06.153406+0000","last_active":"2023-09-23T22:38:06.153406+0000","last_peered":"2023-09-23T22:38:06.153406+0000","last_clean":"2023-09-23T22:38:06.153406+0000","last_became_active":"2023-09-23T22:38:06.153092+0000","last_became_peered":"2023-09-23T22:38:06.153092+0000","last_unstale":"2023-09-23T22:38:06.153406+0000","last_undegraded":"2023-09-23T22:38:06.153406+0000","last_fullsized":"2023-09-23T22:38:06.153406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:53.462379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155404+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:38:06.155404+0000","last_peered":"2023-09-23T22:38:06.155404+0000","last_clean":"2023-09-23T22:38:06.155404+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:38:06.155404+0000","last_undegraded":"2023-09-23T22:38:06.155404+0000","last_fullsized":"2023-09-23T22:38:06.155404+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.142766+0000","last_change":"2023-09-23T22:37:03.800899+0000","last_active":"2023-09-23T22:38:05.142766+0000","last_peered":"2023-09-23T22:38:05.142766+0000","last_clean":"2023-09-23T22:38:05.142766+0000","last_became_active":"2023-09-23T22:37:03.800586+0000","last_became_peered":"2023-09-23T22:37:03.800586+0000","last_unstale":"2023-09-23T22:38:05.142766+0000","last_undegraded":"2023-09-23T22:38:05.142766+0000","last_fullsized":"2023-09-23T22:38:05.142766+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.140171+0000","last_change":"2023-09-23T22:37:35.274945+0000","last_active":"2023-09-23T22:38:05.140171+0000","last_peered":"2023-09-23T22:38:05.140171+0000","last_clean":"2023-09-23T22:38:05.140171+0000","last_became_active":"2023-09-23T22:37:35.274580+0000","last_became_peered":"2023-09-23T22:37:35.274580+0000","last_unstale":"2023-09-23T22:38:05.140171+0000","last_undegraded":"2023-09-23T22:38:05.140171+0000","last_fullsized":"2023-09-23T22:38:05.140171+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141735+0000","last_change":"2023-09-23T22:37:35.276836+0000","last_active":"2023-09-23T22:38:05.141735+0000","last_peered":"2023-09-23T22:38:05.141735+0000","last_clean":"2023-09-23T22:38:05.141735+0000","last_became_active":"2023-09-23T22:37:35.276422+0000","last_became_peered":"2023-09-23T22:37:35.276422+0000","last_unstale":"2023-09-23T22:38:05.141735+0000","last_undegraded":"2023-09-23T22:38:05.141735+0000","last_fullsized":"2023-09-23T22:38:05.141735+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137068+0000","last_change":"2023-09-23T22:37:03.804557+0000","last_active":"2023-09-23T22:38:05.137068+0000","last_peered":"2023-09-23T22:38:05.137068+0000","last_clean":"2023-09-23T22:38:05.137068+0000","last_became_active":"2023-09-23T22:37:03.804229+0000","last_became_peered":"2023-09-23T22:37:03.804229+0000","last_unstale":"2023-09-23T22:38:05.137068+0000","last_undegraded":"2023-09-23T22:38:05.137068+0000","last_fullsized":"2023-09-23T22:38:05.137068+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152590+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:38:06.152590+0000","last_peered":"2023-09-23T22:38:06.152590+0000","last_clean":"2023-09-23T22:38:06.152590+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:38:06.152590+0000","last_undegraded":"2023-09-23T22:38:06.152590+0000","last_fullsized":"2023-09-23T22:38:06.152590+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151577+0000","last_change":"2023-09-23T22:38:06.151577+0000","last_active":"2023-09-23T22:38:06.151577+0000","last_peered":"2023-09-23T22:38:06.151577+0000","last_clean":"2023-09-23T22:38:06.151577+0000","last_became_active":"2023-09-23T22:38:06.151229+0000","last_became_peered":"2023-09-23T22:38:06.151229+0000","last_unstale":"2023-09-23T22:38:06.151577+0000","last_undegraded":"2023-09-23T22:38:06.151577+0000","last_fullsized":"2023-09-23T22:38:06.151577+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:25.864388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.144329+0000","last_change":"2023-09-23T22:37:03.800764+0000","last_active":"2023-09-23T22:38:05.144329+0000","last_peered":"2023-09-23T22:38:05.144329+0000","last_clean":"2023-09-23T22:38:05.144329+0000","last_became_active":"2023-09-23T22:37:03.800407+0000","last_became_peered":"2023-09-23T22:37:03.800407+0000","last_unstale":"2023-09-23T22:38:05.144329+0000","last_undegraded":"2023-09-23T22:38:05.144329+0000","last_fullsized":"2023-09-23T22:38:05.144329+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:10.143602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.155895+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:38:06.155895+0000","last_peered":"2023-09-23T22:38:06.155895+0000","last_clean":"2023-09-23T22:38:06.155895+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:38:06.155895+0000","last_undegraded":"2023-09-23T22:38:06.155895+0000","last_fullsized":"2023-09-23T22:38:06.155895+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136911+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:38:05.136911+0000","last_peered":"2023-09-23T22:38:05.136911+0000","last_clean":"2023-09-23T22:38:05.136911+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:38:05.136911+0000","last_undegraded":"2023-09-23T22:38:05.136911+0000","last_fullsized":"2023-09-23T22:38:05.136911+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141209+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:38:05.141209+0000","last_peered":"2023-09-23T22:38:05.141209+0000","last_clean":"2023-09-23T22:38:05.141209+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:38:05.141209+0000","last_undegraded":"2023-09-23T22:38:05.141209+0000","last_fullsized":"2023-09-23T22:38:05.141209+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154190+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:38:06.154190+0000","last_peered":"2023-09-23T22:38:06.154190+0000","last_clean":"2023-09-23T22:38:06.154190+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:38:06.154190+0000","last_undegraded":"2023-09-23T22:38:06.154190+0000","last_fullsized":"2023-09-23T22:38:06.154190+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.138291+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:38:05.138291+0000","last_peered":"2023-09-23T22:38:05.138291+0000","last_clean":"2023-09-23T22:38:05.138291+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:38:05.138291+0000","last_undegraded":"2023-09-23T22:38:05.138291+0000","last_fullsized":"2023-09-23T22:38:05.138291+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134242+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:38:05.134242+0000","last_peered":"2023-09-23T22:38:05.134242+0000","last_clean":"2023-09-23T22:38:05.134242+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:38:05.134242+0000","last_undegraded":"2023-09-23T22:38:05.134242+0000","last_fullsized":"2023-09-23T22:38:05.134242+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":62,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139635+0000","last_change":"2023-09-23T22:35:51.479873+0000","last_active":"2023-09-23T22:38:05.139635+0000","last_peered":"2023-09-23T22:38:05.139635+0000","last_clean":"2023-09-23T22:38:05.139635+0000","last_became_active":"2023-09-23T22:35:51.479506+0000","last_became_peered":"2023-09-23T22:35:51.479506+0000","last_unstale":"2023-09-23T22:38:05.139635+0000","last_undegraded":"2023-09-23T22:38:05.139635+0000","last_fullsized":"2023-09-23T22:38:05.139635+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154965+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:38:06.154965+0000","last_peered":"2023-09-23T22:38:06.154965+0000","last_clean":"2023-09-23T22:38:06.154965+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:38:06.154965+0000","last_undegraded":"2023-09-23T22:38:06.154965+0000","last_fullsized":"2023-09-23T22:38:06.154965+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":70,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136898+0000","last_change":"2023-09-23T22:35:51.475694+0000","last_active":"2023-09-23T22:38:05.136898+0000","last_peered":"2023-09-23T22:38:05.136898+0000","last_clean":"2023-09-23T22:38:05.136898+0000","last_became_active":"2023-09-23T22:35:51.475326+0000","last_became_peered":"2023-09-23T22:35:51.475326+0000","last_unstale":"2023-09-23T22:38:05.136898+0000","last_undegraded":"2023-09-23T22:38:05.136898+0000","last_fullsized":"2023-09-23T22:38:05.136898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156874+0000","last_change":"2023-09-23T22:38:06.156215+0000","last_active":"2023-09-23T22:38:06.156874+0000","last_peered":"2023-09-23T22:38:06.156874+0000","last_clean":"2023-09-23T22:38:06.156874+0000","last_became_active":"2023-09-23T22:38:06.155344+0000","last_became_peered":"2023-09-23T22:38:06.155344+0000","last_unstale":"2023-09-23T22:38:06.156874+0000","last_undegraded":"2023-09-23T22:38:06.156874+0000","last_fullsized":"2023-09-23T22:38:06.156874+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:06.252155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137434+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:38:05.137434+0000","last_peered":"2023-09-23T22:38:05.137434+0000","last_clean":"2023-09-23T22:38:05.137434+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:38:05.137434+0000","last_undegraded":"2023-09-23T22:38:05.137434+0000","last_fullsized":"2023-09-23T22:38:05.137434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157553+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:38:06.157553+0000","last_peered":"2023-09-23T22:38:06.157553+0000","last_clean":"2023-09-23T22:38:06.157553+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:38:06.157553+0000","last_undegraded":"2023-09-23T22:38:06.157553+0000","last_fullsized":"2023-09-23T22:38:06.157553+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.152547+0000","last_change":"2023-09-23T22:38:06.151401+0000","last_active":"2023-09-23T22:38:06.152547+0000","last_peered":"2023-09-23T22:38:06.152547+0000","last_clean":"2023-09-23T22:38:06.152547+0000","last_became_active":"2023-09-23T22:38:06.151071+0000","last_became_peered":"2023-09-23T22:38:06.151071+0000","last_unstale":"2023-09-23T22:38:06.152547+0000","last_undegraded":"2023-09-23T22:38:06.152547+0000","last_fullsized":"2023-09-23T22:38:06.152547+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:14.260144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.154184+0000","last_change":"2023-09-23T22:38:06.154184+0000","last_active":"2023-09-23T22:38:06.154184+0000","last_peered":"2023-09-23T22:38:06.154184+0000","last_clean":"2023-09-23T22:38:06.154184+0000","last_became_active":"2023-09-23T22:38:06.153914+0000","last_became_peered":"2023-09-23T22:38:06.153914+0000","last_unstale":"2023-09-23T22:38:06.154184+0000","last_undegraded":"2023-09-23T22:38:06.154184+0000","last_fullsized":"2023-09-23T22:38:06.154184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:53.068518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":63,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.137833+0000","last_change":"2023-09-23T22:37:35.268294+0000","last_active":"2023-09-23T22:38:05.137833+0000","last_peered":"2023-09-23T22:38:05.137833+0000","last_clean":"2023-09-23T22:38:05.137833+0000","last_became_active":"2023-09-23T22:37:35.267886+0000","last_became_peered":"2023-09-23T22:37:35.267886+0000","last_unstale":"2023-09-23T22:38:05.137833+0000","last_undegraded":"2023-09-23T22:38:05.137833+0000","last_fullsized":"2023-09-23T22:38:05.137833+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141916+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:38:05.141916+0000","last_peered":"2023-09-23T22:38:05.141916+0000","last_clean":"2023-09-23T22:38:05.141916+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:38:05.141916+0000","last_undegraded":"2023-09-23T22:38:05.141916+0000","last_fullsized":"2023-09-23T22:38:05.141916+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":68,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141885+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:38:05.141885+0000","last_peered":"2023-09-23T22:38:05.141885+0000","last_clean":"2023-09-23T22:38:05.141885+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:38:05.141885+0000","last_undegraded":"2023-09-23T22:38:05.141885+0000","last_fullsized":"2023-09-23T22:38:05.141885+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.139285+0000","last_change":"2023-09-23T22:37:35.272454+0000","last_active":"2023-09-23T22:38:05.139285+0000","last_peered":"2023-09-23T22:38:05.139285+0000","last_clean":"2023-09-23T22:38:05.139285+0000","last_became_active":"2023-09-23T22:37:35.271917+0000","last_became_peered":"2023-09-23T22:37:35.271917+0000","last_unstale":"2023-09-23T22:38:05.139285+0000","last_undegraded":"2023-09-23T22:38:05.139285+0000","last_fullsized":"2023-09-23T22:38:05.139285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:42.398917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157765+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:38:06.157765+0000","last_peered":"2023-09-23T22:38:06.157765+0000","last_clean":"2023-09-23T22:38:06.157765+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:38:06.157765+0000","last_undegraded":"2023-09-23T22:38:06.157765+0000","last_fullsized":"2023-09-23T22:38:06.157765+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.151474+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:38:06.151474+0000","last_peered":"2023-09-23T22:38:06.151474+0000","last_clean":"2023-09-23T22:38:06.151474+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:38:06.151474+0000","last_undegraded":"2023-09-23T22:38:06.151474+0000","last_fullsized":"2023-09-23T22:38:06.151474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.134905+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:38:05.134905+0000","last_peered":"2023-09-23T22:38:05.134905+0000","last_clean":"2023-09-23T22:38:05.134905+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:38:05.134905+0000","last_undegraded":"2023-09-23T22:38:05.134905+0000","last_fullsized":"2023-09-23T22:38:05.134905+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":76,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148885+0000","last_change":"2023-09-23T22:38:06.148885+0000","last_active":"2023-09-23T22:38:06.148885+0000","last_peered":"2023-09-23T22:38:06.148885+0000","last_clean":"2023-09-23T22:38:06.148885+0000","last_became_active":"2023-09-23T22:38:06.148470+0000","last_became_peered":"2023-09-23T22:38:06.148470+0000","last_unstale":"2023-09-23T22:38:06.148885+0000","last_undegraded":"2023-09-23T22:38:06.148885+0000","last_fullsized":"2023-09-23T22:38:06.148885+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:02.793510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.153759+0000","last_change":"2023-09-23T22:38:06.153487+0000","last_active":"2023-09-23T22:38:06.153759+0000","last_peered":"2023-09-23T22:38:06.153759+0000","last_clean":"2023-09-23T22:38:06.153759+0000","last_became_active":"2023-09-23T22:38:06.153156+0000","last_became_peered":"2023-09-23T22:38:06.153156+0000","last_unstale":"2023-09-23T22:38:06.153759+0000","last_undegraded":"2023-09-23T22:38:06.153759+0000","last_fullsized":"2023-09-23T22:38:06.153759+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:49.053379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":61,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.136355+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:38:05.136355+0000","last_peered":"2023-09-23T22:38:05.136355+0000","last_clean":"2023-09-23T22:38:05.136355+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:38:05.136355+0000","last_undegraded":"2023-09-23T22:38:05.136355+0000","last_fullsized":"2023-09-23T22:38:05.136355+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.148692+0000","last_change":"2023-09-23T22:38:06.148692+0000","last_active":"2023-09-23T22:38:06.148692+0000","last_peered":"2023-09-23T22:38:06.148692+0000","last_clean":"2023-09-23T22:38:06.148692+0000","last_became_active":"2023-09-23T22:38:06.148096+0000","last_became_peered":"2023-09-23T22:38:06.148096+0000","last_unstale":"2023-09-23T22:38:06.148692+0000","last_undegraded":"2023-09-23T22:38:06.148692+0000","last_fullsized":"2023-09-23T22:38:06.148692+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:15.042597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.156081+0000","last_change":"2023-09-23T22:38:06.156081+0000","last_active":"2023-09-23T22:38:06.156081+0000","last_peered":"2023-09-23T22:38:06.156081+0000","last_clean":"2023-09-23T22:38:06.156081+0000","last_became_active":"2023-09-23T22:38:06.155768+0000","last_became_peered":"2023-09-23T22:38:06.155768+0000","last_unstale":"2023-09-23T22:38:06.156081+0000","last_undegraded":"2023-09-23T22:38:06.156081+0000","last_fullsized":"2023-09-23T22:38:06.156081+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:21.557265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.157097+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:38:06.157097+0000","last_peered":"2023-09-23T22:38:06.157097+0000","last_clean":"2023-09-23T22:38:06.157097+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:38:06.157097+0000","last_undegraded":"2023-09-23T22:38:06.157097+0000","last_fullsized":"2023-09-23T22:38:06.157097+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-09-23T22:38:06.150667+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:38:06.150667+0000","last_peered":"2023-09-23T22:38:06.150667+0000","last_clean":"2023-09-23T22:38:06.150667+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:38:06.150667+0000","last_undegraded":"2023-09-23T22:38:06.150667+0000","last_fullsized":"2023-09-23T22:38:06.150667+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":77,"reported_epoch":94,"state":"active+clean","last_fresh":"2023-09-23T22:38:05.141602+0000","last_change":"2023-09-23T22:37:35.275167+0000","last_active":"2023-09-23T22:38:05.141602+0000","last_peered":"2023-09-23T22:38:05.141602+0000","last_clean":"2023-09-23T22:38:05.141602+0000","last_became_active":"2023-09-23T22:37:35.274809+0000","last_became_peered":"2023-09-23T22:37:35.274809+0000","last_unstale":"2023-09-23T22:38:05.141602+0000","last_undegraded":"2023-09-23T22:38:05.141602+0000","last_fullsized":"2023-09-23T22:38:05.141602+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430352,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310920,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93434232,"statfs":{"total":95995035648,"available":95676653568,"internally_reserved":0,"allocated":1572864,"data_stored":1111945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57299999999999995,"15min":0.59899999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.60099999999999998,"15min":0.61599999999999999},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.755,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.499}]},{"osd":1,"last update":"Sat Sep 23 22:38:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:37:28 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58399999999999996,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.32400000000000001},"max":{"1min":0.79600000000000004,"5min":0.85399999999999998,"15min":1.4970000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.48499999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.81200000000000006,"5min":0.81299999999999994,"15min":1.087},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47899999999999998,"15min":0.498},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.26300000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":5.3140000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.51800000000000002,"15min":0.502},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.93799999999999994},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.80600000000000005,"5min":0.85799999999999998,"15min":1.0669999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57099999999999995,"15min":0.58299999999999996},"min":{"1min":0.42399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":5.3490000000000002},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56699999999999995,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.95799999999999996},"last":0.438},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61699999999999999,"15min":0.59199999999999997},"min":{"1min":0.504,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":1.038},"last":0.58299999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626580,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1560576,"data_stored":1111799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.57099999999999995,"15min":0.59799999999999998},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.68400000000000005,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.44},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57999999999999996,"15min":0.59799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.47399999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 22:37:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":0.81100000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.61399999999999999,"15min":0.58899999999999997},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.34300000000000003},"max":{"1min":0.79800000000000004,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.61699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51300000000000001,"15min":0.46700000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53600000000000003,"15min":0.47899999999999998},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":0.748,"5min":0.80600000000000005,"15min":0.81299999999999994},"last":0.65100000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54100000000000004,"15min":0.497},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":0.755,"5min":0.77300000000000002,"15min":0.84899999999999998},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57799999999999996,"15min":0.52500000000000002},"min":{"1min":0.38200000000000001,"5min":0.377,"15min":0.32400000000000001},"max":{"1min":0.83899999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.60899999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:38:32 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.59099999999999997,"15min":0.55000000000000004},"min":{"1min":0.46000000000000002,"5min":0.41799999999999998,"15min":0.34799999999999998},"max":{"1min":0.76000000000000001,"5min":0.81999999999999995,"15min":0.83199999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.29399999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":1.357},"last":0.58299999999999996}]}]},{"osd":1,"up_from":84,"seq":360777252890,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32412,"kb_used_data":980,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93712740,"statfs":{"total":95995035648,"available":95961845760,"internally_reserved":0,"allocated":1003520,"data_stored":527042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.313}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.495},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.53300000000000003}]}]},{"osd":0,"up_from":79,"seq":339302416424,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33068,"kb_used_data":1572,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712084,"statfs":{"total":95995035648,"available":95961174016,"internally_reserved":0,"allocated":1609728,"data_stored":1117239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.312},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.27700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.46300000000000002,"15min":0.47099999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.46500000000000002,"15min":0.47399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.625,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66900000000000004,"15min":0.67900000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.66300000000000003,"15min":0.67200000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.98099999999999998,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.63200000000000001},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.64900000000000002,"15min":0.66500000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91100000000000003,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.64800000000000002,"15min":0.66800000000000004},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.879,"5min":1.036,"15min":1.036},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.65500000000000003,"15min":0.67300000000000004},"min":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.67200000000000004},"min":{"1min":0.41199999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.98899999999999999,"5min":1.085,"15min":1.085},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67900000000000005,"15min":0.70899999999999996},"min":{"1min":0.44,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89900000000000002,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.52900000000000003}]}]},{"osd":2,"up_from":89,"seq":382252089364,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36628,"kb_used_data":972,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708524,"statfs":{"total":95995035648,"available":95957528576,"internally_reserved":0,"allocated":995328,"data_stored":524245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.374},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.35099999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.69299999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.622},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.67500000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64300000000000002}]}]},{"osd":3,"up_from":94,"seq":403726925837,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28288,"kb_used_data":952,"kb_used_omap":7,"kb_used_meta":27320,"kb_avail":93716864,"statfs":{"total":95995035648,"available":95966068736,"internally_reserved":0,"allocated":974848,"data_stored":521160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":27976296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25600000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28999999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33300000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019036,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302144,"kb_used_data":952,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93443008,"statfs":{"total":95995035648,"available":95685640192,"internally_reserved":0,"allocated":974848,"data_stored":521507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.58899999999999997,"15min":0.60899999999999999},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.64100000000000001,"15min":0.66300000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69699999999999995,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 22:37:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.36299999999999999},"max":{"1min":0.81699999999999995,"5min":0.90500000000000003,"15min":1.482},"last":0.434},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":0.85999999999999999},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51100000000000001,"15min":0.46999999999999997},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":0.65900000000000003,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.623},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54000000000000004,"15min":0.48799999999999999},"min":{"1min":0.34899999999999998,"5min":0.32800000000000001,"15min":0.29399999999999998},"max":{"1min":0.65600000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:38:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57299999999999995,"15min":0.5},"min":{"1min":0.38600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76400000000000001,"5min":0.875,"15min":0.875},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.52100000000000002},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.79200000000000004,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.60299999999999998,"15min":0.55200000000000005},"min":{"1min":0.41999999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.754,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59999999999999998,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.53600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822808,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306216,"kb_used_data":928,"kb_used_omap":0,"kb_used_meta":305280,"kb_avail":93438936,"statfs":{"total":95995035648,"available":95681470464,"internally_reserved":0,"allocated":950272,"data_stored":516621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312606720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.56299999999999994,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80200000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.63},"min":{"1min":0.34100000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.79000000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.79000000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.77900000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.751}]},{"osd":3,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.34999999999999998},"max":{"1min":0.68400000000000005,"5min":0.79000000000000004,"15min":1.278},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.64000000000000001,"15min":0.61899999999999999},"min":{"1min":0.51700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.3109999999999999},"last":0.75800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:51 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28799999999999998},"max":{"1min":0.72199999999999998,"5min":0.81299999999999994,"15min":0.89000000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.47899999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":0.70099999999999996,"5min":0.73099999999999998,"15min":0.88300000000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.48599999999999999},"min":{"1min":0.35899999999999999,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":0.69499999999999995,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56599999999999995,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":0.79200000000000004,"5min":0.84599999999999997,"15min":0.90300000000000002},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.59099999999999997,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.89500000000000002},"last":0.497},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59399999999999997,"15min":0.55400000000000005},"min":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":0.77000000000000002,"5min":0.78400000000000003,"15min":0.85199999999999998},"last":0.46400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:39:04.859 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:39:04.860 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T22:39:05.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:04 smithi151 ceph-mon[159210]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:05.228 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:05.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:04 smithi052 ceph-mon[180979]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:05.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:05.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:04 smithi052 ceph-mon[187743]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:05 smithi151 ceph-mon[159210]: from='client.34617 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:06.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:05 smithi052 ceph-mon[187743]: from='client.34617 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:06.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:05 smithi052 ceph-mon[180979]: from='client.34617 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi052.stdout:epoch 95 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:created 2023-09-23T22:22:34.777440+0000 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:modified 2023-09-23T22:38:06.133584+0000 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:crush_version 22 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:full_ratio 0.95 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:backfillfull_ratio 0.9 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:nearfull_ratio 0.85 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:require_min_compat_client luminous 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:min_compat_client luminous 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi052.stdout:require_osd_release pacific 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:stretch_mode_enabled false 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 72 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:max_osd 8 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0 up in weight 1 up_from 79 up_thru 94 down_at 76 last_clean_interval [8,75) [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800] [v2:172.21.15.52:6804/570465800,v1:172.21.15.52:6805/570465800] exists,up c382e099-ff66-4b35-af2f-978f52c83cfc 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 up in weight 1 up_from 84 up_thru 94 down_at 81 last_clean_interval [13,80) [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443] [v2:172.21.15.52:6812/1231582443,v1:172.21.15.52:6813/1231582443] exists,up 8704f27e-d1a5-49b1-9382-0ebc689e3f63 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2 up in weight 1 up_from 89 up_thru 94 down_at 86 last_clean_interval [18,85) [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046] [v2:172.21.15.52:6820/1755732046,v1:172.21.15.52:6821/1755732046] exists,up 80877779-5cd3-4a3c-bb3e-ffd41e160fb5 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 up in weight 1 up_from 94 up_thru 94 down_at 91 last_clean_interval [25,90) [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872] [v2:172.21.15.52:6828/133265872,v1:172.21.15.52:6829/133265872] exists,up c94bebc9-a105-4d74-9a03-c88e10cc2876 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 up in weight 1 up_from 30 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857] [v2:172.21.15.151:6802/2146533857,v1:172.21.15.151:6803/2146533857] exists,up a8b1470a-6d28-44d9-9921-6f8fbcb52b59 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 up in weight 1 up_from 36 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865] [v2:172.21.15.151:6810/992496865,v1:172.21.15.151:6811/992496865] exists,up 768d49a4-59ee-43e4-802b-a58f2b77580b 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 up in weight 1 up_from 42 up_thru 91 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922] [v2:172.21.15.151:6818/2869932922,v1:172.21.15.151:6819/2869932922] exists,up 2c295677-55e8-41ff-8733-982844192fbc 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 up in weight 1 up_from 48 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088] [v2:172.21.15.151:6826/2800377088,v1:172.21.15.151:6827/2800377088] exists,up c1b48eff-20d5-4d6d-bb17-0c44ffdf866e 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:pg_upmap_items 4.4 [1,5] 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:pg_upmap_items 4.f [1,2] 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:pg_upmap_items 4.10 [1,2] 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:0/3773142740 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/2401228233 expires 2023-09-24T22:23:02.549172+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:0/1914484619 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/1262640861 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6800/624901514 expires 2023-09-24T22:23:25.849910+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6801/65131393 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:0/3484573808 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3645713772 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/642109542 expires 2023-09-24T22:23:25.849910+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/4018405217 expires 2023-09-24T22:23:02.549172+0000 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6801/3545504609 expires 2023-09-24T22:23:02.549172+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6800/3545504609 expires 2023-09-24T22:23:02.549172+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/1038804029 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6801/624901514 expires 2023-09-24T22:23:25.849910+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/1078027830 expires 2023-09-24T22:30:59.031552+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6801/2011536210 expires 2023-09-24T22:27:29.395575+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3912833337 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3442490676 expires 2023-09-24T22:23:25.849910+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:6835/664659773 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3278569487 expires 2023-09-24T22:27:29.395575+0000 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:6834/664659773 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6800/2011536210 expires 2023-09-24T22:27:29.395575+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:0/483361858 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:0/4181179150 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3629843530 expires 2023-09-24T22:27:29.395575+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3758668631 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3637583595 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.151:0/4148041136 expires 2023-09-24T22:32:09.537934+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6800/65131393 expires 2023-09-24T22:32:49.359696+0000 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6801/2740388443 expires 2023-09-24T22:30:59.031552+0000 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:0/3070502744 expires 2023-09-24T22:30:59.031552+0000 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi052.stdout:blocklist 172.21.15.52:6800/2740388443 expires 2023-09-24T22:30:59.031552+0000 2023-09-23T22:39:07.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:06 smithi151 ceph-mon[159210]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1266532789' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:39:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:06 smithi052 ceph-mon[187743]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:07.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1266532789' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:39:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:06 smithi052 ceph-mon[180979]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1266532789' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:39:07.499 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 79 up_thru 94 down_at 76 last_clean_interval [8,75) [v2:172.21.15.52:6802/570465800,v1:172.21.15.52:6803/570465800] [v2:172.21.15.52:6804/570465800,v1:172.21.15.52:6805/570465800] exists,up c382e099-ff66-4b35-af2f-978f52c83cfc', 'osd.1 up in weight 1 up_from 84 up_thru 94 down_at 81 last_clean_interval [13,80) [v2:172.21.15.52:6810/1231582443,v1:172.21.15.52:6811/1231582443] [v2:172.21.15.52:6812/1231582443,v1:172.21.15.52:6813/1231582443] exists,up 8704f27e-d1a5-49b1-9382-0ebc689e3f63', 'osd.2 up in weight 1 up_from 89 up_thru 94 down_at 86 last_clean_interval [18,85) [v2:172.21.15.52:6818/1755732046,v1:172.21.15.52:6819/1755732046] [v2:172.21.15.52:6820/1755732046,v1:172.21.15.52:6821/1755732046] exists,up 80877779-5cd3-4a3c-bb3e-ffd41e160fb5', 'osd.3 up in weight 1 up_from 94 up_thru 94 down_at 91 last_clean_interval [25,90) [v2:172.21.15.52:6826/133265872,v1:172.21.15.52:6827/133265872] [v2:172.21.15.52:6828/133265872,v1:172.21.15.52:6829/133265872] exists,up c94bebc9-a105-4d74-9a03-c88e10cc2876', 'osd.4 up in weight 1 up_from 30 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6800/2146533857,v1:172.21.15.151:6801/2146533857] [v2:172.21.15.151:6802/2146533857,v1:172.21.15.151:6803/2146533857] exists,up a8b1470a-6d28-44d9-9921-6f8fbcb52b59', 'osd.5 up in weight 1 up_from 36 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6808/992496865,v1:172.21.15.151:6809/992496865] [v2:172.21.15.151:6810/992496865,v1:172.21.15.151:6811/992496865] exists,up 768d49a4-59ee-43e4-802b-a58f2b77580b', 'osd.6 up in weight 1 up_from 42 up_thru 91 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6816/2869932922,v1:172.21.15.151:6817/2869932922] [v2:172.21.15.151:6818/2869932922,v1:172.21.15.151:6819/2869932922] exists,up 2c295677-55e8-41ff-8733-982844192fbc', 'osd.7 up in weight 1 up_from 48 up_thru 94 down_at 0 last_clean_interval [0,0) [v2:172.21.15.151:6824/2800377088,v1:172.21.15.151:6825/2800377088] [v2:172.21.15.151:6826/2800377088,v1:172.21.15.151:6827/2800377088] exists,up c1b48eff-20d5-4d6d-bb17-0c44ffdf866e'] 2023-09-23T22:39:07.500 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T22:39:07.869 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:09.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:08 smithi052 ceph-mon[180979]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:09.113 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:08 smithi052 ceph-mon[187743]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:09.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:08 smithi151 ceph-mon[159210]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "name": "mon.a", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "cluster": "ceph", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "admin_socket_mode": "", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_client_required": "cephx, none", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_cluster_required": "cephx", 2023-09-23T22:39:09.252 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_debug": "true", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_service_required": "cephx", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "auth_supported": "", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_aio": "true", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_aio_reap_max": "16", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_async_discard": "false", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_block_size": "4096", 2023-09-23T22:39:09.253 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_debug_aio": "false", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_enable_discard": "false", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_flock_retry": "3", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_inject_crash": "0", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_ioring": "false", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_ioring_hipri": "false", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T22:39:09.254 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bdev_type": "", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_allocator": "hybrid", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_buffered_io": "true", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T22:39:09.255 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_replay_recovery": "false", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluefs_sync_write": "false", 2023-09-23T22:39:09.256 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_allocator": "hybrid", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T22:39:09.257 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_create": "true", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_db_create": "false", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_db_path": "", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_db_size": "0", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_path": "", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_size": "107374182400", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_wal_create": "false", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_wal_path": "", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_bluefs": "true", 2023-09-23T22:39:09.258 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_autotune": "true", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T22:39:09.259 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_size": "0", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cache_type": "2q", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_clone_cow": "true", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T22:39:09.260 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_mode": "none", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_freelist": "false", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T22:39:09.261 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T22:39:09.262 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T22:39:09.263 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fail_eio": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T22:39:09.265 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_max_blob_size": "0", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T22:39:09.266 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T22:39:09.267 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_spdk_mem": "512", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_tracing": "false", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T22:39:09.268 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T22:39:09.269 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_cluster_require_version": "2", 2023-09-23T22:39:09.270 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_require_signatures": "false", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_require_version": "2", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_service_require_signatures": "false", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_service_require_version": "2", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "cephx_sign_messages": "true", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "chdir": "", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_acl_type": "", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_asio_thread_count": "2", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_cache_mid": "0.750000", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_cache_size": "16384", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_caps_release_delay": "5", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_check_pool_perm": "true", 2023-09-23T22:39:09.271 INFO:teuthology.orchestra.run.smithi052.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_debug_force_sync_read": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_debug_getattr_caps": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_die_on_failed_remount": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_dirsize_rbytes": "true", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_force_lazyio": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_fs": "", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_inject_release_failure": "false", 2023-09-23T22:39:09.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_max_inline_size": "4096", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_mds_namespace": "", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_metadata": "", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_mount_gid": "-1", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_mount_timeout": "300", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_mount_uid": "-1", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_mountpoint": "/", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_notify_timeout": "10", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_oc": "true", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T22:39:09.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_oc_max_objects": "1000", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_oc_size": "209715200", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_permissions": "true", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_quota": "true", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_quota_df": "true", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_readahead_max_bytes": "0", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_readahead_max_periods": "4", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_readahead_min": "131072", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_reconnect_stale": "false", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_shutdown_timeout": "30", 2023-09-23T22:39:09.274 INFO:teuthology.orchestra.run.smithi052.stdout: "client_snapdir": ".snap", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "client_tick_interval": "1", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "client_trace": "", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "client_use_faked_inos": "false", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "client_use_random_mds": "false", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_graylog": "false", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_graylog_port": "12201", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_monitors": "default=true", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_syslog": "false", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T22:39:09.275 INFO:teuthology.orchestra.run.smithi052.stdout: "clog_to_syslog_level": "info", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "cluster_addr": "-", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "cluster_network": "", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "cluster_network_interface": "", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "compressor_zlib_isal": "false", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "compressor_zlib_level": "5", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "compressor_zstd_level": "1", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T22:39:09.276 INFO:teuthology.orchestra.run.smithi052.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T22:39:09.277 INFO:teuthology.orchestra.run.smithi052.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T22:39:09.277 INFO:teuthology.orchestra.run.smithi052.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T22:39:09.277 INFO:teuthology.orchestra.run.smithi052.stdout: "crimson_seastar_smp": "1", 2023-09-23T22:39:09.277 INFO:teuthology.orchestra.run.smithi052.stdout: "crush_location": "", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "crush_location_hook": "", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "crush_location_hook_timeout": "10", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "daemonize": "false", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "daos_pool": "tank", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_alienstore": "0/5", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_asok": "1/5", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_asok_assert_abort": "false", 2023-09-23T22:39:09.278 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_auth": "1/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_bdev": "1/3", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_bluefs": "1/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_bluestore": "1/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_buffer": "0/1", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_cephsqlite": "0/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_client": "0/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_compressor": "1/5", 2023-09-23T22:39:09.279 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_context": "0/1", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_crush": "1/1", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_crypto": "1/5", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_cyanstore": "0/5", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_dpdk": "1/5", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_eventtrace": "1/5", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_filer": "0/1", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_filestore": "1/3", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_finisher": "1/1", 2023-09-23T22:39:09.280 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_fuse": "1/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_javaclient": "1/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_journal": "1/3", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_journaler": "0/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_kstore": "1/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_leveldb": "4/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_lockdep": "0/1", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mclock": "1/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mds": "1/5", 2023-09-23T22:39:09.281 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mds_balancer": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mds_locker": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mds_log": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mds_migrator": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_memstore": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mgr": "2/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mgrc": "1/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_mon": "20/20", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_monc": "0/10", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_ms": "1/1", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_none": "0/5", 2023-09-23T22:39:09.282 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_objclass": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_objectcacher": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_objecter": "0/1", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_optracker": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_osd": "1/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_paxos": "20/20", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_perfcounter": "1/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_prioritycache": "1/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rados": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rbd": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T22:39:09.283 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rbd_replay": "0/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_refs": "0/0", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_reserver": "1/1", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rgw": "1/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rgw_access": "1/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rgw_flight": "1/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rgw_sync": "1/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_rocksdb": "4/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore": "0/5", 2023-09-23T22:39:09.284 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_backref": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_cache": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_device": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_epm": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_journal": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_lba": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_odata": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_omap": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_onode": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_t": "0/5", 2023-09-23T22:39:09.285 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_seastore_tm": "0/5", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_striper": "0/1", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_test": "0/5", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_throttle": "1/1", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_timer": "0/1", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "debug_tp": "0/5", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "device_failure_prediction_mode": "none", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "err_to_graylog": "false", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "err_to_journald": "false", 2023-09-23T22:39:09.286 INFO:teuthology.orchestra.run.smithi052.stdout: "err_to_stderr": "true", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "err_to_syslog": "false", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event_tracing": "false", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "exporter_http_port": "9926", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "exporter_prio_limit": "5", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "exporter_sort_metrics": "true", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "exporter_stats_period": "5", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "fake_statfs_for_testing": "0", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "fatal_signal_handlers": "true", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "filer_max_purge_ops": "10", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "filer_max_truncate_ops": "128", 2023-09-23T22:39:09.287 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_blackhole": "false", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_btrfs_snap": "true", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_caller_concurrency": "10", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_debug_omap_check": "false", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_debug_verify_split": "false", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_dump_file": "", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T22:39:09.288 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fadvise": "true", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fail_eio": "true", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fd_cache_size": "128", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fiemap": "false", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_inject_stall": "0", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_journal_parallel": "false", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_journal_trailing": "false", 2023-09-23T22:39:09.289 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_journal_writeahead": "false", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_kill_at": "0", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T22:39:09.290 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_merge_threshold": "-10", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_odsync_write": "false", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_omap_backend_path": "", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T22:39:09.291 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_op_threads": "2", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_punch_hole": "false", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T22:39:09.292 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_queue_max_ops": "50", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_seek_data_hole": "false", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_sloppy_crc": "false", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_splice": "false", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_split_multiple": "2", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_split_rand_factor": "20", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_update_to": "1000", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T22:39:09.293 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T22:39:09.294 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_xfs_extsize": "false", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "filestore_zfs_snap": "false", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fio_dir": "/tmp/fio", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fsid": "891acd3a-5a5f-11ee-8db4-212e2dc638e7", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_allow_other": "true", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_big_writes": "true", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_debug": "false", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_default_permissions": "false", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_disable_pagecache": "false", 2023-09-23T22:39:09.295 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_max_write": "0", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_multithreaded": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_require_active_mds": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_set_user_groups": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_splice_move": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_splice_read": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_splice_write": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T22:39:09.297 INFO:teuthology.orchestra.run.smithi052.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "gss_target_name": "ceph", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "heartbeat_file": "", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "heartbeat_inject_failure": "0", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "heartbeat_interval": "5", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "host": "smithi052", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T22:39:09.298 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "inject_early_sigterm": "false", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "jaeger_agent_port": "6799", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "jaeger_tracing_enable": "false", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_aio": "true", 2023-09-23T22:39:09.299 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_align_min_size": "65536", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_block_align": "true", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_block_size": "4096", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_dio": "true", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_discard": "false", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_force_aio": "false", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_ignore_corruption": "false", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_max_write_entries": "100", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_replay_from": "0", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T22:39:09.300 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_write_header_frequency": "0", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journal_zero_on_create": "false", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journaler_prefetch_periods": "10", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journaler_prezero_periods": "5", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "journaler_write_head_interval": "15", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "key": "", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "keyfile": "", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_backend": "rocksdb", 2023-09-23T22:39:09.301 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_max_bytes": "67108864", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_max_ops": "512", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_onode_map_size": "1024", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "kstore_sync_transaction": "false", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_block_size": "65536", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_bloom_size": "0", 2023-09-23T22:39:09.302 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_cache_size": "536870912", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_compression": "false", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_log": "", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_max_open_files": "0", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_paranoid": "false", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "librados_thread_count": "2", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "lockdep": "false", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "lockdep_force_backtrace": "false", 2023-09-23T22:39:09.303 INFO:teuthology.orchestra.run.smithi052.stdout: "log_coarse_timestamps": "true", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_flush_on_exit": "false", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_graylog_port": "12201", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_max_new": "1000", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_max_recent": "10000", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_stderr_prefix": "", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_to_file": "true", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_to_graylog": "false", 2023-09-23T22:39:09.304 INFO:teuthology.orchestra.run.smithi052.stdout: "log_to_journald": "false", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "log_to_stderr": "false", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "log_to_syslog": "false", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_action_on_write_error": "1", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_alternate_name_max": "8192", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_asio_thread_count": "2", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_export_pin": "true", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T22:39:09.305 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_interval": "10", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_max": "-1", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_max_until": "-1", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_merge_size": "50", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_mode": "0", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T22:39:09.306 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_split_bits": "3", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_split_size": "10000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T22:39:09.307 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_mid": "0.700000", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_trim_interval": "1", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T22:39:09.308 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_debug_auth_pins": "false", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_debug_frag": "false", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_debug_scatterstat": "false", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_debug_subtrees": "false", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_default_dir_hash": "2", 2023-09-23T22:39:09.309 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_defer_session_stale": "true", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dir_max_entries": "0", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dir_prefetch": "true", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T22:39:09.310 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_early_reply": "true", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_enable_op_tracker": "true", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_enforce_unique_name": "true", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T22:39:09.311 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_inject_health_dummy": "false", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_join_fs": "", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_journal_format": "1", 2023-09-23T22:39:09.312 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_create_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_export_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_import_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_journal_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_link_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_openc_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_rename_at": "0", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T22:39:09.313 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_max_events": "-1", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_max_segments": "128", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_pause": "false", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_segment_size": "0", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_completed_requests": "100000", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_export_size": "20971520", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_file_recover": "32", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_purge_files": "64", 2023-09-23T22:39:09.314 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_purge_ops": "8192", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_metrics_update_interval": "2", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_min_caps_per_client": "100", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_numa_node": "-1", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T22:39:09.315 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_op_history_duration": "600", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_op_history_size": "20", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_op_log_threshold": "5", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_ping_grace": "15", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_ping_interval": "5", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_recall_max_caps": "30000", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T22:39:09.316 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_replay_interval": "1.000000", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_root_ino_gid": "0", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_root_ino_uid": "0", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T22:39:09.317 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_shutdown_check": "0", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_skip_ino": "0", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_snap_min_uid": "0", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_snap_rstat": "false", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T22:39:09.318 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_symlink_recovery": "true", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_thrash_exports": "0", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_thrash_fragments": "0", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_tick_interval": "5.000000", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_valgrind_exit": "false", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_verify_backtrace": "1", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_verify_scatter": "false", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mds_wipe_sessions": "false", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "mempool_debug": "false", 2023-09-23T22:39:09.319 INFO:teuthology.orchestra.run.smithi052.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "memstore_page_set": "false", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "memstore_page_size": "65536", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_client_bytes": "134217728", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_client_messages": "512", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_disabled_modules": "", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T22:39:09.320 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_mds_messages": "128", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_mon_messages": "128", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_osd_messages": "8192", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_pool": "true", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_standby_modules": "true", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_stats_period": "5", 2023-09-23T22:39:09.321 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_stats_threshold": "5", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_tick_period": "2", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_allow_pool_delete": "true", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_bytes": "104857600", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T22:39:09.322 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T22:39:09.323 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T22:39:09.324 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_compact_on_start": "false", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_compact_on_trim": "true", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_cpu_threads": "4", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_data_avail_crit": "5", 2023-09-23T22:39:09.325 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_data_avail_warn": "5", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_dump_json": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_extra_checks": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T22:39:09.326 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_election_default_strategy": "1", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_election_timeout": "5.000000", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T22:39:09.327 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_enable_op_tracker": "true", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_fake_pool_delete": "false", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_force_quorum_join": "false", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_health_log_update_period": "5", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_health_max_detail": "50", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_health_to_clog": "true", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_host": "[v2:172.21.15.52:3300/0,v1:172.21.15.52:6789/0] [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] [v2:172.21.15.52:3301/0,v1:172.21.15.52:6790/0]", 2023-09-23T22:39:09.328 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_host_override": "", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_initial_members": "", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_lease": "5.000000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_log_full_interval": "50", 2023-09-23T22:39:09.329 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_log_max": "10000", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_log_max_summary": "50", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_log_epochs": "500", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_osd": "10000", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T22:39:09.330 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_memory_autotune": "true", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_memory_target": "2147483648", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mgr_digest_period": "5", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T22:39:09.331 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_op_complaint_time": "30", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_op_history_duration": "600", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_op_history_size": "20", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_op_log_threshold": "5", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T22:39:09.332 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_cache_size": "500", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T22:39:09.333 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T22:39:09.334 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_report_timeout": "900", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T22:39:09.335 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T22:39:09.336 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_reweight_max_osds": "4", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_scrub_interval": "86400", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_scrub_max_keys": "100", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_scrub_timeout": "300", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_session_timeout": "300", 2023-09-23T22:39:09.337 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_smart_report_timeout": "5", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_stretch_pool_size": "4", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_sync_debug": "false", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T22:39:09.338 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_tick_interval": "5", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T22:39:09.339 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_older_version": "true", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T22:39:09.340 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "monmap": "", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "motr_tracing_enabled": "false", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_op_threads": "3", 2023-09-23T22:39:09.341 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_cm": "false", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_device_name": "", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T22:39:09.342 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_sl": "3", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_rdma_type": "ib", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_async_reap_threshold": "5", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_before_connect": "false", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_ipv4": "true", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_ipv6": "false", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_msgr1": "true", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_msgr2": "true", 2023-09-23T22:39:09.343 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_port_max": "7568", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_port_min": "6800", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_retry_count": "3", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_bind_retry_delay": "5", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_blackhole_client": "false", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_blackhole_mds": "false", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_blackhole_mgr": "false", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_blackhole_mon": "false", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_blackhole_osd": "false", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_client_mode": "crc secure", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T22:39:09.344 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_cluster_type": "", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_compress_secure": "false", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_crc_data": "true", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_crc_header": "true", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_die_on_bug": "true", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_die_on_old_message": "true", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T22:39:09.345 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_hugepages": "", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_lro": "true", 2023-09-23T22:39:09.346 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_pmd": "", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_port_id": "0", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_dump_on_send": "false", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_inject_delay_type": "", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T22:39:09.347 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_inject_network_congestion": "0", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_inject_socket_failures": "0", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_max_accept_failures": "4", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_max_backoff": "15.000000", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_osd_compress_mode": "none", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_pq_min_cost": "65536", 2023-09-23T22:39:09.348 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_public_type": "", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_service_mode": "crc secure", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_tcp_nodelay": "true", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "ms_type": "async+posix", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "no_config_file": "false", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T22:39:09.349 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_inflight_ops": "1024", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "objecter_timeout": "10.000000", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "objectstore_blackhole": "false", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "openssl_engine_opts": "", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_max_ops": "4", 2023-09-23T22:39:09.350 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_agent_slop": "0.020000", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_asio_thread_count": "2", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_backfill_scan_max": "512", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_backfill_scan_min": "64", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T22:39:09.351 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_backoff_on_peering": "false", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_beacon_report_interval": "300", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_bench_duration": "30", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_blkin_trace_all": "false", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T22:39:09.352 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_class_update_on_start": "true", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_client_message_cap": "256", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_client_op_priority": "63", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_client_watch_timeout": "30", 2023-09-23T22:39:09.353 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_command_max_records": "256", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_compact_on_start": "false", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_crush_update_on_start": "true", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T22:39:09.354 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_op_order": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T22:39:09.355 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_shutdown": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T22:39:09.356 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_default_notify_timeout": "30", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_enable_op_tracker": "true", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T22:39:09.357 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_fast_info": "true", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_fast_shutdown": "true", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_function_tracing": "false", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_grace": "20", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_interval": "6", 2023-09-23T22:39:09.358 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_stale": "600", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T22:39:09.359 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_journal_size": "5120", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_kill_backfill_at": "0", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_map_cache_size": "50", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_map_dedup": "true", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_map_message_max": "40", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_attr_name_len": "100", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_attr_size": "0", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_backfills": "1", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_markdown_count": "5", 2023-09-23T22:39:09.360 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_markdown_period": "600", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_object_name_len": "2048", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_object_size": "134217728", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_pgls": "1024", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_push_cost": "8388608", 2023-09-23T22:39:09.361 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_push_objects": "10", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_scrubs": "1", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_max_write_size": "90", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T22:39:09.362 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_profile": "balanced", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T22:39:09.363 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_base": "805306368", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_target": "4294967296", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_target_autotune": "false", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T22:39:09.364 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_min_recovery_priority": "0", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mon_report_interval": "5", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_num_cache_shards": "32", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_numa_node": "-1", 2023-09-23T22:39:09.365 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_objecter_finishers": "1", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_objectstore": "bluestore", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_objectstore_fuse": "false", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_objectstore_tracing": "false", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_history_duration": "600", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_history_size": "20", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T22:39:09.366 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_log_threshold": "5", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_num_shards": "0", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_op_thread_timeout": "15", 2023-09-23T22:39:09.367 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_open_classes_on_start": "true", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_os_flags": "0", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_peering_op_priority": "255", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_delete_priority": "5", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T22:39:09.368 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_crimson": "false", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T22:39:09.369 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_flags": "0", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_min_size": "0", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T22:39:09.370 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_size": "3", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_default_type": "replicated", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_cost": "20971520", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_max_active": "0", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T22:39:09.371 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_op_priority": "3", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_priority": "5", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T22:39:09.372 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_repair_during_recovery": "false", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T22:39:09.373 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_cost": "52428800", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_end_hour": "0", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T22:39:09.374 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_priority": "5", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_skip_data_digest": "false", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_smart_report_timeout": "5", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T22:39:09.375 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_snap_trim_priority": "5", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_target_transaction_size": "30", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T22:39:09.376 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_tracing": "false", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_use_stale_snap": "false", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T22:39:09.377 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_kill_at": "0", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_max_join_drift": "10", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_min": "500", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_min_wait": "0.050000", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_service_trim_max": "500", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_service_trim_min": "250", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_stash_full_interval": "25", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_trim_max": "500", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "paxos_trim_min": "250", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "perf": "true", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "pid_file": "", 2023-09-23T22:39:09.378 INFO:teuthology.orchestra.run.smithi052.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "public_addr": "-", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "public_addrv": "", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "public_bind_addr": "-", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "public_network": "172.21.0.0/20", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "public_network_interface": "", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "qat_compressor_enabled": "false", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "rados_mon_op_timeout": "0", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "rados_osd_op_timeout": "0", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "rados_tracing": "false", 2023-09-23T22:39:09.379 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_atime_update_interval": "60", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache": "true", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T22:39:09.380 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_policy": "writearound", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_size": "33554432", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_compression_hint": "none", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_clone_format": "auto", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_data_pool": "", 2023-09-23T22:39:09.381 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_features": "61", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_format": "2", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_map_options": "", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_order": "22", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_pool": "rbd", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_stripe_count": "0", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T22:39:09.382 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_io_scheduler": "simple", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T22:39:09.383 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_order": "24", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_pool": "", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_journal_splay_width": "4", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T22:39:09.384 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T22:39:09.385 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T22:39:09.386 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_op_threads": "1", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_plugins": "", 2023-09-23T22:39:09.387 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T22:39:09.388 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T22:39:09.389 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_tracing": "false", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_validate_names": "true", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rbd_validate_pool": "true", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "restapi_base_url": "", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "restapi_log_level": "", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_admin_entry": "admin", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T22:39:09.390 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_backend_store": "rados", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_barbican_url": "", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_beast_enable_async": "true", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_cache_enabled": "true", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T22:39:09.391 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_config_store": "rados", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_content_length_compat": "false", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_cross_domain_policy": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T22:39:09.392 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T22:39:09.393 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T22:39:09.394 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T22:39:09.395 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T22:39:09.396 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_log_window": "30", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T22:39:09.397 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T22:39:09.398 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dns_name": "", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dns_s3website_name": "", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_ops_log": "false", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_static_website": "false", 2023-09-23T22:39:09.399 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enable_usage_log": "false", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_expose_bucket": "false", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_extended_http_attrs": "", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_filter": "none", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_max_objs": "32", 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_init_timeout": "300", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T22:39:09.401 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_password": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_project": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_token": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_admin_user": "", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_api_version": "2", 2023-09-23T22:39:09.402 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_url": "", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T22:39:09.403 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_max_objs": "32", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_max_worker": "3", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T22:39:09.404 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_log_http_headers": "", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_attr_size": "0", 2023-09-23T22:39:09.405 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_listing_results": "1000", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_notify_retries": "10", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T22:39:09.406 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T22:39:09.407 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_num_control_oids": "8", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_numa_node": "-1", 2023-09-23T22:39:09.408 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_op_tracing": "false", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_opa_token": "", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_opa_url": "", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T22:39:09.409 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ops_log_rados": "false", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_print_continue": "true", 2023-09-23T22:39:09.410 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_rados_tracing": "false", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_realm": "", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_realm_id": "", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_region": "", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T22:39:09.411 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_request_uri": "", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_resolve_cname": "false", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_run_sync_thread": "true", 2023-09-23T22:39:09.412 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_script_uri": "", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_service_provider_name": "", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T22:39:09.413 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_client_id": "", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_client_secret": "", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_entry": "sts", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_key": "sts", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_auth_url": "", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_custom_header": "", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T22:39:09.414 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_need_stats": "true", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_tenant_name": "", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_url": "", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_lease_period": "120", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T22:39:09.415 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_thread_pool_size": "512", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_comment": "", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_createby": "", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_encoding": "", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_flag": "false", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_origin": "", 2023-09-23T22:39:09.416 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_torrent_tracker": "", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_usage_max_shards": "32", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_use_opa_authz": "false", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T22:39:09.417 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_user_unique_email": "true", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_verify_ssl": "true", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_zone": "", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_zone_id": "", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_zonegroup": "", 2023-09-23T22:39:09.418 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_zonegroup_id": "", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_block_size": "4096", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T22:39:09.419 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_partition_filters": "false", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_perf": "false", 2023-09-23T22:39:09.420 INFO:teuthology.orchestra.run.smithi052.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "run_dir": "/var/run/ceph", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_block_create": "true", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_device_size": "53687091200", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T22:39:09.421 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_main_device_type": "SSD", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "seastore_segment_size": "67108864", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "set_keepcaps": "false", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "setgroup": "ceph", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "setuser": "ceph", 2023-09-23T22:39:09.422 INFO:teuthology.orchestra.run.smithi052.stdout: "setuser_match_path": "", 2023-09-23T22:39:09.423 INFO:teuthology.orchestra.run.smithi052.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T22:39:09.423 INFO:teuthology.orchestra.run.smithi052.stdout: "thp": "false", 2023-09-23T22:39:09.423 INFO:teuthology.orchestra.run.smithi052.stdout: "threadpool_default_timeout": "60", 2023-09-23T22:39:09.423 INFO:teuthology.orchestra.run.smithi052.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T22:39:09.423 INFO:teuthology.orchestra.run.smithi052.stdout: "throttler_perf_counter": "true" 2023-09-23T22:39:09.423 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:39:09.920 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T22:39:10.286 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:11.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:10 smithi151 ceph-mon[159210]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:11.236 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:10 smithi052 ceph-mon[187743]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:11.236 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:10 smithi052 ceph-mon[180979]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:11.666 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a: {} 2023-09-23T22:39:11.676 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b: {} 2023-09-23T22:39:11.686 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c: {} 2023-09-23T22:39:11.687 INFO:teuthology.orchestra.run.smithi052.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T22:39:11.687 INFO:teuthology.orchestra.run.smithi052.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T22:39:11.687 INFO:teuthology.orchestra.run.smithi052.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T22:39:11.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/540313887' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:39:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/540313887' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:39:12.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/540313887' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:39:12.352 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T22:39:12.352 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T22:39:12.353 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:39:12.354 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:39:12.354 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T22:39:12.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T22:39:12.355 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T22:39:12.355 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:39:12.355 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:12.355 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:39:12.355 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T22:39:12.355 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:39:12.356 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T22:39:12.356 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:12.356 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T22:39:12.356 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:12.357 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T22:39:12.357 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:39:12.357 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T22:39:12.357 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:39:12.401 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.501 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:12.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:12.606 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:12.658 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.707 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:12.744 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:39:12 smithi052 systemd[1]: Stopping Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:39:12.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:12.841 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.883 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:12.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:12.990 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:13.021 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.068 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:13.071 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:12 smithi052 ceph-mon[187743]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:13.071 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:12 smithi052 ceph-mon[187743]: osd.0 marked itself down and dead 2023-09-23T22:39:13.071 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:39:12 smithi052 conmon[197941]: 2023-09-23T22:39:12.804+0000 7feabdcb7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:39:13.071 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:39:12 smithi052 conmon[197941]: 2023-09-23T22:39:12.804+0000 7feabdcb7700 -1 osd.0 95 *** Got signal Terminated *** 2023-09-23T22:39:13.072 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:39:12 smithi052 conmon[197941]: 2023-09-23T22:39:12.804+0000 7feabdcb7700 -1 osd.0 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:39:13.072 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:12 smithi052 ceph-mon[180979]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:13.072 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:12 smithi052 ceph-mon[180979]: osd.0 marked itself down and dead 2023-09-23T22:39:13.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:13.168 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:13.174 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:13.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:12 smithi151 ceph-mon[159210]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:13.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:12 smithi151 ceph-mon[159210]: osd.0 marked itself down and dead 2023-09-23T22:39:13.202 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.218 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:13.244 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:13.295 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:13.323 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:13.351 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:13.376 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.417 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:13.524 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:13.550 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.592 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:13.606 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:39:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:39:13] "GET /metrics HTTP/1.1" 200 33968 "" "Prometheus/2.33.4" 2023-09-23T22:39:13.698 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:13.724 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.764 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:13.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:13.869 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:13.898 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:39:13 smithi052 bash[228815]: b70c0aa9b5abc19f9172fd4758c71d234777969ca7072c6e2d9f9e1ee6bfca6a 2023-09-23T22:39:13.902 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:14.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:39:14.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T22:39:14.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:13 smithi151 ceph-mon[159210]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[180979]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:39:14.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T22:39:14.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:13 smithi052 ceph-mon[187743]: failed to encode map e96 with expected crc 2023-09-23T22:39:14.470 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:14.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:14.575 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:14.606 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:14.653 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:14.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:14.762 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:14.791 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:14.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: pgmap v216: 73 pgs: 1 peering, 11 stale+active+clean, 61 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:14.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:39:14.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:14.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:14 smithi151 ceph-mon[159210]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: pgmap v216: 73 pgs: 1 peering, 11 stale+active+clean, 61 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[187743]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: pgmap v216: 73 pgs: 1 peering, 11 stale+active+clean, 61 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.058 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.058 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.058 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.058 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:15.058 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:14 smithi052 ceph-mon[180979]: failed to encode map e97 with expected crc 2023-09-23T22:39:16.261 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:15 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 11/72 objects degraded (15.278%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:16.261 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:15 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 11/72 objects degraded (15.278%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:16.403 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:16.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:16.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:16.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:15 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 11/72 objects degraded (15.278%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:16.508 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:16.547 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:16.910 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:16.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:17.015 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:17.047 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:16 smithi052 ceph-mon[187743]: pgmap v218: 73 pgs: 6 active+undersized+degraded, 1 peering, 3 stale+active+clean, 14 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11/72 objects degraded (15.278%) 2023-09-23T22:39:17.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:16 smithi052 ceph-mon[180979]: pgmap v218: 73 pgs: 6 active+undersized+degraded, 1 peering, 3 stale+active+clean, 14 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11/72 objects degraded (15.278%) 2023-09-23T22:39:17.255 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:17.358 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:17.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:16 smithi151 ceph-mon[159210]: pgmap v218: 73 pgs: 6 active+undersized+degraded, 1 peering, 3 stale+active+clean, 14 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11/72 objects degraded (15.278%) 2023-09-23T22:39:17.390 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.440 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:17.545 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:17.582 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.603 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:39:17.603 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:39:17.603 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T22:39:17.603 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:39:17.712 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:17.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:17.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.896 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:17.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:18.001 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:18.010 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:39:18.030 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:18.077 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:18.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:18.188 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:18.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1400271561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:18.214 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:18.256 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:18.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:18.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:18.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1400271561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:18.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1400271561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:18.363 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:18.396 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:19.066 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:39:19.294 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: pgmap v219: 73 pgs: 7 active+undersized+degraded, 1 peering, 18 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1400271561' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[187743]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.295 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: pgmap v219: 73 pgs: 7 active+undersized+degraded, 1 peering, 18 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:19.296 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:19.296 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.296 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.296 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1400271561' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:19.296 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T22:39:19.296 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:19 smithi052 ceph-mon[180979]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.300 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:19.319 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:39:19.349 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:19.352 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:19.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:19.373 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:19.394 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:19.425 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.445 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: pgmap v219: 73 pgs: 7 active+undersized+degraded, 1 peering, 18 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1400271561' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T22:39:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:19 smithi151 ceph-mon[159210]: failed to encode map e98 with expected crc 2023-09-23T22:39:19.458 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:19.488 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:19.986 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:19.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:20.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:20.123 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.171 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:20.241 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[187743]: failed to encode map e98 with expected crc 2023-09-23T22:39:20.241 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3048235472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:20.241 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:20.241 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:20.242 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[180979]: failed to encode map e98 with expected crc 2023-09-23T22:39:20.242 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3048235472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:20.242 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:20.242 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:20.248 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T22:39:20.279 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:20.304 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:20 smithi151 ceph-mon[159210]: failed to encode map e98 with expected crc 2023-09-23T22:39:20.305 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3048235472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:20.305 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:20.305 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:20.307 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.351 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:20.382 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:39:20.382 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T22:39:20.456 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:20.483 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:20.501 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:20.526 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:20.589 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:20.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:20.663 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.714 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:20.817 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:20.847 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.894 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:20.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:20.998 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:21.022 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_0' created 2023-09-23T22:39:21.035 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:21.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[187743]: pgmap v221: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:21.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3744245385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:39:21.202 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:39:21.202 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:39:21.202 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[180979]: pgmap v221: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:21.202 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3744245385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:39:21.202 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:39:21.202 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:21 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:39:21.298 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:39:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:21 smithi151 ceph-mon[159210]: pgmap v221: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3744245385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:39:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:21 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:39:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:21 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:39:21.637 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:21.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:21.741 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:21.772 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:21.821 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:21.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:21.928 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:21.957 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:22.002 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:22.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:22.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:22.096 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T22:39:22.110 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:22.171 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:22.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:22 smithi151 ceph-mon[159210]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:22 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:39:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:22 smithi151 ceph-mon[159210]: osdmap e99: 8 total, 7 up, 8 in 2023-09-23T22:39:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:22 smithi151 ceph-mon[159210]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:22 smithi151 ceph-mon[159210]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:22 smithi151 ceph-mon[159210]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.208 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:22.280 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[187743]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[187743]: osdmap e99: 8 total, 7 up, 8 in 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[187743]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[187743]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[187743]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[180979]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[180979]: osdmap e99: 8 total, 7 up, 8 in 2023-09-23T22:39:22.281 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[180979]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.282 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[180979]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.282 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:22 smithi052 ceph-mon[180979]: failed to encode map e99 with expected crc 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:22.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:22.435 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:22.502 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:22.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:22.612 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:22.638 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:22.679 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:22.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:22.786 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:22.815 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.061 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[187743]: pgmap v223: 89 pgs: 16 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[187743]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[187743]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[187743]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[187743]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[187743]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.064 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[180979]: pgmap v223: 89 pgs: 16 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:23.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[180979]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[180979]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T22:39:23.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[180979]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[180979]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:23 smithi052 ceph-mon[180979]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:23.167 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:23.195 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.238 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:23.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:23.343 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:23.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:23 smithi151 ceph-mon[159210]: pgmap v223: 89 pgs: 16 unknown, 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:23.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:23 smithi151 ceph-mon[159210]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:23 smithi151 ceph-mon[159210]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T22:39:23.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:23 smithi151 ceph-mon[159210]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:23 smithi151 ceph-mon[159210]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:23 smithi151 ceph-mon[159210]: failed to encode map e100 with expected crc 2023-09-23T22:39:23.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.411 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:23.516 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:23.542 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.583 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:23.688 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:23.714 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:39:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:39:23] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-09-23T22:39:23.717 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.790 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:39:23.805 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:23.815 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:23.920 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:23.946 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.959 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:23.989 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:23.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:24.096 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:24.138 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:24 smithi151 ceph-mon[159210]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:24 smithi151 ceph-mon[159210]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T22:39:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:24 smithi151 ceph-mon[159210]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:24 smithi151 ceph-mon[159210]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.319 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[187743]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[187743]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[187743]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[187743]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[180979]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[180979]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[180979]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:24 smithi052 ceph-mon[180979]: failed to encode map e101 with expected crc 2023-09-23T22:39:24.942 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:24.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:25.045 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:25.081 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:25.200 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[187743]: pgmap v226: 89 pgs: 2 creating+peering, 13 unknown, 7 active+undersized+degraded, 19 active+undersized, 48 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[187743]: failed to encode map e101 with expected crc 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/588821597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[180979]: pgmap v226: 89 pgs: 2 creating+peering, 13 unknown, 7 active+undersized+degraded, 19 active+undersized, 48 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[180979]: failed to encode map e101 with expected crc 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:25.201 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:25.202 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/588821597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:25.202 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:25 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:25.240 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:25.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:25.343 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:25.371 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:25.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:25 smithi151 ceph-mon[159210]: pgmap v226: 89 pgs: 2 creating+peering, 13 unknown, 7 active+undersized+degraded, 19 active+undersized, 48 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:25.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:25 smithi151 ceph-mon[159210]: failed to encode map e101 with expected crc 2023-09-23T22:39:25.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:25.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:25.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/588821597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:25.376 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:25 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:25.378 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:25.384 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:39:25.384 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:25.384 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:39:25.384 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T22:39:25.384 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:39:25.426 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:25.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:25.456 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:39:25 smithi052 systemd[1]: Stopping Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:39:25.532 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:25.566 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:25.667 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:39:26.078 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:26.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:26.183 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:26.212 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:26.294 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:39:26.294 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:26.294 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[187743]: failed to encode map e102 with expected crc 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[187743]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[180979]: failed to encode map e102 with expected crc 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:26.295 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:26 smithi052 ceph-mon[180979]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T22:39:26.307 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:26.314 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:26 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:39:26.315 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:26 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:26.315 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:26 smithi151 ceph-mon[159210]: failed to encode map e102 with expected crc 2023-09-23T22:39:26.315 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:26.315 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:26 smithi151 ceph-mon[159210]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T22:39:26.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:26.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:26.417 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:26.467 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:27.044 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T22:39:27.243 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.244 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:27 smithi151 ceph-mon[159210]: pgmap v227: 89 pgs: 2 creating+peering, 7 active+undersized+degraded, 23 active+undersized, 57 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:27 smithi151 ceph-mon[159210]: failed to encode map e102 with expected crc 2023-09-23T22:39:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/915954727' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:39:27.493 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:39:27 smithi052 conmon[208457]: 2023-09-23T22:39:27.040+0000 7ffb6ba60700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:39:27.493 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:39:27 smithi052 conmon[208457]: 2023-09-23T22:39:27.040+0000 7ffb6ba60700 -1 osd.2 102 *** Got signal Terminated *** 2023-09-23T22:39:27.494 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:39:27 smithi052 conmon[208457]: 2023-09-23T22:39:27.040+0000 7ffb6ba60700 -1 osd.2 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:39:27.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:27 smithi052 ceph-mon[180979]: pgmap v227: 89 pgs: 2 creating+peering, 7 active+undersized+degraded, 23 active+undersized, 57 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:27.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:27 smithi052 ceph-mon[180979]: failed to encode map e102 with expected crc 2023-09-23T22:39:27.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/915954727' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:39:27.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:27 smithi052 ceph-mon[187743]: pgmap v227: 89 pgs: 2 creating+peering, 7 active+undersized+degraded, 23 active+undersized, 57 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:27.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:27 smithi052 ceph-mon[187743]: failed to encode map e102 with expected crc 2023-09-23T22:39:27.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/915954727' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:39:27.775 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:27.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:27.879 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:27.906 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:27.950 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:27.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:28.056 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:28.088 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: failed to encode map e102 with expected crc 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/915954727' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: osd.2 marked itself down and dead 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/358848648' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:28 smithi151 ceph-mon[159210]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.252 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:28.301 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: failed to encode map e102 with expected crc 2023-09-23T22:39:28.301 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/915954727' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: osd.2 marked itself down and dead 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/358848648' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[187743]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: failed to encode map e102 with expected crc 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/915954727' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:39:28.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T22:39:28.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: osd.2 marked itself down and dead 2023-09-23T22:39:28.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/358848648' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:28.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:28 smithi052 ceph-mon[180979]: failed to encode map e103 with expected crc 2023-09-23T22:39:28.356 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:28.385 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.429 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:28.513 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:28.534 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:28.560 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.607 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:28.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:28.713 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:28.742 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.905 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:28.993 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:39:28 smithi052 bash[233646]: 58cbbf66d491acc74e901bcfbdf626aff8956a6073e1a44f0937efa927caaf50 2023-09-23T22:39:29.009 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:29.040 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.154 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:29.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:29.258 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:29.285 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.328 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:29.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: pgmap v230: 89 pgs: 7 active+undersized+degraded, 23 active+undersized, 59 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:29.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:39:29.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: osdmap e104: 8 total, 6 up, 8 in 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:29.332 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:29.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:29.434 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:29.435 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: pgmap v230: 89 pgs: 7 active+undersized+degraded, 23 active+undersized, 59 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:29.435 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:39:29.435 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.435 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.435 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: osdmap e104: 8 total, 6 up, 8 in 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:29.436 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: pgmap v230: 89 pgs: 7 active+undersized+degraded, 23 active+undersized, 59 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: osdmap e104: 8 total, 6 up, 8 in 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: failed to encode map e104 with expected crc 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:29.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:29.460 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.501 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:29.557 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:39:29.557 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:39:29.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:39:29.558 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [] 2023-09-23T22:39:29.558 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:39:29.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:29.633 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.674 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:29.781 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:29.807 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:29.907 DEBUG:teuthology.orchestra.run.smithi052:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T22:39:29.910 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:39:29.910 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:39:29.955 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:29.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:hints = 1 2023-09-23T22:39:29.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:39:29.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object prefix: benchmark_data_smithi052_234456 2023-09-23T22:39:29.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:39:29.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:39:29.999 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.077 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:30.084 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:39:30.140 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:39:30.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:30.220 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.288 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:30.292 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: osdmap e105: 8 total, 6 up, 8 in 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[187743]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.293 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: osdmap e105: 8 total, 6 up, 8 in 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:30 smithi052 ceph-mon[180979]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:30.396 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: osdmap e105: 8 total, 6 up, 8 in 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:30 smithi151 ceph-mon[159210]: failed to encode map e105 with expected crc 2023-09-23T22:39:30.423 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.466 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:30.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:30.573 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:30.601 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.839 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:30.917 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:30.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 1 16 101 85 5.3122 5.3125 0.00131838 0.00253822 2023-09-23T22:39:31.022 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:31.050 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.095 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:31.201 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:31.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:31 smithi052 ceph-mon[187743]: pgmap v233: 89 pgs: 1 stale+active+clean, 7 active+undersized+degraded, 3 undersized+peered, 24 active+undersized, 54 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:31.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:31 smithi052 ceph-mon[180979]: pgmap v233: 89 pgs: 1 stale+active+clean, 7 active+undersized+degraded, 3 undersized+peered, 24 active+undersized, 54 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:31.233 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.294 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:31.294 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:31.337 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:31.443 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:31.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:31 smithi151 ceph-mon[159210]: pgmap v233: 89 pgs: 1 stale+active+clean, 7 active+undersized+degraded, 3 undersized+peered, 24 active+undersized, 54 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T22:39:31.475 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.522 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:31.627 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:31.655 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.700 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:31.805 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:31.836 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 2 16 101 85 2.65599 0 - 0.00253822 2023-09-23T22:39:32.193 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T22:39:32.194 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:32.247 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:39:32.247 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T22:39:32.298 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:32.329 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.377 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:32.381 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:32 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 54/228 objects degraded (23.684%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:32.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:32.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:32 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 54/228 objects degraded (23.684%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:32.453 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:32 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 54/228 objects degraded (23.684%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:32.483 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:32.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.556 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:32.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:32.661 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:32.688 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.730 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:32.836 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:32.862 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.904 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:32.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:32.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 3 16 101 85 1.77066 0 - 0.00253822 2023-09-23T22:39:33.009 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:33.035 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.076 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:33.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:33.181 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:33.208 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.249 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:33.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:33.354 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:33.377 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:33 smithi151 ceph-mon[159210]: pgmap v234: 89 pgs: 1 stale+active+clean, 1 undersized+degraded+peered, 13 active+undersized+degraded, 4 undersized+peered, 28 active+undersized, 42 active+clean; 3.8 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 572 KiB/s wr, 8 op/s; 54/228 objects degraded (23.684%) 2023-09-23T22:39:33.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.396 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:33 smithi052 ceph-mon[180979]: pgmap v234: 89 pgs: 1 stale+active+clean, 1 undersized+degraded+peered, 13 active+undersized+degraded, 4 undersized+peered, 28 active+undersized, 42 active+clean; 3.8 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 572 KiB/s wr, 8 op/s; 54/228 objects degraded (23.684%) 2023-09-23T22:39:33.396 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:39:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:39:33] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-09-23T22:39:33.397 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:33 smithi052 ceph-mon[187743]: pgmap v234: 89 pgs: 1 stale+active+clean, 1 undersized+degraded+peered, 13 active+undersized+degraded, 4 undersized+peered, 28 active+undersized, 42 active+clean; 3.8 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 572 KiB/s wr, 8 op/s; 54/228 objects degraded (23.684%) 2023-09-23T22:39:33.421 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:33.527 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:33.553 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.594 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:33.700 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:33.731 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.938 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:33.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 4 16 101 85 1.32796 0 - 0.00253822 2023-09-23T22:39:34.043 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:34.073 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.129 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:39:34.156 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:34.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:34.261 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:34.288 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.330 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:34.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:34.346 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:34.365 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:34.384 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:34.402 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:34.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3011445945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:34.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:34.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3011445945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:34.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:34.422 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:34.435 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:34.440 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:39:34.441 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:34.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3011445945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:34.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:34.461 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.503 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:34.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:34.608 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:34.638 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.886 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:34.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 5 16 101 85 1.06237 0 - 0.00253822 2023-09-23T22:39:34.991 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:35.016 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:35.059 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:35.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:35.123 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: pgmap v235: 89 pgs: 1 undersized+degraded+peered, 14 active+undersized+degraded, 5 undersized+peered, 31 active+undersized, 38 active+clean; 4.4 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 606 KiB/s wr, 9 op/s; 56/255 objects degraded (21.961%) 2023-09-23T22:39:35.123 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:35.164 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:35.194 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:35.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: pgmap v235: 89 pgs: 1 undersized+degraded+peered, 14 active+undersized+degraded, 5 undersized+peered, 31 active+undersized, 38 active+clean; 4.4 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 606 KiB/s wr, 9 op/s; 56/255 objects degraded (21.961%) 2023-09-23T22:39:35.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:35.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: failed to encode map e106 with expected crc 2023-09-23T22:39:35.409 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: osdmap e106: 8 total, 6 up, 8 in 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3714809638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[187743]: failed to encode map e106 with expected crc 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: failed to encode map e106 with expected crc 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:35.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: osdmap e106: 8 total, 6 up, 8 in 2023-09-23T22:39:35.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3714809638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:35.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:35.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:35.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:35.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:35 smithi052 ceph-mon[180979]: failed to encode map e106 with expected crc 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: pgmap v235: 89 pgs: 1 undersized+degraded+peered, 14 active+undersized+degraded, 5 undersized+peered, 31 active+undersized, 38 active+clean; 4.4 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 606 KiB/s wr, 9 op/s; 56/255 objects degraded (21.961%) 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: failed to encode map e106 with expected crc 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: osdmap e106: 8 total, 6 up, 8 in 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3714809638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:35.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:35 smithi151 ceph-mon[159210]: failed to encode map e106 with expected crc 2023-09-23T22:39:35.795 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:35.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:35.812 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:35.881 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:35.881 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:35.902 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:35.935 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:35.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 6 16 101 85 0.885315 0 - 0.00253822 2023-09-23T22:39:36.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:36.142 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:36.173 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.220 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:36.326 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:36.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:36 smithi052 ceph-mon[180979]: failed to encode map e106 with expected crc 2023-09-23T22:39:36.353 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:36 smithi052 ceph-mon[180979]: failed to encode map e106 with expected crc 2023-09-23T22:39:36.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:36 smithi052 ceph-mon[187743]: failed to encode map e106 with expected crc 2023-09-23T22:39:36.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:36 smithi052 ceph-mon[187743]: failed to encode map e106 with expected crc 2023-09-23T22:39:36.356 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:36 smithi151 ceph-mon[159210]: failed to encode map e106 with expected crc 2023-09-23T22:39:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:36 smithi151 ceph-mon[159210]: failed to encode map e106 with expected crc 2023-09-23T22:39:36.529 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:39:36.598 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:36.704 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:36.736 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.910 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:36.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 7 16 101 85 0.758844 0 - 0.00253822 2023-09-23T22:39:37.015 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:37.047 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.096 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:37.147 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:37.200 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:39:37.203 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:37.226 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:37 smithi052 ceph-mon[187743]: pgmap v237: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 731 KiB/s wr, 11 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:37.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:37 smithi052 ceph-mon[180979]: pgmap v237: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 731 KiB/s wr, 11 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:37.230 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.248 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0, 2] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:39:37.248 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:37.248 INFO:tasks.thrashosds.thrasher:Reweighting osd 1 to 0.8097551753379263 2023-09-23T22:39:37.248 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 0.8097551753379263 2023-09-23T22:39:37.273 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:37.379 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:37.424 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:37 smithi151 ceph-mon[159210]: pgmap v237: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 731 KiB/s wr, 11 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:37.484 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:37.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:37.592 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:37.621 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.730 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:37.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:37.813 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:39:37.834 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:37.860 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.902 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:37.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 8 16 101 85 0.663989 0 - 0.00253822 2023-09-23T22:39:38.007 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:38.033 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:38.074 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:38.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:38.180 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:38.207 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:38.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:38 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:38 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:38 smithi151 ceph-mon[159210]: pgmap v238: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 645 KiB/s wr, 10 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:38 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:39:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:38 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:39:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:38 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[187743]: pgmap v238: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 645 KiB/s wr, 10 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:39:38.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[180979]: pgmap v238: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 645 KiB/s wr, 10 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:39:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:38 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:39:38.495 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:38.524 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:38.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:38.628 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:38.630 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:38.658 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:38.700 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:38.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:38.805 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:38.831 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:38.872 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:38.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:38.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:38.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 9 16 101 85 0.590212 0 - 0.00253822 2023-09-23T22:39:38.977 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:39.003 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.043 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:39.044 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:39.149 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:39.175 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.216 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:39.321 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:39.347 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.389 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:39.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:39.495 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:39.495 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:39.521 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.562 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:39.667 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:39.693 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.734 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:39.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:39.889 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 10 16 101 85 0.53119 0 - 0.00253822 2023-09-23T22:39:39.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:39.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:40.089 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:40.119 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.160 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:40.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:40.300 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.341 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:40.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:40.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:40.475 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.514 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:40.619 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:40.649 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.690 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:40.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:40.796 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:40.820 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:40 smithi151 ceph-mon[159210]: pgmap v239: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 544 KiB/s wr, 8 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:40.820 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 62/327 objects degraded (18.960%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:40.822 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.863 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:40.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:40.929 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:40 smithi052 ceph-mon[180979]: pgmap v239: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 544 KiB/s wr, 8 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:40.929 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 62/327 objects degraded (18.960%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:40.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:40 smithi052 ceph-mon[187743]: pgmap v239: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 544 KiB/s wr, 8 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:40.930 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 62/327 objects degraded (18.960%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:40.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 11 16 101 85 0.482901 0 - 0.00253822 2023-09-23T22:39:40.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:41.002 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:41.422 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:39:41.544 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:41.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:41.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:41.648 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:41.677 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:41.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 12 16 101 85 0.44266 0 - 0.00253822 2023-09-23T22:39:42.182 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:42.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:42.288 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:42.324 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:42.415 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:42.427 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:42.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:42.507 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:42.527 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:42.531 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:42.549 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:42.550 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 0.809755 (cf4c) 2023-09-23T22:39:42.551 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:39:42.567 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:42.579 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:42.601 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:42.618 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:42.620 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:42.621 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:39:42.621 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:42.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:42.723 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:42.750 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:42.792 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:42.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:42.900 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:42.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[187743]: pgmap v240: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s wr, 3 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:42.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2550074409' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:42.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4030385351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:42.928 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1288746325' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.8097551753379263}]: dispatch 2023-09-23T22:39:42.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[180979]: pgmap v240: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s wr, 3 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:42.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2550074409' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:42.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4030385351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:42.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1288746325' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.8097551753379263}]: dispatch 2023-09-23T22:39:42.930 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:42.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:42 smithi151 ceph-mon[159210]: pgmap v240: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s wr, 3 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:42.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2550074409' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4030385351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1288746325' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.8097551753379263}]: dispatch 2023-09-23T22:39:42.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 13 16 101 85 0.408595 0 - 0.00253822 2023-09-23T22:39:43.437 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:43.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:43.541 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:43.578 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:43.630 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:43.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:43.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:43.735 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:39:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:39:43] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.33.4" 2023-09-23T22:39:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2550074409' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1288746325' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.8097551753379263}]': finished 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: osdmap e107: 8 total, 6 up, 8 in 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[187743]: osdmap e108: 8 total, 6 up, 8 in 2023-09-23T22:39:43.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2550074409' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1288746325' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.8097551753379263}]': finished 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: osdmap e107: 8 total, 6 up, 8 in 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:43 smithi052 ceph-mon[180979]: osdmap e108: 8 total, 6 up, 8 in 2023-09-23T22:39:43.764 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:43.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2550074409' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1288746325' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.8097551753379263}]': finished 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: osdmap e107: 8 total, 6 up, 8 in 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: failed to encode map e107 with expected crc 2023-09-23T22:39:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:43 smithi151 ceph-mon[159210]: osdmap e108: 8 total, 6 up, 8 in 2023-09-23T22:39:43.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 14 16 101 85 0.379411 0 - 0.00253822 2023-09-23T22:39:44.369 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:44.371 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:44.473 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:44.503 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.561 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:44.664 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:44.693 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: pgmap v242: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s wr, 2 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[180979]: osdmap e109: 8 total, 6 up, 8 in 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: pgmap v242: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s wr, 2 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:44 smithi052 ceph-mon[187743]: osdmap e109: 8 total, 6 up, 8 in 2023-09-23T22:39:44.941 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: pgmap v242: 89 pgs: 1 undersized+degraded+peered, 15 active+undersized+degraded, 5 undersized+peered, 30 active+undersized, 38 active+clean; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s wr, 2 op/s; 62/327 objects degraded (18.960%) 2023-09-23T22:39:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: failed to encode map e108 with expected crc 2023-09-23T22:39:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:44 smithi151 ceph-mon[159210]: osdmap e109: 8 total, 6 up, 8 in 2023-09-23T22:39:44.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 15 16 101 85 0.354118 0 - 0.00253822 2023-09-23T22:39:45.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:45.077 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:45.224 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:45.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:45.328 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:45.356 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:45.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:39:45.413 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:45.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:45.451 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:39:45.451 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:39:45.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:2 6.c 6:3004d844:::benchmark_data_smithi052_234456_object1:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.957647+0000", 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.489556782999999, 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030476359999999998, 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.452 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 2 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957647+0000", 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957647+0000", 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957645+0000", 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:45.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957705+0000", 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.067e-05 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957724+0000", 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8258999999999999e-05 2023-09-23T22:39:45.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957733+0000", 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7289999999999995e-06 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957805+0000", 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1192000000000005e-05 2023-09-23T22:39:45.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.958034+0000", 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022901 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.958206+0000", 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017269400000000001 2023-09-23T22:39:45.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.959563+0000", 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001357013 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960572+0000", 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001008531 2023-09-23T22:39:45.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960619+0000", 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6774999999999999e-05 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960665+0000", 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6570000000000003e-05 2023-09-23T22:39:45.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960694+0000", 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9190999999999999e-05 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.460 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:5 6.b 6:d51bfd48:::benchmark_data_smithi052_234456_object4:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.957753+0000", 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.489450490999999, 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028278230000000001, 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5 2023-09-23T22:39:45.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957753+0000", 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957753+0000", 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957752+0000", 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957799+0000", 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6690999999999997e-05 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957807+0000", 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.994e-06 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957813+0000", 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7880000000000003e-06 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957867+0000", 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4107000000000002e-05 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.958000+0000", 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013284200000000001 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.958167+0000", 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016751299999999999 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.959516+0000", 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013489000000000001 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.959991+0000", 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00047472999999999998 2023-09-23T22:39:45.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960486+0000", 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00049541799999999997 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960544+0000", 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7689000000000003e-05 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960581+0000", 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6863999999999998e-05 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.469 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:13 6.f 6:f8ee44f5:::benchmark_data_smithi052_234456_object12:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.957828+0000", 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.48937534, 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002700994, 2023-09-23T22:39:45.470 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957828+0000", 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957828+0000", 2023-09-23T22:39:45.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957828+0000", 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.473 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957872+0000", 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4366999999999997e-05 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957879+0000", 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3509999999999996e-06 2023-09-23T22:39:45.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957885+0000", 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2449999999999998e-06 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.957946+0000", 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1198999999999995e-05 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.958107+0000", 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016123600000000001 2023-09-23T22:39:45.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4", 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.958424+0000", 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031711899999999998 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.959545+0000", 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001121143 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960388+0000", 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00084267299999999995 2023-09-23T22:39:45.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960436+0000", 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8516000000000001e-05 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960529+0000", 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2788000000000007e-05 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.479 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.34680.0:14 6.4 e105/99 6:20a66966:::benchmark_data_smithi052_234456_object13:head v 105'3, mlcod=0'0)", 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.960366+0000", 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.486837403999999, 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001742221, 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.480 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960366+0000", 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960366+0000", 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960366+0000", 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960388+0000", 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2002999999999999e-05 2023-09-23T22:39:45.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960394+0000", 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5560000000000004e-06 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960399+0000", 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9030000000000003e-06 2023-09-23T22:39:45.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960440+0000", 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1142e-05 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.960459+0000", 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9126e-05 2023-09-23T22:39:45.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.962038+0000", 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015783920000000001 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.962108+0000", 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0528e-05 2023-09-23T22:39:45.485 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:25 6.c 6:34d0e523:::benchmark_data_smithi052_234456_object24:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.961282+0000", 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.48592161, 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029347100000000001, 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.486 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 25 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961282+0000", 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961282+0000", 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961281+0000", 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961332+0000", 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0806000000000001e-05 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961344+0000", 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1771e-05 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961351+0000", 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3780000000000003e-06 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961465+0000", 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011403799999999999 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961627+0000", 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016165999999999999 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961777+0000", 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014982400000000001 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.962625+0000", 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00084853999999999995 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.962924+0000", 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029833000000000003 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964127+0000", 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012034159999999999 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964179+0000", 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2207999999999999e-05 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964217+0000", 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7428000000000001e-05 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.494 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:27 6.6 6:683e9930:::benchmark_data_smithi052_234456_object26:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.961471+0000", 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.485732838000001, 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020503869999999999, 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.495 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 27 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961471+0000", 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961471+0000", 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961470+0000", 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961602+0000", 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013154599999999999 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961611+0000", 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4320000000000005e-06 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961620+0000", 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3209999999999994e-06 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961673+0000", 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2743e-05 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961792+0000", 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118658 2023-09-23T22:39:45.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.961912+0000", 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011986599999999999 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.962677+0000", 2023-09-23T22:39:45.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00076537999999999997 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.963097+0000", 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041948799999999998 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.963424+0000", 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032730200000000002 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.963478+0000", 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4388999999999997e-05 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.963521+0000", 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2889999999999998e-05 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.504 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:31 6.6 6:6687bd22:::benchmark_data_smithi052_234456_object30:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.964162+0000", 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.483041236, 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026616209999999999, 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.505 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 31 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964162+0000", 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964162+0000", 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964161+0000", 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964183+0000", 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1546999999999999e-05 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964193+0000", 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8239999999999995e-06 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964199+0000", 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5220000000000004e-06 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964270+0000", 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1027999999999998e-05 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964402+0000", 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013161100000000001 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964569+0000", 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000167629 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965587+0000", 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010172479999999999 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965777+0000", 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019035699999999999 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966745+0000", 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00096823999999999999 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966787+0000", 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1946000000000001e-05 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966824+0000", 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6752999999999999e-05 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.34680.0:32 6.2 e105/104 6:427b73cc:::benchmark_data_smithi052_234456_object31:head v 105'2, mlcod=0'0)", 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.964929+0000", 2023-09-23T22:39:45.514 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.482274944, 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001878439, 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964929+0000", 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964929+0000", 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.964928+0000", 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965169+0000", 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000241227 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965179+0000", 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0132000000000001e-05 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965186+0000", 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6370000000000002e-06 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965230+0000", 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3649000000000002e-05 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.965257+0000", 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7072000000000001e-05 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966779+0000", 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015226020000000001 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966807+0000", 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.783e-05 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:35 6.6 6:68c8db3e:::benchmark_data_smithi052_234456_object34:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.520 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.966026+0000", 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.481177708000001, 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022552050000000001, 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 35 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966026+0000", 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966026+0000", 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966025+0000", 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966048+0000", 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3397e-05 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966064+0000", 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5979000000000001e-05 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966074+0000", 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7869999999999996e-06 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966118+0000", 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4153999999999998e-05 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966255+0000", 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013705499999999999 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966413+0000", 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000157758 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967119+0000", 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00070630599999999997 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967709+0000", 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059013299999999996 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968192+0000", 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00048244899999999997 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968241+0000", 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9901000000000002e-05 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968281+0000", 2023-09-23T22:39:45.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9572e-05 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36 6.b 6:d31918f7:::benchmark_data_smithi052_234456_object35:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.966092+0000", 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.481111586000001, 2023-09-23T22:39:45.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0048499720000000001, 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.530 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966092+0000", 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966092+0000", 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966091+0000", 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966136+0000", 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5062000000000003e-05 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966146+0000", 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6919999999999997e-06 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966153+0000", 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2280000000000001e-06 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966322+0000", 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168367 2023-09-23T22:39:45.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966397+0000", 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5420000000000006e-05 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966526+0000", 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012937100000000001 2023-09-23T22:39:45.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970245+0000", 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037186390000000001 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970254+0000", 2023-09-23T22:39:45.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4109999999999996e-06 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970858+0000", 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00060447699999999997 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970909+0000", 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1066000000000002e-05 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970942+0000", 2023-09-23T22:39:45.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2863e-05 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40 6.f 6:f8e92dcc:::benchmark_data_smithi052_234456_object39:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.966606+0000", 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.480597187000001, 2023-09-23T22:39:45.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018421010000000001, 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.540 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966606+0000", 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966606+0000", 2023-09-23T22:39:45.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966605+0000", 2023-09-23T22:39:45.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966636+0000", 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0858000000000002e-05 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966656+0000", 2023-09-23T22:39:45.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9959e-05 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966677+0000", 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1399999999999998e-05 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966735+0000", 2023-09-23T22:39:45.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7039000000000001e-05 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966841+0000", 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010604500000000001 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4", 2023-09-23T22:39:45.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.966986+0000", 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000145499 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967613+0000", 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00062678699999999996 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968305+0000", 2023-09-23T22:39:45.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069241500000000004 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968359+0000", 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3304999999999999e-05 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968448+0000", 2023-09-23T22:39:45.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9863000000000003e-05 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.34680.0:38 6.7 e105/99 6:efbd6f8a:::benchmark_data_smithi052_234456_object37:head v 105'3, mlcod=105'1)", 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.967164+0000", 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.480039135, 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018510449999999999, 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.548 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967164+0000", 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967164+0000", 2023-09-23T22:39:45.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967164+0000", 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967282+0000", 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011828900000000001 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967295+0000", 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3419e-05 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967303+0000", 2023-09-23T22:39:45.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8299999999999996e-06 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967366+0000", 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3248000000000002e-05 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967386+0000", 2023-09-23T22:39:45.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9882999999999999e-05 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.968986+0000", 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015998919999999999 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969015+0000", 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9261e-05 2023-09-23T22:39:45.553 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.34680.0:36 6.b e105/99 ondisk, result = 0)", 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.967673+0000", 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.479530166, 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002581088, 2023-09-23T22:39:45.554 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967673+0000", 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967673+0000", 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967673+0000", 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967676+0000", 2023-09-23T22:39:45.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6400000000000001e-06 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967682+0000", 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.669e-06 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.967688+0000", 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7080000000000002e-06 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970230+0000", 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025418010000000002 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970239+0000", 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8009999999999993e-06 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970254+0000", 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5727999999999999e-05 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.559 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:47 6.c 6:33226f8d:::benchmark_data_smithi052_234456_object46:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.969341+0000", 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.477862747, 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016960510000000001, 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 47 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969341+0000", 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969341+0000", 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969339+0000", 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:45.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969366+0000", 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6468000000000001e-05 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969383+0000", 2023-09-23T22:39:45.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7550999999999998e-05 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969394+0000", 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0926e-05 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969453+0000", 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8584e-05 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969609+0000", 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015614599999999999 2023-09-23T22:39:45.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969759+0000", 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014952499999999999 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970904+0000", 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001145606 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970913+0000", 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1949999999999992e-06 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970945+0000", 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1767000000000003e-05 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970996+0000", 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0455999999999999e-05 2023-09-23T22:39:45.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.971037+0000", 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1162999999999999e-05 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:48 6.6 6:6b03b937:::benchmark_data_smithi052_234456_object47:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.969967+0000", 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.477236271000001, 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002375729, 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.570 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969967+0000", 2023-09-23T22:39:45.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969967+0000", 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969966+0000", 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.969988+0000", 2023-09-23T22:39:45.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1449000000000001e-05 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970004+0000", 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6447e-05 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970011+0000", 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5400000000000001e-06 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970070+0000", 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9602000000000003e-05 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970182+0000", 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111294 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970316+0000", 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000134513 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.970961+0000", 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00064528299999999997 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.971431+0000", 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00046957800000000001 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972255+0000", 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00082373500000000003 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972304+0000", 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8770999999999998e-05 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972343+0000", 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9436e-05 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:52 6.b 6:d11ada63:::benchmark_data_smithi052_234456_object51:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.971968+0000", 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.475235530999999, 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027486910000000002, 2023-09-23T22:39:45.579 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 52 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.971968+0000", 2023-09-23T22:39:45.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.971968+0000", 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.971967+0000", 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.971988+0000", 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0947000000000001e-05 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972001+0000", 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3346999999999999e-05 2023-09-23T22:39:45.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972008+0000", 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8519999999999998e-06 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972068+0000", 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0591000000000003e-05 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972174+0000", 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105176 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972302+0000", 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012820799999999999 2023-09-23T22:39:45.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972987+0000", 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00068537800000000005 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.973324+0000", 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033627400000000001 2023-09-23T22:39:45.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974629+0000", 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013055849999999999 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974680+0000", 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1190999999999998e-05 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974717+0000", 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6396000000000002e-05 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:56 6.6 6:62c98fb0:::benchmark_data_smithi052_234456_object55:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.587 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.972865+0000", 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.474338339999999, 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001889637, 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 56 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.588 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972865+0000", 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972865+0000", 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972864+0000", 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972892+0000", 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7447e-05 2023-09-23T22:39:45.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972903+0000", 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1209999999999999e-05 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972911+0000", 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8990000000000001e-06 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.972957+0000", 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6131999999999998e-05 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.973105+0000", 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000147668 2023-09-23T22:39:45.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.973256+0000", 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000151562 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.973961+0000", 2023-09-23T22:39:45.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00070440900000000002 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974082+0000", 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121159 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.594 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974657+0000", 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00057535099999999999 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974713+0000", 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5930000000000002e-05 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974755+0000", 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1817999999999998e-05 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:58 6.f 6:f0aa1bc0:::benchmark_data_smithi052_234456_object57:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.974004+0000", 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.473199062999999, 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021495749999999999, 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.597 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 58 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974004+0000", 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974004+0000", 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974004+0000", 2023-09-23T22:39:45.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974027+0000", 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3034e-05 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974038+0000", 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1090000000000001e-05 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974044+0000", 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5350000000000001e-06 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974138+0000", 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3867000000000001e-05 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974228+0000", 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0140000000000001e-05 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4", 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974339+0000", 2023-09-23T22:39:45.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110387 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.974720+0000", 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00038105700000000002 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976027+0000", 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001307695 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976100+0000", 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2818000000000007e-05 2023-09-23T22:39:45.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976154+0000", 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3857000000000003e-05 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.605 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:65 6.f 6:f97a5936:::benchmark_data_smithi052_234456_object64:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.975942+0000", 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.471261095999999, 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00173474, 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.606 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 65 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.607 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.975942+0000", 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.975942+0000", 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.975941+0000", 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.975963+0000", 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1860000000000001e-05 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.975974+0000", 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1003e-05 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.975982+0000", 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8059999999999995e-06 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976180+0000", 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000198402 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976286+0000", 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105536 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4", 2023-09-23T22:39:45.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976368+0000", 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2090999999999997e-05 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.976709+0000", 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00034090599999999998 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.977534+0000", 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00082556700000000003 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.977588+0000", 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3984999999999999e-05 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.977677+0000", 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8893000000000004e-05 2023-09-23T22:39:45.613 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:74 6.b 6:d6eb1735:::benchmark_data_smithi052_234456_object73:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:39:29.978007+0000", 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 15.469196386, 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017263280000000001, 2023-09-23T22:39:45.614 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 74 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:39:45.615 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978007+0000", 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978007+0000", 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978006+0000", 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978030+0000", 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3683e-05 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978040+0000", 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0165e-05 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978048+0000", 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7440000000000007e-06 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.618 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978115+0000", 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7694999999999995e-05 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978228+0000", 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000112908 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:45.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.978377+0000", 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014872099999999999 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.979227+0000", 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00085026800000000005 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:39:45.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.979329+0000", 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101523 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.979653+0000", 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032460099999999999 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.979699+0000", 2023-09-23T22:39:45.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5463999999999997e-05 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:39:29.979733+0000", 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4578e-05 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:39:45.622 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:39:45.623 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:39:45.623 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:45.655 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:45.702 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:45.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:45.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[180979]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[180979]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[180979]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[180979]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[187743]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[187743]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[187743]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:45 smithi052 ceph-mon[187743]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.809 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:45.838 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:45.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:45 smithi151 ceph-mon[159210]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:45 smithi151 ceph-mon[159210]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:45 smithi151 ceph-mon[159210]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:45 smithi151 ceph-mon[159210]: failed to encode map e109 with expected crc 2023-09-23T22:39:45.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 16 16 101 85 0.331985 0 - 0.00253822 2023-09-23T22:39:46.100 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:46.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:39:46.170 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:46.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:46.232 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:46.281 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:46.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:46.372 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:46.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:46.411 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:46.455 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:46.464 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:46.559 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:46.590 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:46.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:46 smithi052 ceph-mon[180979]: failed to encode map e109 with expected crc 2023-09-23T22:39:46.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:46 smithi052 ceph-mon[180979]: pgmap v245: 89 pgs: 4 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 2 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered, 2 remapped+peering; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:46.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:46 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:46.742 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:46.846 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:46.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:46.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:46 smithi151 ceph-mon[159210]: failed to encode map e109 with expected crc 2023-09-23T22:39:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:46 smithi151 ceph-mon[159210]: pgmap v245: 89 pgs: 4 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 2 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered, 2 remapped+peering; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:46 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:46.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 17 16 101 85 0.312458 0 - 0.00253822 2023-09-23T22:39:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:46 smithi052 ceph-mon[187743]: failed to encode map e109 with expected crc 2023-09-23T22:39:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:46 smithi052 ceph-mon[187743]: pgmap v245: 89 pgs: 4 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 2 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered, 2 remapped+peering; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:46.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:46 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:47.072 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:47.176 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:47.203 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.244 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:47.245 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:47.347 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:47.378 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.627 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:47.732 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:47.760 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.807 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:47.910 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:47.938 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 18 16 101 85 0.2951 0 - 0.00253822 2023-09-23T22:39:47.984 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:47.985 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:39:47.985 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:39:47.985 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:39:47.985 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:39:47.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:48.088 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:48.114 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.205 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:48.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:48.309 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:48.339 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.390 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:48.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:48.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:48.520 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:48.571 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:39:48.671 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:48.684 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:39:48.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.911 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:48.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:48.915 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:48.934 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:48 smithi151 ceph-mon[159210]: pgmap v246: 89 pgs: 4 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 2 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered, 2 remapped+peering; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:48.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3418451732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:48.953 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:48.953 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:48.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 19 16 101 85 0.279569 0 - 0.00253822 2023-09-23T22:39:48.971 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:48.989 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:48 smithi052 ceph-mon[187743]: pgmap v246: 89 pgs: 4 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 2 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered, 2 remapped+peering; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:48.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3418451732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:48 smithi052 ceph-mon[180979]: pgmap v246: 89 pgs: 4 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 2 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered, 2 remapped+peering; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3418451732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:49.007 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:39:49.008 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:49.014 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:49.040 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.081 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:49.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:49.184 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0, 2] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:39:49.184 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:49.184 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T22:39:49.185 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:39:49.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:49.212 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.255 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:49.358 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:49.385 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.632 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:49.735 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[187743]: failed to encode map e110 with expected crc 2023-09-23T22:39:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3418451732' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[187743]: osdmap e110: 8 total, 6 up, 8 in 2023-09-23T22:39:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/756758939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[180979]: failed to encode map e110 with expected crc 2023-09-23T22:39:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3418451732' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[180979]: osdmap e110: 8 total, 6 up, 8 in 2023-09-23T22:39:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/756758939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:49.767 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.884 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:49 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:49.884 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:49 smithi151 ceph-mon[159210]: failed to encode map e110 with expected crc 2023-09-23T22:39:49.884 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3418451732' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:49.884 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:49 smithi151 ceph-mon[159210]: osdmap e110: 8 total, 6 up, 8 in 2023-09-23T22:39:49.884 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/756758939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:49.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:49.893 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:49.930 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:49.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:39:49.959435+0000 min lat: 0.00121387 max lat: 0.00607929 avg lat: 0.00253822 2023-09-23T22:39:49.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:39:49.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 20 16 101 85 0.265591 0 - 0.00253822 2023-09-23T22:39:49.997 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:50.033 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:50.061 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:50.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:50.209 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:50.235 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.280 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:50.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:50.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:50.412 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.434 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:50.434 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:50.455 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:50.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:50.559 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:50.589 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.636 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:50.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:50.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:50.766 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:50 smithi151 ceph-mon[159210]: pgmap v248: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 3 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:50 smithi151 ceph-mon[159210]: failed to encode map e110 with expected crc 2023-09-23T22:39:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:50 smithi151 ceph-mon[159210]: failed to encode map e110 with expected crc 2023-09-23T22:39:50.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:50 smithi052 ceph-mon[187743]: pgmap v248: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 3 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:50.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:50 smithi052 ceph-mon[187743]: failed to encode map e110 with expected crc 2023-09-23T22:39:50.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:50 smithi052 ceph-mon[187743]: failed to encode map e110 with expected crc 2023-09-23T22:39:50.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:50 smithi052 ceph-mon[180979]: pgmap v248: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 3 active+undersized+remapped, 13 active+undersized+degraded, 1 undersized+degraded+peered; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 128 KiB/s, 1 objects/s recovering 2023-09-23T22:39:50.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:50 smithi052 ceph-mon[180979]: failed to encode map e110 with expected crc 2023-09-23T22:39:50.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:50 smithi052 ceph-mon[180979]: failed to encode map e110 with expected crc 2023-09-23T22:39:50.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 21 16 101 85 0.252944 0 - 0.00253822 2023-09-23T22:39:50.988 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:51.049 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:39:51.092 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:51.118 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.162 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:51.267 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.401 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:51.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:51.544 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:51.570 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.603 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:39:51.605 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:39:51.666 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:51.769 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:51.797 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:51.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:39:51.923 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:39:51.944 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:51.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 22 16 101 85 0.241447 0 - 0.00253822 2023-09-23T22:39:51.971 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.012 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:52.116 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:52.141 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.270 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":249,"stamp":"2023-09-23T22:39:51.489684+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6166432,"num_objects":109,"num_object_clones":0,"num_object_copies":327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":50,"num_objects_misplaced":19,"num_objects_unfound":0,"num_objects_dirty":109,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":345,"num_write_kb":7885,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":279,"ondisk_log_size":279,"up":204,"acting":212,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":266,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1375120,"kb_used_data":23952,"kb_used_omap":126,"kb_used_meta":1350977,"kb_avail":748586096,"statfs":{"total":767960285184,"available":766552162304,"internally_reserved":0,"allocated":24526848,"data_stored":20703417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1383401083},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-12,"num_objects_misplaced":19,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.946106"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":98,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.552835+0000","last_change":"2023-09-23T22:39:29.060202+0000","last_active":"2023-09-23T22:39:44.552835+0000","last_peered":"2023-09-23T22:39:44.552835+0000","last_clean":"2023-09-23T22:39:27.524383+0000","last_became_active":"2023-09-23T22:39:29.060202+0000","last_became_peered":"2023-09-23T22:39:29.060202+0000","last_unstale":"2023-09-23T22:39:44.552835+0000","last_undegraded":"2023-09-23T22:39:44.552835+0000","last_fullsized":"2023-09-23T22:39:29.057225+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549309+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:39:44.549309+0000","last_peered":"2023-09-23T22:39:44.549309+0000","last_clean":"2023-09-23T22:39:44.549309+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:39:44.549309+0000","last_undegraded":"2023-09-23T22:39:44.549309+0000","last_fullsized":"2023-09-23T22:39:44.549309+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:39:43.547434+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:39:43.547434+0000","last_peered":"2023-09-23T22:39:43.547434+0000","last_clean":"2023-09-23T22:39:43.547434+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:39:43.547434+0000","last_undegraded":"2023-09-23T22:39:43.547434+0000","last_fullsized":"2023-09-23T22:39:43.547434+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.684450+0000","last_change":"2023-09-23T22:39:14.011199+0000","last_active":"2023-09-23T22:39:44.684450+0000","last_peered":"2023-09-23T22:39:44.684450+0000","last_clean":"2023-09-23T22:37:49.457133+0000","last_became_active":"2023-09-23T22:39:14.011199+0000","last_became_peered":"2023-09-23T22:39:14.011199+0000","last_unstale":"2023-09-23T22:39:44.684450+0000","last_undegraded":"2023-09-23T22:39:44.684450+0000","last_fullsized":"2023-09-23T22:39:13.961824+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":109,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:44.552403+0000","last_change":"2023-09-23T22:39:44.552123+0000","last_active":"2023-09-23T22:39:44.552403+0000","last_peered":"2023-09-23T22:39:44.552403+0000","last_clean":"2023-09-23T22:38:06.149335+0000","last_became_active":"2023-09-23T22:39:44.552123+0000","last_became_peered":"2023-09-23T22:39:44.552123+0000","last_unstale":"2023-09-23T22:39:44.552403+0000","last_undegraded":"2023-09-23T22:39:44.552403+0000","last_fullsized":"2023-09-23T22:39:44.548925+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:57.334435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.550659+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:39:44.550659+0000","last_peered":"2023-09-23T22:39:44.550659+0000","last_clean":"2023-09-23T22:39:44.550659+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:39:44.550659+0000","last_undegraded":"2023-09-23T22:39:44.550659+0000","last_fullsized":"2023-09-23T22:39:44.550659+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.549479+0000","last_change":"2023-09-23T22:39:14.011184+0000","last_active":"2023-09-23T22:39:44.549479+0000","last_peered":"2023-09-23T22:39:44.549479+0000","last_clean":"2023-09-23T22:37:35.261066+0000","last_became_active":"2023-09-23T22:39:14.011184+0000","last_became_peered":"2023-09-23T22:39:14.011184+0000","last_unstale":"2023-09-23T22:39:44.549479+0000","last_undegraded":"2023-09-23T22:39:44.549479+0000","last_fullsized":"2023-09-23T22:39:13.959535+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":97,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.548841+0000","last_change":"2023-09-23T22:39:29.052467+0000","last_active":"2023-09-23T22:39:44.548841+0000","last_peered":"2023-09-23T22:39:44.548841+0000","last_clean":"2023-09-23T22:39:27.674947+0000","last_became_active":"2023-09-23T22:39:29.052467+0000","last_became_peered":"2023-09-23T22:39:29.052467+0000","last_unstale":"2023-09-23T22:39:44.548841+0000","last_undegraded":"2023-09-23T22:39:44.548841+0000","last_fullsized":"2023-09-23T22:39:29.049608+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:47.935154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549702+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:39:44.549702+0000","last_peered":"2023-09-23T22:39:44.549702+0000","last_clean":"2023-09-23T22:39:44.549702+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:39:44.549702+0000","last_undegraded":"2023-09-23T22:39:44.549702+0000","last_fullsized":"2023-09-23T22:39:44.549702+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":37,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549112+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:39:44.549112+0000","last_peered":"2023-09-23T22:39:44.549112+0000","last_clean":"2023-09-23T22:39:44.549112+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:39:44.549112+0000","last_undegraded":"2023-09-23T22:39:44.549112+0000","last_fullsized":"2023-09-23T22:39:44.549112+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.684788+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:39:44.684788+0000","last_peered":"2023-09-23T22:39:44.684788+0000","last_clean":"2023-09-23T22:39:44.684788+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:39:44.684788+0000","last_undegraded":"2023-09-23T22:39:44.684788+0000","last_fullsized":"2023-09-23T22:39:44.684788+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:44.553768+0000","last_change":"2023-09-23T22:39:44.553430+0000","last_active":"2023-09-23T22:39:44.553768+0000","last_peered":"2023-09-23T22:39:44.553768+0000","last_clean":"2023-09-23T22:39:27.713280+0000","last_became_active":"2023-09-23T22:39:44.553430+0000","last_became_peered":"2023-09-23T22:39:44.553430+0000","last_unstale":"2023-09-23T22:39:44.553768+0000","last_undegraded":"2023-09-23T22:39:44.553768+0000","last_fullsized":"2023-09-23T22:39:44.550115+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.552086+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:39:44.552086+0000","last_peered":"2023-09-23T22:39:44.552086+0000","last_clean":"2023-09-23T22:39:44.552086+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:39:44.552086+0000","last_undegraded":"2023-09-23T22:39:44.552086+0000","last_fullsized":"2023-09-23T22:39:44.552086+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.553580+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:39:44.553580+0000","last_peered":"2023-09-23T22:39:44.553580+0000","last_clean":"2023-09-23T22:39:44.553580+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:39:44.553580+0000","last_undegraded":"2023-09-23T22:39:44.553580+0000","last_fullsized":"2023-09-23T22:39:44.553580+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":84,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.684548+0000","last_change":"2023-09-23T22:39:29.058721+0000","last_active":"2023-09-23T22:39:44.684548+0000","last_peered":"2023-09-23T22:39:44.684548+0000","last_clean":"2023-09-23T22:39:27.181407+0000","last_became_active":"2023-09-23T22:39:29.058721+0000","last_became_peered":"2023-09-23T22:39:29.058721+0000","last_unstale":"2023-09-23T22:39:44.684548+0000","last_undegraded":"2023-09-23T22:39:29.055623+0000","last_fullsized":"2023-09-23T22:39:29.055483+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:44.685606+0000","last_change":"2023-09-23T22:39:44.685345+0000","last_active":"2023-09-23T22:39:44.685606+0000","last_peered":"2023-09-23T22:39:44.685606+0000","last_clean":"2023-09-23T22:39:44.685606+0000","last_became_active":"2023-09-23T22:39:44.684848+0000","last_became_peered":"2023-09-23T22:39:44.684848+0000","last_unstale":"2023-09-23T22:39:44.685606+0000","last_undegraded":"2023-09-23T22:39:44.685606+0000","last_fullsized":"2023-09-23T22:39:44.685606+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:56.659998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.684338+0000","last_change":"2023-09-23T22:39:14.011235+0000","last_active":"2023-09-23T22:39:44.684338+0000","last_peered":"2023-09-23T22:39:44.684338+0000","last_clean":"2023-09-23T22:37:35.263869+0000","last_became_active":"2023-09-23T22:39:14.011235+0000","last_became_peered":"2023-09-23T22:39:14.011235+0000","last_unstale":"2023-09-23T22:39:44.684338+0000","last_undegraded":"2023-09-23T22:39:44.684338+0000","last_fullsized":"2023-09-23T22:39:13.961292+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549670+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:39:44.549670+0000","last_peered":"2023-09-23T22:39:44.549670+0000","last_clean":"2023-09-23T22:39:44.549670+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:39:44.549670+0000","last_undegraded":"2023-09-23T22:39:44.549670+0000","last_fullsized":"2023-09-23T22:39:44.549670+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.684068+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:39:44.684068+0000","last_peered":"2023-09-23T22:39:44.684068+0000","last_clean":"2023-09-23T22:39:44.684068+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:39:44.684068+0000","last_undegraded":"2023-09-23T22:39:44.684068+0000","last_fullsized":"2023-09-23T22:39:44.684068+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":65,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.549605+0000","last_change":"2023-09-23T22:39:14.010950+0000","last_active":"2023-09-23T22:39:44.549605+0000","last_peered":"2023-09-23T22:39:44.549605+0000","last_clean":"2023-09-23T22:37:35.260733+0000","last_became_active":"2023-09-23T22:39:14.010950+0000","last_became_peered":"2023-09-23T22:39:14.010950+0000","last_unstale":"2023-09-23T22:39:44.549605+0000","last_undegraded":"2023-09-23T22:39:44.549605+0000","last_fullsized":"2023-09-23T22:39:13.954201+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:48.612093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":91,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.549915+0000","last_change":"2023-09-23T22:39:13.960430+0000","last_active":"2023-09-23T22:39:44.549915+0000","last_peered":"2023-09-23T22:39:44.549915+0000","last_clean":"2023-09-23T22:38:06.146629+0000","last_became_active":"2023-09-23T22:39:13.960430+0000","last_became_peered":"2023-09-23T22:39:13.960430+0000","last_unstale":"2023-09-23T22:39:44.549915+0000","last_undegraded":"2023-09-23T22:39:13.956944+0000","last_fullsized":"2023-09-23T22:39:13.956813+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549505+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:39:44.549505+0000","last_peered":"2023-09-23T22:39:44.549505+0000","last_clean":"2023-09-23T22:39:44.549505+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:39:44.549505+0000","last_undegraded":"2023-09-23T22:39:44.549505+0000","last_fullsized":"2023-09-23T22:39:44.549505+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"105'4","reported_seq":20,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.550027+0000","last_change":"2023-09-23T22:39:29.969914+0000","last_active":"2023-09-23T22:39:44.550027+0000","last_peered":"2023-09-23T22:39:44.550027+0000","last_clean":"2023-09-23T22:39:21.005110+0000","last_became_active":"2023-09-23T22:39:22.035933+0000","last_became_peered":"2023-09-23T22:39:22.035933+0000","last_unstale":"2023-09-23T22:39:44.550027+0000","last_undegraded":"2023-09-23T22:39:29.964644+0000","last_fullsized":"2023-09-23T22:39:22.026669+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":83,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.685054+0000","last_change":"2023-09-23T22:39:13.964599+0000","last_active":"2023-09-23T22:39:44.685054+0000","last_peered":"2023-09-23T22:39:44.685054+0000","last_clean":"2023-09-23T22:38:06.151624+0000","last_became_active":"2023-09-23T22:39:13.964599+0000","last_became_peered":"2023-09-23T22:39:13.964599+0000","last_unstale":"2023-09-23T22:39:44.685054+0000","last_undegraded":"2023-09-23T22:39:13.962135+0000","last_fullsized":"2023-09-23T22:39:13.962007+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549187+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:39:44.549187+0000","last_peered":"2023-09-23T22:39:44.549187+0000","last_clean":"2023-09-23T22:39:44.549187+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:39:44.549187+0000","last_undegraded":"2023-09-23T22:39:44.549187+0000","last_fullsized":"2023-09-23T22:39:44.549187+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"105'5","reported_seq":25,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549081+0000","last_change":"2023-09-23T22:39:22.024643+0000","last_active":"2023-09-23T22:39:44.549081+0000","last_peered":"2023-09-23T22:39:44.549081+0000","last_clean":"2023-09-23T22:39:44.549081+0000","last_became_active":"2023-09-23T22:39:22.024328+0000","last_became_peered":"2023-09-23T22:39:22.024328+0000","last_unstale":"2023-09-23T22:39:44.549081+0000","last_undegraded":"2023-09-23T22:39:44.549081+0000","last_fullsized":"2023-09-23T22:39:44.549081+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:57.874188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.684698+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:39:44.684698+0000","last_peered":"2023-09-23T22:39:44.684698+0000","last_clean":"2023-09-23T22:39:44.684698+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:39:44.684698+0000","last_undegraded":"2023-09-23T22:39:44.684698+0000","last_fullsized":"2023-09-23T22:39:44.684698+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.552204+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:39:44.552204+0000","last_peered":"2023-09-23T22:39:44.552204+0000","last_clean":"2023-09-23T22:39:44.552204+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:39:44.552204+0000","last_undegraded":"2023-09-23T22:39:44.552204+0000","last_fullsized":"2023-09-23T22:39:44.552204+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"105'5","reported_seq":30,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.683987+0000","last_change":"2023-09-23T22:39:29.961122+0000","last_active":"2023-09-23T22:39:44.683987+0000","last_peered":"2023-09-23T22:39:44.683987+0000","last_clean":"2023-09-23T22:39:27.180772+0000","last_became_active":"2023-09-23T22:39:29.061752+0000","last_became_peered":"2023-09-23T22:39:29.061752+0000","last_unstale":"2023-09-23T22:39:44.683987+0000","last_undegraded":"2023-09-23T22:39:29.960809+0000","last_fullsized":"2023-09-23T22:39:29.052889+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":5}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.684862+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:39:44.684862+0000","last_peered":"2023-09-23T22:39:44.684862+0000","last_clean":"2023-09-23T22:39:44.684862+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:39:44.684862+0000","last_undegraded":"2023-09-23T22:39:44.684862+0000","last_fullsized":"2023-09-23T22:39:44.684862+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.550380+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:39:44.550380+0000","last_peered":"2023-09-23T22:39:44.550380+0000","last_clean":"2023-09-23T22:39:44.550380+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:39:44.550380+0000","last_undegraded":"2023-09-23T22:39:44.550380+0000","last_fullsized":"2023-09-23T22:39:44.550380+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":21,"reported_epoch":109,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:44.684080+0000","last_change":"2023-09-23T22:39:29.054295+0000","last_active":"2023-09-23T22:39:28.052778+0000","last_peered":"2023-09-23T22:39:44.684080+0000","last_clean":"2023-09-23T22:39:21.005110+0000","last_became_active":"2023-09-23T22:39:22.023737+0000","last_became_peered":"2023-09-23T22:39:29.054295+0000","last_unstale":"2023-09-23T22:39:44.684080+0000","last_undegraded":"2023-09-23T22:39:44.684080+0000","last_fullsized":"2023-09-23T22:39:29.052978+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.551172+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:39:44.551172+0000","last_peered":"2023-09-23T22:39:44.551172+0000","last_clean":"2023-09-23T22:39:44.551172+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:39:44.551172+0000","last_undegraded":"2023-09-23T22:39:44.551172+0000","last_fullsized":"2023-09-23T22:39:44.551172+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":55,"reported_epoch":108,"state":"active+undersized","last_fresh":"2023-09-23T22:39:43.548312+0000","last_change":"2023-09-23T22:39:13.963250+0000","last_active":"2023-09-23T22:39:43.548312+0000","last_peered":"2023-09-23T22:39:43.548312+0000","last_clean":"2023-09-23T22:35:51.474127+0000","last_became_active":"2023-09-23T22:39:13.963250+0000","last_became_peered":"2023-09-23T22:39:13.963250+0000","last_unstale":"2023-09-23T22:39:43.548312+0000","last_undegraded":"2023-09-23T22:39:43.548312+0000","last_fullsized":"2023-09-23T22:39:13.953342+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:14.067612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.e","version":"105'5","reported_seq":52,"reported_epoch":109,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:44.775749+0000","last_change":"2023-09-23T22:39:44.775522+0000","last_active":"2023-09-23T22:39:44.775749+0000","last_peered":"2023-09-23T22:39:44.775749+0000","last_clean":"2023-09-23T22:39:44.775749+0000","last_became_active":"2023-09-23T22:39:44.559115+0000","last_became_peered":"2023-09-23T22:39:44.559115+0000","last_unstale":"2023-09-23T22:39:44.775749+0000","last_undegraded":"2023-09-23T22:39:44.775749+0000","last_fullsized":"2023-09-23T22:39:44.775749+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":5}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":83,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.549388+0000","last_change":"2023-09-23T22:39:14.011801+0000","last_active":"2023-09-23T22:39:44.549388+0000","last_peered":"2023-09-23T22:39:44.549388+0000","last_clean":"2023-09-23T22:38:06.152164+0000","last_became_active":"2023-09-23T22:39:14.011801+0000","last_became_peered":"2023-09-23T22:39:14.011801+0000","last_unstale":"2023-09-23T22:39:44.549388+0000","last_undegraded":"2023-09-23T22:39:13.953983+0000","last_fullsized":"2023-09-23T22:39:13.953867+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:41.277493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.552018+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:39:44.552018+0000","last_peered":"2023-09-23T22:39:44.552018+0000","last_clean":"2023-09-23T22:39:44.552018+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:39:44.552018+0000","last_undegraded":"2023-09-23T22:39:44.552018+0000","last_fullsized":"2023-09-23T22:39:44.552018+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"105'6","reported_seq":20,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.549360+0000","last_change":"2023-09-23T22:39:29.968360+0000","last_active":"2023-09-23T22:39:44.549360+0000","last_peered":"2023-09-23T22:39:44.549360+0000","last_clean":"2023-09-23T22:39:21.005110+0000","last_became_active":"2023-09-23T22:39:29.059001+0000","last_became_peered":"2023-09-23T22:39:29.059001+0000","last_unstale":"2023-09-23T22:39:44.549360+0000","last_undegraded":"2023-09-23T22:39:29.960439+0000","last_fullsized":"2023-09-23T22:39:29.052027+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:04.556801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":6}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.550511+0000","last_change":"2023-09-23T22:39:43.560976+0000","last_active":"2023-09-23T22:39:44.550511+0000","last_peered":"2023-09-23T22:39:44.550511+0000","last_clean":"2023-09-23T22:39:44.550511+0000","last_became_active":"2023-09-23T22:39:43.560543+0000","last_became_peered":"2023-09-23T22:39:43.560543+0000","last_unstale":"2023-09-23T22:39:44.550511+0000","last_undegraded":"2023-09-23T22:39:44.550511+0000","last_fullsized":"2023-09-23T22:39:44.550511+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":96,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.552528+0000","last_change":"2023-09-23T22:39:29.062356+0000","last_active":"2023-09-23T22:39:44.552528+0000","last_peered":"2023-09-23T22:39:44.552528+0000","last_clean":"2023-09-23T22:39:27.524929+0000","last_became_active":"2023-09-23T22:39:29.062356+0000","last_became_peered":"2023-09-23T22:39:29.062356+0000","last_unstale":"2023-09-23T22:39:44.552528+0000","last_undegraded":"2023-09-23T22:39:44.552528+0000","last_fullsized":"2023-09-23T22:39:29.055948+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"105'5","reported_seq":25,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549693+0000","last_change":"2023-09-23T22:39:22.036246+0000","last_active":"2023-09-23T22:39:44.549693+0000","last_peered":"2023-09-23T22:39:44.549693+0000","last_clean":"2023-09-23T22:39:44.549693+0000","last_became_active":"2023-09-23T22:39:22.035914+0000","last_became_peered":"2023-09-23T22:39:22.035914+0000","last_unstale":"2023-09-23T22:39:44.549693+0000","last_undegraded":"2023-09-23T22:39:44.549693+0000","last_fullsized":"2023-09-23T22:39:44.549693+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:16.518970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":82,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.551464+0000","last_change":"2023-09-23T22:39:29.058163+0000","last_active":"2023-09-23T22:39:44.551464+0000","last_peered":"2023-09-23T22:39:44.551464+0000","last_clean":"2023-09-23T22:39:27.524191+0000","last_became_active":"2023-09-23T22:39:29.058163+0000","last_became_peered":"2023-09-23T22:39:29.058163+0000","last_unstale":"2023-09-23T22:39:44.551464+0000","last_undegraded":"2023-09-23T22:39:44.551464+0000","last_fullsized":"2023-09-23T22:39:29.055134+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.552836+0000","last_change":"2023-09-23T22:39:43.559993+0000","last_active":"2023-09-23T22:39:44.552836+0000","last_peered":"2023-09-23T22:39:44.552836+0000","last_clean":"2023-09-23T22:39:44.552836+0000","last_became_active":"2023-09-23T22:39:43.559612+0000","last_became_peered":"2023-09-23T22:39:43.559612+0000","last_unstale":"2023-09-23T22:39:44.552836+0000","last_undegraded":"2023-09-23T22:39:44.552836+0000","last_fullsized":"2023-09-23T22:39:44.552836+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"105'7","reported_seq":30,"reported_epoch":109,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:44.553526+0000","last_change":"2023-09-23T22:39:44.553165+0000","last_active":"2023-09-23T22:39:44.553526+0000","last_peered":"2023-09-23T22:39:44.553526+0000","last_clean":"2023-09-23T22:39:21.005110+0000","last_became_active":"2023-09-23T22:39:44.553165+0000","last_became_peered":"2023-09-23T22:39:44.553165+0000","last_unstale":"2023-09-23T22:39:44.553526+0000","last_undegraded":"2023-09-23T22:39:44.553526+0000","last_fullsized":"2023-09-23T22:39:44.549546+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458752,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":14,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":7}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549524+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:39:44.549524+0000","last_peered":"2023-09-23T22:39:44.549524+0000","last_clean":"2023-09-23T22:39:44.549524+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:39:44.549524+0000","last_undegraded":"2023-09-23T22:39:44.549524+0000","last_fullsized":"2023-09-23T22:39:44.549524+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.549771+0000","last_change":"2023-09-23T22:39:29.058670+0000","last_active":"2023-09-23T22:39:44.549771+0000","last_peered":"2023-09-23T22:39:44.549771+0000","last_clean":"2023-09-23T22:37:35.276501+0000","last_became_active":"2023-09-23T22:39:29.058670+0000","last_became_peered":"2023-09-23T22:39:29.058670+0000","last_unstale":"2023-09-23T22:39:44.549771+0000","last_undegraded":"2023-09-23T22:39:44.549771+0000","last_fullsized":"2023-09-23T22:39:29.051702+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:43.056991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":108,"state":"active+undersized","last_fresh":"2023-09-23T22:39:43.548194+0000","last_change":"2023-09-23T22:39:13.962379+0000","last_active":"2023-09-23T22:39:43.548194+0000","last_peered":"2023-09-23T22:39:43.548194+0000","last_clean":"2023-09-23T22:39:06.811180+0000","last_became_active":"2023-09-23T22:39:13.962379+0000","last_became_peered":"2023-09-23T22:39:13.962379+0000","last_unstale":"2023-09-23T22:39:43.548194+0000","last_undegraded":"2023-09-23T22:39:43.548194+0000","last_fullsized":"2023-09-23T22:39:13.953652+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:12.993551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.549766+0000","last_change":"2023-09-23T22:39:29.062983+0000","last_active":"2023-09-23T22:39:44.549766+0000","last_peered":"2023-09-23T22:39:44.549766+0000","last_clean":"2023-09-23T22:39:27.042499+0000","last_became_active":"2023-09-23T22:39:29.062983+0000","last_became_peered":"2023-09-23T22:39:29.062983+0000","last_unstale":"2023-09-23T22:39:44.549766+0000","last_undegraded":"2023-09-23T22:39:44.549766+0000","last_fullsized":"2023-09-23T22:39:29.057285+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":29,"reported_epoch":109,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:44.556340+0000","last_change":"2023-09-23T22:39:44.555972+0000","last_active":"2023-09-23T22:39:44.556340+0000","last_peered":"2023-09-23T22:39:44.556340+0000","last_clean":"2023-09-23T22:39:44.556340+0000","last_became_active":"2023-09-23T22:39:44.555594+0000","last_became_peered":"2023-09-23T22:39:44.555594+0000","last_unstale":"2023-09-23T22:39:44.556340+0000","last_undegraded":"2023-09-23T22:39:44.556340+0000","last_fullsized":"2023-09-23T22:39:44.556340+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:44.548815+0000","last_change":"2023-09-23T22:39:29.049978+0000","last_active":"2023-09-23T22:39:28.042713+0000","last_peered":"2023-09-23T22:39:44.548815+0000","last_clean":"2023-09-23T22:39:21.005110+0000","last_became_active":"2023-09-23T22:39:22.021483+0000","last_became_peered":"2023-09-23T22:39:29.049978+0000","last_unstale":"2023-09-23T22:39:44.548815+0000","last_undegraded":"2023-09-23T22:39:44.548815+0000","last_fullsized":"2023-09-23T22:39:29.049475+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:26.784903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3],"avail_no_missing":["3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.6","version":"105'8","reported_seq":28,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549311+0000","last_change":"2023-09-23T22:39:22.024738+0000","last_active":"2023-09-23T22:39:44.549311+0000","last_peered":"2023-09-23T22:39:44.549311+0000","last_clean":"2023-09-23T22:39:44.549311+0000","last_became_active":"2023-09-23T22:39:22.024460+0000","last_became_peered":"2023-09-23T22:39:22.024460+0000","last_unstale":"2023-09-23T22:39:44.549311+0000","last_undegraded":"2023-09-23T22:39:44.549311+0000","last_fullsized":"2023-09-23T22:39:44.549311+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:48.071185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":524288,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8,"num_write_kb":512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.550556+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:39:44.550556+0000","last_peered":"2023-09-23T22:39:44.550556+0000","last_clean":"2023-09-23T22:39:44.550556+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:39:44.550556+0000","last_undegraded":"2023-09-23T22:39:44.550556+0000","last_fullsized":"2023-09-23T22:39:44.550556+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":82,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.553743+0000","last_change":"2023-09-23T22:39:13.961647+0000","last_active":"2023-09-23T22:39:44.553743+0000","last_peered":"2023-09-23T22:39:44.553743+0000","last_clean":"2023-09-23T22:38:06.149467+0000","last_became_active":"2023-09-23T22:39:13.961647+0000","last_became_peered":"2023-09-23T22:39:13.961647+0000","last_unstale":"2023-09-23T22:39:44.553743+0000","last_undegraded":"2023-09-23T22:39:44.553743+0000","last_fullsized":"2023-09-23T22:39:13.960162+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.550971+0000","last_change":"2023-09-23T22:39:29.063032+0000","last_active":"2023-09-23T22:39:44.550971+0000","last_peered":"2023-09-23T22:39:44.550971+0000","last_clean":"2023-09-23T22:39:27.041787+0000","last_became_active":"2023-09-23T22:39:29.063032+0000","last_became_peered":"2023-09-23T22:39:29.063032+0000","last_unstale":"2023-09-23T22:39:44.550971+0000","last_undegraded":"2023-09-23T22:39:44.550971+0000","last_fullsized":"2023-09-23T22:39:29.058510+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"105'9","reported_seq":28,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.684691+0000","last_change":"2023-09-23T22:39:22.036407+0000","last_active":"2023-09-23T22:39:44.684691+0000","last_peered":"2023-09-23T22:39:44.684691+0000","last_clean":"2023-09-23T22:39:44.684691+0000","last_became_active":"2023-09-23T22:39:22.036087+0000","last_became_peered":"2023-09-23T22:39:22.036087+0000","last_unstale":"2023-09-23T22:39:44.684691+0000","last_undegraded":"2023-09-23T22:39:44.684691+0000","last_fullsized":"2023-09-23T22:39:44.684691+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":589824,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":99,"reported_epoch":109,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:44.548938+0000","last_change":"2023-09-23T22:39:29.053182+0000","last_active":"2023-09-23T22:39:12.952973+0000","last_peered":"2023-09-23T22:39:44.548938+0000","last_clean":"2023-09-23T22:38:06.149164+0000","last_became_active":"2023-09-23T22:38:06.148636+0000","last_became_peered":"2023-09-23T22:39:29.053182+0000","last_unstale":"2023-09-23T22:39:44.548938+0000","last_undegraded":"2023-09-23T22:39:44.548938+0000","last_fullsized":"2023-09-23T22:39:29.050081+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:10.502340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3],"avail_no_missing":["3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+undersized","last_fresh":"2023-09-23T22:39:43.547650+0000","last_change":"2023-09-23T22:39:29.056454+0000","last_active":"2023-09-23T22:39:43.547650+0000","last_peered":"2023-09-23T22:39:43.547650+0000","last_clean":"2023-09-23T22:39:27.713269+0000","last_became_active":"2023-09-23T22:39:29.056454+0000","last_became_peered":"2023-09-23T22:39:29.056454+0000","last_unstale":"2023-09-23T22:39:43.547650+0000","last_undegraded":"2023-09-23T22:39:43.547650+0000","last_fullsized":"2023-09-23T22:39:29.053813+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:46.051852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:44.685839+0000","last_change":"2023-09-23T22:39:44.685468+0000","last_active":"2023-09-23T22:39:44.685839+0000","last_peered":"2023-09-23T22:39:44.685839+0000","last_clean":"2023-09-23T22:39:44.685839+0000","last_became_active":"2023-09-23T22:39:44.685187+0000","last_became_peered":"2023-09-23T22:39:44.685187+0000","last_unstale":"2023-09-23T22:39:44.685839+0000","last_undegraded":"2023-09-23T22:39:44.685839+0000","last_fullsized":"2023-09-23T22:39:44.685839+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,1],"avail_no_missing":["5","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"105'5","reported_seq":24,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549323+0000","last_change":"2023-09-23T22:39:22.028829+0000","last_active":"2023-09-23T22:39:44.549323+0000","last_peered":"2023-09-23T22:39:44.549323+0000","last_clean":"2023-09-23T22:39:44.549323+0000","last_became_active":"2023-09-23T22:39:22.028521+0000","last_became_peered":"2023-09-23T22:39:22.028521+0000","last_unstale":"2023-09-23T22:39:44.549323+0000","last_undegraded":"2023-09-23T22:39:44.549323+0000","last_fullsized":"2023-09-23T22:39:44.549323+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.549335+0000","last_change":"2023-09-23T22:39:29.059026+0000","last_active":"2023-09-23T22:39:44.549335+0000","last_peered":"2023-09-23T22:39:44.549335+0000","last_clean":"2023-09-23T22:39:27.041585+0000","last_became_active":"2023-09-23T22:39:29.059026+0000","last_became_peered":"2023-09-23T22:39:29.059026+0000","last_unstale":"2023-09-23T22:39:44.549335+0000","last_undegraded":"2023-09-23T22:39:44.549335+0000","last_fullsized":"2023-09-23T22:39:29.056607+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":318,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.684123+0000","last_change":"2023-09-23T22:39:13.964464+0000","last_active":"2023-09-23T22:39:44.684123+0000","last_peered":"2023-09-23T22:39:44.684123+0000","last_clean":"2023-09-23T22:38:06.152331+0000","last_became_active":"2023-09-23T22:39:13.964464+0000","last_became_peered":"2023-09-23T22:39:13.964464+0000","last_unstale":"2023-09-23T22:39:44.684123+0000","last_undegraded":"2023-09-23T22:39:13.961786+0000","last_fullsized":"2023-09-23T22:39:13.961665+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":37,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549005+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:39:44.549005+0000","last_peered":"2023-09-23T22:39:44.549005+0000","last_clean":"2023-09-23T22:39:44.549005+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:39:44.549005+0000","last_undegraded":"2023-09-23T22:39:44.549005+0000","last_fullsized":"2023-09-23T22:39:44.549005+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":61,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.550410+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:39:44.550410+0000","last_peered":"2023-09-23T22:39:44.550410+0000","last_clean":"2023-09-23T22:39:44.550410+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:39:44.550410+0000","last_undegraded":"2023-09-23T22:39:44.550410+0000","last_fullsized":"2023-09-23T22:39:44.550410+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":101,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.684157+0000","last_change":"2023-09-23T22:39:13.963187+0000","last_active":"2023-09-23T22:39:44.684157+0000","last_peered":"2023-09-23T22:39:44.684157+0000","last_clean":"2023-09-23T22:38:06.152257+0000","last_became_active":"2023-09-23T22:39:13.963187+0000","last_became_peered":"2023-09-23T22:39:13.963187+0000","last_unstale":"2023-09-23T22:39:44.684157+0000","last_undegraded":"2023-09-23T22:39:44.684157+0000","last_fullsized":"2023-09-23T22:39:13.961291+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.552372+0000","last_change":"2023-09-23T22:39:14.012294+0000","last_active":"2023-09-23T22:39:44.552372+0000","last_peered":"2023-09-23T22:39:44.552372+0000","last_clean":"2023-09-23T22:37:54.457558+0000","last_became_active":"2023-09-23T22:39:14.012294+0000","last_became_peered":"2023-09-23T22:39:14.012294+0000","last_unstale":"2023-09-23T22:39:44.552372+0000","last_undegraded":"2023-09-23T22:39:44.552372+0000","last_fullsized":"2023-09-23T22:39:13.958799+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":79,"reported_epoch":108,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:43.548716+0000","last_change":"2023-09-23T22:39:29.056752+0000","last_active":"2023-09-23T22:39:28.046973+0000","last_peered":"2023-09-23T22:39:43.548716+0000","last_clean":"2023-09-23T22:37:03.801144+0000","last_became_active":"2023-09-23T22:39:13.956823+0000","last_became_peered":"2023-09-23T22:39:29.056752+0000","last_unstale":"2023-09-23T22:39:43.548716+0000","last_undegraded":"2023-09-23T22:39:43.548716+0000","last_fullsized":"2023-09-23T22:39:29.056009+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:27.786871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.4","version":"105'7","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.550877+0000","last_change":"2023-09-23T22:39:43.906603+0000","last_active":"2023-09-23T22:39:44.550877+0000","last_peered":"2023-09-23T22:39:44.550877+0000","last_clean":"2023-09-23T22:39:44.550877+0000","last_became_active":"2023-09-23T22:39:43.566382+0000","last_became_peered":"2023-09-23T22:39:43.566382+0000","last_unstale":"2023-09-23T22:39:44.550877+0000","last_undegraded":"2023-09-23T22:39:44.550877+0000","last_fullsized":"2023-09-23T22:39:44.550877+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458752,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.684843+0000","last_change":"2023-09-23T22:39:13.964961+0000","last_active":"2023-09-23T22:39:44.684843+0000","last_peered":"2023-09-23T22:39:44.684843+0000","last_clean":"2023-09-23T22:38:06.152904+0000","last_became_active":"2023-09-23T22:39:13.964961+0000","last_became_peered":"2023-09-23T22:39:13.964961+0000","last_unstale":"2023-09-23T22:39:44.684843+0000","last_undegraded":"2023-09-23T22:39:44.684843+0000","last_fullsized":"2023-09-23T22:39:13.961730+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+undersized","last_fresh":"2023-09-23T22:39:43.547978+0000","last_change":"2023-09-23T22:39:29.060771+0000","last_active":"2023-09-23T22:39:43.547978+0000","last_peered":"2023-09-23T22:39:43.547978+0000","last_clean":"2023-09-23T22:39:27.713294+0000","last_became_active":"2023-09-23T22:39:29.060771+0000","last_became_peered":"2023-09-23T22:39:29.060771+0000","last_unstale":"2023-09-23T22:39:43.547978+0000","last_undegraded":"2023-09-23T22:39:43.547978+0000","last_fullsized":"2023-09-23T22:39:29.055720+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:00.634869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":36,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.550695+0000","last_change":"2023-09-23T22:39:13.963671+0000","last_active":"2023-09-23T22:39:44.550695+0000","last_peered":"2023-09-23T22:39:44.550695+0000","last_clean":"2023-09-23T22:28:42.648793+0000","last_became_active":"2023-09-23T22:39:13.963671+0000","last_became_peered":"2023-09-23T22:39:13.963671+0000","last_unstale":"2023-09-23T22:39:44.550695+0000","last_undegraded":"2023-09-23T22:39:44.550695+0000","last_fullsized":"2023-09-23T22:39:13.959328+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"105'2","reported_seq":27,"reported_epoch":108,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:43.547234+0000","last_change":"2023-09-23T22:39:29.972434+0000","last_active":"2023-09-23T22:39:43.547234+0000","last_peered":"2023-09-23T22:39:43.547234+0000","last_clean":"2023-09-23T22:39:27.713425+0000","last_became_active":"2023-09-23T22:39:29.062252+0000","last_became_peered":"2023-09-23T22:39:29.062252+0000","last_unstale":"2023-09-23T22:39:43.547234+0000","last_undegraded":"2023-09-23T22:39:29.970208+0000","last_fullsized":"2023-09-23T22:39:29.053806+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:00.825319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":2}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":83,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.549618+0000","last_change":"2023-09-23T22:39:14.012323+0000","last_active":"2023-09-23T22:39:44.549618+0000","last_peered":"2023-09-23T22:39:44.549618+0000","last_clean":"2023-09-23T22:38:06.156690+0000","last_became_active":"2023-09-23T22:39:14.012323+0000","last_became_peered":"2023-09-23T22:39:14.012323+0000","last_unstale":"2023-09-23T22:39:44.549618+0000","last_undegraded":"2023-09-23T22:39:14.010333+0000","last_fullsized":"2023-09-23T22:39:14.010220+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:15.822831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":47,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:39:43.548824+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:39:43.548824+0000","last_peered":"2023-09-23T22:39:43.548824+0000","last_clean":"2023-09-23T22:39:43.548824+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:39:43.548824+0000","last_undegraded":"2023-09-23T22:39:43.548824+0000","last_fullsized":"2023-09-23T22:39:43.548824+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.548951+0000","last_change":"2023-09-23T22:39:14.011039+0000","last_active":"2023-09-23T22:39:44.548951+0000","last_peered":"2023-09-23T22:39:44.548951+0000","last_clean":"2023-09-23T22:38:06.156666+0000","last_became_active":"2023-09-23T22:39:14.011039+0000","last_became_peered":"2023-09-23T22:39:14.011039+0000","last_unstale":"2023-09-23T22:39:44.548951+0000","last_undegraded":"2023-09-23T22:39:44.548951+0000","last_fullsized":"2023-09-23T22:39:13.952932+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:51.047004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"105'10","reported_seq":35,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.552675+0000","last_change":"2023-09-23T22:39:29.967283+0000","last_active":"2023-09-23T22:39:44.552675+0000","last_peered":"2023-09-23T22:39:44.552675+0000","last_clean":"2023-09-23T22:39:27.524308+0000","last_became_active":"2023-09-23T22:39:29.064458+0000","last_became_peered":"2023-09-23T22:39:29.064458+0000","last_unstale":"2023-09-23T22:39:44.552675+0000","last_undegraded":"2023-09-23T22:39:29.960772+0000","last_fullsized":"2023-09-23T22:39:29.055043+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":10}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":87,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.550532+0000","last_change":"2023-09-23T22:39:29.053805+0000","last_active":"2023-09-23T22:39:44.550532+0000","last_peered":"2023-09-23T22:39:44.550532+0000","last_clean":"2023-09-23T22:39:27.042610+0000","last_became_active":"2023-09-23T22:39:29.053805+0000","last_became_peered":"2023-09-23T22:39:29.053805+0000","last_unstale":"2023-09-23T22:39:44.550532+0000","last_undegraded":"2023-09-23T22:39:44.550532+0000","last_fullsized":"2023-09-23T22:39:29.051786+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":41,"reported_epoch":108,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:43.548984+0000","last_change":"2023-09-23T22:39:13.960587+0000","last_active":"2023-09-23T22:39:43.548984+0000","last_peered":"2023-09-23T22:39:43.548984+0000","last_clean":"2023-09-23T22:28:44.694773+0000","last_became_active":"2023-09-23T22:39:13.960587+0000","last_became_peered":"2023-09-23T22:39:13.960587+0000","last_unstale":"2023-09-23T22:39:43.548984+0000","last_undegraded":"2023-09-23T22:39:13.953922+0000","last_fullsized":"2023-09-23T22:39:13.953813+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:38.388801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.551599+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:39:44.551599+0000","last_peered":"2023-09-23T22:39:44.551599+0000","last_clean":"2023-09-23T22:39:44.551599+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:39:44.551599+0000","last_undegraded":"2023-09-23T22:39:44.551599+0000","last_fullsized":"2023-09-23T22:39:44.551599+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"105'7","reported_seq":32,"reported_epoch":109,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:44.684358+0000","last_change":"2023-09-23T22:39:29.968678+0000","last_active":"2023-09-23T22:39:44.684358+0000","last_peered":"2023-09-23T22:39:44.684358+0000","last_clean":"2023-09-23T22:39:27.181215+0000","last_became_active":"2023-09-23T22:39:29.059135+0000","last_became_peered":"2023-09-23T22:39:29.059135+0000","last_unstale":"2023-09-23T22:39:44.684358+0000","last_undegraded":"2023-09-23T22:39:29.959849+0000","last_fullsized":"2023-09-23T22:39:29.054777+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":99,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:39:21.005110+0000","last_clean_scrub_stamp":"2023-09-23T22:39:21.005110+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458752,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":7}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":31,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.553901+0000","last_change":"2023-09-23T22:39:13.963821+0000","last_active":"2023-09-23T22:39:44.553901+0000","last_peered":"2023-09-23T22:39:44.553901+0000","last_clean":"2023-09-23T22:28:41.643121+0000","last_became_active":"2023-09-23T22:39:13.963821+0000","last_became_peered":"2023-09-23T22:39:13.963821+0000","last_unstale":"2023-09-23T22:39:44.553901+0000","last_undegraded":"2023-09-23T22:39:44.553901+0000","last_fullsized":"2023-09-23T22:39:13.960951+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":86,"reported_epoch":108,"state":"active+undersized","last_fresh":"2023-09-23T22:39:43.548041+0000","last_change":"2023-09-23T22:39:29.055352+0000","last_active":"2023-09-23T22:39:43.548041+0000","last_peered":"2023-09-23T22:39:43.548041+0000","last_clean":"2023-09-23T22:37:35.276266+0000","last_became_active":"2023-09-23T22:39:29.055352+0000","last_became_peered":"2023-09-23T22:39:29.055352+0000","last_unstale":"2023-09-23T22:39:43.548041+0000","last_undegraded":"2023-09-23T22:39:43.548041+0000","last_fullsized":"2023-09-23T22:39:29.053834+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:09.480112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.551317+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:39:44.551317+0000","last_peered":"2023-09-23T22:39:44.551317+0000","last_clean":"2023-09-23T22:39:44.551317+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:39:44.551317+0000","last_undegraded":"2023-09-23T22:39:44.551317+0000","last_fullsized":"2023-09-23T22:39:44.551317+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":100,"reported_epoch":109,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:44.556854+0000","last_change":"2023-09-23T22:39:44.554953+0000","last_active":"2023-09-23T22:39:44.556854+0000","last_peered":"2023-09-23T22:39:44.556854+0000","last_clean":"2023-09-23T22:39:44.556854+0000","last_became_active":"2023-09-23T22:39:44.554475+0000","last_became_peered":"2023-09-23T22:39:44.554475+0000","last_unstale":"2023-09-23T22:39:44.556854+0000","last_undegraded":"2023-09-23T22:39:44.556854+0000","last_fullsized":"2023-09-23T22:39:44.556854+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.549897+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:39:44.549897+0000","last_peered":"2023-09-23T22:39:44.549897+0000","last_clean":"2023-09-23T22:39:44.549897+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:39:44.549897+0000","last_undegraded":"2023-09-23T22:39:44.549897+0000","last_fullsized":"2023-09-23T22:39:44.549897+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":96,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.553031+0000","last_change":"2023-09-23T22:39:14.012434+0000","last_active":"2023-09-23T22:39:44.553031+0000","last_peered":"2023-09-23T22:39:44.553031+0000","last_clean":"2023-09-23T22:38:06.153072+0000","last_became_active":"2023-09-23T22:39:14.012434+0000","last_became_peered":"2023-09-23T22:39:14.012434+0000","last_unstale":"2023-09-23T22:39:44.553031+0000","last_undegraded":"2023-09-23T22:39:44.553031+0000","last_fullsized":"2023-09-23T22:39:13.959310+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":90,"reported_epoch":109,"state":"active+undersized","last_fresh":"2023-09-23T22:39:44.549428+0000","last_change":"2023-09-23T22:39:29.052477+0000","last_active":"2023-09-23T22:39:44.549428+0000","last_peered":"2023-09-23T22:39:44.549428+0000","last_clean":"2023-09-23T22:39:27.675497+0000","last_became_active":"2023-09-23T22:39:29.052477+0000","last_became_peered":"2023-09-23T22:39:29.052477+0000","last_unstale":"2023-09-23T22:39:44.549428+0000","last_undegraded":"2023-09-23T22:39:44.549428+0000","last_fullsized":"2023-09-23T22:39:29.050344+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:07.201825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":108,"reported_epoch":109,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:44.549289+0000","last_change":"2023-09-23T22:39:29.053279+0000","last_active":"2023-09-23T22:39:28.044406+0000","last_peered":"2023-09-23T22:39:44.549289+0000","last_clean":"2023-09-23T22:38:06.149179+0000","last_became_active":"2023-09-23T22:39:14.012358+0000","last_became_peered":"2023-09-23T22:39:29.053279+0000","last_unstale":"2023-09-23T22:39:44.549289+0000","last_undegraded":"2023-09-23T22:39:44.549289+0000","last_fullsized":"2023-09-23T22:39:29.050274+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:53.717246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3],"acting":[3],"avail_no_missing":["3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":47,"reported_epoch":109,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:44.549626+0000","last_change":"2023-09-23T22:39:29.058429+0000","last_active":"2023-09-23T22:39:28.056910+0000","last_peered":"2023-09-23T22:39:44.549626+0000","last_clean":"2023-09-23T22:28:44.694413+0000","last_became_active":"2023-09-23T22:39:13.961530+0000","last_became_peered":"2023-09-23T22:39:29.058429+0000","last_unstale":"2023-09-23T22:39:44.549626+0000","last_undegraded":"2023-09-23T22:39:29.057202+0000","last_fullsized":"2023-09-23T22:39:29.057120+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5],"acting":[5],"avail_no_missing":["5"],"object_location_counts":[{"shards":"5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:39:44.551424+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:39:44.551424+0000","last_peered":"2023-09-23T22:39:44.551424+0000","last_clean":"2023-09-23T22:39:44.551424+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:39:44.551424+0000","last_undegraded":"2023-09-23T22:39:44.551424+0000","last_fullsized":"2023-09-23T22:39:44.551424+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5570560,"num_objects":85,"num_object_clones":0,"num_object_copies":255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34,"num_objects_misplaced":19,"num_objects_unfound":0,"num_objects_dirty":85,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85,"num_write_kb":5440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14024704,"data_stored":14024704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85,"ondisk_log_size":85,"up":35,"acting":37,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":80,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":75,"acting":76,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":17,"acting":17,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430362,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313208,"kb_used_data":3824,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93431944,"statfs":{"total":95995035648,"available":95674310656,"internally_reserved":0,"allocated":3915776,"data_stored":3449642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57299999999999995,"15min":0.59899999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.60099999999999998,"15min":0.61599999999999999},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.755,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.626,"15min":0.63},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81299999999999994,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.45100000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48499999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48199999999999998,"15min":0.498},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.26300000000000001},"max":{"1min":0.71799999999999997,"5min":0.79200000000000004,"15min":5.3140000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.51800000000000002,"15min":0.503},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.70099999999999996,"5min":0.77900000000000003,"15min":0.93799999999999994},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.33000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.67200000000000004,"5min":0.85799999999999998,"15min":1.0669999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.36899999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.77700000000000002,"5min":0.83299999999999996,"15min":5.3490000000000002},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56599999999999995,"15min":0.55200000000000005},"min":{"1min":0.39300000000000002,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":0.68500000000000005,"5min":0.81999999999999995,"15min":0.95799999999999996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.622,"15min":0.59699999999999998},"min":{"1min":0.443,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":0.81899999999999995,"5min":0.86299999999999999,"15min":1.038},"last":0.32500000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626590,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305004,"kb_used_data":3812,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93440148,"statfs":{"total":95995035648,"available":95682711552,"internally_reserved":0,"allocated":3903488,"data_stored":3449496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.57099999999999995,"15min":0.59799999999999998},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.68400000000000005,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57999999999999996,"15min":0.59799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.63900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41699999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.46200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:39:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.52200000000000002,"15min":0.47899999999999998},"min":{"1min":0.41799999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70799999999999996,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53100000000000003,"15min":0.48699999999999999},"min":{"1min":0.38600000000000001,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.81299999999999994},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:39:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.54600000000000004,"15min":0.50700000000000001},"min":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.84899999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.53300000000000003},"min":{"1min":0.47199999999999998,"5min":0.377,"15min":0.32400000000000001},"max":{"1min":0.79500000000000004,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.58399999999999996,"15min":0.55500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34799999999999998},"max":{"1min":0.79700000000000004,"5min":0.81999999999999995,"15min":0.83199999999999996},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":0.69799999999999995,"5min":0.82699999999999996,"15min":1.357},"last":0.36899999999999999}]}]},{"osd":1,"up_from":84,"seq":360777252899,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33780,"kb_used_data":2348,"kb_used_omap":9,"kb_used_meta":31414,"kb_avail":93711372,"statfs":{"total":95995035648,"available":95960444928,"internally_reserved":0,"allocated":2404352,"data_stored":1887592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":32168937},"hb_peers":[3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.35899999999999999,"15min":0.36599999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.51100000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.40000000000000002,"15min":0.39100000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":2.9199999999999999,"5min":2.9199999999999999,"15min":2.9199999999999999},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.57399999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.20999999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.221}]},{"osd":4,"last update":"Sat Sep 23 22:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.628,"15min":0.63300000000000001},"min":{"1min":0.47499999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.84899999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65600000000000003,"15min":0.66000000000000003},"min":{"1min":0.48699999999999999,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81999999999999995,"5min":0.87,"15min":0.87},"last":0.313}]},{"osd":5,"last update":"Sat Sep 23 22:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.65000000000000002,"15min":0.64100000000000001},"min":{"1min":0.46000000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":3.024,"5min":3.024,"15min":3.024},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.63300000000000001,"15min":0.63500000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.377},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.65000000000000002,"15min":0.63800000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":2.9980000000000002,"5min":2.9980000000000002,"15min":2.9980000000000002},"last":0.78300000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.63300000000000001,"15min":0.63700000000000001},"min":{"1min":0.45000000000000001,"5min":0.437,"15min":0.437},"max":{"1min":0.80900000000000005,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65900000000000003,"15min":0.66200000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.40500000000000003}]}]},{"osd":0,"up_from":79,"seq":339302416426,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33068,"kb_used_data":1572,"kb_used_omap":28,"kb_used_meta":31459,"kb_avail":93712084,"statfs":{"total":95995035648,"available":95961174016,"internally_reserved":0,"allocated":1609728,"data_stored":1117239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":32214864},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.35399999999999998,"15min":0.35599999999999998},"min":{"1min":0.24299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.22900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:38:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.29699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66900000000000004,"15min":0.67900000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.66300000000000003,"15min":0.67200000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.98099999999999998,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.63200000000000001},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.64900000000000002,"15min":0.66500000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91100000000000003,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.64800000000000002,"15min":0.66800000000000004},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.879,"5min":1.036,"15min":1.036},"last":0.621},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.65500000000000003,"15min":0.67300000000000004},"min":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.67200000000000004},"min":{"1min":0.41199999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.98899999999999999,"5min":1.085,"15min":1.085},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.67900000000000005,"15min":0.70899999999999996},"min":{"1min":0.44,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89900000000000002,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.52900000000000003}]}]},{"osd":2,"up_from":89,"seq":382252089368,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36684,"kb_used_data":1028,"kb_used_omap":81,"kb_used_meta":35566,"kb_avail":93708468,"statfs":{"total":95995035648,"available":95957471232,"internally_reserved":0,"allocated":1052672,"data_stored":564848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36419802},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.30199999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.56100000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.40699999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.49399999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925847,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36400,"kb_used_data":4968,"kb_used_omap":7,"kb_used_meta":31416,"kb_avail":93708752,"statfs":{"total":95995035648,"available":95957762048,"internally_reserved":0,"allocated":5087232,"data_stored":4578498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":32170600},"hb_peers":[1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"last":0.376},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.438,"5min":0.438,"15min":0.438},"last":0.31}]},{"osd":1,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.20000000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":0.27000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":2.2050000000000001,"5min":2.2050000000000001,"15min":2.2050000000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.177,"5min":2.177,"15min":2.177},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.308}]},{"osd":6,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.1360000000000001,"5min":2.1360000000000001,"15min":2.1360000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.372}]},{"osd":7,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.345,"5min":1.345,"15min":1.345},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.39700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019046,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304432,"kb_used_data":3240,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93440720,"statfs":{"total":95995035648,"available":95683297280,"internally_reserved":0,"allocated":3317760,"data_stored":2859204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.58899999999999997,"15min":0.60899999999999999},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.64100000000000001,"15min":0.66300000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69699999999999995,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.54900000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59399999999999997,"15min":0.59899999999999998},"min":{"1min":0.44,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77300000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.54400000000000004,"15min":0.55100000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.65800000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.9239999999999999,"5min":2.9239999999999999,"15min":2.9239999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51100000000000001,"15min":0.46999999999999997},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":0.65900000000000003,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.54000000000000004,"15min":0.48799999999999999},"min":{"1min":0.34899999999999998,"5min":0.32800000000000001,"15min":0.29399999999999998},"max":{"1min":0.65600000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56100000000000005,"15min":0.50800000000000001},"min":{"1min":0.379,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70599999999999996,"5min":0.875,"15min":0.875},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.58899999999999997,"15min":0.53100000000000003},"min":{"1min":0.39900000000000002,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.73199999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.311}]},{"osd":7,"last update":"Sat Sep 23 22:39:38 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.59299999999999997,"15min":0.55500000000000005},"min":{"1min":0.39200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.72199999999999998,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.58899999999999997,"15min":0.54500000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71699999999999997,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.39300000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822818,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312544,"kb_used_data":3160,"kb_used_omap":0,"kb_used_meta":309376,"kb_avail":93432608,"statfs":{"total":95995035648,"available":95674990592,"internally_reserved":0,"allocated":3235840,"data_stored":2796898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316801024},"hb_peers":[1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.56299999999999994,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80200000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.63},"min":{"1min":0.34100000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.79000000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53000000000000003,"15min":0.53100000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.443},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:38:51 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28799999999999998},"max":{"1min":0.72199999999999998,"5min":0.81299999999999994,"15min":0.89000000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.47899999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":0.70099999999999996,"5min":0.73099999999999998,"15min":0.88300000000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.52700000000000002,"15min":0.49299999999999999},"min":{"1min":0.42299999999999999,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":0.80900000000000005,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.56999999999999995,"15min":0.52400000000000002},"min":{"1min":0.42499999999999999,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":0.78800000000000003,"5min":0.84599999999999997,"15min":0.90300000000000002},"last":0.33300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:39:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.58499999999999996,"15min":0.55200000000000005},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.80600000000000005,"5min":0.83699999999999997,"15min":0.89500000000000002},"last":0.378},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.58399999999999996,"15min":0.55800000000000005},"min":{"1min":0.36799999999999999,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":0.77900000000000003,"5min":0.78400000000000003,"15min":0.85199999999999998},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1245184,"data_stored":1245184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3932160,"data_stored":3932160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2228224,"data_stored":2228224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2162688,"data_stored":2162688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2228224,"data_stored":2228224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2228224,"data_stored":2228224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:39:52.276 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-09-23T22:39:52.334 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:52.351 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:52.408 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:52.437 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:52.464 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.508 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:52.611 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:52.640 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.738 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:52.739 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:52 smithi151 ceph-mon[159210]: pgmap v249: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 97 KiB/s, 1 objects/s recovering 2023-09-23T22:39:52.739 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:52 smithi151 ceph-mon[159210]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:52.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:52.782 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:52.782 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:52.841 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:52.869 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.887 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:52 smithi052 ceph-mon[187743]: pgmap v249: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 97 KiB/s, 1 objects/s recovering 2023-09-23T22:39:52.887 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:52 smithi052 ceph-mon[187743]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:52.888 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:52 smithi052 ceph-mon[180979]: pgmap v249: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 97 KiB/s, 1 objects/s recovering 2023-09-23T22:39:52.888 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:52 smithi052 ceph-mon[180979]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:52.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 23 16 101 85 0.230949 0 - 0.00253822 2023-09-23T22:39:53.064 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:53.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:53.168 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:53.198 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:53.382 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:39:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:39:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:39:53] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.33.4" 2023-09-23T22:39:53.728 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:53.765 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:53.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:53.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:53.869 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:39:53.903 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:53.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 24 16 101 85 0.221327 0 - 0.00253822 2023-09-23T22:39:53.974 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:39:53.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:53.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:39:53.993 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:54.080 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:54.117 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:54.633 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:54.652 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:54.673 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:54.687 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:54.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:54.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:54.693 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:39:54.697 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:54.716 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:54.735 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:54.735 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:54.735 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:54.735 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:39:54.736 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:39:54.736 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:54.736 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:54.736 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:39:54.736 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:54.791 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:54.821 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:54.916 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:54.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:54 smithi151 ceph-mon[159210]: pgmap v250: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 86 KiB/s, 1 objects/s recovering 2023-09-23T22:39:54.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3275784174' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:54.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:54 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:54.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/17234237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:54.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:54.930 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[187743]: pgmap v250: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 86 KiB/s, 1 objects/s recovering 2023-09-23T22:39:54.930 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3275784174' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:54.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:54.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/17234237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:54.931 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[180979]: pgmap v250: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%); 86 KiB/s, 1 objects/s recovering 2023-09-23T22:39:54.931 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3275784174' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:54.931 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:54.931 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/17234237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:54.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 25 16 101 85 0.212474 0 - 0.00253822 2023-09-23T22:39:55.019 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:55.047 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.089 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:39:55.089 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:39:55.089 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:39:55.089 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:39:55.090 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:55.193 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:55.221 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.264 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:55.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:55.367 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:55.393 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.435 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:55.538 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:55.584 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.637 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:55.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:55.677 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:55.677 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:55.699 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:39:55.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:55.771 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.819 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:55.926 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:55.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:55 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:55 smithi151 ceph-mon[159210]: failed to encode map e111 with expected crc 2023-09-23T22:39:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:55 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:55 smithi151 ceph-mon[159210]: osdmap e111: 8 total, 6 up, 8 in 2023-09-23T22:39:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:55 smithi151 ceph-mon[159210]: failed to encode map e111 with expected crc 2023-09-23T22:39:55.954 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 26 16 101 85 0.204302 0 - 0.00253822 2023-09-23T22:39:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[180979]: failed to encode map e111 with expected crc 2023-09-23T22:39:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[180979]: osdmap e111: 8 total, 6 up, 8 in 2023-09-23T22:39:55.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[180979]: failed to encode map e111 with expected crc 2023-09-23T22:39:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[187743]: failed to encode map e111 with expected crc 2023-09-23T22:39:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[187743]: osdmap e111: 8 total, 6 up, 8 in 2023-09-23T22:39:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:55 smithi052 ceph-mon[187743]: failed to encode map e111 with expected crc 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:56.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:56.206 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:56.234 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:56.281 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:56.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:56.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:56.413 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:56.656 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:56.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:56.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:56 smithi052 ceph-mon[180979]: pgmap v252: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:39:56.728 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T22:39:56.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:56.786 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:56 smithi151 ceph-mon[159210]: pgmap v252: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:39:56.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 27 16 101 85 0.196735 0 - 0.00253822 2023-09-23T22:39:56.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:56 smithi052 ceph-mon[187743]: pgmap v252: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:39:57.008 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:57.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:57.114 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:57.146 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.305 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T22:39:57.408 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:57.418 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:39:57.419 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:39:57.419 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:39:57.419 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:39:57.419 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:39:57.437 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.482 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:57.586 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:57.615 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.658 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:57.679 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:57.761 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:57.787 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 28 16 101 85 0.189709 0 - 0.00253822 2023-09-23T22:39:57.986 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:57.989 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/80107968' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:39:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/80107968' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:39:57.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/80107968' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:39:58.007 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:39:58.041 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:58.092 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:58.118 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.162 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:58.267 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:58.296 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.338 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:39:58.342 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:58.409 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:58.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:58.473 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.536 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:58.640 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:58.666 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:58.813 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:58.840 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.883 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:58.884 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:58 smithi151 ceph-mon[159210]: pgmap v253: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:39:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:58.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 29 16 101 85 0.183168 0 - 0.00253822 2023-09-23T22:39:58.986 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:39:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:58 smithi052 ceph-mon[187743]: pgmap v253: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:39:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:58 smithi052 ceph-mon[180979]: pgmap v253: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:39:59.013 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.055 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:39:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:39:59.059 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:59.158 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:59.184 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.348 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:59.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:59.454 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:59.482 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.489 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:39:59.510 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:39:59.532 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:39:59.551 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:39:59.569 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:39:59.576 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:59.587 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:39:59.588 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:59.680 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:59.709 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.754 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:59.755 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:39:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1367373392' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:59.766 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:59.766 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:59.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:39:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1367373392' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:59.813 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:39:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1367373392' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:59.857 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:59.886 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 30 16 101 85 0.177062 0 - 0.00253822 2023-09-23T22:39:59.979 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:00.083 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:00.111 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.154 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:00.258 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:00.292 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.341 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:00.439 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:40:00.445 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:00.508 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.552 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:00.644 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:00.645 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:00.658 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:00.687 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.735 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:00.794 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:00.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:00.868 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 31 16 101 85 0.171351 0 - 0.00253822 2023-09-23T22:40:00.964 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:00.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pgmap v254: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:00.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 2 osds down; Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded 2023-09-23T22:40:00.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T22:40:00.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T22:40:00.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: osd.0 (root=default,host=smithi052) is down 2023-09-23T22:40:00.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: osd.2 (root=default,host=smithi052) is down 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: [WRN] PG_DEGRADED: Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.13 is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 3.1f is undersized+degraded+peered, acting [5] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 6.1 is active+undersized+degraded, acting [1,6] 2023-09-23T22:40:00.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 6.2 is active+undersized+degraded, acting [4,3] 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 6.3 is active+undersized+degraded, acting [7,6] 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 6.8 is active+undersized+degraded, acting [7,3] 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 6.a is active+undersized+degraded, acting [5,6] 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[187743]: pg 6.f is active+undersized+degraded, acting [3,4] 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pgmap v254: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 2 osds down; Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T22:40:00.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: osd.0 (root=default,host=smithi052) is down 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: osd.2 (root=default,host=smithi052) is down 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: [WRN] PG_DEGRADED: Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.13 is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 3.1f is undersized+degraded+peered, acting [5] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 6.1 is active+undersized+degraded, acting [1,6] 2023-09-23T22:40:00.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 6.2 is active+undersized+degraded, acting [4,3] 2023-09-23T22:40:00.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 6.3 is active+undersized+degraded, acting [7,6] 2023-09-23T22:40:00.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 6.8 is active+undersized+degraded, acting [7,3] 2023-09-23T22:40:00.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 6.a is active+undersized+degraded, acting [5,6] 2023-09-23T22:40:00.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:00 smithi052 ceph-mon[180979]: pg 6.f is active+undersized+degraded, acting [3,4] 2023-09-23T22:40:00.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:00.974 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pgmap v254: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:00.974 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 2 osds down; Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded 2023-09-23T22:40:00.974 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T22:40:00.974 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: osd.0 (root=default,host=smithi052) is down 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: osd.2 (root=default,host=smithi052) is down 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: [WRN] PG_DEGRADED: Degraded data redundancy: 50/327 objects degraded (15.291%), 14 pgs degraded 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.13 is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 3.1f is undersized+degraded+peered, acting [5] 2023-09-23T22:40:00.975 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 6.1 is active+undersized+degraded, acting [1,6] 2023-09-23T22:40:00.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 6.2 is active+undersized+degraded, acting [4,3] 2023-09-23T22:40:00.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 6.3 is active+undersized+degraded, acting [7,6] 2023-09-23T22:40:00.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 6.8 is active+undersized+degraded, acting [7,3] 2023-09-23T22:40:00.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 6.a is active+undersized+degraded, acting [5,6] 2023-09-23T22:40:00.976 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:00 smithi151 ceph-mon[159210]: pg 6.f is active+undersized+degraded, acting [3,4] 2023-09-23T22:40:01.071 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:01.100 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.142 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:01.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:01.246 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:01.273 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.365 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:01.469 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:01.495 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.536 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:01.639 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:01.666 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.718 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:40:01.759 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:01.864 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:01.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2424835809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:01.892 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 32 16 101 85 0.165996 0 - 0.00253822 2023-09-23T22:40:01.988 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:01.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:01.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2424835809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:01.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2424835809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:02.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:02.120 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.163 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:02.167 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:02.167 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:02.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:02.269 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:02.295 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.379 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0, 2] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:40:02.379 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:40:02.379 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:40:02.379 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T22:40:02.388 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:02.492 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:02.519 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.646 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:02.686 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:02.749 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:02.755 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:40:02.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:02 smithi151 ceph-mon[159210]: pgmap v255: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:02.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:02 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:02.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2424835809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:02.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:02 smithi151 ceph-mon[159210]: osdmap e112: 8 total, 6 up, 8 in 2023-09-23T22:40:02.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:02 smithi151 ceph-mon[159210]: failed to encode map e112 with expected crc 2023-09-23T22:40:02.790 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:02.816 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.859 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:02.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 33 16 101 85 0.160966 0 - 0.00253822 2023-09-23T22:40:02.968 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:02.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[187743]: pgmap v255: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:02.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:02.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2424835809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:02.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[187743]: osdmap e112: 8 total, 6 up, 8 in 2023-09-23T22:40:02.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[187743]: failed to encode map e112 with expected crc 2023-09-23T22:40:02.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[180979]: pgmap v255: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:02.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:02.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2424835809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:02.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[180979]: osdmap e112: 8 total, 6 up, 8 in 2023-09-23T22:40:02.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:02 smithi052 ceph-mon[180979]: failed to encode map e112 with expected crc 2023-09-23T22:40:02.994 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.038 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:03.144 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:03.164 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:03.171 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.218 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:03.323 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:03.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.398 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:03.502 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:03.530 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.574 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:03.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:03.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:03.705 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.716 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:40:03] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.33.4" 2023-09-23T22:40:03.718 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:03 smithi052 ceph-mon[187743]: failed to encode map e112 with expected crc 2023-09-23T22:40:03.718 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:03 smithi052 ceph-mon[187743]: failed to encode map e112 with expected crc 2023-09-23T22:40:03.718 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:03 smithi052 ceph-mon[187743]: failed to encode map e112 with expected crc 2023-09-23T22:40:03.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:03 smithi052 ceph-mon[180979]: failed to encode map e112 with expected crc 2023-09-23T22:40:03.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:03 smithi052 ceph-mon[180979]: failed to encode map e112 with expected crc 2023-09-23T22:40:03.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:03 smithi052 ceph-mon[180979]: failed to encode map e112 with expected crc 2023-09-23T22:40:03.719 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:03.748 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:03.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:03.851 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:03.877 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.918 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:03.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 34 16 101 85 0.156231 0 - 0.00253822 2023-09-23T22:40:04.022 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:04.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:03 smithi151 ceph-mon[159210]: failed to encode map e112 with expected crc 2023-09-23T22:40:04.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:03 smithi151 ceph-mon[159210]: failed to encode map e112 with expected crc 2023-09-23T22:40:04.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:03 smithi151 ceph-mon[159210]: failed to encode map e112 with expected crc 2023-09-23T22:40:04.048 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.144 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:04.251 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:04.279 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.374 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:04.416 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:40:04.478 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:04.508 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.525 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:04.525 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:04.559 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:04.619 INFO:teuthology.orchestra.run.smithi052.stdout:moved 5 / 267 (1.87266%) 2023-09-23T22:40:04.619 INFO:teuthology.orchestra.run.smithi052.stdout:avg 44.5 2023-09-23T22:40:04.619 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 12.1209 -> 10.6575 (expected baseline 6.08961) 2023-09-23T22:40:04.619 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.1 with 26 -> 37 pgs (0.58427 -> 0.831461 * mean) 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.3 with 43 -> 41 pgs (0.966292 -> 0.921348 * mean) 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 1 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0018 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.0183 2023-09-23T22:40:04.620 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 weight 0.8098 -> 1.0000 2023-09-23T22:40:04.622 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T22:40:04.662 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:04.691 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.717 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:04 smithi052 conmon[173965]: 2023-09-23T22:40:04.618+0000 7f39bb05d700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T22:40:04.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:04 smithi052 ceph-mon[180979]: pgmap v257: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:04.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:04.785 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:04.889 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:04.915 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:04 smithi151 ceph-mon[159210]: pgmap v257: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:04.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 35 16 101 85 0.151768 0 - 0.00253822 2023-09-23T22:40:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:04 smithi052 ceph-mon[187743]: pgmap v257: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:05.015 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:05.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:05.100 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:40:05.119 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:05.148 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.295 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:05.401 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:05.430 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.440 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:05.477 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:05.493 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:40:05.493 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T22:40:05.582 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:05.608 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.656 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:05.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:05.782 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:05.785 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.890 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:05.911 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:05.929 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:05.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:05.933 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:05.952 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:05.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 36 16 101 85 0.147552 0 - 0.00253822 2023-09-23T22:40:05.970 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:05.986 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:05.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:05.987 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:05.988 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.988 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.988 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.988 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.988 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:05 smithi052 ceph-mon[180979]: from='client.34764 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:05 smithi052 ceph-mon[187743]: from='client.34764 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:06.033 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:06.058 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:05 smithi151 ceph-mon[159210]: from='client.34764 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:06.061 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.102 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:06.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:06.206 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:06.233 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.363 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:06.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:06.466 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:06.497 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.539 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:06.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:06.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:06.549 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:06.642 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:06.684 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.825 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:40:06.825 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(mds.0.4:43 3.14 e73/56 3:2e2fa760:::200.00000001:head v 73'10, mlcod=70'8)", 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:09.548717+0000", 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 477.27464958600001, 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031029299999999998, 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548717+0000", 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548717+0000", 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548716+0000", 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1308000000000001e-05 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548727+0000", 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4081e-05 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548742+0000", 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2549999999999997e-06 2023-09-23T22:40:06.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548750+0000", 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6973999999999997e-05 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548847+0000", 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.268e-05 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548869+0000", 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029233509999999998 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.551793+0000", 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6866e-05 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.551820+0000", 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031029299999999998 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(mds.0.4:44 3.14 e73/56 3:292cf221:::200.00000000:head v 73'11, mlcod=70'8)", 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:09.549370+0000", 2023-09-23T22:40:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 477.273996039, 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002738613, 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549370+0000", 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549370+0000", 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549369+0000", 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1262e-05 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549381+0000", 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5679999999999999e-05 2023-09-23T22:40:06.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549396+0000", 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2309999999999999e-06 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549404+0000", 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022311050000000002 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.551635+0000", 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3657e-05 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.551648+0000", 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00042950600000000002 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552078+0000", 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1001999999999999e-05 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552109+0000", 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002738613 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:1 6.3 e105/104 6:cd623993:::benchmark_data_smithi052_234456_object0:head v 105'1, mlcod=0'0)", 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.958494+0000", 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.864872767999998, 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011155690000000001, 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958494+0000", 2023-09-23T22:40:06.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958494+0000", 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958492+0000", 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1406999999999999e-05 2023-09-23T22:40:06.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958524+0000", 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1699999999999999e-05 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958546+0000", 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4479999999999997e-06 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958553+0000", 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0310999999999997e-05 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958633+0000", 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3536999999999998e-05 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958657+0000", 2023-09-23T22:40:06.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00089729999999999996 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959554+0000", 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4985000000000003e-05 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959609+0000", 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011155690000000001 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.841 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:2 6.c e105/99 6:3004d844:::benchmark_data_smithi052_234456_object1:head v 105'1, mlcod=0'0)", 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.959545+0000", 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.863820939999997, 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074654699999999997, 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959545+0000", 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959545+0000", 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959544+0000", 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020055800000000001 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959745+0000", 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5739999999999998e-05 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959761+0000", 2023-09-23T22:40:06.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2820000000000002e-06 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959767+0000", 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6490999999999999e-05 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959814+0000", 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8856e-05 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959832+0000", 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00042496 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960257+0000", 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4554000000000002e-05 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960292+0000", 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074654699999999997 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:18 6.5 e105/99 6:a2058bc9:::benchmark_data_smithi052_234456_object17:head v 105'1, mlcod=0'0)", 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.961626+0000", 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.861740808, 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012405140000000001, 2023-09-23T22:40:06.847 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961626+0000", 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961626+0000", 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961625+0000", 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6418999999999999e-05 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961651+0000", 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7373000000000001e-05 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961669+0000", 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2400000000000001e-06 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961676+0000", 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6859999999999993e-05 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961743+0000", 2023-09-23T22:40:06.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.857e-05 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961761+0000", 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010775909999999999 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962839+0000", 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7259000000000001e-05 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962866+0000", 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012405140000000001 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:25 6.c e105/99 6:34d0e523:::benchmark_data_smithi052_234456_object24:head v 105'2, mlcod=0'0)", 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.962111+0000", 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.861254832999997, 2023-09-23T22:40:06.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00079725, 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962111+0000", 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962111+0000", 2023-09-23T22:40:06.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962111+0000", 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000137102 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962248+0000", 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3251999999999999e-05 2023-09-23T22:40:06.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962261+0000", 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8169999999999998e-06 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962266+0000", 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4093e-05 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962300+0000", 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2435999999999999e-05 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962312+0000", 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00056696800000000005 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962879+0000", 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9374e-05 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962909+0000", 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00079725 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.857 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:29 6.a e105/99 6:55ee2183:::benchmark_data_smithi052_234456_object28:head v 105'1, mlcod=0'0)", 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.963958+0000", 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.859408266000003, 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00051541800000000002, 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.963958+0000", 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.963958+0000", 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.963958+0000", 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6643000000000001e-05 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.963994+0000", 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.414e-05 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964008+0000", 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3210000000000001e-06 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964015+0000", 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2206999999999997e-05 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964067+0000", 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0649000000000002e-05 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964087+0000", 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000354194 2023-09-23T22:40:06.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964442+0000", 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1803e-05 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964473+0000", 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00051541800000000002 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.862 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:42 6.1 e105/104 6:88c2eceb:::benchmark_data_smithi052_234456_object41:head v 105'1, mlcod=0'0)", 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.968799+0000", 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.854566837, 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00087266500000000001, 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.968799+0000", 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.968799+0000", 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.968799+0000", 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000279943 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969079+0000", 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2534000000000001e-05 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969092+0000", 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3960000000000003e-06 2023-09-23T22:40:06.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969098+0000", 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8105000000000001e-05 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969146+0000", 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0475e-05 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969167+0000", 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00046730300000000002 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969634+0000", 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8260999999999998e-05 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969672+0000", 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00087266500000000001 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:43 6.a e105/99 6:57c264ad:::benchmark_data_smithi052_234456_object42:head v 105'2, mlcod=0'0)", 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.969191+0000", 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.854175396999999, 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00053837600000000002, 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.868 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969191+0000", 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969191+0000", 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969190+0000", 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3045e-05 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969213+0000", 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0172e-05 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969223+0000", 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9630000000000003e-06 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969229+0000", 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6805000000000003e-05 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969276+0000", 2023-09-23T22:40:06.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3866000000000002e-05 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969300+0000", 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00039209599999999998 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969692+0000", 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7320000000000002e-05 2023-09-23T22:40:06.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969729+0000", 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00053837600000000002 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:44 6.5 e105/99 6:a3334369:::benchmark_data_smithi052_234456_object43:head v 105'2, mlcod=0'0)", 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.969480+0000", 2023-09-23T22:40:06.873 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.853885935999998, 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00060549300000000003, 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969480+0000", 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969480+0000", 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969480+0000", 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3186000000000001e-05 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969503+0000", 2023-09-23T22:40:06.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3822000000000001e-05 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969517+0000", 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5460000000000002e-06 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969521+0000", 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1005999999999998e-05 2023-09-23T22:40:06.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969582+0000", 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3864000000000001e-05 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.969606+0000", 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045408599999999998 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.970060+0000", 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5622e-05 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.970086+0000", 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00060549300000000003 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.878 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:50 6.1 e105/104 6:87293069:::benchmark_data_smithi052_234456_object49:head v 105'2, mlcod=0'0)", 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.971816+0000", 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.851550377000002, 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054433499999999996, 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971816+0000", 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971816+0000", 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971815+0000", 2023-09-23T22:40:06.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012550300000000001 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971941+0000", 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0338e-05 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971951+0000", 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5290000000000002e-06 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971956+0000", 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6551999999999998e-05 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.971992+0000", 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7506000000000001e-05 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972010+0000", 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000312128 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972322+0000", 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8420000000000001e-05 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972360+0000", 2023-09-23T22:40:06.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054433499999999996 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:51 6.5 e105/99 6:af5895e7:::benchmark_data_smithi052_234456_object50:head v 105'4, mlcod=105'2)", 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.972754+0000", 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.850612077000001, 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054137300000000005, 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.884 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972754+0000", 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972754+0000", 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972753+0000", 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7271e-05 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972781+0000", 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.428e-05 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972795+0000", 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5060000000000001e-06 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972801+0000", 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7212000000000006e-05 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972869+0000", 2023-09-23T22:40:06.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9602e-05 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972888+0000", 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037336599999999998 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.973262+0000", 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3983999999999999e-05 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.973296+0000", 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054137300000000005 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:57 6.a e105/99 6:5049fdad:::benchmark_data_smithi052_234456_object56:head v 105'4, mlcod=105'2)", 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.974461+0000", 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.848905344999999, 2023-09-23T22:40:06.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00057597400000000004, 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974461+0000", 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974461+0000", 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974460+0000", 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4138000000000002e-05 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974484+0000", 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3791000000000001e-05 2023-09-23T22:40:06.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974498+0000", 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5300000000000002e-06 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974505+0000", 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4098999999999999e-05 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974549+0000", 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3092000000000001e-05 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974582+0000", 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000425432 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975007+0000", 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9703e-05 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975037+0000", 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00057597400000000004 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.894 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:62 6.3 e105/104 6:ce9cfbe9:::benchmark_data_smithi052_234456_object61:head v 105'3, mlcod=105'1)", 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.975723+0000", 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.847643462999997, 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00050352999999999997, 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975723+0000", 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975723+0000", 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975722+0000", 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9270999999999999e-05 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975741+0000", 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.136e-05 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975753+0000", 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0439999999999998e-06 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975760+0000", 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3395e-05 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975803+0000", 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4530999999999999e-05 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975828+0000", 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035869500000000002 2023-09-23T22:40:06.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976186+0000", 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0012999999999998e-05 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976226+0000", 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00050352999999999997 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.899 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:66 6.3 e105/104 6:ce3b4e8d:::benchmark_data_smithi052_234456_object65:head v 105'4, mlcod=105'2)", 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.976947+0000", 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.846419373000003, 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00059672999999999996, 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976947+0000", 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976947+0000", 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976946+0000", 2023-09-23T22:40:06.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6235000000000001e-05 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976983+0000", 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1392e-05 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976994+0000", 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6600000000000003e-06 2023-09-23T22:40:06.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976999+0000", 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8347999999999999e-05 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977037+0000", 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8766999999999998e-05 2023-09-23T22:40:06.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977056+0000", 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045905700000000002 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977515+0000", 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8929e-05 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977544+0000", 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00059672999999999996 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:69 6.5 e105/99 6:a56ebbfe:::benchmark_data_smithi052_234456_object68:head v 105'5, mlcod=105'3)", 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.978016+0000", 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.845350617999998, 2023-09-23T22:40:06.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054608100000000002, 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978016+0000", 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978016+0000", 2023-09-23T22:40:06.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978015+0000", 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5685000000000001e-05 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978061+0000", 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2597e-05 2023-09-23T22:40:06.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978073+0000", 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6949999999999999e-06 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978078+0000", 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9398999999999999e-05 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978127+0000", 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3079999999999999e-05 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978150+0000", 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00036906800000000001 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978519+0000", 2023-09-23T22:40:06.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2444999999999998e-05 2023-09-23T22:40:06.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978562+0000", 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054608100000000002 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:71 6.3 e105/104 6:cd89b350:::benchmark_data_smithi052_234456_object70:head v 105'5, mlcod=105'3)", 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.978464+0000", 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.844902572000002, 2023-09-23T22:40:06.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00057942599999999996, 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978464+0000", 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978464+0000", 2023-09-23T22:40:06.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978463+0000", 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7790999999999999e-05 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978491+0000", 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4806e-05 2023-09-23T22:40:06.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978506+0000", 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2330000000000003e-06 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978512+0000", 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0609000000000002e-05 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978572+0000", 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2604000000000001e-05 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.978615+0000", 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000389956 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.979005+0000", 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8155000000000003e-05 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.979043+0000", 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00057942599999999996 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:82 6.5 e105/99 6:aeceea60:::benchmark_data_smithi052_234456_object81:head v 105'7, mlcod=105'5)", 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.981467+0000", 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 36.841899509999998, 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00051491000000000004, 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981467+0000", 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981467+0000", 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981466+0000", 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3671999999999998e-05 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981510+0000", 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9663000000000001e-05 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981529+0000", 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1389999999999998e-06 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981536+0000", 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8304000000000001e-05 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981615+0000", 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2867000000000001e-05 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981628+0000", 2023-09-23T22:40:06.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00032396200000000001 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981952+0000", 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0094e-05 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.981982+0000", 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00051491000000000004 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.921 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPush(6.4 108/107 [PushOp(6:258101c1:::benchmark_data_smithi052_234456_object5:head, version: 105'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:258101c1:::benchmark_data_smithi052_234456_object5:head@105'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:43.606380+0000", 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 23.216986148, 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0004283, 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606380+0000", 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606380+0000", 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606379+0000", 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4196000000000003e-05 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606413+0000", 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9369999999999999e-05 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606433+0000", 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5419999999999994e-06 2023-09-23T22:40:06.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606442+0000", 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8658999999999993e-05 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606521+0000", 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1418e-05 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606532+0000", 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000276259 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.606808+0000", 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0004283 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.926 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPush(6.4 108/107 [PushOp(6:2677d073:::benchmark_data_smithi052_234456_object69:head, version: 105'7, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:2677d073:::benchmark_data_smithi052_234456_object69:head@105'7, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:43.864158+0000", 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 22.959207957, 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037920299999999999, 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:06.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864158+0000", 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864158+0000", 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864156+0000", 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115975 2023-09-23T22:40:06.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864272+0000", 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4733e-05 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864317+0000", 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5775e-05 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864343+0000", 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8258999999999999e-05 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864401+0000", 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7199999999999997e-06 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:06.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864406+0000", 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013184700000000001 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:43.864538+0000", 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037920299999999999 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:06.931 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:06.932 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:40:06.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:06 smithi151 ceph-mon[159210]: pgmap v258: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:06.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1642361709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:06.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 37 16 101 85 0.143564 0 - 0.00253822 2023-09-23T22:40:06.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:06 smithi052 ceph-mon[180979]: pgmap v258: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:06.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1642361709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:06 smithi052 ceph-mon[187743]: pgmap v258: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:06.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1642361709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:07.150 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:07.150 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:07.192 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:07.296 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:07.326 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.387 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:07.431 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:40:07.491 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:07.518 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.562 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:07.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:07.669 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:07.698 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.744 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:07.793 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:07.853 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:07.884 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 38 16 101 85 0.139786 0 - 0.00253822 2023-09-23T22:40:08.081 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:08.185 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:08.213 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.310 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:08.414 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:08.442 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.485 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:08.589 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:08.620 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.667 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:08.734 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:40:08.770 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:08.797 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.899 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:08.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:08.910 INFO:teuthology.orchestra.run.smithi052.stdout:moved 48 / 267 (17.9775%) 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:avg 44.5 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 12.1209 -> 18.1911 (expected baseline 6.08961) 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.1 with 26 -> 47 pgs (0.58427 -> 1.05618 * mean) 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.3 with 43 -> 49 pgs (0.966292 -> 1.10112 * mean) 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:oload 110 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 3 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0018 2023-09-23T22:40:08.911 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.0020 2023-09-23T22:40:08.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 weight 1.0000 -> 0.5488 2023-09-23T22:40:08.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 weight 1.0000 -> 0.5500 2023-09-23T22:40:08.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 weight 0.8098 -> 1.0000 2023-09-23T22:40:08.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 weight 1.0000 -> 0.5636 2023-09-23T22:40:08.913 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T22:40:08.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 39 16 101 85 0.136202 0 - 0.00253822 2023-09-23T22:40:08.993 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:08 smithi052 conmon[173965]: 2023-09-23T22:40:08.909+0000 7f39bb05d700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T22:40:08.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:08 smithi052 ceph-mon[180979]: pgmap v259: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:08.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4041049127' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:08.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:08 smithi052 ceph-mon[187743]: pgmap v259: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4041049127' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:08.994 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:08 smithi151 ceph-mon[159210]: pgmap v259: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:08.995 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4041049127' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:08.995 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:09.002 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:09.029 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:09.152 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:09.193 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:40:09.193 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:40:09.193 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:40:09.194 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:40:09.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:09.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:09.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: osdmap e113: 8 total, 6 up, 8 in 2023-09-23T22:40:09.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: from='client.44690 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:09.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: failed to encode map e113 with expected crc 2023-09-23T22:40:09.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: failed to encode map e113 with expected crc 2023-09-23T22:40:09.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[180979]: failed to encode map e113 with expected crc 2023-09-23T22:40:09.777 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:09.779 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:40:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:09.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:09.848 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T22:40:09.848 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T22:40:09.882 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:09.888 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:09.908 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:09.955 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:09.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:40:09.961500+0000 min lat: 0.00121387 max lat: 0.00607929 avg lat: 0.00253822 2023-09-23T22:40:09.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:40:09.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 40 16 101 85 0.132797 0 - 0.00253822 2023-09-23T22:40:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:09.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: osdmap e113: 8 total, 6 up, 8 in 2023-09-23T22:40:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: from='client.44690 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: failed to encode map e113 with expected crc 2023-09-23T22:40:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: failed to encode map e113 with expected crc 2023-09-23T22:40:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:09 smithi052 ceph-mon[187743]: failed to encode map e113 with expected crc 2023-09-23T22:40:10.062 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: osdmap e113: 8 total, 6 up, 8 in 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: from='client.44690 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: failed to encode map e113 with expected crc 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: failed to encode map e113 with expected crc 2023-09-23T22:40:10.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:09 smithi151 ceph-mon[159210]: failed to encode map e113 with expected crc 2023-09-23T22:40:10.089 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:10.143 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:10.185 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:10.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:10.289 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:10.316 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:10.363 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:10.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:10.467 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:10.496 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:10.792 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:10.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:10.799 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:10.896 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:10.923 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:10.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 41 16 101 85 0.129558 0 - 0.00253822 2023-09-23T22:40:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:10 smithi052 ceph-mon[187743]: pgmap v261: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:10 smithi052 ceph-mon[187743]: failed to encode map e113 with expected crc 2023-09-23T22:40:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:10 smithi052 ceph-mon[180979]: pgmap v261: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:10 smithi052 ceph-mon[180979]: failed to encode map e113 with expected crc 2023-09-23T22:40:11.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:10 smithi151 ceph-mon[159210]: pgmap v261: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:11.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:10 smithi151 ceph-mon[159210]: failed to encode map e113 with expected crc 2023-09-23T22:40:11.102 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:11.208 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:11.241 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.338 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:11.441 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:11.470 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.514 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:11.548 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:40:11.549 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:40:11.549 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:40:11.549 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:40:11.549 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:40:11.618 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:11.646 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.692 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:11.763 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:40:11.795 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:11.821 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.863 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:11.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:11.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 42 16 101 85 0.126473 0 - 0.00253822 2023-09-23T22:40:11.967 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:11.979 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:11.992 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.004 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:12.025 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:12.043 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:12.062 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:12.080 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:40:12.081 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:12.174 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:12.228 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:40:12.277 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:12.304 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.347 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:12.452 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:12.484 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.529 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:12.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:12.577 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:12.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:12.659 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.707 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:12.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:12.749 INFO:teuthology.orchestra.run.smithi052.stdout:moved 5 / 267 (1.87266%) 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:avg 44.5 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 12.1209 -> 10.6575 (expected baseline 6.08961) 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.1 with 26 -> 37 pgs (0.58427 -> 0.831461 * mean) 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.3 with 43 -> 41 pgs (0.966292 -> 0.921348 * mean) 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 3 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0018 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.0183 2023-09-23T22:40:12.750 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 weight 0.8098 -> 1.0000 2023-09-23T22:40:12.810 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:12.839 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.884 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:12 smithi151 ceph-mon[159210]: pgmap v262: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2867464908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'65536'}"}]: dispatch 2023-09-23T22:40:12.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 43 16 101 85 0.123532 0 - 0.00253822 2023-09-23T22:40:12.987 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:13.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:12 smithi052 ceph-mon[187743]: pgmap v262: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:13.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2867464908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:13.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'65536'}"}]: dispatch 2023-09-23T22:40:13.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:12 smithi052 ceph-mon[180979]: pgmap v262: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:13.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2867464908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:13.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'65536'}"}]: dispatch 2023-09-23T22:40:13.026 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.071 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:13.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:13.201 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.238 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:13.263 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:13.451 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T22:40:13.452 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T22:40:13.465 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:13.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:13.569 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:13.596 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:40:13] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.33.4" 2023-09-23T22:40:13.599 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.655 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:13.758 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:13.789 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:13.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'65536'}"}]': finished 2023-09-23T22:40:13.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: osdmap e114: 8 total, 6 up, 8 in 2023-09-23T22:40:13.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.873 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:13 smithi151 ceph-mon[159210]: osdmap e115: 8 total, 6 up, 8 in 2023-09-23T22:40:13.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:13.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'65536'}"}]': finished 2023-09-23T22:40:13.953 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: osdmap e114: 8 total, 6 up, 8 in 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[180979]: osdmap e115: 8 total, 6 up, 8 in 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'65536'}"}]': finished 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: osdmap e114: 8 total, 6 up, 8 in 2023-09-23T22:40:13.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.955 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.955 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.955 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: failed to encode map e114 with expected crc 2023-09-23T22:40:13.955 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:13 smithi052 ceph-mon[187743]: osdmap e115: 8 total, 6 up, 8 in 2023-09-23T22:40:13.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 44 16 101 85 0.120725 0 - 0.00253822 2023-09-23T22:40:13.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:14.060 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:14.060 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:14.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:14.125 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:14.325 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:14.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:14.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:14.428 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:14.461 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:14.553 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:14.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:14.657 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:14.691 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:14.779 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:40:14.809 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:14.846 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:14.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:14.950 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:14.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 45 16 101 85 0.118042 0 - 0.00253822 2023-09-23T22:40:14.979 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: pgmap v264: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[187743]: osdmap e116: 8 total, 6 up, 8 in 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: pgmap v264: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: failed to encode map e115 with expected crc 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:40:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:14 smithi052 ceph-mon[180979]: osdmap e116: 8 total, 6 up, 8 in 2023-09-23T22:40:15.021 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: pgmap v264: 89 pgs: 5 active+clean+remapped, 35 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 3 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50/327 objects degraded (15.291%); 19/327 objects misplaced (5.810%) 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: failed to encode map e115 with expected crc 2023-09-23T22:40:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:40:15.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:14 smithi151 ceph-mon[159210]: osdmap e116: 8 total, 6 up, 8 in 2023-09-23T22:40:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:15.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:15.156 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.198 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:15.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:15.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:15.304 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:15.330 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.372 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:15.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:15.477 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:15.503 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.545 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:15.650 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:15.676 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.717 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:15.764 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:40:15.822 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:15.853 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.907 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:15 smithi151 ceph-mon[159210]: failed to encode map e116 with expected crc 2023-09-23T22:40:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:15 smithi151 ceph-mon[159210]: failed to encode map e116 with expected crc 2023-09-23T22:40:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:15 smithi151 ceph-mon[159210]: failed to encode map e116 with expected crc 2023-09-23T22:40:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:15 smithi151 ceph-mon[159210]: failed to encode map e116 with expected crc 2023-09-23T22:40:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3890679713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:15.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 46 16 101 85 0.115476 0 - 0.00253822 2023-09-23T22:40:16.011 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:16.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[187743]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[187743]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[187743]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[187743]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3890679713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[180979]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[180979]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[180979]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[180979]: failed to encode map e116 with expected crc 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3890679713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:16.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:16.039 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:16.495 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:16.754 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:16.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:16.823 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:16.859 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:16.887 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:16.928 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:16.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:16.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 47 16 101 85 0.113019 0 - 0.00253822 2023-09-23T22:40:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: pgmap v267: 89 pgs: 1 peering, 3 active+clean+remapped, 36 active+clean, 28 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:16.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 57/327 objects degraded (17.431%), 15 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: failed to encode map e117 with expected crc 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: failed to encode map e117 with expected crc 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: osdmap e117: 8 total, 6 up, 8 in 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[187743]: failed to encode map e117 with expected crc 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: pgmap v267: 89 pgs: 1 peering, 3 active+clean+remapped, 36 active+clean, 28 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 57/327 objects degraded (17.431%), 15 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:16.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: failed to encode map e117 with expected crc 2023-09-23T22:40:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: failed to encode map e117 with expected crc 2023-09-23T22:40:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: osdmap e117: 8 total, 6 up, 8 in 2023-09-23T22:40:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:16 smithi052 ceph-mon[180979]: failed to encode map e117 with expected crc 2023-09-23T22:40:17.035 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:17.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: pgmap v267: 89 pgs: 1 peering, 3 active+clean+remapped, 36 active+clean, 28 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:17.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 57/327 objects degraded (17.431%), 15 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:17.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:17.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: failed to encode map e117 with expected crc 2023-09-23T22:40:17.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: failed to encode map e117 with expected crc 2023-09-23T22:40:17.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:17.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: osdmap e117: 8 total, 6 up, 8 in 2023-09-23T22:40:17.060 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:16 smithi151 ceph-mon[159210]: failed to encode map e117 with expected crc 2023-09-23T22:40:17.062 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:17.104 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:17.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:17.210 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:17.236 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:17.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:17.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:17.383 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:17.415 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:moved 4 / 267 (1.49813%) 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:avg 44.5 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 10.6575 -> 10.9506 (expected baseline 6.08961) 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.5 with 26 -> 26 pgs (0.58427 -> 0.58427 * mean) 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.3 with 41 -> 43 pgs (0.921348 -> 0.966292 * mean) 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:oload 110 2023-09-23T22:40:17.766 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 0.05 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0018 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.0020 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7 weight 1.0000 -> 0.9500 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5 weight 1.0000 -> 0.9500 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6 weight 1.0000 -> 0.9500 2023-09-23T22:40:17.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4 weight 1.0000 -> 0.9500 2023-09-23T22:40:17.939 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:17.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:17.961 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 48 16 101 85 0.110665 0 - 0.00253822 2023-09-23T22:40:18.044 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:18.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:17 smithi151 ceph-mon[159210]: failed to encode map e117 with expected crc 2023-09-23T22:40:18.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:17 smithi151 ceph-mon[159210]: from='client.34806 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:18.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:17 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]: dispatch 2023-09-23T22:40:18.073 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:18.116 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:18.222 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:18.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:17 smithi052 ceph-mon[180979]: failed to encode map e117 with expected crc 2023-09-23T22:40:18.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:17 smithi052 ceph-mon[180979]: from='client.34806 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:18.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:17 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]: dispatch 2023-09-23T22:40:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:17 smithi052 ceph-mon[187743]: failed to encode map e117 with expected crc 2023-09-23T22:40:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:17 smithi052 ceph-mon[187743]: from='client.34806 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:17 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]: dispatch 2023-09-23T22:40:18.248 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:18.289 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:18.396 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:18.428 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:18.750 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:40:18.891 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:18.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:18.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 49 16 101 85 0.108406 0 - 0.00253822 2023-09-23T22:40:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[180979]: pgmap v269: 89 pgs: 1 peering, 2 active+clean+remapped, 37 active+clean, 29 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[180979]: failed to encode map e118 with expected crc 2023-09-23T22:40:18.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[180979]: failed to encode map e118 with expected crc 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]': finished 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[180979]: osdmap e118: 8 total, 6 up, 8 in 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[187743]: pgmap v269: 89 pgs: 1 peering, 2 active+clean+remapped, 37 active+clean, 29 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[187743]: failed to encode map e118 with expected crc 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[187743]: failed to encode map e118 with expected crc 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]': finished 2023-09-23T22:40:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:18 smithi052 ceph-mon[187743]: osdmap e118: 8 total, 6 up, 8 in 2023-09-23T22:40:18.996 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:19.031 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:19.080 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T22:40:19.080 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T22:40:19.195 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:19.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:18 smithi151 ceph-mon[159210]: pgmap v269: 89 pgs: 1 peering, 2 active+clean+remapped, 37 active+clean, 29 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:18 smithi151 ceph-mon[159210]: failed to encode map e118 with expected crc 2023-09-23T22:40:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:18 smithi151 ceph-mon[159210]: failed to encode map e118 with expected crc 2023-09-23T22:40:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:18 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260','6':'62260','7':'62260'}"}]': finished 2023-09-23T22:40:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:18 smithi151 ceph-mon[159210]: osdmap e118: 8 total, 6 up, 8 in 2023-09-23T22:40:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:19.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:19.228 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:19.228 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:19.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:19.351 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:19.505 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:19.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:19.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:19.547 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:19.566 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:19.584 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:19.603 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:19.609 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:19.621 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:19.638 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:19.639 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:40:19.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:19.684 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:19.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:19.776 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: osdmap e119: 8 total, 6 up, 8 in 2023-09-23T22:40:19.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:19.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:19.778 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4176739288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:19.790 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:19.850 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:19.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: osdmap e119: 8 total, 6 up, 8 in 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: failed to encode map e119 with expected crc 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4176739288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:19.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 50 16 101 85 0.106238 0 - 0.00253822 2023-09-23T22:40:20.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: failed to encode map e119 with expected crc 2023-09-23T22:40:20.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: failed to encode map e119 with expected crc 2023-09-23T22:40:20.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: osdmap e119: 8 total, 6 up, 8 in 2023-09-23T22:40:20.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: failed to encode map e119 with expected crc 2023-09-23T22:40:20.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: failed to encode map e119 with expected crc 2023-09-23T22:40:20.036 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:20.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:20.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4176739288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:20.044 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:20.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:20.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:20.147 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:20.173 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:20.214 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:20.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:20.319 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:20.349 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:20.442 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:40:20.468 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:20.469 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:20.497 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:20.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:20.525 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:20.541 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:20.602 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:20.629 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:20.670 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:20.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:20.775 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:20.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:20.868 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:20.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:20 smithi151 ceph-mon[159210]: pgmap v272: 89 pgs: 2 remapped+peering, 2 active+clean+remapped, 38 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:20 smithi151 ceph-mon[159210]: failed to encode map e120 with expected crc 2023-09-23T22:40:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:20 smithi151 ceph-mon[159210]: osdmap e120: 8 total, 6 up, 8 in 2023-09-23T22:40:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:20 smithi151 ceph-mon[159210]: failed to encode map e120 with expected crc 2023-09-23T22:40:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:20 smithi151 ceph-mon[159210]: failed to encode map e120 with expected crc 2023-09-23T22:40:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:20 smithi151 ceph-mon[159210]: Health check update: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:40:20.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 51 16 101 85 0.104155 0 - 0.00253822 2023-09-23T22:40:20.973 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:21.003 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[187743]: pgmap v272: 89 pgs: 2 remapped+peering, 2 active+clean+remapped, 38 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[187743]: failed to encode map e120 with expected crc 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[187743]: osdmap e120: 8 total, 6 up, 8 in 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[187743]: failed to encode map e120 with expected crc 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[187743]: failed to encode map e120 with expected crc 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[187743]: Health check update: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:40:21.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[180979]: pgmap v272: 89 pgs: 2 remapped+peering, 2 active+clean+remapped, 38 active+clean, 27 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 14 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57/327 objects degraded (17.431%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:21.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[180979]: failed to encode map e120 with expected crc 2023-09-23T22:40:21.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[180979]: osdmap e120: 8 total, 6 up, 8 in 2023-09-23T22:40:21.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[180979]: failed to encode map e120 with expected crc 2023-09-23T22:40:21.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[180979]: failed to encode map e120 with expected crc 2023-09-23T22:40:21.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:20 smithi052 ceph-mon[180979]: Health check update: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:40:21.098 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:21.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:21.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:21.203 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:21.237 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:21.734 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:21.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:21.837 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:21.869 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:21.918 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:21.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:21.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 52 16 101 85 0.102152 0 - 0.00253822 2023-09-23T22:40:21.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:21 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:21.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:21 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:22.023 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:22.050 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.094 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:22.097 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:21 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:22.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:22.199 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:22.229 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.284 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:22.293 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:22.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:22.402 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:22.429 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.470 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:22.472 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:22.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:22.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:22.577 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:22.603 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.645 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:22.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:22.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:22.752 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:22.786 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.842 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:22.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:22.946 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:22.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 53 16 101 85 0.100225 0 - 0.00253822 2023-09-23T22:40:22.973 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.017 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:23.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:22 smithi052 ceph-mon[187743]: pgmap v274: 89 pgs: 1 activating+remapped, 2 remapped+peering, 3 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:23.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:22 smithi052 ceph-mon[180979]: pgmap v274: 89 pgs: 1 activating+remapped, 2 remapped+peering, 3 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:23.122 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:23.146 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:22 smithi151 ceph-mon[159210]: pgmap v274: 89 pgs: 1 activating+remapped, 2 remapped+peering, 3 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:23.149 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.190 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:23.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:23.279 INFO:teuthology.orchestra.run.smithi052.stdout:moved 0 / 267 (0%) 2023-09-23T22:40:23.280 INFO:teuthology.orchestra.run.smithi052.stdout:avg 44.5 2023-09-23T22:40:23.280 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 10.9506 -> 10.9506 (expected baseline 6.08961) 2023-09-23T22:40:23.280 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.5 with 26 -> 26 pgs (0.58427 -> 0.58427 * mean) 2023-09-23T22:40:23.280 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.3 with 43 -> 43 pgs (0.966292 -> 0.966292 * mean) 2023-09-23T22:40:23.280 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:40:23.281 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T22:40:23.281 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 3 2023-09-23T22:40:23.281 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T22:40:23.281 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0018 2023-09-23T22:40:23.281 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.0183 2023-09-23T22:40:23.282 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T22:40:23.295 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:23.321 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.363 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:23.468 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:23.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:40:23] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.33.4" 2023-09-23T22:40:23.498 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.839 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:40:23.893 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:23.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:23.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 54 16 101 85 0.0983689 0 - 0.00253822 2023-09-23T22:40:23.998 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:24.025 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.068 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:24.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4255143835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:24.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:24.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4255143835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:24.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:24.173 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:24.197 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4255143835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:24.197 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:24.199 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:24.347 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:24.373 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.415 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:24.522 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:24.546 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.591 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:24.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:24.697 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:24.723 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.765 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:24.863 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:24.873 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:24.899 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.940 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:24.953 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:40:24.953 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:40:24.953 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:40:24.953 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:40:24.953 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:40:24.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 55 16 101 85 0.0965804 0 - 0.00253822 2023-09-23T22:40:24.978 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[187743]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:24.978 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[187743]: pgmap v275: 89 pgs: 1 activating+remapped, 2 remapped+peering, 3 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[187743]: osdmap e121: 8 total, 6 up, 8 in 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[180979]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[180979]: pgmap v275: 89 pgs: 1 activating+remapped, 2 remapped+peering, 3 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:24.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:24 smithi052 ceph-mon[180979]: osdmap e121: 8 total, 6 up, 8 in 2023-09-23T22:40:25.046 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:25.076 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:25.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:24 smithi151 ceph-mon[159210]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:40:25.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:24 smithi151 ceph-mon[159210]: pgmap v275: 89 pgs: 1 activating+remapped, 2 remapped+peering, 3 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:24 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:24 smithi151 ceph-mon[159210]: osdmap e121: 8 total, 6 up, 8 in 2023-09-23T22:40:25.476 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:25.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:25.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:25.579 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:25.612 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:25.662 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:25.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:25.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:25.742 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:40:25.768 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:25.813 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:25.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:25 smithi151 ceph-mon[159210]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:25 smithi151 ceph-mon[159210]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:25 smithi151 ceph-mon[159210]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:25 smithi151 ceph-mon[159210]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 56 16 101 85 0.0948558 0 - 0.00253822 2023-09-23T22:40:25.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[180979]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[180979]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[180979]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[180979]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[187743]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[187743]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[187743]: failed to encode map e121 with expected crc 2023-09-23T22:40:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:25 smithi052 ceph-mon[187743]: failed to encode map e121 with expected crc 2023-09-23T22:40:26.019 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:26.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:26.122 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:26.151 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:26.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:26.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:26.531 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:26.578 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:26.610 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:26.660 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:26.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:26.765 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:26.794 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:26.851 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:26.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[187743]: failed to encode map e121 with expected crc 2023-09-23T22:40:26.867 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[187743]: pgmap v277: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:26.867 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[187743]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:40:26.867 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1487876280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:26.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[180979]: failed to encode map e121 with expected crc 2023-09-23T22:40:26.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[180979]: pgmap v277: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:26.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[180979]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:40:26.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1487876280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:26.872 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:26.882 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:40:26.892 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:26.903 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:26.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:26.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:26.914 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:26.933 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:26 smithi151 ceph-mon[159210]: failed to encode map e121 with expected crc 2023-09-23T22:40:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:26 smithi151 ceph-mon[159210]: pgmap v277: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:26 smithi151 ceph-mon[159210]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:40:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1487876280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:26.951 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:26.951 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:26.951 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:26.951 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:26.951 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:40:26.952 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:26.952 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:26.952 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:40:26.952 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:26.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 57 16 101 85 0.0931916 0 - 0.00253822 2023-09-23T22:40:27.006 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:27.038 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:27.082 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:27.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:27.187 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:27.215 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:27.258 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:27.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:27.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:27.363 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:27.390 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:27.432 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:27.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:27.539 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:27.568 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:27.864 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:27.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:27.940 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:27.962 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 58 16 101 85 0.0915849 0 - 0.00253822 2023-09-23T22:40:27.968 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:27.994 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:28.034 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:28.036 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:28.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:28.141 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:28.170 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:28.267 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:28.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:28.373 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:28.403 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:28.507 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:28.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:28.522 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:28.522 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:28.613 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:28.642 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:28.963 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 59 16 101 85 0.0900326 0 - 0.00253822 2023-09-23T22:40:29.125 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0, 2] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:40:29.125 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:40:29.126 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T22:40:29.126 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:40:29.126 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:40:29.126 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T22:40:29.157 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:28 smithi052 ceph-mon[187743]: pgmap v278: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:29.157 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:28 smithi052 ceph-mon[180979]: pgmap v278: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:29.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:28 smithi151 ceph-mon[159210]: pgmap v278: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:29.294 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:29.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:29.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:40:29.397 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:29.428 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:29.477 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:29.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:29.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:29.584 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:29.617 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:29.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:29.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:29.874 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:29.904 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:29.906 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:29.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:29.963 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:40:29.964020+0000 min lat: 0.00121387 max lat: 0.00607929 avg lat: 0.00253822 2023-09-23T22:40:29.963 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:40:29.964 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 60 16 101 85 0.0885311 0 - 0.00253822 2023-09-23T22:40:29.964 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:30.023 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:30.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:29 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:30.025 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:29 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:30.127 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:30.160 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:30.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:29 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:30.476 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:30.580 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:30.612 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:30.659 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:30.766 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:30.795 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:30.838 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:30.856 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:40:30.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:30.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:30 smithi151 ceph-mon[159210]: pgmap v279: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:30.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1365608676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:30.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:30 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:30.963 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 61 16 101 85 0.0870798 0 - 0.00253822 2023-09-23T22:40:30.974 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:31.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:30 smithi052 ceph-mon[180979]: pgmap v279: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:31.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1365608676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:31.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:30 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:31.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:30 smithi052 ceph-mon[187743]: pgmap v279: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%); 0 B/s, 0 objects/s recovering 2023-09-23T22:40:31.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1365608676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:31.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:30 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:31.621 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:31.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:31.725 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:31.754 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:31.855 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:31.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:31.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:31.963 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 62 16 101 85 0.0856753 0 - 0.00253822 2023-09-23T22:40:31.988 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:32.034 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:32.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:32.066 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:32.066 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[180979]: failed to encode map e122 with expected crc 2023-09-23T22:40:32.066 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[180979]: failed to encode map e122 with expected crc 2023-09-23T22:40:32.066 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:32.066 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[180979]: osdmap e122: 8 total, 6 up, 8 in 2023-09-23T22:40:32.066 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:32.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[187743]: failed to encode map e122 with expected crc 2023-09-23T22:40:32.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[187743]: failed to encode map e122 with expected crc 2023-09-23T22:40:32.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:32.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:31 smithi052 ceph-mon[187743]: osdmap e122: 8 total, 6 up, 8 in 2023-09-23T22:40:32.139 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:32.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:31 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:32.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:31 smithi151 ceph-mon[159210]: failed to encode map e122 with expected crc 2023-09-23T22:40:32.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:31 smithi151 ceph-mon[159210]: failed to encode map e122 with expected crc 2023-09-23T22:40:32.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:31 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:32.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:31 smithi151 ceph-mon[159210]: osdmap e122: 8 total, 6 up, 8 in 2023-09-23T22:40:32.166 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:32.208 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:32.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:32.285 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:32.313 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:32.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:32.380 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:32.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:32.485 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:32.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:32.552 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:32.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:32.601 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:40:32.601 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:40:32.601 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:40:32.660 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:32.695 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:32.964 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 63 16 101 85 0.0843154 0 - 0.00253822 2023-09-23T22:40:32.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:32 smithi052 ceph-mon[180979]: pgmap v281: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:32 smithi052 ceph-mon[187743]: pgmap v281: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:33.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:32 smithi151 ceph-mon[159210]: pgmap v281: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:33.586 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:33.587 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:33.604 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:40:33] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.33.4" 2023-09-23T22:40:33.848 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:33.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:33.952 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:33.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 64 16 101 85 0.082998 0 - 0.00253822 2023-09-23T22:40:33.984 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.031 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:34.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:34.084 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:40:34.136 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:34.164 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.328 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:34.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:34.432 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:34.461 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.510 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:40:34.716 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:34.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:34.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:34.768 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:34.814 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:34.820 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:34.838 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:34.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.868 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:34.892 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:34.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:34.932 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:40:34.933 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:34.964 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 65 16 101 85 0.0817207 0 - 0.00253822 2023-09-23T22:40:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[187743]: pgmap v282: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[187743]: Health check update: Reduced data availability: 6 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/888605110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[180979]: pgmap v282: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[180979]: Health check update: Reduced data availability: 6 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/888605110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:35.171 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:35.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:34 smithi151 ceph-mon[159210]: pgmap v282: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:34 smithi151 ceph-mon[159210]: Health check update: Reduced data availability: 6 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:34 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 53/327 objects degraded (16.208%), 14 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/888605110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:35.274 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:35.306 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:35.354 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:35.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:35.461 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:35.492 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:35.721 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:35.741 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:35.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:35.763 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:35.845 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:35.877 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:35.925 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:35.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:35.964 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 66 16 101 85 0.0804825 0 - 0.00253822 2023-09-23T22:40:36.033 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:36.061 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:36.213 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:36.240 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.281 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:36.386 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:36.411 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.453 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:36.560 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:36.586 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.627 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:36.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:36.732 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:36.758 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:36.907 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:36.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.964 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 67 16 101 85 0.0792812 0 - 0.00253822 2023-09-23T22:40:36.984 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:37.088 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:37.114 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.155 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:37.158 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:36 smithi151 ceph-mon[159210]: pgmap v283: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:37.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:37.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:36 smithi052 ceph-mon[187743]: pgmap v283: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:37.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:36 smithi052 ceph-mon[180979]: pgmap v283: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:37.260 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:37.286 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.327 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:37.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:37.435 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:37.464 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.559 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:37.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:37.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:37.691 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.722 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:37.740 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:37.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:37.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:37.749 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:40:37.749 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:40:37.749 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:40:37.749 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:40:37.749 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:40:37.844 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:37.881 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.885 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:40:37.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 68 16 101 85 0.0781154 0 - 0.00253822 2023-09-23T22:40:38.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/692430771' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:38.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:40:38.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/692430771' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:38.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:40:38.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/692430771' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:38.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:40:38.641 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:38.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:38.736 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:40:38.745 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:38.774 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:38.819 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:38.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:38.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:38.949 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:38.952 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:38.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 69 16 101 85 0.0769833 0 - 0.00253822 2023-09-23T22:40:38.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[187743]: pgmap v284: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:38.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:38.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[187743]: failed to encode map e123 with expected crc 2023-09-23T22:40:38.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/692430771' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:38.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[187743]: osdmap e123: 8 total, 6 up, 8 in 2023-09-23T22:40:38.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[187743]: failed to encode map e123 with expected crc 2023-09-23T22:40:38.994 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:38.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[180979]: pgmap v284: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:38.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:38.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[180979]: failed to encode map e123 with expected crc 2023-09-23T22:40:38.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/692430771' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:38.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[180979]: osdmap e123: 8 total, 6 up, 8 in 2023-09-23T22:40:38.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:38 smithi052 ceph-mon[180979]: failed to encode map e123 with expected crc 2023-09-23T22:40:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:38.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:39.097 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:39.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:38 smithi151 ceph-mon[159210]: pgmap v284: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:39.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:38 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:39.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:38 smithi151 ceph-mon[159210]: failed to encode map e123 with expected crc 2023-09-23T22:40:39.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/692430771' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:39.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:38 smithi151 ceph-mon[159210]: osdmap e123: 8 total, 6 up, 8 in 2023-09-23T22:40:39.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:38 smithi151 ceph-mon[159210]: failed to encode map e123 with expected crc 2023-09-23T22:40:39.123 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.167 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:39.272 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:39.299 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.393 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:39.396 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0, 2] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:40:39.397 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-09-23T22:40:39.397 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-09-23T22:40:39.397 DEBUG:teuthology.orchestra.run.smithi052:osd.0> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T22:40:39.399 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl start ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:40:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:39.429 INFO:journalctl@ceph.osd.0.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:40:39.501 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:39.528 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.681 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:39.743 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:39 smithi052 systemd[1]: Starting Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:40:39.784 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:39.811 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.855 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:39.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:39 smithi151 ceph-mon[159210]: failed to encode map e123 with expected crc 2023-09-23T22:40:39.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:39 smithi151 ceph-mon[159210]: failed to encode map e123 with expected crc 2023-09-23T22:40:39.962 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:39.966 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 70 16 101 85 0.0758836 0 - 0.00253822 2023-09-23T22:40:39.996 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:40.054 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:40.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:40.085 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:40.159 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:40.186 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:39 smithi052 ceph-mon[187743]: failed to encode map e123 with expected crc 2023-09-23T22:40:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:39 smithi052 ceph-mon[187743]: failed to encode map e123 with expected crc 2023-09-23T22:40:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:39 smithi052 ceph-mon[180979]: failed to encode map e123 with expected crc 2023-09-23T22:40:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:39 smithi052 ceph-mon[180979]: failed to encode map e123 with expected crc 2023-09-23T22:40:40.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:40.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:40.383 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:40.415 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:40.557 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:40.575 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:40.576 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:40.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:40.596 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:40.616 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:40.642 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:40.669 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:40.670 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:40:40.670 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:40.683 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:40.712 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:40.754 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:40.757 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:40:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:40.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:40.859 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:40.886 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:40.928 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:40.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:40.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:40.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 71 16 101 85 0.0748149 0 - 0.00253822 2023-09-23T22:40:41.034 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:41.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:40 smithi151 ceph-mon[159210]: pgmap v286: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:41.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/694989419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:41.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:41.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:41.062 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:41.110 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:41.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:41.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:41.215 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[187743]: pgmap v286: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/694989419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[180979]: pgmap v286: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/694989419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:41.246 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:41.444 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:41.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:41.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:41.548 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:41.575 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:41.616 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:41.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:41.722 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:41.748 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:41.790 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:41.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:41.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:41.896 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:41.923 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:40:41.924 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:41 smithi052 bash[257227]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T22:40:41.927 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:41.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 72 16 101 85 0.0737758 0 - 0.00253822 2023-09-23T22:40:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:42.219 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:42.377 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:42.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:42.470 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:42.481 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:42.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:42.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:42.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:42.723 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:42.755 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:42.803 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:42.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:42.823 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:42.823 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:42.912 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:42.940 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:42.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 73 16 101 85 0.0727652 0 - 0.00253822 2023-09-23T22:40:42.984 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:42.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:42.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:43.089 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:43.113 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:42 smithi151 ceph-mon[159210]: pgmap v287: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:43.115 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:43.157 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:43.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:43.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:42 smithi052 ceph-mon[187743]: pgmap v287: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:43.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:42 smithi052 ceph-mon[180979]: pgmap v287: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:43.265 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:43.291 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:43.333 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:43.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:43.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:43.465 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:43.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:40:43] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-09-23T22:40:43.506 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:43.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:43.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:43.613 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:43.646 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:43.906 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:43.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:43.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 74 16 101 85 0.0717819 0 - 0.00253822 2023-09-23T22:40:44.010 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:44.037 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.080 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:44.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:44.188 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:40:44.211 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.220 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:44.254 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:44.359 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:44.395 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.501 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:44.532 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:44.595 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:40:44.605 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:44.631 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.725 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:44.828 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:44.855 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.882 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:44 smithi052 bash[258815]: 0aacb3cd0674f8ffe1cf727b0cdc62f981aec69d4dec948502b26fb6b232f206 2023-09-23T22:40:44.882 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-osd[259144]: -- 172.21.15.52:0/3501286901 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5562c9e3c4e0 con 0x5562c9e0f000 2023-09-23T22:40:44.882 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:44 smithi052 systemd[1]: Started Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:40:44.950 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:44.965 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 75 16 101 85 0.0708248 0 - 0.00253822 2023-09-23T22:40:44.985 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:45.053 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:45.079 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.123 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:45.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:45.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:44 smithi151 ceph-mon[159210]: pgmap v288: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1277231344' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:45.227 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:45.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:45 smithi052 conmon[258962]: 2023-09-23T22:40:45.119+0000 7f4f2b2ff640 -1 Falling back to public interface 2023-09-23T22:40:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-mon[187743]: pgmap v288: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1277231344' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-mon[180979]: pgmap v288: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1277231344' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:45.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.311 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:45.414 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:45.441 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.485 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:45.646 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:45.710 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:40:45.752 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:45.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.873 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:45.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:45.966 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 76 16 101 85 0.0698929 0 - 0.00253822 2023-09-23T22:40:45.976 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:45 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:45 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:45 smithi052 ceph-mon[187743]: osdmap e124: 8 total, 6 up, 8 in 2023-09-23T22:40:45.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:45 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:45.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:45 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:45.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:45 smithi052 ceph-mon[180979]: osdmap e124: 8 total, 6 up, 8 in 2023-09-23T22:40:46.005 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:46.051 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:46.053 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:45 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:46.053 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:45 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:46.053 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:45 smithi151 ceph-mon[159210]: osdmap e124: 8 total, 6 up, 8 in 2023-09-23T22:40:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:46.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:46.095 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:46.096 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:46.155 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:46.184 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:46.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:46.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:46.382 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:46.409 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:46.452 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:46.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:46.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:46.555 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:46.590 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:46.638 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:46.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:46.690 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:40:46.742 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:46.772 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:46.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: pgmap v289: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: failed to encode map e124 with expected crc 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: failed to encode map e124 with expected crc 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: failed to encode map e124 with expected crc 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: failed to encode map e124 with expected crc 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:40:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:46.966 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 77 16 101 85 0.0689849 0 - 0.00253822 2023-09-23T22:40:47.037 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:47.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: pgmap v289: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:47.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: pgmap v289: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: failed to encode map e124 with expected crc 2023-09-23T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:47.373 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:47.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:47.508 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:47.706 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:47.810 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:47.843 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:47.950 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:47.956 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:47.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:47.966 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 78 16 101 85 0.0681003 0 - 0.00253822 2023-09-23T22:40:47.970 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:47.988 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:48.008 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:48.026 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:48.043 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.044 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.059 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:47 smithi052 ceph-mon[187743]: pgmap v291: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:48.059 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:47 smithi052 ceph-mon[187743]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T22:40:48.059 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:47 smithi052 conmon[258962]: 2023-09-23T22:40:47.891+0000 7f4f2b2ff640 -1 osd.0 95 log_to_monitors true 2023-09-23T22:40:48.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:47 smithi052 ceph-mon[180979]: pgmap v291: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:48.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:47 smithi052 ceph-mon[180979]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T22:40:48.060 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:48.090 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:48.178 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:47 smithi151 ceph-mon[159210]: pgmap v291: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:48.178 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:47 smithi151 ceph-mon[159210]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T22:40:48.185 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:48.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:48.269 INFO:teuthology.orchestra.run.smithi052.stderr:no valid command found; 10 closest matches: 2023-09-23T22:40:48.269 INFO:teuthology.orchestra.run.smithi052.stderr:0 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:1 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:2 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:abort 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:assert 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs debug_inject_read_zeros 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs files list 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs stats 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:bluestore allocator dump block 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:bluestore allocator fragmentation block 2023-09-23T22:40:48.270 INFO:teuthology.orchestra.run.smithi052.stderr:admin_socket: invalid command 2023-09-23T22:40:48.288 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:48.319 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:48.392 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:40:48.392 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:40:48.392 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:40:48.392 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:40:48.392 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:40:48.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:48.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:48.836 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:48.874 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:48.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:48 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:40:48.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4121634055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:48.905 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:48.967 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 79 16 101 85 0.0672382 0 - 0.00253822 2023-09-23T22:40:48.998 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:40:49.012 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:49.118 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:49.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:48 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:40:49.144 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4121634055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:49.144 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:48 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:40:49.144 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4121634055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:49.145 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:40:48 smithi052 conmon[258962]: 2023-09-23T22:40:48.906+0000 7f4f19766700 -1 osd.0 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:40:49.147 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.295 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:49.353 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:40:49.363 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2023-09-23T22:40:49.399 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:49.426 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.430 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:40:49.470 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:49.574 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:49.601 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.651 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:49.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:49.758 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:49.786 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.802 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:49.967 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:40:49.967393+0000 min lat: 0.00121387 max lat: 0.00607929 avg lat: 0.00253822 2023-09-23T22:40:49.967 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:40:49.967 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 80 16 101 85 0.0663975 0 - 0.00253822 2023-09-23T22:40:49.982 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:50.085 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:50.115 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: pgmap v292: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: osdmap e125: 8 total, 6 up, 8 in 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[180979]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: pgmap v292: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: osdmap e125: 8 total, 6 up, 8 in 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:50.194 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:50.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:49 smithi052 ceph-mon[187743]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: pgmap v292: 89 pgs: 5 active+clean+remapped, 38 active+clean, 26 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53/327 objects degraded (16.208%); 9/327 objects misplaced (2.752%) 2023-09-23T22:40:50.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: osdmap e125: 8 total, 6 up, 8 in 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:49 smithi151 ceph-mon[159210]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.210 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:50.314 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:50.341 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.417 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:50.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:50.520 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:50.548 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.600 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:50.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:50.704 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:50.733 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.827 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:50.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:50.838 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:50.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: failed to encode map e125 with expected crc 2023-09-23T22:40:50.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:40:50.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: failed to encode map e126 with expected crc 2023-09-23T22:40:50.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: failed to encode map e126 with expected crc 2023-09-23T22:40:50.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080] boot 2023-09-23T22:40:50.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T22:40:50.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: failed to encode map e126 with expected crc 2023-09-23T22:40:50.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: failed to encode map e126 with expected crc 2023-09-23T22:40:50.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: failed to encode map e126 with expected crc 2023-09-23T22:40:50.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:40:50.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:50 smithi151 ceph-mon[159210]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:40 3.14 3:2e2fa760:::200.00000001:head [write 892~392 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e70)", 2023-09-23T22:40:50.930 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:31:02.957079+0000", 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 587.96858300300005, 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043268810000000003, 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957079+0000", 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957079+0000", 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957076+0000", 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8077e-05 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957094+0000", 2023-09-23T22:40:50.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5945e-05 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957120+0000", 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0684e-05 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957131+0000", 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7071999999999995e-05 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957228+0000", 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3030000000000005e-06 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957237+0000", 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018836770000000001 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.959121+0000", 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0195e-05 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.959211+0000", 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3368000000000005e-05 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.959305+0000", 2023-09-23T22:40:50.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011998569999999999 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.960504+0000", 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074673100000000004 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961251+0000", 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4182000000000003e-05 2023-09-23T22:40:50.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961315+0000", 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2167999999999999e-05 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961367+0000", 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8475e-05 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961406+0000", 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043268810000000003 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:41 3.14 3:292cf221:::200.00000000:head [writefull 0~90 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e70)", 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:31:02.957152+0000", 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 587.96851019999997, 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043600519999999997, 2023-09-23T22:40:50.938 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957152+0000", 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957152+0000", 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:50.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957151+0000", 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2639999999999997e-06 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957157+0000", 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1659e-05 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957169+0000", 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0618999999999999e-05 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957180+0000", 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045094500000000001 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957631+0000", 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5889999999999999e-06 2023-09-23T22:40:50.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.957637+0000", 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020815090000000001 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.959719+0000", 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6348999999999997e-05 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.959795+0000", 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012457200000000001 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.959920+0000", 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00058789900000000002 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.960508+0000", 2023-09-23T22:40:50.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000778741 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961286+0000", 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015301400000000001 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961439+0000", 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9539999999999998e-05 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961479+0000", 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3000000000000003e-05 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961512+0000", 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043600519999999997 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:42 3.14 3:2e2fa760:::200.00000001:head [write 1284~392 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e70)", 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:31:02.958346+0000", 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 587.96731560700005, 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003405267, 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958346+0000", 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958346+0000", 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958345+0000", 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0319999999999992e-06 2023-09-23T22:40:50.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958354+0000", 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2602999999999999e-05 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958367+0000", 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9169999999999995e-06 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958376+0000", 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8331000000000003e-05 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958414+0000", 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5339999999999999e-06 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:40:50.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.958420+0000", 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001715692 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.960135+0000", 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2062000000000002e-05 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.960207+0000", 2023-09-23T22:40:50.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013177999999999999 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.960339+0000", 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034557300000000001 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.960685+0000", 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00094013899999999999 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961625+0000", 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3893999999999997e-05 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961669+0000", 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9122999999999997e-05 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961718+0000", 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3812000000000001e-05 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:31:02.961752+0000", 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003405267 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:43 3.14 3:2e2fa760:::200.00000001:head [write 1676~141 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e73)", 2023-09-23T22:40:50.953 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:09.541121+0000", 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 521.38454130599996, 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010924265000000001, 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541121+0000", 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541121+0000", 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999895 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541110+0000", 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4786999999999997e-05 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541145+0000", 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5936000000000002e-05 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541191+0000", 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7750000000000001e-05 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541209+0000", 2023-09-23T22:40:50.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000167145 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541376+0000", 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1005999999999998e-05 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541397+0000", 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068469840000000004 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548244+0000", 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103303 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548347+0000", 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021690199999999999 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.548564+0000", 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013603140000000001 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549924+0000", 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9870000000000002e-06 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549931+0000", 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020436870000000002 2023-09-23T22:40:50.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.551975+0000", 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1854000000000003e-05 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552017+0000", 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8220000000000001e-05 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552045+0000", 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010924265000000001 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:44 3.14 3:292cf221:::200.00000000:head [writefull 0~90 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e73)", 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:09.541242+0000", 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 521.38442011999996, 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011064210999999999, 2023-09-23T22:40:50.961 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541242+0000", 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541242+0000", 2023-09-23T22:40:50.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541241+0000", 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1103e-05 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541252+0000", 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5523999999999999e-05 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541268+0000", 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3090000000000001e-06 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541277+0000", 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00044255 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541719+0000", 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1110000000000008e-06 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541727+0000", 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073478609999999998 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549075+0000", 2023-09-23T22:40:50.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7069000000000004e-05 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549142+0000", 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013171299999999999 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549274+0000", 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00066209899999999998 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549936+0000", 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000778596 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.550715+0000", 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015017870000000001 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552217+0000", 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0352999999999998e-05 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552257+0000", 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9203000000000003e-05 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552306+0000", 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011064210999999999 2023-09-23T22:40:50.968 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:45 3.14 3:2e2fa760:::200.00000001:head [write 1817~141 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e73)", 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:09.541439+0000", 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 521.38422270199999, 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010996471000000001, 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541439+0000", 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541439+0000", 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541438+0000", 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3482999999999999e-05 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541461+0000", 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4122e-05 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541485+0000", 2023-09-23T22:40:50.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0293999999999999e-05 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.541506+0000", 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00067068599999999998 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.542176+0000", 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1070000000000001e-06 2023-09-23T22:40:50.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.542179+0000", 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.007332631 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549512+0000", 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0240000000000005e-05 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549582+0000", 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013484400000000001 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.549717+0000", 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00038439099999999999 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.550102+0000", 2023-09-23T22:40:50.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000773349 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.550875+0000", 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015024890000000001 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552377+0000", 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4904999999999999e-05 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552412+0000", 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.355e-05 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:09.552436+0000", 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010996471000000001 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:48 3.14 3:2e2fa760:::200.00000001:head [write 2742~141 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e75)", 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:49.366658+0000", 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 481.55900387100002, 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011250532000000001, 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366658+0000", 2023-09-23T22:40:50.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366658+0000", 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366655+0000", 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5832e-05 2023-09-23T22:40:50.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366671+0000", 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6478999999999998e-05 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366698+0000", 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3823e-05 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366711+0000", 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8838000000000006e-05 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366790+0000", 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8669999999999999e-06 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366796+0000", 2023-09-23T22:40:50.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0091129590000000003 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.375909+0000", 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1158999999999999e-05 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.376000+0000", 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011885 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.376119+0000", 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016104069999999999 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377729+0000", 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4287999999999999e-05 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377744+0000", 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7764999999999999e-05 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377811+0000", 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5795999999999999e-05 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377867+0000", 2023-09-23T22:40:50.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1439000000000001e-05 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377909+0000", 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011250532000000001 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:49 3.14 3:292cf221:::200.00000000:head [writefull 0~90 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e75)", 2023-09-23T22:40:50.984 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:49.366736+0000", 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 481.55892652300003, 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011652747, 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366736+0000", 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366736+0000", 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366735+0000", 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1311999999999999e-05 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366746+0000", 2023-09-23T22:40:50.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9289999999999998e-06 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366754+0000", 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2499999999999997e-06 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366759+0000", 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00044191400000000001 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.367201+0000", 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4340000000000001e-06 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.367205+0000", 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.009510526 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.376715+0000", 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3979000000000004e-05 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.376789+0000", 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013548499999999999 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.376925+0000", 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00082715800000000004 2023-09-23T22:40:50.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377752+0000", 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00036682199999999999 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.378119+0000", 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016872999999999999 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.378287+0000", 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8869000000000002e-05 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.378346+0000", 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2058000000000003e-05 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.378388+0000", 2023-09-23T22:40:50.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011652747 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(mds.0.4:50 3.14 3:2e2fa760:::200.00000001:head [write 2883~141 [fadvise_dontneed]] snapc 0=[] ondisk+write+known_if_redirected+full_force e75)", 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:32:49.366780+0000", 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 481.55888176399998, 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012255416999999999, 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:50.992 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366780+0000", 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366780+0000", 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:50.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366780+0000", 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8460000000000002e-06 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366787+0000", 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7340000000000008e-06 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366794+0000", 2023-09-23T22:40:50.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1600000000000003e-06 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.366800+0000", 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00086442099999999998 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.367665+0000", 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8890000000000002e-06 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.367668+0000", 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095544210000000004 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377222+0000", 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0848999999999998e-05 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:50.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377303+0000", 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014450199999999999 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377448+0000", 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052343099999999998 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.377971+0000", 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097354899999999999 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.378945+0000", 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4403000000000002e-05 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.378979+0000", 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2563e-05 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.379011+0000", 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4244999999999999e-05 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:32:49.379036+0000", 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012255416999999999 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:50.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:9 6.2 6:4f4f63f6:::benchmark_data_smithi052_234456_object8:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.957986+0000", 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.967675646000004, 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028698249999999999, 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 9 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.957986+0000", 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.957986+0000", 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.957985+0000", 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00032033300000000001 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958305+0000", 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6512999999999999e-05 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958322+0000", 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2779999999999996e-06 2023-09-23T22:40:51.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958329+0000", 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000109211 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958438+0000", 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014654100000000001 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958585+0000", 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000149152 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958734+0000", 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001004451 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959738+0000", 2023-09-23T22:40:51.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0009842710000000001 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960723+0000", 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9027999999999999e-05 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960772+0000", 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4575999999999998e-05 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960856+0000", 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028698249999999999 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:15 6.e 6:794f52d6:::benchmark_data_smithi052_234456_object14:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.958348+0000", 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.967314262000002, 2023-09-23T22:40:51.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021853559999999998, 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 15 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958348+0000", 2023-09-23T22:40:51.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958348+0000", 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958347+0000", 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000190507 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958538+0000", 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4538999999999999e-05 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958552+0000", 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7229999999999998e-06 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958559+0000", 2023-09-23T22:40:51.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7259999999999997e-05 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958616+0000", 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4877999999999998e-05 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958681+0000", 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100481 2023-09-23T22:40:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1", 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.958781+0000", 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010025279999999999 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.959784+0000", 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00055585200000000002 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960340+0000", 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8041999999999994e-05 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960438+0000", 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5353999999999995e-05 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.960533+0000", 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021853559999999998 2023-09-23T22:40:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:21 6.e 6:76e46e7e:::benchmark_data_smithi052_234456_object20:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.961055+0000", 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.964607478000005, 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048081900000000004, 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 21 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961055+0000", 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961055+0000", 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961053+0000", 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1559999999999999e-05 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961125+0000", 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4202999999999999e-05 2023-09-23T22:40:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961139+0000", 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9949999999999999e-06 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961146+0000", 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0810999999999998e-05 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961207+0000", 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017680699999999999 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961384+0000", 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000210836 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1", 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961595+0000", 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00090154299999999998 2023-09-23T22:40:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962496+0000", 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032532300000000002 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965749+0000", 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7448000000000003e-05 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965807+0000", 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6042000000000003e-05 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965863+0000", 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048081900000000004 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:26 6.e 6:7d2b294c:::benchmark_data_smithi052_234456_object25:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.961542+0000", 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.964119957999998, 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044336260000000004, 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 26 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961542+0000", 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961542+0000", 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961541+0000", 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00022915199999999999 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961770+0000", 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5335000000000001e-05 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961786+0000", 2023-09-23T22:40:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5899999999999996e-06 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961792+0000", 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6161000000000005e-05 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.961868+0000", 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013634699999999999 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962005+0000", 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000203802 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1", 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962208+0000", 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047572800000000002 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.962684+0000", 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032157169999999999 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965900+0000", 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9544999999999999e-05 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965929+0000", 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6258999999999999e-05 2023-09-23T22:40:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965976+0000", 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044336260000000004 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:32 6.2 6:427b73cc:::benchmark_data_smithi052_234456_object31:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.964461+0000", 2023-09-23T22:40:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.961201087000006, 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028979589999999999, 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 32 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964461+0000", 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964461+0000", 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964460+0000", 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000137346 2023-09-23T22:40:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964597+0000", 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3981999999999999e-05 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964611+0000", 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6919999999999996e-06 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964618+0000", 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0689999999999998e-05 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964658+0000", 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9105999999999996e-05 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964757+0000", 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000159999 2023-09-23T22:40:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.964917+0000", 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00061631100000000003 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.965534+0000", 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001705453 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.967239+0000", 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3566000000000003e-05 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.967283+0000", 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6137999999999995e-05 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.967359+0000", 2023-09-23T22:40:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028979589999999999 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:53 6.2 6:4c9d277a:::benchmark_data_smithi052_234456_object52:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.972196+0000", 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.953465651000002, 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002076465, 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 53 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972196+0000", 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972196+0000", 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972195+0000", 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020228600000000001 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972397+0000", 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4097000000000001e-05 2023-09-23T22:40:51.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972411+0000", 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4649999999999997e-06 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972419+0000", 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6076999999999999e-05 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972475+0000", 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000142158 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972617+0000", 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014646599999999999 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.972763+0000", 2023-09-23T22:40:51.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00042609100000000002 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.973189+0000", 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000935085 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974125+0000", 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5492999999999999e-05 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974180+0000", 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2844999999999996e-05 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974273+0000", 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002076465 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:59 6.2 6:4062cc28:::benchmark_data_smithi052_234456_object58:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.974211+0000", 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.951450820999995, 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021444979999999999, 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 59 2023-09-23T22:40:51.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974211+0000", 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974211+0000", 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.041 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974210+0000", 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000101964 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974312+0000", 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0924000000000001e-05 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.042 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974323+0000", 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0309999999999998e-06 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974328+0000", 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1768000000000003e-05 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974380+0000", 2023-09-23T22:40:51.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017175799999999999 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974552+0000", 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016554699999999999 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.974717+0000", 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047423799999999998 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975192+0000", 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010247819999999999 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976216+0000", 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3640000000000001e-05 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976270+0000", 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5607999999999998e-05 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976356+0000", 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021444979999999999 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.046 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:63 6.2 6:4ffea29d:::benchmark_data_smithi052_234456_object62:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e105)", 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:39:29.975195+0000", 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.950467275999998, 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021721969999999998, 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 63 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.047 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975195+0000", 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975195+0000", 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.048 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975194+0000", 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2029999999999999e-05 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975276+0000", 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2361000000000001e-05 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975288+0000", 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0079999999999996e-06 2023-09-23T22:40:51.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975295+0000", 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9798e-05 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975345+0000", 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110713 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975456+0000", 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019008999999999999 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.975646+0000", 2023-09-23T22:40:51.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00038242499999999998 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.976028+0000", 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001217489 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977246+0000", 2023-09-23T22:40:51.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8331999999999997e-05 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977294+0000", 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3093e-05 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:39:29.977367+0000", 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021721969999999998 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.053 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:104 6.7 e127/99 6:edc6c5e9:::benchmark_data_smithi052_234456_object103:head v 127'6, mlcod=105'4)", 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:50.920866+0000", 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0047957349999999998, 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000504895, 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920866+0000", 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920866+0000", 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920866+0000", 2023-09-23T22:40:51.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4559e-05 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920891+0000", 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0773e-05 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920901+0000", 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7010000000000001e-06 2023-09-23T22:40:51.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920907+0000", 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0927000000000003e-05 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920948+0000", 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8093000000000001e-05 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.920966+0000", 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000379668 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921346+0000", 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5524000000000002e-05 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921371+0000", 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000504895 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:110 6.e 6:75d63eae:::benchmark_data_smithi052_234456_object109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:50.921397+0000", 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0042649189999999998, 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022887210000000001, 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 110 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921397+0000", 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921397+0000", 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921396+0000", 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013730700000000001 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921533+0000", 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2157000000000001e-05 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921545+0000", 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.502e-06 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921551+0000", 2023-09-23T22:40:51.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9282e-05 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921600+0000", 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2385999999999994e-05 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921683+0000", 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000104917 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.921787+0000", 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000431381 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922219+0000", 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00040297500000000002 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922622+0000", 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000860277 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.923482+0000", 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6056000000000006e-05 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.923548+0000", 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013772799999999999 2023-09-23T22:40:51.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.923686+0000", 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022887210000000001 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:107 6.7 e127/99 6:e46c5406:::benchmark_data_smithi052_234456_object106:head v 127'7, mlcod=105'4)", 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:50.922825+0000", 2023-09-23T22:40:51.066 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0028375520000000001, 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00049918, 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922825+0000", 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:40:51.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922825+0000", 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922824+0000", 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8104000000000001e-05 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922852+0000", 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1855000000000001e-05 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922864+0000", 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9680000000000003e-06 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922870+0000", 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1746999999999999e-05 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:40:51.069 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922911+0000", 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9324999999999999e-05 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.922931+0000", 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035397 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.923285+0000", 2023-09-23T22:40:51.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9072000000000001e-05 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:50.923324+0000", 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00049918 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:40:51.071 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:40:51.072 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:51.072 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 81 16 284 268 0.206762 0.142969 0.001941 4.8363 2023-09-23T22:40:51.106 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: failed to encode map e125 with expected crc 2023-09-23T22:40:51.106 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:40:51.106 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080] boot 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[180979]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: failed to encode map e125 with expected crc 2023-09-23T22:40:51.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: osd.0 [v2:172.21.15.52:6802/1943329080,v1:172.21.15.52:6803/1943329080] boot 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: failed to encode map e126 with expected crc 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:40:51.108 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:50 smithi052 ceph-mon[187743]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:51.322 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:51.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:51.427 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:51.446 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:51.455 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:51.530 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:51.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:51.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:51.634 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:51.680 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:51.814 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:40:51.829 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:51.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:51.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:51.912 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:40:51.937 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:51.967 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 82 16 3069 3053 2.32666 174.062 0.00809445 0.429784 2023-09-23T22:40:51.995 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:52.099 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: pgmap v295: 89 pgs: 1 remapped+peering, 7 peering, 4 active+clean+remapped, 38 active+clean, 23 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 39/327 objects degraded (11.927%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:52.099 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.099 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 39/327 objects degraded (11.927%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/181832415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/181832415' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[187743]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T22:40:52.100 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: pgmap v295: 89 pgs: 1 remapped+peering, 7 peering, 4 active+clean+remapped, 38 active+clean, 23 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 39/327 objects degraded (11.927%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 39/327 objects degraded (11.927%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/181832415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:52.101 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/181832415' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:52.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:51 smithi052 ceph-mon[180979]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T22:40:52.110 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:52.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: pgmap v295: 89 pgs: 1 remapped+peering, 7 peering, 4 active+clean+remapped, 38 active+clean, 23 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 5.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 39/327 objects degraded (11.927%); 5/327 objects misplaced (1.529%) 2023-09-23T22:40:52.121 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: failed to encode map e127 with expected crc 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 39/327 objects degraded (11.927%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/181832415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/181832415' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:52.122 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:51 smithi151 ceph-mon[159210]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T22:40:52.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:52.224 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:52.278 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:52.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:52.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:52.497 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:52.548 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:52.614 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:52.703 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:52.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:52.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:52.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:52.862 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:52.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 83 16 6795 6779 5.10397 232.875 0.00326224 0.195916 2023-09-23T22:40:52.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:52 smithi151 ceph-mon[159210]: failed to encode map e128 with expected crc 2023-09-23T22:40:52.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:52 smithi151 ceph-mon[159210]: failed to encode map e128 with expected crc 2023-09-23T22:40:52.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:52 smithi151 ceph-mon[159210]: failed to encode map e128 with expected crc 2023-09-23T22:40:52.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:52 smithi151 ceph-mon[159210]: failed to encode map e128 with expected crc 2023-09-23T22:40:52.972 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:52 smithi151 ceph-mon[159210]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:53.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[187743]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[187743]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[187743]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[187743]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[187743]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[180979]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[180979]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[180979]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[180979]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:52 smithi052 ceph-mon[180979]: failed to encode map e128 with expected crc 2023-09-23T22:40:53.307 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:53.349 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:53.390 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:53.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:53.494 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:53.547 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:53.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:40:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:40:53] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T22:40:53.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 84 16 9398 9382 6.9797 162.688 0.00360477 0.143266 2023-09-23T22:40:54.007 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:54.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:54.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:54.112 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:54.159 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:54.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:53 smithi151 ceph-mon[159210]: pgmap v298: 89 pgs: 1 remapped+peering, 8 peering, 3 active+clean+remapped, 39 active+clean, 22 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 24 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s wr, 72 op/s; 173/1194 objects degraded (14.489%); 5/1194 objects misplaced (0.419%) 2023-09-23T22:40:54.230 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:54.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:54.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:54.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:53 smithi052 ceph-mon[180979]: pgmap v298: 89 pgs: 1 remapped+peering, 8 peering, 3 active+clean+remapped, 39 active+clean, 22 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 24 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s wr, 72 op/s; 173/1194 objects degraded (14.489%); 5/1194 objects misplaced (0.419%) 2023-09-23T22:40:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:53 smithi052 ceph-mon[187743]: pgmap v298: 89 pgs: 1 remapped+peering, 8 peering, 3 active+clean+remapped, 39 active+clean, 22 active+undersized, 5 undersized+peered, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 24 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s wr, 72 op/s; 173/1194 objects degraded (14.489%); 5/1194 objects misplaced (0.419%) 2023-09-23T22:40:54.333 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:54.364 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:40:54.366 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:54.521 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:54.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:54.594 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:40:54.614 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:40:54.626 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:54.637 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:40:54.666 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:54.673 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:40:54.711 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:40:54.747 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:40:54.785 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.786 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.914 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:54.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:54.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 85 16 11440 11424 8.39885 127.625 0.0047979 0.119056 2023-09-23T22:40:55.019 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:55.069 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:55.182 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.193 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3982866551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:55.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:55.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3982866551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3982866551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:55.294 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:55.346 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:55.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:55.537 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:40:55.568 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:55.588 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:55.588 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:55.611 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:40:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:40:55.715 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:55.758 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:55.858 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:55.937 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:55 smithi052 ceph-mon[180979]: pgmap v299: 89 pgs: 1 remapped+peering, 8 peering, 3 active+clean+remapped, 52 active+clean, 16 active+undersized, 9 active+undersized+degraded; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 1948/11985 objects degraded (16.254%); 5/11985 objects misplaced (0.042%) 2023-09-23T22:40:55.937 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:55 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive) 2023-09-23T22:40:55.937 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:55 smithi151 ceph-mon[159210]: pgmap v299: 89 pgs: 1 remapped+peering, 8 peering, 3 active+clean+remapped, 52 active+clean, 16 active+undersized, 9 active+undersized+degraded; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 1948/11985 objects degraded (16.254%); 5/11985 objects misplaced (0.042%) 2023-09-23T22:40:55.937 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:55 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive) 2023-09-23T22:40:55.963 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:55.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 86 16 14157 14141 10.2755 169.812 0.00508117 0.0973125 2023-09-23T22:40:56.014 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:56.111 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:56.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:56.160 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:40:56.175 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:56.202 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:55 smithi052 ceph-mon[187743]: pgmap v299: 89 pgs: 1 remapped+peering, 8 peering, 3 active+clean+remapped, 52 active+clean, 16 active+undersized, 9 active+undersized+degraded; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 1948/11985 objects degraded (16.254%); 5/11985 objects misplaced (0.042%) 2023-09-23T22:40:56.202 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:55 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive) 2023-09-23T22:40:56.203 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:56.204 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:56.217 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:56.272 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:56.324 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:56.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:56.427 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:56.467 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:56.518 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:56.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:56.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:56.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:56.680 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:56.883 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:56.888 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:40:56.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:56.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 87 16 16245 16229 11.6572 130.5 0.00673293 0.0857788 2023-09-23T22:40:56.990 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:40:57.052 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:56 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4446/31440 objects degraded (14.141%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:57.203 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:40:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:57.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:40:57.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:56 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4446/31440 objects degraded (14.141%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:57.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:56 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4446/31440 objects degraded (14.141%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:57.309 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:57.336 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.368 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:57.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:57.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:57.490 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:57.604 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:57.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:57.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:57.787 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:57.813 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:57.866 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 88 15 18488 18473 13.1182 140.25 0.0062153 0.0762238 2023-09-23T22:40:58.070 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:58.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:58.087 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:57 smithi052 ceph-mon[180979]: pgmap v300: 89 pgs: 1 peering, 3 active+clean+remapped, 60 active+clean, 16 active+undersized, 9 active+undersized+degraded; 654 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 4446/31440 objects degraded (14.141%); 835/31440 objects misplaced (2.656%); 102 KiB/s, 1 objects/s recovering 2023-09-23T22:40:58.087 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:57 smithi052 ceph-mon[187743]: pgmap v300: 89 pgs: 1 peering, 3 active+clean+remapped, 60 active+clean, 16 active+undersized, 9 active+undersized+degraded; 654 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 4446/31440 objects degraded (14.141%); 835/31440 objects misplaced (2.656%); 102 KiB/s, 1 objects/s recovering 2023-09-23T22:40:58.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:58.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:57 smithi151 ceph-mon[159210]: pgmap v300: 89 pgs: 1 peering, 3 active+clean+remapped, 60 active+clean, 16 active+undersized, 9 active+undersized+degraded; 654 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 4446/31440 objects degraded (14.141%); 835/31440 objects misplaced (2.656%); 102 KiB/s, 1 objects/s recovering 2023-09-23T22:40:58.205 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:58.233 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:58.946 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:58.946 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:58.968 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 89 16 20396 20380 14.3099 119.188 0.00182234 0.0698144 2023-09-23T22:40:59.026 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:59.134 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:59.174 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:59.221 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:59.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:59.325 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:59.352 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:59.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:59.547 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:59.615 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:59.735 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:59.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.740 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:59.741 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:59.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:59.842 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:59.871 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total time run: 90.0024 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total writes made: 22141 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Write size: 65536 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object size: 65536 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Bandwidth (MB/sec): 15.3753 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Bandwidth: 46.3615 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max bandwidth (MB/sec): 232.875 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:40:59.969 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average IOPS: 246 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev IOPS: 741.786 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max IOPS: 3726 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min IOPS: 0 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average Latency(s): 0.0650355 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Latency(s): 2.17533 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max latency(s): 80.9656 2023-09-23T22:40:59.970 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min latency(s): 0.00105021 2023-09-23T22:40:59.971 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:41:00.272 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:40:59 smithi052 ceph-mon[180979]: pgmap v301: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 758 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5343/36450 objects degraded (14.658%); 835/36450 objects misplaced (2.291%); 88 KiB/s, 1 objects/s recovering 2023-09-23T22:41:00.273 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:40:59 smithi052 ceph-mon[187743]: pgmap v301: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 758 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5343/36450 objects degraded (14.658%); 835/36450 objects misplaced (2.291%); 88 KiB/s, 1 objects/s recovering 2023-09-23T22:41:00.349 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:00.356 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T22:41:00.356 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T22:41:00.356 DEBUG:teuthology.orchestra.run.smithi052:osd.2> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T22:41:00.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:00.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:00.405 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl start ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:41:00.434 INFO:journalctl@ceph.osd.2.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:41:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:40:59 smithi151 ceph-mon[159210]: pgmap v301: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 758 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5343/36450 objects degraded (14.658%); 835/36450 objects misplaced (2.291%); 88 KiB/s, 1 objects/s recovering 2023-09-23T22:41:00.460 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:00.464 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:00.486 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:00.528 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:00.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:00.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:00.535 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:00 smithi052 systemd[1]: Starting Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:41:00.632 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:00.680 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:00.950 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:00.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:00.964 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:41:01.063 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:01.114 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.222 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:01.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:01.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:01.333 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:01.388 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.638 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:02.020 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:41:02.110 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:02.182 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:02 smithi151 ceph-mon[159210]: pgmap v302: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 103 B/s rd, 131 MiB/s wr, 2.15k op/s; 9018/61260 objects degraded (14.721%); 1303/61260 objects misplaced (2.127%); 71 KiB/s, 1 objects/s recovering 2023-09-23T22:41:02.182 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3134849023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:02.182 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:02 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:02.214 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:02.261 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.289 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:01 smithi052 ceph-mon[187743]: pgmap v302: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 103 B/s rd, 131 MiB/s wr, 2.15k op/s; 9018/61260 objects degraded (14.721%); 1303/61260 objects misplaced (2.127%); 71 KiB/s, 1 objects/s recovering 2023-09-23T22:41:02.290 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3134849023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:02.290 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:02.290 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:01 smithi052 ceph-mon[180979]: pgmap v302: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 103 B/s rd, 131 MiB/s wr, 2.15k op/s; 9018/61260 objects degraded (14.721%); 1303/61260 objects misplaced (2.127%); 71 KiB/s, 1 objects/s recovering 2023-09-23T22:41:02.290 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3134849023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:02.290 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:02.351 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:02.455 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:02.506 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.591 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:02.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:02.698 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:02.730 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:03.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[180979]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.346 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:03 smithi052 ceph-mon[187743]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T22:41:03.346 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:41:03.347 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:03 smithi052 bash[263849]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T22:41:03.411 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:03.411 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.411 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:03.411 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T22:41:03.412 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.412 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.412 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:03 smithi151 ceph-mon[159210]: failed to encode map e129 with expected crc 2023-09-23T22:41:03.534 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:03.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:03.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:03.642 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:03.673 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:41:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:41:03] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T22:41:03.768 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:03.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:03.872 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:03.905 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.945 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:03.945 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:04.227 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:04.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:04 smithi052 ceph-mon[180979]: pgmap v304: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 133 MiB/s wr, 2.21k op/s; 9193/62313 objects degraded (14.753%); 1303/62313 objects misplaced (2.091%); 70 KiB/s, 1 objects/s recovering 2023-09-23T22:41:04.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:04 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 9193/62313 objects degraded (14.753%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:04.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:04 smithi052 ceph-mon[187743]: pgmap v304: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 133 MiB/s wr, 2.21k op/s; 9193/62313 objects degraded (14.753%); 1303/62313 objects misplaced (2.091%); 70 KiB/s, 1 objects/s recovering 2023-09-23T22:41:04.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:04 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 9193/62313 objects degraded (14.753%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:04.333 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:04.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:04 smithi151 ceph-mon[159210]: pgmap v304: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 133 MiB/s wr, 2.21k op/s; 9193/62313 objects degraded (14.753%); 1303/62313 objects misplaced (2.091%); 70 KiB/s, 1 objects/s recovering 2023-09-23T22:41:04.374 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:04 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 9193/62313 objects degraded (14.753%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:04.381 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.453 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:04.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:04.559 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:04.585 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.637 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:04.637 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:41:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:04.657 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:41:04.678 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:41:04.700 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:41:04.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:04.741 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:41:04.774 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:41:04.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.818 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:41:04.819 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:04.819 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:04.819 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:41:04.820 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:41:04.820 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:04.820 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:04.820 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:41:04.820 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:05.012 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:05.022 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:41:05.122 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:05.209 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:05.336 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:05.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:05.516 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:05.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:05.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:05.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2345835135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:05.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[187743]: pgmap v305: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 114 MiB/s wr, 2.22k op/s; 7960/54381 objects degraded (14.637%); 1303/54381 objects misplaced (2.396%); 70 KiB/s, 1 objects/s recovering 2023-09-23T22:41:05.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:05.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:05.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2345835135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:05.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:05 smithi052 ceph-mon[180979]: pgmap v305: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 114 MiB/s wr, 2.22k op/s; 7960/54381 objects degraded (14.637%); 1303/54381 objects misplaced (2.396%); 70 KiB/s, 1 objects/s recovering 2023-09-23T22:41:05.631 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:05.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:05.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:05.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2345835135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:05.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:05 smithi151 ceph-mon[159210]: pgmap v305: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 102 B/s rd, 114 MiB/s wr, 2.22k op/s; 7960/54381 objects degraded (14.637%); 1303/54381 objects misplaced (2.396%); 70 KiB/s, 1 objects/s recovering 2023-09-23T22:41:05.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:05.742 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:05.745 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:05.784 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:05.883 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:05.949 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:05 smithi052 bash[264647]: d77ee82fce9a6952037c717636df1ebd96f68fc4c493d1b73c852d6dac94f0d6 2023-09-23T22:41:05.949 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:05 smithi052 conmon[264731]: 2023-09-23T22:41:05.945+0000 7fef14818640 -1 Falling back to public interface 2023-09-23T22:41:05.986 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:06.045 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:06.048 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:41:06.074 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:41:06.243 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:06 smithi052 systemd[1]: Started Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:41:06.444 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:06.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:06.454 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:06.552 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:06.594 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:06.663 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:06.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:06.733 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:06.768 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:06.802 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:06.807 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:06.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:06.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:06.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:06.965 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:06.995 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:07.048 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:07.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:07.151 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:07.192 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:07.480 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:07.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:07.588 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:07.650 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:07.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:07.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:07.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:07.912 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:07.942 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:37916 6.1 e129/104 6:8e9239fa:::benchmark_data_smithi052_234456_object15772:head v 129'2438, mlcod=129'2436)", 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:08.042319+0000", 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.001365387, 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013925229999999999, 2023-09-23T22:41:08.050 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042319+0000", 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042319+0000", 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042318+0000", 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1471000000000001e-05 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042329+0000", 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1514999999999999e-05 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042361+0000", 2023-09-23T22:41:08.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4309999999999997e-06 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042368+0000", 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1161000000000006e-05 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042429+0000", 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1937999999999999e-05 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:08.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.042451+0000", 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132368 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:37919 6.a e129/126 6:5fb9def3:::benchmark_data_smithi052_234456_object15775:head v 129'2416, mlcod=129'2414)", 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:08.043073+0000", 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.00061165700000000004, 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071842599999999998, 2023-09-23T22:41:08.054 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043073+0000", 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043073+0000", 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:08.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043072+0000", 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5589999999999994e-06 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043081+0000", 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3293e-05 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:08.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043094+0000", 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7360000000000004e-06 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043101+0000", 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3182000000000001e-05 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043144+0000", 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8065999999999999e-05 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:08.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.043162+0000", 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9839000000000005e-05 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 2 2023-09-23T22:41:08.058 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:41:08.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:07 smithi151 ceph-mon[159210]: pgmap v306: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 805 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 73 MiB/s wr, 2.09k op/s; 5991/38700 objects degraded (15.481%); 653/38700 objects misplaced (1.687%) 2023-09-23T22:41:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:08.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:08.160 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:08.188 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:07 smithi052 ceph-mon[187743]: pgmap v306: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 805 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 73 MiB/s wr, 2.09k op/s; 5991/38700 objects degraded (15.481%); 653/38700 objects misplaced (1.687%) 2023-09-23T22:41:08.188 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:07 smithi052 ceph-mon[180979]: pgmap v306: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 805 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 73 MiB/s wr, 2.09k op/s; 5991/38700 objects degraded (15.481%); 653/38700 objects misplaced (1.687%) 2023-09-23T22:41:08.191 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:08.454 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:08.836 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:08 smithi052 conmon[264731]: 2023-09-23T22:41:08.693+0000 7fef14818640 -1 osd.2 103 log_to_monitors true 2023-09-23T22:41:09.008 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:09.105 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:08 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:41:09.105 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:08 smithi151 ceph-mon[159210]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:41:09.114 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:09.131 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:41:09.159 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:08 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:41:09.159 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:08 smithi052 ceph-mon[180979]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:41:09.160 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:08 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:41:09.160 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:08 smithi052 ceph-mon[187743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:41:09.160 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:41:08 smithi052 conmon[264731]: 2023-09-23T22:41:08.921+0000 7fef02c7f700 -1 osd.2 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:41:09.163 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.464 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:09.574 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:09.641 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.718 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:09.763 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:09.859 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:09.872 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:09.898 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: pgmap v307: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 723 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 63 MiB/s wr, 2.06k op/s; 5282/34743 objects degraded (15.203%); 653/34743 objects misplaced (1.880%) 2023-09-23T22:41:09.898 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5282/34743 objects degraded (15.203%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: failed to encode map e130 with expected crc 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: osdmap e130: 8 total, 7 up, 8 in 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: failed to encode map e130 with expected crc 2023-09-23T22:41:09.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:09 smithi151 ceph-mon[159210]: failed to encode map e130 with expected crc 2023-09-23T22:41:09.901 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.995 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40792 6.f 6.8b4b57ef (undecoded) ondisk+retry+write+known_if_redirected e131)", 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.134351584, 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.13438618099999999, 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "delayed", 2023-09-23T22:41:09.997 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40792 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:09.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861820+0000", 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:09.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861828+0000", 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861841+0000", 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2709e-05 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861851+0000", 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861957+0000", 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106518 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861972+0000", 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5226e-05 2023-09-23T22:41:10.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875022+0000", 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013050091999999999 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875034+0000", 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1232e-05 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:10.002 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40806 6.f 6.e80b2dbf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.13429247999999999, 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.13443370399999999, 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "delayed", 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:10.003 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40806 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861880+0000", 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861885+0000", 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.561e-06 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:10.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862005+0000", 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120701 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862017+0000", 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862139+0000", 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012248000000000001 2023-09-23T22:41:10.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862145+0000", 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881337+0000", 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019192607 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:10.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881351+0000", 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3879999999999999e-05 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40815 6.f 6.8b4fc53f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.11191134, 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.112146334, 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:10.008 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "delayed", 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40815 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884260+0000", 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884266+0000", 2023-09-23T22:41:10.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6060000000000002e-06 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884276+0000", 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7180000000000008e-06 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884284+0000", 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6170000000000007e-06 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884403+0000", 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011853099999999999 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884415+0000", 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1683e-05 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40816 6.f 6.4f46a35f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.111870474, 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.112182562, 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "delayed", 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40816 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.013 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884302+0000", 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884309+0000", 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884317+0000", 2023-09-23T22:41:10.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5660000000000007e-06 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884324+0000", 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0859999999999996e-06 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884441+0000", 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011671 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884449+0000", 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.571e-06 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.017 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40818 6.f 6.431951cf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.105953149, 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.106341331, 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "delayed", 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40818 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.018 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890219+0000", 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890224+0000", 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7840000000000003e-06 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890235+0000", 2023-09-23T22:41:10.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0899e-05 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890243+0000", 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.225e-06 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890301+0000", 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890312+0000", 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1235e-05 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40826 6.f 6.3e4c7bff (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.093767669999999997, 2023-09-23T22:41:10.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.094230046999999997, 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "delayed", 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40826 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:41:10.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902404+0000", 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902413+0000", 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5449999999999992e-06 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902432+0000", 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8488999999999999e-05 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902441+0000", 2023-09-23T22:41:10.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902502+0000", 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1064999999999999e-05 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902511+0000", 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7350000000000002e-06 2023-09-23T22:41:10.026 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.027 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:10.027 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.027 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:10.027 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:41:10.027 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 6 2023-09-23T22:41:10.027 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:41:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:10.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:10.096 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: pgmap v307: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 723 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 63 MiB/s wr, 2.06k op/s; 5282/34743 objects degraded (15.203%); 653/34743 objects misplaced (1.880%) 2023-09-23T22:41:10.096 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5282/34743 objects degraded (15.203%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:10.096 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: failed to encode map e130 with expected crc 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: osdmap e130: 8 total, 7 up, 8 in 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: failed to encode map e130 with expected crc 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[187743]: failed to encode map e130 with expected crc 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: pgmap v307: 89 pgs: 3 active+clean+remapped, 61 active+clean, 16 active+undersized, 9 active+undersized+degraded; 723 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 102 B/s rd, 63 MiB/s wr, 2.06k op/s; 5282/34743 objects degraded (15.203%); 653/34743 objects misplaced (1.880%) 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5282/34743 objects degraded (15.203%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: failed to encode map e130 with expected crc 2023-09-23T22:41:10.097 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:41:10.098 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: osdmap e130: 8 total, 7 up, 8 in 2023-09-23T22:41:10.098 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:41:10.098 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:41:10.098 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: failed to encode map e130 with expected crc 2023-09-23T22:41:10.098 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:09 smithi052 ceph-mon[180979]: failed to encode map e130 with expected crc 2023-09-23T22:41:10.128 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:10.159 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.255 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:10.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:10.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:10.359 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:10.387 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.483 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:10.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:10.587 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:10.614 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.862 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847] boot 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.124 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3338574763' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:11.125 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:11.154 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:41:11.154 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847] boot 2023-09-23T22:41:11.154 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3338574763' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: osd.2 [v2:172.21.15.52:6818/596807847,v1:172.21.15.52:6819/596807847] boot 2023-09-23T22:41:11.155 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: failed to encode map e131 with expected crc 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3338574763' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:11.156 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:11.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:11.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:11.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:11.292 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:11.558 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:41:11.558 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:41:11.558 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:41:11.558 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:41:11.558 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:41:11.841 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:11.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:11.991 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:41:12.021 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:12.103 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: pgmap v310: 89 pgs: 2 remapped+peering, 10 peering, 1 active+clean+remapped, 61 active+clean, 10 active+undersized, 5 active+undersized+degraded; 284 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 2.15k op/s; 1454/13683 objects degraded (10.626%) 2023-09-23T22:41:12.103 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:11 smithi151 ceph-mon[159210]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.135 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: pgmap v310: 89 pgs: 2 remapped+peering, 10 peering, 1 active+clean+remapped, 61 active+clean, 10 active+undersized, 5 active+undersized+degraded; 284 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 2.15k op/s; 1454/13683 objects degraded (10.626%) 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[180979]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: pgmap v310: 89 pgs: 2 remapped+peering, 10 peering, 1 active+clean+remapped, 61 active+clean, 10 active+undersized, 5 active+undersized+degraded; 284 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 2.15k op/s; 1454/13683 objects degraded (10.626%) 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T22:41:12.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.251 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:11 smithi052 ceph-mon[187743]: failed to encode map e132 with expected crc 2023-09-23T22:41:12.368 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:12.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:12.480 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:12.528 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:12.603 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:12.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:12.627 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:12.708 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:12.728 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:41:12.742 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:12.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:41:12.804 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:12.819 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:41:12.869 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:41:12.902 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:12.903 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:41:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:12.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:12.935 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:41:12.969 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:41:13.003 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:13.012 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:13.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2661858991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.041 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T22:41:13.042 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.042 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.042 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.042 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2661858991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:13.062 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: failed to encode map e133 with expected crc 2023-09-23T22:41:13.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2661858991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:13.231 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:13.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:13.339 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:13.348 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:41:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:41:13] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.33.4" 2023-09-23T22:41:13.416 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:14.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:13 smithi151 ceph-mon[159210]: pgmap v313: 89 pgs: 3 remapped+peering, 10 peering, 1 active+clean+remapped, 64 active+clean, 7 active+undersized, 4 active+undersized+degraded; 247 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.90k op/s; 955/11898 objects degraded (8.027%) 2023-09-23T22:41:14.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:13 smithi052 ceph-mon[180979]: pgmap v313: 89 pgs: 3 remapped+peering, 10 peering, 1 active+clean+remapped, 64 active+clean, 7 active+undersized, 4 active+undersized+degraded; 247 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.90k op/s; 955/11898 objects degraded (8.027%) 2023-09-23T22:41:14.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:13 smithi052 ceph-mon[187743]: pgmap v313: 89 pgs: 3 remapped+peering, 10 peering, 1 active+clean+remapped, 64 active+clean, 7 active+undersized, 4 active+undersized+degraded; 247 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.90k op/s; 955/11898 objects degraded (8.027%) 2023-09-23T22:41:14.285 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:41:14.471 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:14.471 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:14.892 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Removed 22141 objects 2023-09-23T22:41:14.892 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Clean up completed and total clean up time :14.9213 2023-09-23T22:41:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:14 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 955/11898 objects degraded (8.027%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:15.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:14 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 955/11898 objects degraded (8.027%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:15.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:14 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 955/11898 objects degraded (8.027%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:15.448 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:41:15.796 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:15.915 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:15.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:16.019 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:16.048 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.083 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:16.093 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:16.197 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:15 smithi151 ceph-mon[159210]: pgmap v314: 89 pgs: 3 remapped+peering, 10 peering, 73 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 156 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.53k op/s; 2792/7518 objects degraded (37.138%); 2.2 MiB/s, 38 objects/s recovering 2023-09-23T22:41:16.227 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:15 smithi052 ceph-mon[187743]: pgmap v314: 89 pgs: 3 remapped+peering, 10 peering, 73 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 156 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.53k op/s; 2792/7518 objects degraded (37.138%); 2.2 MiB/s, 38 objects/s recovering 2023-09-23T22:41:16.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:15 smithi052 ceph-mon[180979]: pgmap v314: 89 pgs: 3 remapped+peering, 10 peering, 73 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 156 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.53k op/s; 2792/7518 objects degraded (37.138%); 2.2 MiB/s, 38 objects/s recovering 2023-09-23T22:41:16.230 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.390 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:16.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:16.496 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:16.524 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:16.669 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:16.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.758 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:16.862 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:16.893 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.899 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:41:16.945 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:17.048 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:17.076 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/365927511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:41:17.076 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:41:17.080 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:17.180 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:17.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:17.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/365927511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:41:17.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:41:17.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/365927511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:41:17.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:41:17.283 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:17.285 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:17.285 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:17.311 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:17.791 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:17.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:17 smithi151 ceph-mon[159210]: pgmap v315: 89 pgs: 1 remapped+peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8392/2127 objects degraded (394.546%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T22:41:17.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:17 smithi151 ceph-mon[159210]: failed to encode map e134 with expected crc 2023-09-23T22:41:17.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:41:17.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:17 smithi151 ceph-mon[159210]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T22:41:17.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:17 smithi151 ceph-mon[159210]: failed to encode map e134 with expected crc 2023-09-23T22:41:17.894 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[180979]: pgmap v315: 89 pgs: 1 remapped+peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8392/2127 objects degraded (394.546%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T22:41:17.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[180979]: failed to encode map e134 with expected crc 2023-09-23T22:41:17.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:41:17.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[180979]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T22:41:17.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[180979]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.080 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:41:18.114 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:18.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:18.220 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[187743]: pgmap v315: 89 pgs: 1 remapped+peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8392/2127 objects degraded (394.546%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T22:41:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[187743]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:41:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[187743]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T22:41:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:17 smithi052 ceph-mon[187743]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.251 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.396 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:41:18.424 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:18.456 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:18.560 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:18.595 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:18.761 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:18.805 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:18.833 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.839 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:18.893 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:18.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: failed to encode map e135 with expected crc 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: failed to encode map e135 with expected crc 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: failed to encode map e135 with expected crc 2023-09-23T22:41:18.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1130510970' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: failed to encode map e134 with expected crc 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: failed to encode map e135 with expected crc 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: failed to encode map e135 with expected crc 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: failed to encode map e135 with expected crc 2023-09-23T22:41:18.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1130510970' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:18.910 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:41:18.996 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:19.028 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: failed to encode map e134 with expected crc 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: failed to encode map e134 with expected crc 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: failed to encode map e135 with expected crc 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: failed to encode map e135 with expected crc 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: failed to encode map e135 with expected crc 2023-09-23T22:41:19.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1130510970' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:19.549 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:19.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:19.653 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:19.686 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:19.733 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T22:41:19.733 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e135 with expected crc 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: pgmap v318: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 18 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 9801/912 objects degraded (1074.671%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1130510970' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 9801/912 objects degraded (1074.671%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:41:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:19 smithi151 ceph-mon[159210]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T22:41:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e135 with expected crc 2023-09-23T22:41:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: pgmap v318: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 18 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 9801/912 objects degraded (1074.671%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T22:41:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1130510970' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 9801/912 objects degraded (1074.671%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[187743]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T22:41:20.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e135 with expected crc 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: pgmap v318: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 18 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 9801/912 objects degraded (1074.671%); 1.9 MiB/s, 33 objects/s recovering 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1130510970' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: failed to encode map e136 with expected crc 2023-09-23T22:41:20.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:20.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:20.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 9801/912 objects degraded (1074.671%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:41:20.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:19 smithi052 ceph-mon[180979]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T22:41:20.737 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:20.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:20.742 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:20.742 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:20.841 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:20.879 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:20.947 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:20.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[187743]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[187743]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[187743]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[187743]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[187743]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T22:41:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[180979]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[180979]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[180979]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[180979]: failed to encode map e137 with expected crc 2023-09-23T22:41:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:20 smithi052 ceph-mon[180979]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T22:41:21.052 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:21.093 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:20 smithi151 ceph-mon[159210]: failed to encode map e137 with expected crc 2023-09-23T22:41:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:20 smithi151 ceph-mon[159210]: failed to encode map e137 with expected crc 2023-09-23T22:41:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:20 smithi151 ceph-mon[159210]: failed to encode map e137 with expected crc 2023-09-23T22:41:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:20 smithi151 ceph-mon[159210]: failed to encode map e137 with expected crc 2023-09-23T22:41:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:20 smithi151 ceph-mon[159210]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T22:41:21.252 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:21.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:21.356 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:21.388 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.411 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:41:21.433 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:21.539 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:21.566 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.609 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:21.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:21.716 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:21.749 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.792 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:21.802 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:41:21.802 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:41:21.802 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:41:21.802 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:41:21.802 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:41:21.803 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:41:21.803 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:41:21.813 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.815 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:21.907 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:22.011 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:22.041 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.084 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:22.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:22.187 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:22.192 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:21 smithi151 ceph-mon[159210]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.192 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:21 smithi151 ceph-mon[159210]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.192 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:21 smithi151 ceph-mon[159210]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.192 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:21 smithi151 ceph-mon[159210]: pgmap v322: 89 pgs: 1 peering, 82 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 8403/72 objects degraded (11670.833%); 0 B/s, 660 objects/s recovering 2023-09-23T22:41:22.192 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2996064235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[187743]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[187743]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[187743]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[187743]: pgmap v322: 89 pgs: 1 peering, 82 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 8403/72 objects degraded (11670.833%); 0 B/s, 660 objects/s recovering 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2996064235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[180979]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[180979]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[180979]: failed to encode map e138 with expected crc 2023-09-23T22:41:22.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[180979]: pgmap v322: 89 pgs: 1 peering, 82 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 8403/72 objects degraded (11670.833%); 0 B/s, 660 objects/s recovering 2023-09-23T22:41:22.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2996064235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:22.219 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.936 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:41:23.025 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:23.050 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:22 smithi151 ceph-mon[159210]: failed to encode map e139 with expected crc 2023-09-23T22:41:23.050 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:22 smithi151 ceph-mon[159210]: failed to encode map e139 with expected crc 2023-09-23T22:41:23.051 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:22 smithi151 ceph-mon[159210]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T22:41:23.128 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:23.160 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.207 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:23.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:23.217 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:41:23.217 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:41:23.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:41:23.217 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:41:23.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1177 6.8 6:1fae9040:::benchmark_data_smithi052_234456_object1176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.902197743999999, 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16501295899999999, 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.218 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1177 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312080+0000", 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6277999999999999e-05 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312137+0000", 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9989999999999997e-06 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312146+0000", 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6670000000000004e-06 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312150+0000", 2023-09-23T22:41:23.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8374999999999998e-05 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312189+0000", 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8557e-05 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312267+0000", 2023-09-23T22:41:23.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011574900000000001 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312383+0000", 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16315368599999999 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475537+0000", 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013937680000000001 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476930+0000", 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1847000000000006e-05 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476992+0000", 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010188600000000001 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477094+0000", 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16501295899999999 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1180 6.3 6:c4de4eba:::benchmark_data_smithi052_234456_object1179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:41:23.225 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:41:23.226 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.901715807999999, 2023-09-23T22:41:23.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.165229389, 2023-09-23T22:41:23.226 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.226 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.226 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.226 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1180 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312562+0000", 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000111852 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312674+0000", 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3390999999999999e-05 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312687+0000", 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312694+0000", 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9179999999999999e-05 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312753+0000", 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16226790299999999 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475021+0000", 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018092 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475202+0000", 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020914169999999999 2023-09-23T22:41:23.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477293+0000", 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035115500000000002 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477644+0000", 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2717999999999999e-05 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477697+0000", 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5310999999999996e-05 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477792+0000", 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.165229389 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1182 6.8 6:12f46353:::benchmark_data_smithi052_234456_object1181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.901567971999999, 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16475775100000001, 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1182 2023-09-23T22:41:23.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312710+0000", 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5873e-05 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312746+0000", 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4910000000000005e-06 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312755+0000", 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8200000000000002e-06 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312760+0000", 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9837000000000002e-05 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312830+0000", 2023-09-23T22:41:23.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.162295933 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475126+0000", 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015127000000000001 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475277+0000", 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001836475 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477114+0000", 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000174838 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477289+0000", 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9021000000000003e-05 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477348+0000", 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120896 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477469+0000", 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16475775100000001 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.240 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1184 6.5 6:ad9e7e6c:::benchmark_data_smithi052_234456_object1183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.9015044, 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16539384100000001, 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1184 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6648000000000001e-05 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312861+0000", 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2448e-05 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312873+0000", 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7630000000000003e-06 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312880+0000", 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16255894300000001 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475439+0000", 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129219 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475568+0000", 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000164919 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475733+0000", 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022748180000000001 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478008+0000", 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478015+0000", 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4327999999999997e-05 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478089+0000", 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2700000000000001e-05 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478132+0000", 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6504999999999997e-05 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478168+0000", 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16539384100000001 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.247 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:1191 6.6 e127/99 6:684d433b:::benchmark_data_smithi052_234456_object1190:head v 127'68, mlcod=127'65)", 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.901086109000001, 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16367246499999999, 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313192+0000", 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8267000000000001e-05 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313270+0000", 2023-09-23T22:41:23.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16176015999999999 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475030+0000", 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7800000000000006e-06 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475039+0000", 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00056310300000000002 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475602+0000", 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0098e-05 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475622+0000", 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012034210000000001 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476826+0000", 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9504e-05 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476865+0000", 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16367246499999999 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.252 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:9163 6.8 6:1645402e:::benchmark_data_smithi052_234456_object9162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:53.806373+0000", 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.407905572000001, 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.104994164, 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 9163 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806373+0000", 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806373+0000", 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806372+0000", 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1971999999999996e-05 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806434+0000", 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5475e-05 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806450+0000", 2023-09-23T22:41:23.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3209999999999999e-06 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806457+0000", 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0683999999999998e-05 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806508+0000", 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155582 2023-09-23T22:41:23.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806663+0000", 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000184255 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.806847+0000", 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083925999999999998 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807687+0000", 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10346137599999999 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911148+0000", 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4869999999999994e-05 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911213+0000", 2023-09-23T22:41:23.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000154551 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911367+0000", 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.104994164 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:9165 6.5 6:a483c5b5:::benchmark_data_smithi052_234456_object9164:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.259 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:53.807322+0000", 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.406956660999999, 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.103799976, 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 9165 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807322+0000", 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807322+0000", 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807321+0000", 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021192800000000001 2023-09-23T22:41:23.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807533+0000", 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5007e-05 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807558+0000", 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0917999999999999e-05 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807569+0000", 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5293000000000001e-05 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807624+0000", 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120915 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807745+0000", 2023-09-23T22:41:23.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000156043 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.807901+0000", 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00072947699999999997 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808630+0000", 2023-09-23T22:41:23.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00077184599999999997 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.809402+0000", 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10160754800000001 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911010+0000", 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5969000000000005e-05 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911076+0000", 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6334000000000001e-05 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911122+0000", 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.103799976 2023-09-23T22:41:23.266 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:9170 6.8 6:12a09f13:::benchmark_data_smithi052_234456_object9169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:53.808589+0000", 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.405689887000001, 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.103042256, 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.267 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 9170 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808589+0000", 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808589+0000", 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808588+0000", 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014925499999999999 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808737+0000", 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3824e-05 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808751+0000", 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.229e-06 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808757+0000", 2023-09-23T22:41:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1373999999999999e-05 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808818+0000", 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015354799999999999 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.808972+0000", 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000201783 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.809174+0000", 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00065857399999999999 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.809832+0000", 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.101590638 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911423+0000", 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7362999999999999e-05 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911480+0000", 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015099800000000001 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:53.911631+0000", 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.103042256 2023-09-23T22:41:23.273 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10178 6.3 6:cbb8686c:::benchmark_data_smithi052_234456_object10177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.014664385, 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26379366399999998, 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.274 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10178 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8393999999999994e-05 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199702+0000", 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0538e-05 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199713+0000", 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5840000000000002e-06 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199718+0000", 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7531000000000001e-05 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199786+0000", 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012609700000000001 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199912+0000", 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024177399999999999 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200154+0000", 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029848309999999999 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203139+0000", 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.260079748 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463218+0000", 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3586999999999999e-05 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463292+0000", 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116289 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463408+0000", 2023-09-23T22:41:23.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26379366399999998 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10181 6.3 6:c0620c37:::benchmark_data_smithi052_234456_object10180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.014461547, 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.265707154, 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.282 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10181 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8221e-05 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199845+0000", 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3889999999999999e-06 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199852+0000", 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9130000000000002e-06 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199857+0000", 2023-09-23T22:41:23.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085247700000000003 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200710+0000", 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001666491 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202376+0000", 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124387 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202501+0000", 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00086751799999999996 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203368+0000", 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26193364000000002 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465302+0000", 2023-09-23T22:41:23.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106265 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465408+0000", 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116496 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465524+0000", 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.265707154 2023-09-23T22:41:23.288 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10191 6.5 6:a7cca2de:::benchmark_data_smithi052_234456_object10190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.010404817000001, 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26015834900000001, 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.289 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10191 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203873+0000", 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010616300000000001 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203979+0000", 2023-09-23T22:41:23.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2459999999999992e-06 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203989+0000", 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129999999999996e-06 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203993+0000", 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5849000000000002e-05 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204029+0000", 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105581 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204135+0000", 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000160994 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:41:23.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204296+0000", 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00127407 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205570+0000", 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052499199999999995 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206095+0000", 2023-09-23T22:41:23.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25784317800000001 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463938+0000", 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5868000000000001e-05 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463994+0000", 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8639000000000001e-05 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464032+0000", 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26015834900000001 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10193 6.5 6:a177b7dd:::benchmark_data_smithi052_234456_object10192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:41:23.296 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.010039564, 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26049174200000003, 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10193 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:41:23.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6399999999999997e-05 2023-09-23T22:41:23.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204275+0000", 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1413000000000001e-05 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204286+0000", 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9870000000000002e-06 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204293+0000", 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054160700000000005 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204835+0000", 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013009600000000001 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204965+0000", 2023-09-23T22:41:23.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016865000000000001 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205134+0000", 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00065124099999999999 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205785+0000", 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037879500000000002 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206164+0000", 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25846449399999999 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464628+0000", 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9333000000000003e-05 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464688+0000", 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3455999999999999e-05 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464731+0000", 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26049174200000003 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10206 6.3 6:cca2051f:::benchmark_data_smithi052_234456_object10205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.005205204999999, 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25668903900000001, 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10206 2023-09-23T22:41:23.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209072+0000", 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1259000000000002e-05 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209154+0000", 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3302000000000001e-05 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209167+0000", 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3980000000000002e-06 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209174+0000", 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4868000000000003e-05 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209239+0000", 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018571700000000001 2023-09-23T22:41:23.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209425+0000", 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018386100000000001 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209609+0000", 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00066804500000000003 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.210277+0000", 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.255318247 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465595+0000", 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6978999999999998e-05 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465652+0000", 2023-09-23T22:41:23.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110548 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465763+0000", 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25668903900000001 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.310 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10215 6.3 6:c04bb586:::benchmark_data_smithi052_234456_object10214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 29.000971236000002, 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25266233500000002, 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10215 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213306+0000", 2023-09-23T22:41:23.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3293000000000001e-05 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213340+0000", 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0866000000000001e-05 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213350+0000", 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2249999999999997e-06 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213357+0000", 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1418999999999997e-05 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213408+0000", 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6898999999999995e-05 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213505+0000", 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132287 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213637+0000", 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00064820999999999998 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214285+0000", 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25152426300000003 2023-09-23T22:41:23.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465810+0000", 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4280999999999997e-05 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465864+0000", 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105955 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465970+0000", 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25266233500000002 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10217 6.3 6:c781daf6:::benchmark_data_smithi052_234456_object10216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 28.999422517999999, 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25134704800000002, 2023-09-23T22:41:23.317 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10217 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214855+0000", 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8257999999999994e-05 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214944+0000", 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1997999999999999e-05 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214956+0000", 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1810000000000004e-06 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214963+0000", 2023-09-23T22:41:23.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9869e-05 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215013+0000", 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015027000000000001 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215163+0000", 2023-09-23T22:41:23.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021892400000000001 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215382+0000", 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073708599999999997 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.216119+0000", 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.249917901 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466037+0000", 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8545999999999998e-05 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466095+0000", 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010809199999999999 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466203+0000", 2023-09-23T22:41:23.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25134704800000002 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10224 6.3 6:c3676ada:::benchmark_data_smithi052_234456_object10223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 28.993875012, 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.246054935, 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.324 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10224 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220402+0000", 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7369e-05 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220449+0000", 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6140000000000001e-05 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220466+0000", 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8979999999999996e-06 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220473+0000", 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2861000000000002e-05 2023-09-23T22:41:23.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220516+0000", 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011724500000000001 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220634+0000", 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013401900000000001 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220768+0000", 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00059656599999999998 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.221364+0000", 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24489038599999999 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466255+0000", 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6962e-05 2023-09-23T22:41:23.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466321+0000", 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013730400000000001 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466459+0000", 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.246054935 2023-09-23T22:41:23.330 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:37700 6.5 6:aafb5f10:::benchmark_data_smithi052_234456_object15556:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:07.831516+0000", 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 15.382763268, 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.106200691, 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.331 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 37700 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831516+0000", 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831516+0000", 2023-09-23T22:41:23.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831515+0000", 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5659999999999997e-06 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831520+0000", 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0523000000000001e-05 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831531+0000", 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5369999999999999e-06 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831535+0000", 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1248999999999997e-05 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831587+0000", 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8263000000000004e-05 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831685+0000", 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2556e-05 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:41:23.335 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.831767+0000", 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001190572 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.832958+0000", 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00060083999999999995 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.833559+0000", 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.104061365 2023-09-23T22:41:23.336 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.937620+0000", 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5441e-05 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.937676+0000", 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0725000000000001e-05 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.337 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.937716+0000", 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.106200691 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:38887 6.5 6:ac5fd0b5:::benchmark_data_smithi052_234456_object16743:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:08.791810+0000", 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 14.422468798000001, 2023-09-23T22:41:23.338 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.122386063, 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 38887 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.339 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791810+0000", 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791810+0000", 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.340 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791809+0000", 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9989999999999997e-06 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791815+0000", 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2398000000000001e-05 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.341 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791827+0000", 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2839999999999997e-06 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791833+0000", 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7528000000000003e-05 2023-09-23T22:41:23.342 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791880+0000", 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0001171 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.791997+0000", 2023-09-23T22:41:23.343 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120228 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.792117+0000", 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001339968 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.344 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.793457+0000", 2023-09-23T22:41:23.345 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000553179 2023-09-23T22:41:23.345 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.345 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.345 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.345 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.794011+0000", 2023-09-23T22:41:23.345 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12009497700000001 2023-09-23T22:41:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.914106+0000", 2023-09-23T22:41:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4372000000000001e-05 2023-09-23T22:41:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.914150+0000", 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6149000000000001e-05 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:08.914196+0000", 2023-09-23T22:41:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.122386063 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40799 6.8 6:14756be4:::benchmark_data_smithi052_234456_object18655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 13.352709415, 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2385245110000001, 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40799 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861568+0000", 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9182999999999999e-05 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861587+0000", 2023-09-23T22:41:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3654e-05 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861600+0000", 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.588e-06 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861608+0000", 2023-09-23T22:41:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114259 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861722+0000", 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.251e-06 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861731+0000", 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028264639999999999 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864558+0000", 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4350000000000003e-06 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864567+0000", 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99719855899999998 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861766+0000", 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9449999999999998e-05 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861785+0000", 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2282697419999999 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090055+0000", 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000194957 2023-09-23T22:41:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090250+0000", 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000107145 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090357+0000", 2023-09-23T22:41:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030987689999999999 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.093456+0000", 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023293989999999998 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.095785+0000", 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038349410000000001 2023-09-23T22:41:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.099620+0000", 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1438000000000004e-05 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.099682+0000", 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00041221600000000003 2023-09-23T22:41:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.100094+0000", 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2385245110000001 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40800 6.3 6:c0d13c02:::benchmark_data_smithi052_234456_object18656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 13.352645388999999, 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.236326756, 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40800 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.693e-06 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861637+0000", 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6470000000000001e-06 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861643+0000", 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.408e-06 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861648+0000", 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4624999999999998e-05 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861722+0000", 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8899999999999996e-06 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861731+0000", 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003144053 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864875+0000", 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6930000000000002e-06 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864885+0000", 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99906347699999998 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.863948+0000", 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2204999999999999e-05 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.863961+0000", 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2265645000000001 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090525+0000", 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016863800000000001 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090694+0000", 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018472599999999999 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090878+0000", 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018197459999999999 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.092698+0000", 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.679e-06 2023-09-23T22:41:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.092708+0000", 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048923589999999998 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097600+0000", 2023-09-23T22:41:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7243000000000007e-05 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097667+0000", 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00029281000000000001 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:41:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097960+0000", 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.236326756 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:41:23.370 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:41:23.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:23.372 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:22 smithi052 ceph-mon[187743]: failed to encode map e139 with expected crc 2023-09-23T22:41:23.372 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:22 smithi052 ceph-mon[187743]: failed to encode map e139 with expected crc 2023-09-23T22:41:23.372 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:22 smithi052 ceph-mon[187743]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T22:41:23.372 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:22 smithi052 ceph-mon[180979]: failed to encode map e139 with expected crc 2023-09-23T22:41:23.372 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:22 smithi052 ceph-mon[180979]: failed to encode map e139 with expected crc 2023-09-23T22:41:23.372 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:22 smithi052 ceph-mon[180979]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T22:41:23.407 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.448 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:23.551 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:23.579 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.625 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:41:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:41:23] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-09-23T22:41:23.713 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:23.722 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:23.776 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:41:23.809 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:23.825 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:23.862 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.936 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:24.041 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:24.066 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: pgmap v324: 89 pgs: 1 peering, 82 active+clean, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 6994/72 objects degraded (9713.889%); 0 B/s, 994 objects/s recovering 2023-09-23T22:41:24.066 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T22:41:24.066 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.066 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.066 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:23 smithi151 ceph-mon[159210]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.069 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.111 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:24.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:24.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: pgmap v324: 89 pgs: 1 peering, 82 active+clean, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 6994/72 objects degraded (9713.889%); 0 B/s, 994 objects/s recovering 2023-09-23T22:41:24.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T22:41:24.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[187743]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: pgmap v324: 89 pgs: 1 peering, 82 active+clean, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 6994/72 objects degraded (9713.889%); 0 B/s, 994 objects/s recovering 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:23 smithi052 ceph-mon[180979]: failed to encode map e140 with expected crc 2023-09-23T22:41:24.220 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:24.246 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.288 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:24.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:24.393 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:24.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.715 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:41:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/928895397' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6994/72 objects degraded (9713.889%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[180979]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/928895397' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6994/72 objects degraded (9713.889%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:41:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:24 smithi052 ceph-mon[187743]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T22:41:25.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/928895397' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:41:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:41:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:24 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6994/72 objects degraded (9713.889%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:41:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:41:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:24 smithi151 ceph-mon[159210]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T22:41:25.446 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:25.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:25.549 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:25.576 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.618 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:25.687 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:25.725 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:25.750 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.793 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:25.899 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:25.924 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.966 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:25.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[187743]: failed to encode map e141 with expected crc 2023-09-23T22:41:25.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[187743]: failed to encode map e141 with expected crc 2023-09-23T22:41:25.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[187743]: pgmap v327: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5794/72 objects degraded (8047.222%); 0 B/s, 604 objects/s recovering 2023-09-23T22:41:25.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[187743]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T22:41:25.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[180979]: failed to encode map e141 with expected crc 2023-09-23T22:41:25.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[180979]: failed to encode map e141 with expected crc 2023-09-23T22:41:25.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[180979]: pgmap v327: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5794/72 objects degraded (8047.222%); 0 B/s, 604 objects/s recovering 2023-09-23T22:41:25.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:25 smithi052 ceph-mon[180979]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T22:41:25.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:26.074 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:26.099 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.142 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:26.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:26.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:25 smithi151 ceph-mon[159210]: failed to encode map e141 with expected crc 2023-09-23T22:41:26.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:25 smithi151 ceph-mon[159210]: failed to encode map e141 with expected crc 2023-09-23T22:41:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:25 smithi151 ceph-mon[159210]: pgmap v327: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5794/72 objects degraded (8047.222%); 0 B/s, 604 objects/s recovering 2023-09-23T22:41:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:25 smithi151 ceph-mon[159210]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T22:41:26.248 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:26.273 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.318 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:26.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:26.407 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:26.425 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:26.437 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:41:26.450 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.491 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:26.596 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:26.621 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.667 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:26.733 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:26.774 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:26.800 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:26.946 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:26.978 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.128 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[187743]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.128 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[187743]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.128 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[187743]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.128 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[187743]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.129 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[180979]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.129 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[180979]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.129 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[180979]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.129 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:26 smithi052 ceph-mon[180979]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:26 smithi151 ceph-mon[159210]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:26 smithi151 ceph-mon[159210]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:26 smithi151 ceph-mon[159210]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:26 smithi151 ceph-mon[159210]: failed to encode map e142 with expected crc 2023-09-23T22:41:27.335 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:27.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:27.464 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.507 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:27.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:27.615 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:27.642 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.683 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:27.788 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:27.821 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.922 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:27 smithi052 ceph-mon[180979]: pgmap v329: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4612/72 objects degraded (6405.556%); 0 B/s, 219 objects/s recovering 2023-09-23T22:41:27.922 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4090753874' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:41:27.941 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_0' removed 2023-09-23T22:41:28.099 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:28.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:28.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:27 smithi151 ceph-mon[159210]: pgmap v329: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4612/72 objects degraded (6405.556%); 0 B/s, 219 objects/s recovering 2023-09-23T22:41:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4090753874' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:41:28.205 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:28.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:27 smithi052 ceph-mon[187743]: pgmap v329: 89 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4612/72 objects degraded (6405.556%); 0 B/s, 219 objects/s recovering 2023-09-23T22:41:28.232 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4090753874' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:41:28.235 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:28.241 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:28.797 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:41:28.797 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:41:28.797 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T22:41:28.797 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:41:28.942 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:41:29.094 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[187743]: failed to encode map e143 with expected crc 2023-09-23T22:41:29.094 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4090753874' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:41:29.094 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[187743]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T22:41:29.094 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1389148597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.095 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[180979]: failed to encode map e143 with expected crc 2023-09-23T22:41:29.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4090753874' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:41:29.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[180979]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T22:41:29.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1389148597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:28 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.154 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:29.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:29.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:29.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:28 smithi151 ceph-mon[159210]: failed to encode map e143 with expected crc 2023-09-23T22:41:29.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4090753874' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:41:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:28 smithi151 ceph-mon[159210]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T22:41:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1389148597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:28 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.258 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:29.289 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:29.340 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:29.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:29.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:29.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:29.475 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:29.519 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:29.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:29.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:29.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:29.658 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:30.152 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:41:30.152 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:41:30.152 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T22:41:30.152 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T22:41:30.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:30.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4612/72 objects degraded (6405.556%), 5 pgs degraded) 2023-09-23T22:41:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T22:41:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: failed to encode map e144 with expected crc 2023-09-23T22:41:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: failed to encode map e144 with expected crc 2023-09-23T22:41:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:29 smithi151 ceph-mon[159210]: failed to encode map e143 with expected crc 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4612/72 objects degraded (6405.556%), 5 pgs degraded) 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: failed to encode map e144 with expected crc 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: failed to encode map e144 with expected crc 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[180979]: failed to encode map e143 with expected crc 2023-09-23T22:41:30.219 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4612/72 objects degraded (6405.556%), 5 pgs degraded) 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: failed to encode map e144 with expected crc 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: failed to encode map e144 with expected crc 2023-09-23T22:41:30.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:29 smithi052 ceph-mon[187743]: failed to encode map e143 with expected crc 2023-09-23T22:41:30.511 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:30.614 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:30.646 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:30.696 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:30.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:30.802 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:30.831 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:30.876 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:30.982 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:31.008 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.049 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:31.155 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:31.180 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:30 smithi052 ceph-mon[187743]: failed to encode map e144 with expected crc 2023-09-23T22:41:31.181 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:30 smithi052 ceph-mon[180979]: failed to encode map e144 with expected crc 2023-09-23T22:41:31.183 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:30 smithi151 ceph-mon[159210]: failed to encode map e144 with expected crc 2023-09-23T22:41:31.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:31.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:31.245 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:41:31.334 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:31.367 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.425 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:31.530 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:31.531 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:31.566 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.648 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:41:31.667 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:41:31.688 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:41:31.715 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:41:31.739 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:41:31.760 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:41:31.782 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:31.787 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:41:31.806 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:31.886 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:31.916 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.961 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:31.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:32.067 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:32.096 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:31 smithi052 ceph-mon[187743]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:32.097 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1897481589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:32.097 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:31 smithi052 ceph-mon[180979]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:32.097 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1897481589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:32.100 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:32.188 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:32.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:31 smithi151 ceph-mon[159210]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:32.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1897481589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:32.340 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:41:32.866 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:32.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:32.950 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:41:32.970 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:32.985 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/823914035' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:41:32.985 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:32 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:41:32.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/823914035' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:41:32.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:32 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:41:33.002 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/823914035' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:41:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:32 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:41:33.452 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:33.556 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:33.583 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:41:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:41:33] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.33.4" 2023-09-23T22:41:33.767 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:33.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:33.904 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.943 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T22:41:33.954 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.024 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[180979]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:33 smithi052 ceph-mon[187743]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.060 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:34.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:33 smithi151 ceph-mon[159210]: failed to encode map e145 with expected crc 2023-09-23T22:41:34.899 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:34.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:34.920 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:34.958 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:35.006 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:35.036 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.141 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[187743]: failed to encode map e145 with expected crc 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[180979]: failed to encode map e145 with expected crc 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:35.169 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:34 smithi151 ceph-mon[159210]: failed to encode map e145 with expected crc 2023-09-23T22:41:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:35.246 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:35.278 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.326 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:35.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:35.357 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:35.434 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:35.462 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.506 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:35.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:35.638 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.962 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_1' created 2023-09-23T22:41:36.089 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:36.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:36.196 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:36.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:35 smithi151 ceph-mon[159210]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:36.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/286822115' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:41:36.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:35 smithi052 ceph-mon[187743]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:36.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/286822115' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:41:36.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:35 smithi052 ceph-mon[180979]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:36.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/286822115' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:41:36.225 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:36.314 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:36.314 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[187743]: failed to encode map e146 with expected crc 2023-09-23T22:41:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[187743]: failed to encode map e146 with expected crc 2023-09-23T22:41:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/286822115' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:41:37.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[187743]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T22:41:37.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[180979]: failed to encode map e146 with expected crc 2023-09-23T22:41:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[180979]: failed to encode map e146 with expected crc 2023-09-23T22:41:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/286822115' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:41:37.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:36 smithi052 ceph-mon[180979]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T22:41:37.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:36 smithi151 ceph-mon[159210]: failed to encode map e146 with expected crc 2023-09-23T22:41:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:36 smithi151 ceph-mon[159210]: failed to encode map e146 with expected crc 2023-09-23T22:41:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/286822115' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:41:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:36 smithi151 ceph-mon[159210]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T22:41:37.578 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:41:37.914 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:37.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:37.933 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T22:41:38.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:38.050 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.134 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:38.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[187743]: pgmap v338: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:38.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[187743]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[187743]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[187743]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[187743]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[187743]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[180979]: pgmap v338: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[180979]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[180979]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[180979]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[180979]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:37 smithi052 ceph-mon[180979]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.156 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:38.261 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:38.290 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.345 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:38.406 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:38.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:37 smithi151 ceph-mon[159210]: pgmap v338: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:37 smithi151 ceph-mon[159210]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:37 smithi151 ceph-mon[159210]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T22:41:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:37 smithi151 ceph-mon[159210]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:37 smithi151 ceph-mon[159210]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:37 smithi151 ceph-mon[159210]: failed to encode map e147 with expected crc 2023-09-23T22:41:38.510 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:38.540 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.932 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:38.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:38.974 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:41:39.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:39.072 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.106 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:39.121 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:39.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:39.227 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[187743]: failed to encode map e148 with expected crc 2023-09-23T22:41:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[187743]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T22:41:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/965185608' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:39.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[180979]: failed to encode map e148 with expected crc 2023-09-23T22:41:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[180979]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T22:41:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/965185608' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:39.252 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:38 smithi151 ceph-mon[159210]: failed to encode map e148 with expected crc 2023-09-23T22:41:39.252 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:38 smithi151 ceph-mon[159210]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T22:41:39.252 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/965185608' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:39.252 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:39.254 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.299 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:39.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:39.404 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:39.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.682 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:39.785 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:39.813 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.936 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:39.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:39.976 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T22:41:40.039 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:40.069 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: pgmap v341: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:40.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2312034496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: failed to encode map e149 with expected crc 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[187743]: failed to encode map e149 with expected crc 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: pgmap v341: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2312034496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: failed to encode map e149 with expected crc 2023-09-23T22:41:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:39 smithi052 ceph-mon[180979]: failed to encode map e149 with expected crc 2023-09-23T22:41:40.414 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:40.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: pgmap v341: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2312034496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: failed to encode map e149 with expected crc 2023-09-23T22:41:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:39 smithi151 ceph-mon[159210]: failed to encode map e149 with expected crc 2023-09-23T22:41:40.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:40.547 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.240 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:41:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2312034496' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:41:41.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[180979]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2312034496' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:40 smithi052 ceph-mon[187743]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.260 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:41:41.278 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:41:41.300 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:41:41.318 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:41:41.337 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:41:41.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:41:41.373 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.375 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e149 with expected crc 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2312034496' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:40 smithi151 ceph-mon[159210]: failed to encode map e150 with expected crc 2023-09-23T22:41:41.648 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:41.752 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:41.784 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.833 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:41.940 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:41.969 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:42.014 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:42.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:42.089 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:41:42.123 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:42.147 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:41 smithi151 ceph-mon[159210]: pgmap v344: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:42.147 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3672660220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:42.149 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:42.192 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:42.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:42.227 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:41 smithi052 ceph-mon[187743]: pgmap v344: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:42.227 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3672660220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:42.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:41 smithi052 ceph-mon[180979]: pgmap v344: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:42.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3672660220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:42.240 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:3 6.9 6:977e6172:::benchmark_data_smithi052_234456_object2:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.899530+0000", 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.338501631, 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.016604193, 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.241 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 3 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899530+0000", 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899530+0000", 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899527+0000", 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899606+0000", 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8311000000000003e-05 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899620+0000", 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3997e-05 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899633+0000", 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3023000000000001e-05 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921844+0000", 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022211709 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921861+0000", 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6385999999999999e-05 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.903617+0000", 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98175606299999996 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.903777+0000", 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015961799999999999 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.903919+0000", 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000142418 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912920+0000", 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090012289999999995 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.915972+0000", 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030520180000000001 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916033+0000", 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0594999999999998e-05 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916134+0000", 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000100994 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:4 6.9 6:966b311b:::benchmark_data_smithi052_234456_object3:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.899651+0000", 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.33838042, 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0166748489999999, 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.250 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 4 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899651+0000", 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899651+0000", 2023-09-23T22:41:42.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899650+0000", 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899721+0000", 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0710999999999994e-05 2023-09-23T22:41:42.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899731+0000", 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9599999999999995e-06 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899740+0000", 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0289999999999994e-06 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921883+0000", 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022143121000000002 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921888+0000", 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7880000000000003e-06 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904432+0000", 2023-09-23T22:41:42.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98254400399999997 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904566+0000", 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013371999999999999 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904707+0000", 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014122700000000001 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912936+0000", 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0082280860000000008 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916177+0000", 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003241945 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916244+0000", 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6903999999999995e-05 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916326+0000", 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1295e-05 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10 6.9 6:93a199a5:::benchmark_data_smithi052_234456_object9:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.338017911000001, 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0164958529999999, 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.258 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900062+0000", 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9654000000000003e-05 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900074+0000", 2023-09-23T22:41:42.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1392999999999999e-05 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900088+0000", 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4338999999999999e-05 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921910+0000", 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021822426999999998 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921916+0000", 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3680000000000001e-06 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904977+0000", 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98306100500000004 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905090+0000", 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113242 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905219+0000", 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012862999999999999 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912941+0000", 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0077225569999999997 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916368+0000", 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003426751 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916415+0000", 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6869000000000001e-05 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916509+0000", 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4359999999999998e-05 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:16 6.9 6:9e1b8017:::benchmark_data_smithi052_234456_object15:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900101+0000", 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337930593000003, 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0165771290000001, 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.265 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 16 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900101+0000", 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900101+0000", 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900100+0000", 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900145+0000", 2023-09-23T22:41:42.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5096999999999999e-05 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900153+0000", 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7009999999999996e-06 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900158+0000", 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6999999999999999e-06 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921938+0000", 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021780167 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921943+0000", 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5969999999999999e-06 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905456+0000", 2023-09-23T22:41:42.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98351275599999999 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905557+0000", 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010088 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905686+0000", 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0001292 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912946+0000", 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072599969999999998 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916550+0000", 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036041070000000001 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916592+0000", 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1462999999999999e-05 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916678+0000", 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6054e-05 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:22 6.0 6:02bff156:::benchmark_data_smithi052_234456_object21:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337863925000001, 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.025782167, 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 22 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.273 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900206+0000", 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8720000000000002e-05 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900210+0000", 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9269999999999996e-06 2023-09-23T22:41:42.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900214+0000", 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8650000000000003e-06 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923636+0000", 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023421252 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923641+0000", 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5260000000000001e-06 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920207+0000", 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.996566222 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920350+0000", 2023-09-23T22:41:42.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014280299999999999 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920498+0000", 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014755399999999999 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925223+0000", 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047253130000000001 2023-09-23T22:41:42.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925814+0000", 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059094599999999996 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925870+0000", 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6121e-05 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925949+0000", 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9439999999999998e-05 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:23 6.0 6:0341baf4:::benchmark_data_smithi052_234456_object22:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900224+0000", 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337807560000002, 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0258131610000001, 2023-09-23T22:41:42.280 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 23 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900224+0000", 2023-09-23T22:41:42.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900224+0000", 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900223+0000", 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900269+0000", 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5688e-05 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900275+0000", 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9619999999999998e-06 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900280+0000", 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.575e-06 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923664+0000", 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023383731000000001 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923668+0000", 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.484e-06 2023-09-23T22:41:42.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920864+0000", 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.997196254 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920973+0000", 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000108769 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921106+0000", 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013302600000000001 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925254+0000", 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041479220000000001 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925959+0000", 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00070547000000000003 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925993+0000", 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3612000000000003e-05 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926037+0000", 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4141999999999999e-05 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.287 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:24 6.0 6:0b99a54d:::benchmark_data_smithi052_234456_object23:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337740146000002, 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.026286555, 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.288 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 24 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900328+0000", 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7772999999999999e-05 2023-09-23T22:41:42.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900334+0000", 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2070000000000003e-06 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900337+0000", 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6550000000000002e-06 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.291 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923686+0000", 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023348694 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923692+0000", 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.818e-06 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921474+0000", 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.997782736 2023-09-23T22:41:42.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921590+0000", 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011517 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921733+0000", 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143277 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925283+0000", 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035501679999999998 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926440+0000", 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011565950000000001 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926524+0000", 2023-09-23T22:41:42.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4742999999999998e-05 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926578+0000", 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3273999999999998e-05 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:28 6.9 6:9a0dcc21:::benchmark_data_smithi052_234456_object27:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:41:42.295 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337685821000001, 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0164962040000001, 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 28 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.296 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900385+0000", 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0535999999999999e-05 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900391+0000", 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1619999999999999e-06 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900394+0000", 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6370000000000001e-06 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921968+0000", 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021573569000000001 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921975+0000", 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6569999999999997e-06 2023-09-23T22:41:42.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905926+0000", 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98395083999999999 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906026+0000", 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9716000000000005e-05 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906162+0000", 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000136558 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912952+0000", 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0067891499999999999 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916720+0000", 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037686159999999998 2023-09-23T22:41:42.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916761+0000", 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1146999999999998e-05 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916842+0000", 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0220000000000001e-05 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.302 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:45 6.9 6:93672be9:::benchmark_data_smithi052_234456_object44:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900401+0000", 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337630169000001, 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0188872680000001, 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.303 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 45 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900401+0000", 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900401+0000", 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900400+0000", 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900442+0000", 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1789999999999998e-05 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900447+0000", 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8130000000000003e-06 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900456+0000", 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9109999999999999e-06 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921997+0000", 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021541227 2023-09-23T22:41:42.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922586+0000", 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000589358 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906367+0000", 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98378031200000005 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906447+0000", 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0572e-05 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906575+0000", 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000127472 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912957+0000", 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063821770000000002 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919146+0000", 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0061887330000000001 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919195+0000", 2023-09-23T22:41:42.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9317000000000002e-05 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919288+0000", 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3289999999999996e-05 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:72 6.0 6:029c92c4:::benchmark_data_smithi052_234456_object71:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.310 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337567808000003, 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0262414900000001, 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:41:42.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900507+0000", 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4363000000000002e-05 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900515+0000", 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6760000000000007e-06 2023-09-23T22:41:42.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900520+0000", 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6700000000000002e-06 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923713+0000", 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023193394999999999 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923716+0000", 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.083e-06 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921995+0000", 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99827871099999999 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922138+0000", 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014364600000000001 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922288+0000", 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014937699999999999 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925313+0000", 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030248839999999998 2023-09-23T22:41:42.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926586+0000", 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012736679999999999 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926632+0000", 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6100999999999998e-05 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.317 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926705+0000", 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2418999999999999e-05 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:75 6.0 6:0ddd5ad2:::benchmark_data_smithi052_234456_object74:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337500200000001, 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.027114453, 2023-09-23T22:41:42.318 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.319 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.319 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.319 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.319 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.319 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75 2023-09-23T22:41:42.319 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900613+0000", 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2361999999999995e-05 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900618+0000", 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5040000000000002e-06 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900622+0000", 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9149999999999998e-06 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923738+0000", 2023-09-23T22:41:42.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023115244 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923741+0000", 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0400000000000001e-06 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922543+0000", 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.998801936 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922656+0000", 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011299 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922796+0000", 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014071299999999999 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926713+0000", 2023-09-23T22:41:42.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039166870000000003 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927511+0000", 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079828500000000003 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927558+0000", 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6536000000000003e-05 2023-09-23T22:41:42.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927645+0000", 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7645000000000005e-05 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:87 6.9 6:9eaa18f5:::benchmark_data_smithi052_234456_object86:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:41:42.327 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337400743000003, 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0188088259999999, 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 87 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.328 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:41:42.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900674+0000", 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3816000000000002e-05 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900681+0000", 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5780000000000001e-06 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900686+0000", 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2640000000000002e-06 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922616+0000", 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021929976 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922623+0000", 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5390000000000002e-06 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906760+0000", 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.984136699 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906861+0000", 2023-09-23T22:41:42.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010134200000000001 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906990+0000", 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012864899999999999 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912962+0000", 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059724019999999999 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919330+0000", 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.006368243 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919371+0000", 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0352000000000003e-05 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919439+0000", 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8473000000000007e-05 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:90 6.0 6:030f5829:::benchmark_data_smithi052_234456_object89:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900695+0000", 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337336413999999, 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0271018009999999, 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.335 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 90 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900695+0000", 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900695+0000", 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900694+0000", 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900734+0000", 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9341999999999997e-05 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900738+0000", 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7639999999999996e-06 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900742+0000", 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2909999999999999e-06 2023-09-23T22:41:42.338 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923763+0000", 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023021491000000002 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923766+0000", 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0070000000000002e-06 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.339 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923164+0000", 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99939733799999997 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923299+0000", 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000135239 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.340 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923470+0000", 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017162899999999999 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926718+0000", 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032476250000000001 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927680+0000", 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00096186999999999998 2023-09-23T22:41:42.341 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927718+0000", 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7710000000000003e-05 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927797+0000", 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9010000000000004e-05 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.342 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:92 6.9 6:9807eb97:::benchmark_data_smithi052_234456_object91:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337264769999997, 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.01884157, 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.343 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:41:42.344 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900824+0000", 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8517999999999998e-05 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.345 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900831+0000", 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0739999999999998e-06 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900835+0000", 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3209999999999998e-06 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.346 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922653+0000", 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021818273999999999 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922660+0000", 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9940000000000003e-06 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907201+0000", 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98454060200000004 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.347 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907305+0000", 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103848 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907388+0000", 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3669000000000001e-05 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.348 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912967+0000", 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055790249999999996 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919483+0000", 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065161849999999999 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919528+0000", 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4042000000000003e-05 2023-09-23T22:41:42.349 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919608+0000", 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0524000000000004e-05 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:93 6.0 6:0f027d47:::benchmark_data_smithi052_234456_object92:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900843+0000", 2023-09-23T22:41:42.350 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337188173000001, 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0271189679999999, 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 93 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900843+0000", 2023-09-23T22:41:42.351 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900843+0000", 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900842+0000", 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.352 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900881+0000", 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8917e-05 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900903+0000", 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1769000000000001e-05 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.353 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900909+0000", 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0510000000000002e-06 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923785+0000", 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022875933000000001 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923788+0000", 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9119999999999998e-06 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923720+0000", 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99993241899999996 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923845+0000", 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012472899999999999 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.355 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.924026+0000", 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000180886 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926726+0000", 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026996870000000001 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927834+0000", 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011082279999999999 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.356 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927883+0000", 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9249999999999998e-05 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927962+0000", 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8947000000000002e-05 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.357 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:101 6.9 6:9ca0677b:::benchmark_data_smithi052_234456_object100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900930+0000", 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.337101509, 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0190877739999999, 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101 2023-09-23T22:41:42.358 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900930+0000", 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900930+0000", 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.359 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900929+0000", 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900998+0000", 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8745000000000007e-05 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901031+0000", 2023-09-23T22:41:42.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3470999999999999e-05 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901062+0000", 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0997000000000002e-05 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922689+0000", 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021626510000000002 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922695+0000", 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.913e-06 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907555+0000", 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98486037800000004 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907675+0000", 2023-09-23T22:41:42.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119653 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907764+0000", 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8918000000000005e-05 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912972+0000", 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052083060000000002 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919650+0000", 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0066783859999999997 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919692+0000", 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1408000000000001e-05 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920017+0000", 2023-09-23T22:41:42.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032568300000000001 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:34028 6.9 6:9641f3a3:::benchmark_data_smithi052_234456_object11884:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:05.745624+0000", 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 36.492407665999998, 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14995703899999999, 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.365 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 34028 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745624+0000", 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745624+0000", 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745622+0000", 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745628+0000", 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8030000000000002e-06 2023-09-23T22:41:42.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745640+0000", 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1544e-05 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745647+0000", 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4900000000000003e-06 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745697+0000", 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9425999999999997e-05 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.889966+0000", 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14426931900000001 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.890067+0000", 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101331 2023-09-23T22:41:42.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.894361+0000", 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042942420000000002 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.895456+0000", 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010947369999999999 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.895501+0000", 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5087999999999999e-05 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.895581+0000", 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9313000000000004e-05 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:34029 6.0 6:0bacd20f:::benchmark_data_smithi052_234456_object11885:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:05.745663+0000", 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 36.492368540999998, 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14832375, 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 34029 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745663+0000", 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745663+0000", 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745662+0000", 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745665+0000", 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8109999999999999e-06 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745671+0000", 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1999999999999999e-06 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.745676+0000", 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7729999999999999e-06 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.889896+0000", 2023-09-23T22:41:42.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14421958800000001 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.890003+0000", 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107223 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.890126+0000", 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012283199999999999 2023-09-23T22:41:42.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.892544+0000", 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024179969999999999 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.893836+0000", 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012921530000000001 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.893868+0000", 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2257e-05 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:05.893986+0000", 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118433 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40807 6.9 6:9345ac67:::benchmark_data_smithi052_234456_object18663:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861640+0000", 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 32.376391720999997, 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2337652800000001, 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40807 2023-09-23T22:41:42.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861640+0000", 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861640+0000", 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861637+0000", 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861648+0000", 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0575e-05 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861672+0000", 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4164000000000001e-05 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861684+0000", 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2551999999999999e-05 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.382 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861747+0000", 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2843999999999997e-05 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861755+0000", 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2349999999999999e-06 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.868387+0000", 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0066316300000000003 2023-09-23T22:41:42.383 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.868400+0000", 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.29e-05 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858560+0000", 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99016036799999996 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.384 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858575+0000", 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4430000000000001e-05 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.086138+0000", 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2275634580000001 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.086287+0000", 2023-09-23T22:41:42.385 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014845 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.086427+0000", 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014055599999999999 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.088326+0000", 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018989790000000001 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.386 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.094860+0000", 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065336420000000001 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.095008+0000", 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014823399999999999 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.387 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.095077+0000", 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8437999999999998e-05 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.095405+0000", 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032823699999999999 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.388 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40811 6.0 6:02d50e34:::benchmark_data_smithi052_234456_object18667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.877262+0000", 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 32.36076937, 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.226087551, 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40811 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:42.389 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877262+0000", 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877262+0000", 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:42.390 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877260+0000", 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877267+0000", 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1940000000000001e-06 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877280+0000", 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2565000000000001e-05 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.391 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877288+0000", 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2060000000000007e-06 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877521+0000", 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000232687 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.392 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877533+0000", 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1844999999999999e-05 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.923517+0000", 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0459847 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.923529+0000", 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1555e-05 2023-09-23T22:41:42.393 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.097078+0000", 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.173549138 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.097281+0000", 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000202773 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.394 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.097491+0000", 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000210132 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.101496+0000", 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040048030000000004 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:42.395 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.101536+0000", 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0089000000000002e-05 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.103009+0000", 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014735950000000001 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.103064+0000", 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4772999999999999e-05 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:42.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.103349+0000", 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00028529100000000002 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:42.397 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:41:42.397 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:42.423 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:42.465 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:42.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:42.570 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:42.598 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.073 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:41:43.348 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:43.451 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:43.544 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:41:43.615 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:43.637 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:43.661 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:43.667 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:41:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:41:43] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.33.4" 2023-09-23T22:41:43.671 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.916 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:43.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:44.021 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:44.063 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.115 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:44.199 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:43 smithi052 ceph-mon[187743]: pgmap v345: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:44.199 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:43 smithi052 ceph-mon[180979]: pgmap v345: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:44.220 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:44.249 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.398 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:44.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:43 smithi151 ceph-mon[159210]: pgmap v345: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:44.501 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:44.532 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.581 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:44.686 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:44.714 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.761 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:44.867 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:44.894 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.936 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:45.002 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T22:41:45.041 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:45.070 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/255906665' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:45.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:45.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:45.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/255906665' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/255906665' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:45.333 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:45.361 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:45.446 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:45.450 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:45.911 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:45.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:46.015 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:46.047 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:46.095 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:46.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: pgmap v346: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.190 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[180979]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: pgmap v346: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:45 smithi052 ceph-mon[187743]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.201 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:46.229 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:46.273 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:46.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: pgmap v346: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:46.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:41:46.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T22:41:46.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:45 smithi151 ceph-mon[159210]: failed to encode map e151 with expected crc 2023-09-23T22:41:46.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:46.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:46.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:46.414 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.010 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:41:47.263 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3025019387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:47.263 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:47.264 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:41:47.264 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3025019387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:47.264 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:47.264 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:41:47.341 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:47.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:47.422 DEBUG:teuthology.orchestra.run.smithi052:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T22:41:47.425 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:41:47.425 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:41:47.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:47.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3025019387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:47.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:47.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:41:47.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:hints = 1 2023-09-23T22:41:47.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:41:47.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object prefix: benchmark_data_smithi052_277165 2023-09-23T22:41:47.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:41:47.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:41:47.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.607 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:47.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:47.753 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.827 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:47.935 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:47.964 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.009 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:48.010 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: pgmap v348: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.013 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/298941110' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:41:48.013 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:48 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:41:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:48.015 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:41:48.015 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:41:48.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.015 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1067 7.e 7:798da2d2:::benchmark_data_smithi052_277165_object1066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T22:41:48.015 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:47.917313+0000", 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.001821072, 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001858104, 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1067 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917313+0000", 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917313+0000", 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917311+0000", 2023-09-23T22:41:48.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917335+0000", 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4049999999999998e-05 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917346+0000", 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1365e-05 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917352+0000", 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1090000000000003e-06 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917403+0000", 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0983999999999998e-05 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:48.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917541+0000", 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000137875 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917693+0000", 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015155999999999999 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.020 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1069 7.8 7:11d40e9e:::benchmark_data_smithi052_277165_object1068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:47.917364+0000", 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.001769273, 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019278800000000001, 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1069 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:48.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917364+0000", 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917364+0000", 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:48.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917364+0000", 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917383+0000", 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9673e-05 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917391+0000", 2023-09-23T22:41:48.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9889999999999992e-06 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917397+0000", 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7180000000000001e-06 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.917997+0000", 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00060030599999999999 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918135+0000", 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000137794 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918272+0000", 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000136451 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:1068 7.a e152/146 7:55977206:::benchmark_data_smithi052_277165_object1067:head v 152'73, mlcod=152'71)", 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:47.918789+0000", 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.00034445999999999999, 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00061713399999999998, 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:48.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918789+0000", 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918789+0000", 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918788+0000", 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918815+0000", 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6656e-05 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918826+0000", 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0584000000000001e-05 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:48.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918833+0000", 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8639999999999998e-06 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918875+0000", 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2551e-05 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.918897+0000", 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1438999999999999e-05 2023-09-23T22:41:48.029 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1076 7.8 7.2a1e8388 (undecoded) ondisk+write+known_if_redirected e152)", 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:47.919427+0000", 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4294967295.9997072, 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0508999999999999e-05, 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:41:48.030 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1076 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.919427+0000", 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.919427+0000", 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.919425+0000", 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:41:48.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.919450+0000", 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4791e-05 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.919463+0000", 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2661e-05 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.919471+0000", 2023-09-23T22:41:48.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9619999999999993e-06 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 4 2023-09-23T22:41:48.034 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:41:48.116 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:48.155 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:48.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: pgmap v348: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:48.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/298941110' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: pgmap v348: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T22:41:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: failed to encode map e152 with expected crc 2023-09-23T22:41:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/298941110' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:41:48.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:48 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:41:48.305 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:48.375 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.476 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 1 16 2035 2019 126.182 126.188 0.00391776 0.00789869 2023-09-23T22:41:48.852 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:48.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:48.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:49.010 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:49.140 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:49.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:49.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[180979]: failed to encode map e152 with expected crc 2023-09-23T22:41:49.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[180979]: failed to encode map e152 with expected crc 2023-09-23T22:41:49.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[180979]: failed to encode map e153 with expected crc 2023-09-23T22:41:49.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[180979]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[180979]: failed to encode map e153 with expected crc 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[187743]: failed to encode map e152 with expected crc 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[187743]: failed to encode map e152 with expected crc 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[187743]: failed to encode map e153 with expected crc 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:41:49.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[187743]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T22:41:49.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:49 smithi052 ceph-mon[187743]: failed to encode map e153 with expected crc 2023-09-23T22:41:49.249 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:41:49.315 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:49 smithi151 ceph-mon[159210]: failed to encode map e152 with expected crc 2023-09-23T22:41:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:49 smithi151 ceph-mon[159210]: failed to encode map e152 with expected crc 2023-09-23T22:41:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:49 smithi151 ceph-mon[159210]: failed to encode map e153 with expected crc 2023-09-23T22:41:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:49 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:41:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:49 smithi151 ceph-mon[159210]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T22:41:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:49 smithi151 ceph-mon[159210]: failed to encode map e153 with expected crc 2023-09-23T22:41:49.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 2 16 4868 4852 151.613 177.062 0.00151666 0.0062653 2023-09-23T22:41:49.817 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: pgmap v351: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:50.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:50.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: pgmap v351: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:50.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T22:41:50.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:50.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:50.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:50.296 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:41:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:50.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:41:50.402 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: pgmap v351: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: failed to encode map e154 with expected crc 2023-09-23T22:41:50.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:50.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:50.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:50.447 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:50.475 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 3 16 6836 6820 142.071 123 0.00359792 0.007029 2023-09-23T22:41:50.515 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:50.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:50.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:50.636 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:41:50.688 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:50.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:41:50.895 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:41:50.970 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:50.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:51.082 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:51.116 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:51.167 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:51.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:51 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:51 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:51 smithi052 ceph-mon[180979]: pgmap v353: 89 pgs: 1 peering, 88 active+clean; 296 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T22:41:51.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:51 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:51.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:51 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:51.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:51 smithi052 ceph-mon[187743]: pgmap v353: 89 pgs: 1 peering, 88 active+clean; 296 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T22:41:51.275 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:51.395 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:51.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:51 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:51 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:41:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:51 smithi151 ceph-mon[159210]: pgmap v353: 89 pgs: 1 peering, 88 active+clean; 296 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T22:41:51.476 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 4 16 8669 8653 135.191 114.562 0.00370528 0.00738643 2023-09-23T22:41:52.476 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 5 16 10506 10490 131.113 114.812 0.00448136 0.00761852 2023-09-23T22:41:52.762 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:52.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:52.867 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:52.901 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:52.950 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:52.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:52.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:52.987 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:53.006 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:53.055 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:53.091 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:53.126 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:53.239 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:53.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:53.346 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:53.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:53.391 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:41:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:41:53] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-09-23T22:41:53.437 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:53.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:53.477 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 6 16 12528 12512 130.311 126.375 0.00499698 0.00766637 2023-09-23T22:41:53.542 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:53.576 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:53.607 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:41:53.639 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:53.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:53.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:53.653 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:41:53.693 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:41:53.738 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:41:53.744 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:53.787 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:41:53.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:53.840 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:41:53.889 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:41:53.943 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:41:53.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:53 smithi151 ceph-mon[159210]: pgmap v354: 89 pgs: 1 peering, 88 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-09-23T22:41:53.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2298061384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:41:53.962 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:54.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:53 smithi052 ceph-mon[187743]: pgmap v354: 89 pgs: 1 peering, 88 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-09-23T22:41:54.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2298061384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:54.150 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:53 smithi052 ceph-mon[180979]: pgmap v354: 89 pgs: 1 peering, 88 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-09-23T22:41:54.150 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2298061384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:54.476 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 7 16 14339 14323 127.863 113.188 0.00572575 0.00781256 2023-09-23T22:41:55.358 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:55.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:55.463 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:55.477 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 8 16 16291 16275 127.129 122 0.00607166 0.00785984 2023-09-23T22:41:55.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:55.567 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:55.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:55.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:55.674 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:55.710 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:55.727 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:55 smithi052 ceph-mon[180979]: pgmap v355: 89 pgs: 89 active+clean; 674 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2023-09-23T22:41:55.770 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:55.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:55 smithi151 ceph-mon[159210]: pgmap v355: 89 pgs: 89 active+clean; 674 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2023-09-23T22:41:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:55.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:55.876 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:55.916 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:55.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:55 smithi052 ceph-mon[187743]: pgmap v355: 89 pgs: 89 active+clean; 674 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2023-09-23T22:41:56.477 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 9 16 17554 17538 121.774 78.9375 0.00147755 0.00811857 2023-09-23T22:41:56.484 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:56.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:56.589 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:56.622 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:56.680 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:56.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:56.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:56.787 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:56.822 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:56.887 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:56.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:56.993 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:57.035 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:57.091 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:57.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:57.197 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:57.284 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:57.478 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 10 16 19534 19518 121.951 123.75 0.01007 0.00818928 2023-09-23T22:41:57.851 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:57 smithi052 ceph-mon[180979]: pgmap v356: 89 pgs: 89 active+clean; 898 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T22:41:58.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:57 smithi151 ceph-mon[159210]: pgmap v356: 89 pgs: 89 active+clean; 898 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T22:41:58.202 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:58.210 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:57 smithi052 ceph-mon[187743]: pgmap v356: 89 pgs: 89 active+clean; 898 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-09-23T22:41:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:58.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:58.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:58.349 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:58.413 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:58.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:58.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:58.478 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 11 16 21562 21546 122.385 126.75 0.00160819 0.00813675 2023-09-23T22:41:58.519 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:58.544 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:58.613 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:58.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:58.719 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:41:58.756 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:58.819 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:41:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:58.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:41:58.827 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:58.827 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:58.886 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:41:58.926 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3792183580' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:41:58.988 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.164 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:59.228 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3792183580' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:41:59.229 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3792183580' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:41:59.413 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:59.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:59.479 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 12 16 23408 23392 121.8 115.375 0.0121164 0.00820395 2023-09-23T22:41:59.523 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:59.556 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.624 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:59.729 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:59.755 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.815 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:59.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:59.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:59.947 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.990 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:00.096 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:00.129 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:59 smithi151 ceph-mon[159210]: pgmap v357: 89 pgs: 89 active+clean; 930 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T22:42:00.129 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:59 smithi151 ceph-mon[159210]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.129 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3792183580' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:42:00.130 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:59 smithi151 ceph-mon[159210]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T22:42:00.130 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:59 smithi151 ceph-mon[159210]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.130 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:41:59 smithi151 ceph-mon[159210]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.133 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:00.179 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:00.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[187743]: pgmap v357: 89 pgs: 89 active+clean; 930 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T22:42:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[187743]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3792183580' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:42:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[187743]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T22:42:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[187743]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[187743]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[180979]: pgmap v357: 89 pgs: 89 active+clean; 930 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T22:42:00.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[180979]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3792183580' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:42:00.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[180979]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T22:42:00.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[180979]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:41:59 smithi052 ceph-mon[180979]: failed to encode map e155 with expected crc 2023-09-23T22:42:00.286 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:00.355 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:00.480 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 13 16 24814 24798 119.175 87.875 0.00771781 0.00838456 2023-09-23T22:42:00.566 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:42:00.763 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:42:00.780 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:00.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:00.899 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:01.043 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:00 smithi151 ceph-mon[159210]: failed to encode map e156 with expected crc 2023-09-23T22:42:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:00 smithi151 ceph-mon[159210]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T22:42:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:00 smithi151 ceph-mon[159210]: failed to encode map e156 with expected crc 2023-09-23T22:42:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:00 smithi052 ceph-mon[187743]: failed to encode map e156 with expected crc 2023-09-23T22:42:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:00 smithi052 ceph-mon[187743]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T22:42:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:00 smithi052 ceph-mon[187743]: failed to encode map e156 with expected crc 2023-09-23T22:42:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:00 smithi052 ceph-mon[180979]: failed to encode map e156 with expected crc 2023-09-23T22:42:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:00 smithi052 ceph-mon[180979]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T22:42:01.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:00 smithi052 ceph-mon[180979]: failed to encode map e156 with expected crc 2023-09-23T22:42:01.260 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:01.474 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:01.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:01.482 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 14 16 26233 26217 116.997 88.6875 0.00899215 0.00854172 2023-09-23T22:42:01.579 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:01.678 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:01.921 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:01.936 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:01.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:01.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:01 smithi052 ceph-mon[187743]: pgmap v360: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:01.950 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:01 smithi052 ceph-mon[180979]: pgmap v360: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:02.042 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:02.112 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:02.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:01 smithi151 ceph-mon[159210]: pgmap v360: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:02.481 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 15 16 27860 27844 115.973 101.688 0.00401899 0.00861677 2023-09-23T22:42:02.908 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:42:03.165 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4259403450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:03.166 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:02 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:03.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4259403450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:03.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:02 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4259403450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:02 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:03.417 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:03.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:03.423 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:42:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:42:03] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-09-23T22:42:03.481 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 16 16 29534 29518 115.263 104.625 0.00643506 0.00867027 2023-09-23T22:42:03.524 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:03.557 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:03.614 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:03.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:03.721 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:03.775 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: pgmap v361: 89 pgs: 89 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:03.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[180979]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: pgmap v361: 89 pgs: 89 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: failed to encode map e157 with expected crc 2023-09-23T22:42:03.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:03 smithi052 ceph-mon[187743]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: pgmap v361: 89 pgs: 89 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:04.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T22:42:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:03 smithi151 ceph-mon[159210]: failed to encode map e157 with expected crc 2023-09-23T22:42:04.481 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 17 16 30842 30826 113.292 81.75 0.00308032 0.00879741 2023-09-23T22:42:04.863 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:04.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:04.976 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:05.008 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:05.064 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:05.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:05.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:05.172 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:05.203 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:05.481 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 18 16 32015 31999 111.071 73.3125 0.00205493 0.00893356 2023-09-23T22:42:05.598 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[187743]: pgmap v363: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[180979]: pgmap v363: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.831 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:05.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:05.938 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:05 smithi151 ceph-mon[159210]: pgmap v363: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:05.972 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:06.031 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:06.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:06.137 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:06.225 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:06.482 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 19 16 33446 33430 109.932 89.4375 0.012242 0.0090893 2023-09-23T22:42:06.703 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:42:06.703 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:42:06.704 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:06.709 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:42:06.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:06.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:06.753 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:42:06.795 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:42:06.820 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:06.838 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:42:06.876 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:42:06.894 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:42:06.894 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40792 6.f 6.8b4b57ef (undecoded) ondisk+retry+write+known_if_redirected e131)", 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 57.028119005999997, 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602794099999999, 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.895 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40792 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861820+0000", 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861828+0000", 2023-09-23T22:42:06.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861841+0000", 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2709e-05 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861851+0000", 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:42:06.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861957+0000", 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106518 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861972+0000", 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5226e-05 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875022+0000", 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013050091999999999 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875034+0000", 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1232e-05 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922087+0000", 2023-09-23T22:42:06.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047052836 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922101+0000", 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4476e-05 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40806 6.f 6.e80b2dbf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 57.028059902000003, 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602518750000001, 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40806 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861880+0000", 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861885+0000", 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.561e-06 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862005+0000", 2023-09-23T22:42:06.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120701 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862017+0000", 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862139+0000", 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012248000000000001 2023-09-23T22:42:06.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862145+0000", 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881337+0000", 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019192607 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881351+0000", 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3879999999999999e-05 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922124+0000", 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0407723719999999 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922133+0000", 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40815 6.f 6.8b4fc53f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 57.005678762000002, 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037899828, 2023-09-23T22:42:06.908 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40815 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884260+0000", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884266+0000", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6060000000000002e-06 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884276+0000", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7180000000000008e-06 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884284+0000", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6170000000000007e-06 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884403+0000", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011853099999999999 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884415+0000", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1683e-05 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922153+0000", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037738136 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922162+0000", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9279999999999999e-06 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40816 6.f 6.4f46a35f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 57.005637896000003, 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378988650000001, 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40816 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884302+0000", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884309+0000", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884317+0000", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5660000000000007e-06 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884324+0000", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0859999999999996e-06 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884441+0000", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011671 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884449+0000", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.571e-06 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922191+0000", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037741888 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922202+0000", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.092e-05 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40818 6.f 6.431951cf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 56.999720570999997, 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.032011485, 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40818 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890219+0000", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890224+0000", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7840000000000003e-06 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890235+0000", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0899e-05 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890243+0000", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.225e-06 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890301+0000", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890312+0000", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1235e-05 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922221+0000", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.031909277 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922232+0000", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0465000000000001e-05 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40826 6.f 6.3e4c7bff (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 56.987535092000002, 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.019876078, 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40826 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902404+0000", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902413+0000", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5449999999999992e-06 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902432+0000", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8488999999999999e-05 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902441+0000", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902502+0000", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1064999999999999e-05 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902511+0000", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7350000000000002e-06 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922270+0000", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0197598290000001 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922282+0000", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGRecoveryDelete(6.9 e140,132 6:9a60a6d7:::benchmark_data_smithi052_234456_object11263:head,129'2137)", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:23.018282+0000", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 43.871658224999997, 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.125639062, 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.018282+0000", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.018282+0000", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.018282+0000", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.018286+0000", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3660000000000002e-06 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.143023+0000", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.124736601 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.143046+0000", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3363999999999999e-05 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.143111+0000", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4626000000000001e-05 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.143122+0000", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1606999999999999e-05 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:23.143921+0000", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079901899999999999 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1103 7.e 7:705be4fa:::benchmark_data_smithi052_277165_object1102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:47.927898+0000", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 18.962042678, 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095867864999999997, 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1103 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927898+0000", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927898+0000", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927897+0000", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927920+0000", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3118000000000001e-05 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927934+0000", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4494e-05 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927944+0000", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5380000000000008e-06 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.927991+0000", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7240999999999997e-05 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.928140+0000", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014842800000000001 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.928286+0000", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014632600000000001 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929086+0000", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079973100000000003 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929442+0000", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00035610600000000002 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.023664+0000", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.094222604000000001 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.023725+0000", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0630000000000001e-05 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.023766+0000", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0759000000000001e-05 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1108 7.e 7:772cfe25:::benchmark_data_smithi052_277165_object1107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:47.929472+0000", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 18.960469099000001, 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096601888999999996, 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1108 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929472+0000", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929472+0000", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929470+0000", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929491+0000", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0602999999999999e-05 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929503+0000", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1980999999999999e-05 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929508+0000", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.356e-06 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929561+0000", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2735000000000002e-05 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929712+0000", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015143700000000001 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.929856+0000", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014322400000000001 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.930347+0000", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00049113 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:47.930931+0000", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00058404099999999999 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.025972+0000", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095041 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.026026+0000", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4716999999999998e-05 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.026073+0000", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6974999999999997e-05 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1401 7.e 7:712e9500:::benchmark_data_smithi052_277165_object1400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:48.161045+0000", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 18.728896026000001, 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10460401599999999, 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1401 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161045+0000", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161045+0000", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161043+0000", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161079+0000", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5701999999999998e-05 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161092+0000", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3008e-05 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161103+0000", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1314e-05 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161168+0000", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4708000000000005e-05 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161288+0000", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120617 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.161471+0000", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018233100000000001 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.162156+0000", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000684776 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.162849+0000", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069310599999999997 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.265555+0000", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10270621200000001 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.265608+0000", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3514999999999998e-05 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.265649+0000", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0228999999999997e-05 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:1412 7.e 7:707d5ee6:::benchmark_data_smithi052_277165_object1411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:48.164442+0000", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 18.725498626, 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.101925879, 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1412 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164442+0000", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164442+0000", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164441+0000", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164465+0000", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4085999999999999e-05 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164479+0000", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3838e-05 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164486+0000", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0419999999999997e-06 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164540+0000", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3936e-05 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164640+0000", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010047000000000001 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.164738+0000", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7491999999999994e-05 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.165141+0000", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040297800000000002 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.165618+0000", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00047708400000000002 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.266274+0000", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10065639999999999 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.266328+0000", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3763e-05 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:48.266368+0000", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9956999999999997e-05 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:6014 7.8 7:1b4220de:::benchmark_data_smithi052_277165_object6013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.138427+0000", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.751513815999999, 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071865859000000004, 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 6014 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138427+0000", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138427+0000", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138425+0000", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138449+0000", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3938999999999999e-05 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138458+0000", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.636e-06 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138463+0000", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4779999999999999e-06 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140115+0000", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001651851 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140244+0000", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012921100000000001 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140362+0000", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000117157 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141809+0000", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014471620000000001 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210158+0000", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.068349334999999997 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210219+0000", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1002000000000003e-05 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210258+0000", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8717999999999998e-05 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210293+0000", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4770000000000001e-05 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:6016 7.8 7:144d2f31:::benchmark_data_smithi052_277165_object6015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.138469+0000", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.751471809999998, 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071920755000000003, 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 6016 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138469+0000", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138469+0000", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138468+0000", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138526+0000", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7639000000000002e-05 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138535+0000", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8370000000000003e-06 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.138541+0000", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2369999999999997e-06 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140869+0000", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023280029999999999 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141003+0000", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013407499999999999 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141148+0000", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014448600000000001 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209214+0000", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.068066591999999995 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210189+0000", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097533299999999999 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210332+0000", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014250100000000001 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210369+0000", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7506000000000003e-05 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210390+0000", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0072999999999999e-05 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:6029 7.8 7:1514cb46:::benchmark_data_smithi052_277165_object6028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.139650+0000", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.750290394, 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.072807552999999997, 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 6029 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139650+0000", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139650+0000", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139649+0000", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139684+0000", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5157000000000002e-05 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139696+0000", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1697e-05 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139704+0000", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5129999999999996e-06 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209387+0000", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.069683142000000003 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209546+0000", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000158904 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209699+0000", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015315399999999999 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.211602+0000", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019030679999999999 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.212345+0000", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00074265099999999999 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.212376+0000", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0474e-05 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.212416+0000", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0905000000000002e-05 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.212458+0000", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1288000000000002e-05 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:6030 7.e 7:772d25fe:::benchmark_data_smithi052_277165_object6029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.139711+0000", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.750229717, 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070272642999999996, 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 6030 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139711+0000", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139711+0000", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139710+0000", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139732+0000", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1501e-05 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139740+0000", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3089999999999996e-06 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139746+0000", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2169999999999999e-06 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139787+0000", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1103999999999998e-05 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139900+0000", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011268999999999999 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140032+0000", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013179399999999999 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141112+0000", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010801879999999999 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209392+0000", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.068280023999999995 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209886+0000", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00049415099999999997 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209939+0000", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3109999999999998e-05 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209984+0000", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4362e-05 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:6032 7.e 7:7705d1b5:::benchmark_data_smithi052_277165_object6031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.139879+0000", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.750061856999999, 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070233724999999997, 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 6032 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139879+0000", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139879+0000", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139878+0000", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139900+0000", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2370000000000001e-05 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139910+0000", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9969999999999993e-06 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.139918+0000", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4159999999999998e-06 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140474+0000", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000556739 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140594+0000", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119926 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.140721+0000", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012656000000000001 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209188+0000", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.068467412000000005 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209776+0000", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00058767800000000001 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210026+0000", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025014699999999999 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210078+0000", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1657999999999999e-05 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210112+0000", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4740999999999998e-05 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:6036 7.e 7:7803beb1:::benchmark_data_smithi052_277165_object6035:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.141118+0000", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.748822904000001, 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.069118911000000005, 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 6036 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141118+0000", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141118+0000", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141117+0000", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141151+0000", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4539999999999998e-05 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141163+0000", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1871e-05 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141170+0000", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3470000000000001e-06 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141254+0000", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4007999999999999e-05 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141377+0000", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012227000000000001 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141465+0000", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8719000000000002e-05 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209254+0000", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067788047000000004 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209818+0000", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00056493699999999995 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210161+0000", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00034224599999999999 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210203+0000", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2784999999999998e-05 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210237+0000", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3127000000000003e-05 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:6033 7.3 e154/146 7:c274f55a:::benchmark_data_smithi052_277165_object6032:head v 154'379, mlcod=154'377)", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.141291+0000", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.748649573000002, 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.068800645999999993, 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141291+0000", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141291+0000", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141290+0000", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141321+0000", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1093999999999997e-05 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141332+0000", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.012e-05 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141338+0000", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1970000000000001e-06 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141529+0000", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019081600000000001 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141663+0000", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013395099999999999 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210050+0000", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.068387042999999995 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210092+0000", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2116000000000001e-05 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:6034 7.3 e154/146 7:c65884eb:::benchmark_data_smithi052_277165_object6033:head v 154'380, mlcod=154'377)", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:50.141352+0000", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.748588185999999, 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.069199277000000003, 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141352+0000", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141352+0000", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141352+0000", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141372+0000", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0315000000000001e-05 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141379+0000", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7819999999999996e-06 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.141384+0000", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5990000000000004e-06 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209232+0000", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067847809999999995 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.209274+0000", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2283000000000002e-05 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210522+0000", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012476810000000001 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:50.210552+0000", 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0071999999999999e-05 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:10452 7.e 7:752052f0:::benchmark_data_smithi052_277165_object10451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.312563+0000", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.577377243000001, 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10681325999999999, 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10452 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312563+0000", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312563+0000", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312562+0000", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312589+0000", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7698000000000001e-05 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312606+0000", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6767000000000001e-05 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312614+0000", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1340000000000004e-06 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312845+0000", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023026399999999999 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.312933+0000", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8918000000000005e-05 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.313089+0000", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015522299999999999 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.315021+0000", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001932006 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.315026+0000", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.237e-06 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.419286+0000", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.104260243 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.419339+0000", 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3211000000000003e-05 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.419377+0000", 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7230000000000001e-05 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:07.027 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:07.027 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:42:07.027 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:42:07.028 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:42:07.029 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[180979]: pgmap v364: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3429102537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[187743]: pgmap v364: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:07.030 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3429102537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:07.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:07.132 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:07.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:42:07.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:42:07.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:06 smithi151 ceph-mon[159210]: pgmap v364: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:07.166 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:07.166 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3429102537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:07.168 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.227 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:07.333 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:07.375 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.434 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:07.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:07.482 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:42:07.482860+0000 min lat: 0.00113345 max lat: 0.263266 avg lat: 0.00916821 2023-09-23T22:42:07.483 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:42:07.483 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 20 15 34911 34896 109.013 91.625 0.00942622 0.00916821 2023-09-23T22:42:07.538 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:07.563 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.617 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:07.655 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:42:07.724 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:07.750 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.801 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:07.907 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:07.940 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:07 smithi052 ceph-mon[180979]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T22:42:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:07 smithi052 ceph-mon[187743]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T22:42:08.003 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:08.007 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:07 smithi151 ceph-mon[159210]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T22:42:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:08.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:08.109 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:08.149 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:08.483 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 21 16 36160 36144 107.536 78 0.00598298 0.00929438 2023-09-23T22:42:08.717 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:08.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:08.825 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:08.864 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:08 smithi052 ceph-mon[187743]: pgmap v365: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:08.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:08 smithi052 ceph-mon[180979]: pgmap v365: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:09.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:08 smithi151 ceph-mon[159210]: pgmap v365: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T22:42:09.444 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:09.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:09.482 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 22 16 37646 37630 106.869 92.875 0.0126237 0.00935233 2023-09-23T22:42:09.550 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:09.554 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:42:09.587 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:09.645 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:09.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:09.752 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:09.785 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:09.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:09.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:09.923 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:09.943 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:09.971 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.025 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:10.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:10.131 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:10.210 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.359 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:10.467 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:10.483 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 23 16 38916 38900 105.672 79.375 0.00853666 0.00945713 2023-09-23T22:42:10.542 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.563 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:10.720 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:42:10.757 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:10.762 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:42:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:10.790 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:10 smithi052 ceph-mon[187743]: pgmap v366: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T22:42:10.790 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1954326781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:42:10.790 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:42:10.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:10 smithi052 ceph-mon[180979]: pgmap v366: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T22:42:10.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1954326781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:42:10.790 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:42:10.867 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:10.910 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.960 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:11.065 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:11.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:11.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:10 smithi151 ceph-mon[159210]: pgmap v366: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T22:42:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1954326781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:42:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:42:11.483 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 24 16 39462 39446 102.691 34.125 0.00258825 0.00947237 2023-09-23T22:42:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[180979]: failed to encode map e158 with expected crc 2023-09-23T22:42:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:42:11.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[180979]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T22:42:11.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[180979]: failed to encode map e158 with expected crc 2023-09-23T22:42:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[187743]: failed to encode map e158 with expected crc 2023-09-23T22:42:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:42:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[187743]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T22:42:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:11 smithi052 ceph-mon[187743]: failed to encode map e158 with expected crc 2023-09-23T22:42:12.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:11 smithi151 ceph-mon[159210]: failed to encode map e158 with expected crc 2023-09-23T22:42:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:42:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:11 smithi151 ceph-mon[159210]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T22:42:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:11 smithi151 ceph-mon[159210]: failed to encode map e158 with expected crc 2023-09-23T22:42:12.211 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:12.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:12.316 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:12.347 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:12.394 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:12.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:12.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:12.484 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 25 16 39462 39446 98.5841 0 - 0.00947237 2023-09-23T22:42:12.499 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:12.515 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T22:42:12.527 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:12.571 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:12.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:12.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:12.676 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:12.705 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:12.799 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:12 smithi052 ceph-mon[180979]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T22:42:12.800 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:12 smithi052 ceph-mon[180979]: failed to encode map e159 with expected crc 2023-09-23T22:42:12.800 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:12 smithi052 ceph-mon[180979]: pgmap v369: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T22:42:12.800 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:12 smithi052 ceph-mon[187743]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T22:42:12.800 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:12 smithi052 ceph-mon[187743]: failed to encode map e159 with expected crc 2023-09-23T22:42:12.800 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:12 smithi052 ceph-mon[187743]: pgmap v369: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T22:42:12.807 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:12.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:12.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:12.911 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:12.939 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:13.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:12 smithi151 ceph-mon[159210]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T22:42:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:12 smithi151 ceph-mon[159210]: failed to encode map e159 with expected crc 2023-09-23T22:42:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:12 smithi151 ceph-mon[159210]: pgmap v369: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T22:42:13.369 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:13.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:13.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:13.462 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:13.482 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:13.484 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 26 16 39672 39656 95.2945 6.5625 0.01806 0.0104873 2023-09-23T22:42:13.521 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:13.553 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:13.585 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:13.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:13.694 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:13.746 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:42:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:42:13] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T22:42:13.767 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:13.806 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:42:13.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:13 smithi151 ceph-mon[159210]: failed to encode map e160 with expected crc 2023-09-23T22:42:13.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:13 smithi151 ceph-mon[159210]: failed to encode map e160 with expected crc 2023-09-23T22:42:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:13 smithi151 ceph-mon[159210]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T22:42:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:13 smithi151 ceph-mon[159210]: failed to encode map e160 with expected crc 2023-09-23T22:42:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3151429952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:14.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[187743]: failed to encode map e160 with expected crc 2023-09-23T22:42:14.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[187743]: failed to encode map e160 with expected crc 2023-09-23T22:42:14.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[187743]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T22:42:14.012 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[187743]: failed to encode map e160 with expected crc 2023-09-23T22:42:14.013 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3151429952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:14.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[180979]: failed to encode map e160 with expected crc 2023-09-23T22:42:14.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[180979]: failed to encode map e160 with expected crc 2023-09-23T22:42:14.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[180979]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T22:42:14.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[180979]: failed to encode map e160 with expected crc 2023-09-23T22:42:14.013 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3151429952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:40510 7.4 e161/146 7:28a27070:::benchmark_data_smithi052_277165_object40509:head v 161'2515, mlcod=161'2512)", 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:14.136576+0000", 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.002420682, 2023-09-23T22:42:14.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024504790000000002, 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136576+0000", 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:14.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136576+0000", 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136576+0000", 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136602+0000", 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6313e-05 2023-09-23T22:42:14.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136615+0000", 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3179999999999999e-05 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136621+0000", 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9939999999999997e-06 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136676+0000", 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4564000000000001e-05 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136698+0000", 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1971999999999999e-05 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:14.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:40512 7.4 e161/146)", 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:14.136633+0000", 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0023634120000000001, 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024660340000000002, 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:14.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136633+0000", 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136633+0000", 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136633+0000", 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136659+0000", 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6106e-05 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136666+0000", 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8630000000000002e-06 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.136671+0000", 2023-09-23T22:42:14.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.853e-06 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 2 2023-09-23T22:42:14.161 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:42:14.484 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 27 16 41040 41024 94.9316 85.5 0.00269839 0.010528 2023-09-23T22:42:15.020 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3151429952' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.021 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[187743]: pgmap v372: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3151429952' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:14 smithi052 ceph-mon[180979]: pgmap v372: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3151429952' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: failed to encode map e161 with expected crc 2023-09-23T22:42:15.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:14 smithi151 ceph-mon[159210]: pgmap v372: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T22:42:15.484 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 28 16 42207 42191 94.146 72.9375 0.0069439 0.0106166 2023-09-23T22:42:15.780 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:15.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:15.890 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:15.922 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:16.011 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:16.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:16.119 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:16.165 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:16.252 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:16.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:16.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:16.359 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:16.405 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:16.468 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:16.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:16.476 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:42:16.486 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 29 16 43615 43599 93.9287 88 0.00818626 0.010639 2023-09-23T22:42:16.575 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:16.613 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:16.668 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:16.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:16.775 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:16.860 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:17.074 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:17 smithi052 ceph-mon[187743]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 2243/123063 objects misplaced (1.823%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:42:17.074 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:17 smithi052 ceph-mon[180979]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 2243/123063 objects misplaced (1.823%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:42:17.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:17 smithi151 ceph-mon[159210]: pgmap v373: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 2243/123063 objects misplaced (1.823%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:42:17.486 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 30 16 44896 44880 93.4656 80.0625 0.00416114 0.0106943 2023-09-23T22:42:17.645 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:17.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:17.758 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:17.793 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:17.807 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:42:17.859 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:17.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:17.966 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:18.053 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:18.100 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:42:18.107 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:18.141 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:42:18.191 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:42:18.238 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:42:18.272 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:42:18.280 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:18.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:18.312 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:42:18.332 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1404753460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:18.334 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1404753460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:18.347 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:42:18.387 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:42:18.389 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:42:18.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1404753460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:18.461 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:18.487 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 31 16 46061 46045 92.7981 72.8125 0.0189542 0.0107681 2023-09-23T22:42:18.947 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:18.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:19.059 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:19.105 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:19.122 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T22:42:19.186 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:19.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:19.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:19 smithi052 ceph-mon[187743]: pgmap v374: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 38 MiB/s wr, 609 op/s; 2243/123966 objects misplaced (1.809%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:42:19.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4105456610' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:42:19.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:19 smithi052 ceph-mon[180979]: pgmap v374: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 38 MiB/s wr, 609 op/s; 2243/123966 objects misplaced (1.809%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:42:19.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4105456610' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:42:19.294 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:19.322 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:19.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:19 smithi151 ceph-mon[159210]: pgmap v374: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 38 MiB/s wr, 609 op/s; 2243/123966 objects misplaced (1.809%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:42:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4105456610' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:42:19.487 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 32 16 47340 47324 92.396 79.9375 0.0250489 0.0108156 2023-09-23T22:42:19.810 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:19.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:19.915 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:19.985 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:20.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4105456610' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:42:20.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T22:42:20.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:20.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:20 smithi151 ceph-mon[159210]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T22:42:20.487 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 33 16 48525 48509 91.8403 74.0625 0.0131235 0.0108824 2023-09-23T22:42:20.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4105456610' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:42:20.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T22:42:20.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[180979]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4105456610' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T22:42:20.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: failed to encode map e162 with expected crc 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:20.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:20 smithi052 ceph-mon[187743]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T22:42:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:20.684 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:20.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:20.785 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:20.830 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:20.890 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:20.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:21.001 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:21.039 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:21.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:21.212 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:21.239 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:21 smithi052 ceph-mon[187743]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 2243/134619 objects misplaced (1.666%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:42:21.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:21 smithi052 ceph-mon[187743]: failed to encode map e163 with expected crc 2023-09-23T22:42:21.410 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:21 smithi052 ceph-mon[187743]: failed to encode map e163 with expected crc 2023-09-23T22:42:21.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:21 smithi052 ceph-mon[180979]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 2243/134619 objects misplaced (1.666%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:42:21.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:21 smithi052 ceph-mon[180979]: failed to encode map e163 with expected crc 2023-09-23T22:42:21.411 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:21 smithi052 ceph-mon[180979]: failed to encode map e163 with expected crc 2023-09-23T22:42:21.412 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:42:21.412 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:42:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:21 smithi151 ceph-mon[159210]: pgmap v376: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 2243/134619 objects misplaced (1.666%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:42:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:21 smithi151 ceph-mon[159210]: failed to encode map e163 with expected crc 2023-09-23T22:42:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:21 smithi151 ceph-mon[159210]: failed to encode map e163 with expected crc 2023-09-23T22:42:21.487 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 34 16 49556 49540 91.0344 64.4375 0.0104405 0.0109789 2023-09-23T22:42:21.493 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:21.583 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:21.604 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:21.689 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.936 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:22.045 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:22.081 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:22.152 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:22.265 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:22.305 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:22.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:42:22.366 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:22.475 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:22.494 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 35 16 50688 50672 90.4391 70.75 0.0103087 0.0110517 2023-09-23T22:42:22.509 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:22.510 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:22.510 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:22.573 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:22.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:22.793 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:23.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:23 smithi151 ceph-mon[159210]: pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 1667/143112 objects misplaced (1.165%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T22:42:23.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:42:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:42:23] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.33.4" 2023-09-23T22:42:23.494 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 36 16 51753 51737 89.776 66.5625 0.00396563 0.0111336 2023-09-23T22:42:23.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:23 smithi052 ceph-mon[180979]: pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 1667/143112 objects misplaced (1.165%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T22:42:23.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:23 smithi052 ceph-mon[187743]: pgmap v378: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 1667/143112 objects misplaced (1.165%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T22:42:24.097 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:24.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:24.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:24.192 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:24.206 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:24.237 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:42:24.315 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:24.461 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:24.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:24.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:24.494 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 37 16 52728 52712 88.9967 60.9375 0.00972284 0.0112309 2023-09-23T22:42:24.570 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:24.611 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:24.639 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:24.677 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:24.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:24.785 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:24.818 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:24.883 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:24.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:24.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:25.060 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:25.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:25 smithi151 ceph-mon[159210]: pgmap v379: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 1667/149175 objects misplaced (1.117%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:42:25.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3333556562' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:25.145 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:25 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:25.323 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:25 smithi052 ceph-mon[187743]: pgmap v379: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 1667/149175 objects misplaced (1.117%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:42:25.323 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3333556562' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:25.323 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:25 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:25.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:25 smithi052 ceph-mon[180979]: pgmap v379: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 1667/149175 objects misplaced (1.117%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:42:25.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3333556562' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:25.324 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:25 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:25.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 38 16 53796 53780 88.4071 66.75 0.0106458 0.0113059 2023-09-23T22:42:25.756 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:42:26.292 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:26.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:26.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:26.403 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:26.443 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:26 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:26.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:42:26.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:26 smithi151 ceph-mon[159210]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T22:42:26.447 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:26.492 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:26.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:26 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:26.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:42:26.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:26 smithi052 ceph-mon[187743]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T22:42:26.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:26 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:26.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:42:26.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:26 smithi052 ceph-mon[180979]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T22:42:26.495 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 39 16 55058 55042 88.1626 78.875 0.0043815 0.0113204 2023-09-23T22:42:26.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:26.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:26.597 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:26.664 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:27.112 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:27.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:27.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:27.226 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:27.267 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:27.350 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.352 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[180979]: pgmap v381: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 890/158337 objects misplaced (0.562%); 12 MiB/s, 195 objects/s recovering 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.353 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:27 smithi052 ceph-mon[187743]: pgmap v381: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 890/158337 objects misplaced (0.562%); 12 MiB/s, 195 objects/s recovering 2023-09-23T22:42:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:27.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: failed to encode map e164 with expected crc 2023-09-23T22:42:27.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:27 smithi151 ceph-mon[159210]: pgmap v381: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 890/158337 objects misplaced (0.562%); 12 MiB/s, 195 objects/s recovering 2023-09-23T22:42:27.454 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:27.481 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:27.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:42:27.496334+0000 min lat: 0.00113345 max lat: 2.5358 avg lat: 0.0113765 2023-09-23T22:42:27.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:42:27.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 40 16 56052 56036 87.5118 62.125 0.00264887 0.0113765 2023-09-23T22:42:27.525 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:27.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:27.628 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:27.666 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:27.750 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:27.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:27.865 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:27.920 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:27.990 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:28.102 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:28.138 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.200 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:28.307 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:28.379 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 41 16 57175 57159 87.0893 70.1875 0.0145946 0.0114769 2023-09-23T22:42:28.566 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:28.609 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:28.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:28.687 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:42:28.725 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:28.731 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:42:28.756 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.769 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:42:28.806 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:42:28.824 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:28.854 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:42:28.885 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:42:28.928 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:42:28.930 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:28.970 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.976 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.981 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:42:28.989 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:42:29.014 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:42:29.036 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:29.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:29.144 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:29.180 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:29.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:29 smithi052 ceph-mon[180979]: pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 890/159432 objects misplaced (0.558%); 11 MiB/s, 169 objects/s recovering 2023-09-23T22:42:29.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1271680274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:29.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:29 smithi052 ceph-mon[187743]: pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 890/159432 objects misplaced (0.558%); 11 MiB/s, 169 objects/s recovering 2023-09-23T22:42:29.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1271680274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:29.249 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:29.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:29.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:29.419 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:29.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:29 smithi151 ceph-mon[159210]: pgmap v382: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 890/159432 objects misplaced (0.558%); 11 MiB/s, 169 objects/s recovering 2023-09-23T22:42:29.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1271680274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:29.498 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 42 16 58267 58251 86.6408 68.25 0.00899452 0.0115363 2023-09-23T22:42:30.165 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:42:30.249 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:30.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:30.359 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:30.417 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:30 smithi052 ceph-mon[187743]: from='client.54832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:30.419 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:30 smithi052 ceph-mon[180979]: from='client.54832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:30.420 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:30.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:30 smithi151 ceph-mon[159210]: from='client.54832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:30.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 43 16 59329 59313 86.1695 66.375 0.0116399 0.0115972 2023-09-23T22:42:31.142 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":382,"stamp":"2023-09-23T22:42:28.024533+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3481868192,"num_objects":53144,"num_object_clones":0,"num_object_copies":159432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":890,"num_objects_unfound":0,"num_objects_dirty":53144,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":53380,"num_write_kb":3402125,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1590,"num_bytes_recovered":103677952,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53314,"ondisk_log_size":53314,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11780148,"kb_used_data":10234292,"kb_used_omap":126,"kb_used_meta":1545665,"kb_avail":738181068,"statfs":{"total":767960285184,"available":755897413632,"internally_reserved":0,"allocated":10479915008,"data_stored":10475232070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1582761595},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":542048256,"num_objects":8271,"num_object_clones":0,"num_object_copies":24813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1353,"num_objects_unfound":0,"num_objects_dirty":8271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8271,"num_write_kb":529344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1353,"num_bytes_recovered":88670208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005150"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":160,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114488+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:42:19.114488+0000","last_peered":"2023-09-23T22:42:19.114488+0000","last_clean":"2023-09-23T22:42:19.114488+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:42:19.114488+0000","last_undegraded":"2023-09-23T22:42:19.114488+0000","last_fullsized":"2023-09-23T22:42:19.114488+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.110064+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:42:19.110064+0000","last_peered":"2023-09-23T22:42:19.110064+0000","last_clean":"2023-09-23T22:42:19.110064+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:42:19.110064+0000","last_undegraded":"2023-09-23T22:42:19.110064+0000","last_fullsized":"2023-09-23T22:42:19.110064+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154424+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:42:20.154424+0000","last_peered":"2023-09-23T22:42:20.154424+0000","last_clean":"2023-09-23T22:42:20.154424+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:42:20.154424+0000","last_undegraded":"2023-09-23T22:42:20.154424+0000","last_fullsized":"2023-09-23T22:42:20.154424+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.107450+0000","last_change":"2023-09-23T22:40:50.920332+0000","last_active":"2023-09-23T22:42:19.107450+0000","last_peered":"2023-09-23T22:42:19.107450+0000","last_clean":"2023-09-23T22:42:19.107450+0000","last_became_active":"2023-09-23T22:40:50.919888+0000","last_became_peered":"2023-09-23T22:40:50.919888+0000","last_unstale":"2023-09-23T22:42:19.107450+0000","last_undegraded":"2023-09-23T22:42:19.107450+0000","last_fullsized":"2023-09-23T22:42:19.107450+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:24.536141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":176,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.110945+0000","last_change":"2023-09-23T22:40:50.916258+0000","last_active":"2023-09-23T22:42:19.110945+0000","last_peered":"2023-09-23T22:42:19.110945+0000","last_clean":"2023-09-23T22:42:19.110945+0000","last_became_active":"2023-09-23T22:40:50.915948+0000","last_became_peered":"2023-09-23T22:40:50.915948+0000","last_unstale":"2023-09-23T22:42:19.110945+0000","last_undegraded":"2023-09-23T22:42:19.110945+0000","last_fullsized":"2023-09-23T22:42:19.110945+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:32.210743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.117289+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:42:19.117289+0000","last_peered":"2023-09-23T22:42:19.117289+0000","last_clean":"2023-09-23T22:42:19.117289+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:42:19.117289+0000","last_undegraded":"2023-09-23T22:42:19.117289+0000","last_fullsized":"2023-09-23T22:42:19.117289+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.108345+0000","last_change":"2023-09-23T22:40:50.926694+0000","last_active":"2023-09-23T22:42:19.108345+0000","last_peered":"2023-09-23T22:42:19.108345+0000","last_clean":"2023-09-23T22:42:19.108345+0000","last_became_active":"2023-09-23T22:40:50.926142+0000","last_became_peered":"2023-09-23T22:40:50.926142+0000","last_unstale":"2023-09-23T22:42:19.108345+0000","last_undegraded":"2023-09-23T22:42:19.108345+0000","last_fullsized":"2023-09-23T22:42:19.108345+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:53.874735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":159,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.111013+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:42:19.111013+0000","last_peered":"2023-09-23T22:42:19.111013+0000","last_clean":"2023-09-23T22:42:19.111013+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:42:19.111013+0000","last_undegraded":"2023-09-23T22:42:19.111013+0000","last_fullsized":"2023-09-23T22:42:19.111013+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.138227+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:42:19.138227+0000","last_peered":"2023-09-23T22:42:19.138227+0000","last_clean":"2023-09-23T22:42:19.138227+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:42:19.138227+0000","last_undegraded":"2023-09-23T22:42:19.138227+0000","last_fullsized":"2023-09-23T22:42:19.138227+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":90,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.110406+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:42:19.110406+0000","last_peered":"2023-09-23T22:42:19.110406+0000","last_clean":"2023-09-23T22:42:19.110406+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:42:19.110406+0000","last_undegraded":"2023-09-23T22:42:19.110406+0000","last_fullsized":"2023-09-23T22:42:19.110406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.123220+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:42:19.123220+0000","last_peered":"2023-09-23T22:42:19.123220+0000","last_clean":"2023-09-23T22:42:19.123220+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:42:19.123220+0000","last_undegraded":"2023-09-23T22:42:19.123220+0000","last_fullsized":"2023-09-23T22:42:19.123220+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154596+0000","last_change":"2023-09-23T22:41:10.927399+0000","last_active":"2023-09-23T22:42:20.154596+0000","last_peered":"2023-09-23T22:42:20.154596+0000","last_clean":"2023-09-23T22:42:20.154596+0000","last_became_active":"2023-09-23T22:41:10.926366+0000","last_became_peered":"2023-09-23T22:41:10.926366+0000","last_unstale":"2023-09-23T22:42:20.154596+0000","last_undegraded":"2023-09-23T22:42:20.154596+0000","last_fullsized":"2023-09-23T22:42:20.154596+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:54.104169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.115056+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:42:19.115056+0000","last_peered":"2023-09-23T22:42:19.115056+0000","last_clean":"2023-09-23T22:42:19.115056+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:42:19.115056+0000","last_undegraded":"2023-09-23T22:42:19.115056+0000","last_fullsized":"2023-09-23T22:42:19.115056+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114717+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:42:19.114717+0000","last_peered":"2023-09-23T22:42:19.114717+0000","last_clean":"2023-09-23T22:42:19.114717+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:42:19.114717+0000","last_undegraded":"2023-09-23T22:42:19.114717+0000","last_fullsized":"2023-09-23T22:42:19.114717+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":146,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.122437+0000","last_change":"2023-09-23T22:41:10.925900+0000","last_active":"2023-09-23T22:42:19.122437+0000","last_peered":"2023-09-23T22:42:19.122437+0000","last_clean":"2023-09-23T22:42:19.122437+0000","last_became_active":"2023-09-23T22:41:10.925510+0000","last_became_peered":"2023-09-23T22:41:10.925510+0000","last_unstale":"2023-09-23T22:42:19.122437+0000","last_undegraded":"2023-09-23T22:42:19.122437+0000","last_fullsized":"2023-09-23T22:42:19.122437+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":152,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.110573+0000","last_change":"2023-09-23T22:40:15.114667+0000","last_active":"2023-09-23T22:42:19.110573+0000","last_peered":"2023-09-23T22:42:19.110573+0000","last_clean":"2023-09-23T22:42:19.110573+0000","last_became_active":"2023-09-23T22:40:15.114241+0000","last_became_peered":"2023-09-23T22:40:15.114241+0000","last_unstale":"2023-09-23T22:42:19.110573+0000","last_undegraded":"2023-09-23T22:42:19.110573+0000","last_fullsized":"2023-09-23T22:42:19.110573+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:13.807087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.102579+0000","last_change":"2023-09-23T22:40:50.920182+0000","last_active":"2023-09-23T22:42:19.102579+0000","last_peered":"2023-09-23T22:42:19.102579+0000","last_clean":"2023-09-23T22:42:19.102579+0000","last_became_active":"2023-09-23T22:40:50.919699+0000","last_became_peered":"2023-09-23T22:40:50.919699+0000","last_unstale":"2023-09-23T22:42:19.102579+0000","last_undegraded":"2023-09-23T22:42:19.102579+0000","last_fullsized":"2023-09-23T22:42:19.102579+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:56.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137939+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:42:19.137939+0000","last_peered":"2023-09-23T22:42:19.137939+0000","last_clean":"2023-09-23T22:42:19.137939+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:42:19.137939+0000","last_undegraded":"2023-09-23T22:42:19.137939+0000","last_fullsized":"2023-09-23T22:42:19.137939+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.121905+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:42:19.121905+0000","last_peered":"2023-09-23T22:42:19.121905+0000","last_clean":"2023-09-23T22:42:19.121905+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:42:19.121905+0000","last_undegraded":"2023-09-23T22:42:19.121905+0000","last_fullsized":"2023-09-23T22:42:19.121905+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.104451+0000","last_change":"2023-09-23T22:40:50.920158+0000","last_active":"2023-09-23T22:42:19.104451+0000","last_peered":"2023-09-23T22:42:19.104451+0000","last_clean":"2023-09-23T22:42:19.104451+0000","last_became_active":"2023-09-23T22:40:50.919445+0000","last_became_peered":"2023-09-23T22:40:50.919445+0000","last_unstale":"2023-09-23T22:42:19.104451+0000","last_undegraded":"2023-09-23T22:42:19.104451+0000","last_fullsized":"2023-09-23T22:42:19.104451+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:33.679699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":192,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.115708+0000","last_change":"2023-09-23T22:42:00.067286+0000","last_active":"2023-09-23T22:42:19.115708+0000","last_peered":"2023-09-23T22:42:19.115708+0000","last_clean":"2023-09-23T22:42:19.115708+0000","last_became_active":"2023-09-23T22:41:59.891905+0000","last_became_peered":"2023-09-23T22:41:59.891905+0000","last_unstale":"2023-09-23T22:42:19.115708+0000","last_undegraded":"2023-09-23T22:42:19.115708+0000","last_fullsized":"2023-09-23T22:42:19.115708+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.112930+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:42:19.112930+0000","last_peered":"2023-09-23T22:42:19.112930+0000","last_clean":"2023-09-23T22:42:19.112930+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:42:19.112930+0000","last_undegraded":"2023-09-23T22:42:19.112930+0000","last_fullsized":"2023-09-23T22:42:19.112930+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"164'3482","reported_seq":3509,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-09-23T22:42:26.823839+0000","last_change":"2023-09-23T22:41:36.969870+0000","last_active":"2023-09-23T22:42:26.823839+0000","last_peered":"2023-09-23T22:42:26.823839+0000","last_clean":"2023-09-23T22:42:26.823839+0000","last_became_active":"2023-09-23T22:41:36.969535+0000","last_became_peered":"2023-09-23T22:41:36.969535+0000","last_unstale":"2023-09-23T22:42:26.823839+0000","last_undegraded":"2023-09-23T22:42:26.823839+0000","last_fullsized":"2023-09-23T22:42:26.823839+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3482,"log_dups_size":0,"ondisk_log_size":3482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:24.978626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228196352,"num_objects":3482,"num_object_clones":0,"num_object_copies":10446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3482,"num_write_kb":222848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.122702+0000","last_change":"2023-09-23T22:40:50.909706+0000","last_active":"2023-09-23T22:42:19.122702+0000","last_peered":"2023-09-23T22:42:19.122702+0000","last_clean":"2023-09-23T22:42:19.122702+0000","last_became_active":"2023-09-23T22:40:50.909320+0000","last_became_peered":"2023-09-23T22:40:50.909320+0000","last_unstale":"2023-09-23T22:42:19.122702+0000","last_undegraded":"2023-09-23T22:42:19.122702+0000","last_fullsized":"2023-09-23T22:42:19.122702+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137231+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:42:19.137231+0000","last_peered":"2023-09-23T22:42:19.137231+0000","last_clean":"2023-09-23T22:42:19.137231+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:42:19.137231+0000","last_undegraded":"2023-09-23T22:42:19.137231+0000","last_fullsized":"2023-09-23T22:42:19.137231+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"163'3252","reported_seq":3278,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:24.356295+0000","last_change":"2023-09-23T22:41:36.967245+0000","last_active":"2023-09-23T22:42:24.356295+0000","last_peered":"2023-09-23T22:42:24.356295+0000","last_clean":"2023-09-23T22:42:24.356295+0000","last_became_active":"2023-09-23T22:41:36.966938+0000","last_became_peered":"2023-09-23T22:41:36.966938+0000","last_unstale":"2023-09-23T22:42:24.356295+0000","last_undegraded":"2023-09-23T22:42:24.356295+0000","last_fullsized":"2023-09-23T22:42:24.356295+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3252,"log_dups_size":0,"ondisk_log_size":3252,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:31.285489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213123072,"num_objects":3252,"num_object_clones":0,"num_object_copies":9756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3252,"num_write_kb":208128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.122807+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:42:19.122807+0000","last_peered":"2023-09-23T22:42:19.122807+0000","last_clean":"2023-09-23T22:42:19.122807+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:42:19.122807+0000","last_undegraded":"2023-09-23T22:42:19.122807+0000","last_fullsized":"2023-09-23T22:42:19.122807+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":143,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116986+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:42:19.116986+0000","last_peered":"2023-09-23T22:42:19.116986+0000","last_clean":"2023-09-23T22:42:19.116986+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:42:19.116986+0000","last_undegraded":"2023-09-23T22:42:19.116986+0000","last_fullsized":"2023-09-23T22:42:19.116986+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"163'3260","reported_seq":3286,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:23.903887+0000","last_change":"2023-09-23T22:41:36.969607+0000","last_active":"2023-09-23T22:42:23.903887+0000","last_peered":"2023-09-23T22:42:23.903887+0000","last_clean":"2023-09-23T22:42:23.903887+0000","last_became_active":"2023-09-23T22:41:36.969311+0000","last_became_peered":"2023-09-23T22:41:36.969311+0000","last_unstale":"2023-09-23T22:42:23.903887+0000","last_undegraded":"2023-09-23T22:42:23.903887+0000","last_fullsized":"2023-09-23T22:42:23.903887+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3260,"log_dups_size":0,"ondisk_log_size":3260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:27.916919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213647360,"num_objects":3260,"num_object_clones":0,"num_object_copies":9780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3260,"num_write_kb":208640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":115,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.123395+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:42:19.123395+0000","last_peered":"2023-09-23T22:42:19.123395+0000","last_clean":"2023-09-23T22:42:19.123395+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:42:19.123395+0000","last_undegraded":"2023-09-23T22:42:19.123395+0000","last_fullsized":"2023-09-23T22:42:19.123395+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116068+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:42:19.116068+0000","last_peered":"2023-09-23T22:42:19.116068+0000","last_clean":"2023-09-23T22:42:19.116068+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:42:19.116068+0000","last_undegraded":"2023-09-23T22:42:19.116068+0000","last_fullsized":"2023-09-23T22:42:19.116068+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"163'3282","reported_seq":3306,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:23.911253+0000","last_change":"2023-09-23T22:41:36.966000+0000","last_active":"2023-09-23T22:42:23.911253+0000","last_peered":"2023-09-23T22:42:23.911253+0000","last_clean":"2023-09-23T22:42:23.911253+0000","last_became_active":"2023-09-23T22:41:36.965691+0000","last_became_peered":"2023-09-23T22:41:36.965691+0000","last_unstale":"2023-09-23T22:42:23.911253+0000","last_undegraded":"2023-09-23T22:42:23.911253+0000","last_fullsized":"2023-09-23T22:42:23.911253+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3282,"log_dups_size":0,"ondisk_log_size":3282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:59.028709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215089152,"num_objects":3282,"num_object_clones":0,"num_object_copies":9846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3282,"num_write_kb":210048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137727+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:42:19.137727+0000","last_peered":"2023-09-23T22:42:19.137727+0000","last_clean":"2023-09-23T22:42:19.137727+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:42:19.137727+0000","last_undegraded":"2023-09-23T22:42:19.137727+0000","last_fullsized":"2023-09-23T22:42:19.137727+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":92,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.104379+0000","last_change":"2023-09-23T22:40:50.917176+0000","last_active":"2023-09-23T22:42:19.104379+0000","last_peered":"2023-09-23T22:42:19.104379+0000","last_clean":"2023-09-23T22:42:19.104379+0000","last_became_active":"2023-09-23T22:40:50.915836+0000","last_became_peered":"2023-09-23T22:40:50.915836+0000","last_unstale":"2023-09-23T22:42:19.104379+0000","last_undegraded":"2023-09-23T22:42:19.104379+0000","last_fullsized":"2023-09-23T22:42:19.104379+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:37.211725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"163'3229","reported_seq":3255,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:23.929776+0000","last_change":"2023-09-23T22:41:36.966198+0000","last_active":"2023-09-23T22:42:23.929776+0000","last_peered":"2023-09-23T22:42:23.929776+0000","last_clean":"2023-09-23T22:42:23.929776+0000","last_became_active":"2023-09-23T22:41:36.965865+0000","last_became_peered":"2023-09-23T22:41:36.965865+0000","last_unstale":"2023-09-23T22:42:23.929776+0000","last_undegraded":"2023-09-23T22:42:23.929776+0000","last_fullsized":"2023-09-23T22:42:23.929776+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3229,"log_dups_size":0,"ondisk_log_size":3229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:03.020464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211615744,"num_objects":3229,"num_object_clones":0,"num_object_copies":9687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3229,"num_write_kb":206656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.112603+0000","last_change":"2023-09-23T22:40:50.914941+0000","last_active":"2023-09-23T22:42:19.112603+0000","last_peered":"2023-09-23T22:42:19.112603+0000","last_clean":"2023-09-23T22:42:19.112603+0000","last_became_active":"2023-09-23T22:40:50.914602+0000","last_became_peered":"2023-09-23T22:40:50.914602+0000","last_unstale":"2023-09-23T22:42:19.112603+0000","last_undegraded":"2023-09-23T22:42:19.112603+0000","last_fullsized":"2023-09-23T22:42:19.112603+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:04.998198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114928+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:42:19.114928+0000","last_peered":"2023-09-23T22:42:19.114928+0000","last_clean":"2023-09-23T22:42:19.114928+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:42:19.114928+0000","last_undegraded":"2023-09-23T22:42:19.114928+0000","last_fullsized":"2023-09-23T22:42:19.114928+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"163'3205","reported_seq":3231,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:23.892061+0000","last_change":"2023-09-23T22:41:36.972030+0000","last_active":"2023-09-23T22:42:23.892061+0000","last_peered":"2023-09-23T22:42:23.892061+0000","last_clean":"2023-09-23T22:42:23.892061+0000","last_became_active":"2023-09-23T22:41:36.971693+0000","last_became_peered":"2023-09-23T22:41:36.971693+0000","last_unstale":"2023-09-23T22:42:23.892061+0000","last_undegraded":"2023-09-23T22:42:23.892061+0000","last_fullsized":"2023-09-23T22:42:23.892061+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3205,"log_dups_size":0,"ondisk_log_size":3205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:42:52.762839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210042880,"num_objects":3205,"num_object_clones":0,"num_object_copies":9615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3205,"num_write_kb":205120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":147,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116206+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:42:19.116206+0000","last_peered":"2023-09-23T22:42:19.116206+0000","last_clean":"2023-09-23T22:42:19.116206+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:42:19.116206+0000","last_undegraded":"2023-09-23T22:42:19.116206+0000","last_fullsized":"2023-09-23T22:42:19.116206+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":158,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114887+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:42:19.114887+0000","last_peered":"2023-09-23T22:42:19.114887+0000","last_clean":"2023-09-23T22:42:19.114887+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:42:19.114887+0000","last_undegraded":"2023-09-23T22:42:19.114887+0000","last_fullsized":"2023-09-23T22:42:19.114887+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"163'3359","reported_seq":3383,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:25.353049+0000","last_change":"2023-09-23T22:41:36.971145+0000","last_active":"2023-09-23T22:42:25.353049+0000","last_peered":"2023-09-23T22:42:25.353049+0000","last_clean":"2023-09-23T22:42:25.353049+0000","last_became_active":"2023-09-23T22:41:36.970815+0000","last_became_peered":"2023-09-23T22:41:36.970815+0000","last_unstale":"2023-09-23T22:42:25.353049+0000","last_undegraded":"2023-09-23T22:42:25.353049+0000","last_fullsized":"2023-09-23T22:42:25.353049+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3359,"log_dups_size":0,"ondisk_log_size":3359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220135424,"num_objects":3359,"num_object_clones":0,"num_object_copies":10077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3359,"num_write_kb":214976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116749+0000","last_change":"2023-09-23T22:41:10.925985+0000","last_active":"2023-09-23T22:42:19.116749+0000","last_peered":"2023-09-23T22:42:19.116749+0000","last_clean":"2023-09-23T22:42:19.116749+0000","last_became_active":"2023-09-23T22:41:10.925598+0000","last_became_peered":"2023-09-23T22:41:10.925598+0000","last_unstale":"2023-09-23T22:42:19.116749+0000","last_undegraded":"2023-09-23T22:42:19.116749+0000","last_fullsized":"2023-09-23T22:42:19.116749+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":146,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.115030+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:42:19.115030+0000","last_peered":"2023-09-23T22:42:19.115030+0000","last_clean":"2023-09-23T22:42:19.115030+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:42:19.115030+0000","last_undegraded":"2023-09-23T22:42:19.115030+0000","last_fullsized":"2023-09-23T22:42:19.115030+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"163'3321","reported_seq":3346,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:25.393457+0000","last_change":"2023-09-23T22:41:36.970313+0000","last_active":"2023-09-23T22:42:25.393457+0000","last_peered":"2023-09-23T22:42:25.393457+0000","last_clean":"2023-09-23T22:42:25.393457+0000","last_became_active":"2023-09-23T22:41:36.969917+0000","last_became_peered":"2023-09-23T22:41:36.969917+0000","last_unstale":"2023-09-23T22:42:25.393457+0000","last_undegraded":"2023-09-23T22:42:25.393457+0000","last_fullsized":"2023-09-23T22:42:25.393457+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3321,"log_dups_size":0,"ondisk_log_size":3321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217645056,"num_objects":3321,"num_object_clones":0,"num_object_copies":9963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3321,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":134,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.251174+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:42:20.251174+0000","last_peered":"2023-09-23T22:42:20.251174+0000","last_clean":"2023-09-23T22:42:20.251174+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:42:20.251174+0000","last_undegraded":"2023-09-23T22:42:20.251174+0000","last_fullsized":"2023-09-23T22:42:20.251174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":86,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.118260+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:42:19.118260+0000","last_peered":"2023-09-23T22:42:19.118260+0000","last_clean":"2023-09-23T22:42:19.118260+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:42:19.118260+0000","last_undegraded":"2023-09-23T22:42:19.118260+0000","last_fullsized":"2023-09-23T22:42:19.118260+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"163'3423","reported_seq":6621,"reported_epoch":163,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:42:25.429151+0000","last_change":"2023-09-23T22:42:13.306141+0000","last_active":"2023-09-23T22:42:25.429151+0000","last_peered":"2023-09-23T22:42:25.429151+0000","last_clean":"2023-09-23T22:42:10.731723+0000","last_became_active":"2023-09-23T22:42:13.283644+0000","last_became_peered":"2023-09-23T22:42:13.283644+0000","last_unstale":"2023-09-23T22:42:25.429151+0000","last_undegraded":"2023-09-23T22:42:25.429151+0000","last_fullsized":"2023-09-23T22:42:12.752570+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3423,"log_dups_size":0,"ondisk_log_size":3423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224329728,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":890,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3423,"num_write_kb":219072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1582,"num_bytes_recovered":103677952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":890},{"shards":"3,4","objects":2533}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":159,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154640+0000","last_change":"2023-09-23T22:40:50.918358+0000","last_active":"2023-09-23T22:42:20.154640+0000","last_peered":"2023-09-23T22:42:20.154640+0000","last_clean":"2023-09-23T22:42:20.154640+0000","last_became_active":"2023-09-23T22:40:50.918016+0000","last_became_peered":"2023-09-23T22:40:50.918016+0000","last_unstale":"2023-09-23T22:42:20.154640+0000","last_undegraded":"2023-09-23T22:42:20.154640+0000","last_fullsized":"2023-09-23T22:42:20.154640+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:14.593638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":159,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.138006+0000","last_change":"2023-09-23T22:41:10.929497+0000","last_active":"2023-09-23T22:42:19.138006+0000","last_peered":"2023-09-23T22:42:19.138006+0000","last_clean":"2023-09-23T22:42:19.138006+0000","last_became_active":"2023-09-23T22:41:10.929128+0000","last_became_peered":"2023-09-23T22:41:10.929128+0000","last_unstale":"2023-09-23T22:42:19.138006+0000","last_undegraded":"2023-09-23T22:42:19.138006+0000","last_fullsized":"2023-09-23T22:42:19.138006+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":104,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154502+0000","last_change":"2023-09-23T22:40:14.755608+0000","last_active":"2023-09-23T22:42:20.154502+0000","last_peered":"2023-09-23T22:42:20.154502+0000","last_clean":"2023-09-23T22:42:20.154502+0000","last_became_active":"2023-09-23T22:40:14.755335+0000","last_became_peered":"2023-09-23T22:40:14.755335+0000","last_unstale":"2023-09-23T22:42:20.154502+0000","last_undegraded":"2023-09-23T22:42:20.154502+0000","last_fullsized":"2023-09-23T22:42:20.154502+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:31.014369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"163'3393","reported_seq":3418,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:25.401641+0000","last_change":"2023-09-23T22:41:36.972228+0000","last_active":"2023-09-23T22:42:25.401641+0000","last_peered":"2023-09-23T22:42:25.401641+0000","last_clean":"2023-09-23T22:42:25.401641+0000","last_became_active":"2023-09-23T22:41:36.971904+0000","last_became_peered":"2023-09-23T22:41:36.971904+0000","last_unstale":"2023-09-23T22:42:25.401641+0000","last_undegraded":"2023-09-23T22:42:25.401641+0000","last_fullsized":"2023-09-23T22:42:25.401641+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3393,"log_dups_size":0,"ondisk_log_size":3393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222363648,"num_objects":3393,"num_object_clones":0,"num_object_copies":10179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3393,"num_write_kb":217152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137383+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:42:19.137383+0000","last_peered":"2023-09-23T22:42:19.137383+0000","last_clean":"2023-09-23T22:42:19.137383+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:42:19.137383+0000","last_undegraded":"2023-09-23T22:42:19.137383+0000","last_fullsized":"2023-09-23T22:42:19.137383+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114860+0000","last_change":"2023-09-23T22:40:50.916179+0000","last_active":"2023-09-23T22:42:19.114860+0000","last_peered":"2023-09-23T22:42:19.114860+0000","last_clean":"2023-09-23T22:42:19.114860+0000","last_became_active":"2023-09-23T22:40:50.915799+0000","last_became_peered":"2023-09-23T22:40:50.915799+0000","last_unstale":"2023-09-23T22:42:19.114860+0000","last_undegraded":"2023-09-23T22:42:19.114860+0000","last_fullsized":"2023-09-23T22:42:19.114860+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137590+0000","last_change":"2023-09-23T22:41:10.929964+0000","last_active":"2023-09-23T22:42:19.137590+0000","last_peered":"2023-09-23T22:42:19.137590+0000","last_clean":"2023-09-23T22:42:19.137590+0000","last_became_active":"2023-09-23T22:41:10.929689+0000","last_became_peered":"2023-09-23T22:41:10.929689+0000","last_unstale":"2023-09-23T22:42:19.137590+0000","last_undegraded":"2023-09-23T22:42:19.137590+0000","last_fullsized":"2023-09-23T22:42:19.137590+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"163'3229","reported_seq":3255,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:23.919275+0000","last_change":"2023-09-23T22:41:36.968996+0000","last_active":"2023-09-23T22:42:23.919275+0000","last_peered":"2023-09-23T22:42:23.919275+0000","last_clean":"2023-09-23T22:42:23.919275+0000","last_became_active":"2023-09-23T22:41:36.968694+0000","last_became_peered":"2023-09-23T22:41:36.968694+0000","last_unstale":"2023-09-23T22:42:23.919275+0000","last_undegraded":"2023-09-23T22:42:23.919275+0000","last_fullsized":"2023-09-23T22:42:23.919275+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3229,"log_dups_size":0,"ondisk_log_size":3229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:14.636304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211615744,"num_objects":3229,"num_object_clones":0,"num_object_copies":9687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3229,"num_write_kb":206656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":135,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116655+0000","last_change":"2023-09-23T22:41:10.929848+0000","last_active":"2023-09-23T22:42:19.116655+0000","last_peered":"2023-09-23T22:42:19.116655+0000","last_clean":"2023-09-23T22:42:19.116655+0000","last_became_active":"2023-09-23T22:41:10.929552+0000","last_became_peered":"2023-09-23T22:41:10.929552+0000","last_unstale":"2023-09-23T22:42:19.116655+0000","last_undegraded":"2023-09-23T22:42:19.116655+0000","last_fullsized":"2023-09-23T22:42:19.116655+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:09.783138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":134,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154743+0000","last_change":"2023-09-23T22:41:10.927089+0000","last_active":"2023-09-23T22:42:20.154743+0000","last_peered":"2023-09-23T22:42:20.154743+0000","last_clean":"2023-09-23T22:42:20.154743+0000","last_became_active":"2023-09-23T22:41:10.926767+0000","last_became_peered":"2023-09-23T22:41:10.926767+0000","last_unstale":"2023-09-23T22:42:20.154743+0000","last_undegraded":"2023-09-23T22:42:20.154743+0000","last_fullsized":"2023-09-23T22:42:20.154743+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:03.097039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":155,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154789+0000","last_change":"2023-09-23T22:40:51.909811+0000","last_active":"2023-09-23T22:42:20.154789+0000","last_peered":"2023-09-23T22:42:20.154789+0000","last_clean":"2023-09-23T22:42:20.154789+0000","last_became_active":"2023-09-23T22:40:51.909464+0000","last_became_peered":"2023-09-23T22:40:51.909464+0000","last_unstale":"2023-09-23T22:42:20.154789+0000","last_undegraded":"2023-09-23T22:42:20.154789+0000","last_fullsized":"2023-09-23T22:42:20.154789+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.517012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"163'3314","reported_seq":3340,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:23.905643+0000","last_change":"2023-09-23T22:41:36.963752+0000","last_active":"2023-09-23T22:42:23.905643+0000","last_peered":"2023-09-23T22:42:23.905643+0000","last_clean":"2023-09-23T22:42:23.905643+0000","last_became_active":"2023-09-23T22:41:36.963461+0000","last_became_peered":"2023-09-23T22:41:36.963461+0000","last_unstale":"2023-09-23T22:42:23.905643+0000","last_undegraded":"2023-09-23T22:42:23.905643+0000","last_fullsized":"2023-09-23T22:42:23.905643+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3314,"log_dups_size":0,"ondisk_log_size":3314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:02.507496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3314,"num_write_kb":212096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":148,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137549+0000","last_change":"2023-09-23T22:41:10.925966+0000","last_active":"2023-09-23T22:42:19.137549+0000","last_peered":"2023-09-23T22:42:19.137549+0000","last_clean":"2023-09-23T22:42:19.137549+0000","last_became_active":"2023-09-23T22:41:10.925674+0000","last_became_peered":"2023-09-23T22:41:10.925674+0000","last_unstale":"2023-09-23T22:42:19.137549+0000","last_undegraded":"2023-09-23T22:42:19.137549+0000","last_fullsized":"2023-09-23T22:42:19.137549+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":380,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.123338+0000","last_change":"2023-09-23T22:40:50.904717+0000","last_active":"2023-09-23T22:42:19.123338+0000","last_peered":"2023-09-23T22:42:19.123338+0000","last_clean":"2023-09-23T22:42:19.123338+0000","last_became_active":"2023-09-23T22:40:50.904259+0000","last_became_peered":"2023-09-23T22:40:50.904259+0000","last_unstale":"2023-09-23T22:42:19.123338+0000","last_undegraded":"2023-09-23T22:42:19.123338+0000","last_fullsized":"2023-09-23T22:42:19.123338+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.111848+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:42:19.111848+0000","last_peered":"2023-09-23T22:42:19.111848+0000","last_clean":"2023-09-23T22:42:19.111848+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:42:19.111848+0000","last_undegraded":"2023-09-23T22:42:19.111848+0000","last_fullsized":"2023-09-23T22:42:19.111848+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116412+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:42:19.116412+0000","last_peered":"2023-09-23T22:42:19.116412+0000","last_clean":"2023-09-23T22:42:19.116412+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:42:19.116412+0000","last_undegraded":"2023-09-23T22:42:19.116412+0000","last_fullsized":"2023-09-23T22:42:19.116412+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"163'3460","reported_seq":3485,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:25.391153+0000","last_change":"2023-09-23T22:41:36.966223+0000","last_active":"2023-09-23T22:42:25.391153+0000","last_peered":"2023-09-23T22:42:25.391153+0000","last_clean":"2023-09-23T22:42:25.391153+0000","last_became_active":"2023-09-23T22:41:36.965848+0000","last_became_peered":"2023-09-23T22:41:36.965848+0000","last_unstale":"2023-09-23T22:42:25.391153+0000","last_undegraded":"2023-09-23T22:42:25.391153+0000","last_fullsized":"2023-09-23T22:42:25.391153+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3460,"log_dups_size":0,"ondisk_log_size":3460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3460,"num_write_kb":221440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":163,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.123227+0000","last_change":"2023-09-23T22:40:50.912406+0000","last_active":"2023-09-23T22:42:19.123227+0000","last_peered":"2023-09-23T22:42:19.123227+0000","last_clean":"2023-09-23T22:42:19.123227+0000","last_became_active":"2023-09-23T22:40:50.912085+0000","last_became_peered":"2023-09-23T22:40:50.912085+0000","last_unstale":"2023-09-23T22:42:19.123227+0000","last_undegraded":"2023-09-23T22:42:19.123227+0000","last_fullsized":"2023-09-23T22:42:19.123227+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.102657+0000","last_change":"2023-09-23T22:41:49.035892+0000","last_active":"2023-09-23T22:42:19.102657+0000","last_peered":"2023-09-23T22:42:19.102657+0000","last_clean":"2023-09-23T22:42:19.102657+0000","last_became_active":"2023-09-23T22:41:49.035318+0000","last_became_peered":"2023-09-23T22:41:49.035318+0000","last_unstale":"2023-09-23T22:42:19.102657+0000","last_undegraded":"2023-09-23T22:42:19.102657+0000","last_fullsized":"2023-09-23T22:42:19.102657+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:06.155982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.105333+0000","last_change":"2023-09-23T22:41:10.928059+0000","last_active":"2023-09-23T22:42:19.105333+0000","last_peered":"2023-09-23T22:42:19.105333+0000","last_clean":"2023-09-23T22:42:19.105333+0000","last_became_active":"2023-09-23T22:41:10.927749+0000","last_became_peered":"2023-09-23T22:41:10.927749+0000","last_unstale":"2023-09-23T22:42:19.105333+0000","last_undegraded":"2023-09-23T22:42:19.105333+0000","last_fullsized":"2023-09-23T22:42:19.105333+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:53.307028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"163'3325","reported_seq":3350,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:25.614687+0000","last_change":"2023-09-23T22:41:36.969378+0000","last_active":"2023-09-23T22:42:25.614687+0000","last_peered":"2023-09-23T22:42:25.614687+0000","last_clean":"2023-09-23T22:42:25.614687+0000","last_became_active":"2023-09-23T22:41:36.969153+0000","last_became_peered":"2023-09-23T22:41:36.969153+0000","last_unstale":"2023-09-23T22:42:25.614687+0000","last_undegraded":"2023-09-23T22:42:25.614687+0000","last_fullsized":"2023-09-23T22:42:25.614687+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3325,"log_dups_size":0,"ondisk_log_size":3325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217907200,"num_objects":3325,"num_object_clones":0,"num_object_copies":9975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3325,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":148,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.123306+0000","last_change":"2023-09-23T22:40:50.904566+0000","last_active":"2023-09-23T22:42:19.123306+0000","last_peered":"2023-09-23T22:42:19.123306+0000","last_clean":"2023-09-23T22:42:19.123306+0000","last_became_active":"2023-09-23T22:40:50.904230+0000","last_became_peered":"2023-09-23T22:40:50.904230+0000","last_unstale":"2023-09-23T22:42:19.123306+0000","last_undegraded":"2023-09-23T22:42:19.123306+0000","last_fullsized":"2023-09-23T22:42:19.123306+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":134,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154824+0000","last_change":"2023-09-23T22:41:10.930375+0000","last_active":"2023-09-23T22:42:20.154824+0000","last_peered":"2023-09-23T22:42:20.154824+0000","last_clean":"2023-09-23T22:42:20.154824+0000","last_became_active":"2023-09-23T22:41:10.930050+0000","last_became_peered":"2023-09-23T22:41:10.930050+0000","last_unstale":"2023-09-23T22:42:20.154824+0000","last_undegraded":"2023-09-23T22:42:20.154824+0000","last_fullsized":"2023-09-23T22:42:20.154824+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:29.593309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":72,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.104291+0000","last_change":"2023-09-23T22:40:50.915078+0000","last_active":"2023-09-23T22:42:19.104291+0000","last_peered":"2023-09-23T22:42:19.104291+0000","last_clean":"2023-09-23T22:42:19.104291+0000","last_became_active":"2023-09-23T22:40:50.914759+0000","last_became_peered":"2023-09-23T22:40:50.914759+0000","last_unstale":"2023-09-23T22:42:19.104291+0000","last_undegraded":"2023-09-23T22:42:19.104291+0000","last_fullsized":"2023-09-23T22:42:19.104291+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:58.569943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"163'3308","reported_seq":3334,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:24.356228+0000","last_change":"2023-09-23T22:41:36.966638+0000","last_active":"2023-09-23T22:42:24.356228+0000","last_peered":"2023-09-23T22:42:24.356228+0000","last_clean":"2023-09-23T22:42:24.356228+0000","last_became_active":"2023-09-23T22:41:36.966433+0000","last_became_peered":"2023-09-23T22:41:36.966433+0000","last_unstale":"2023-09-23T22:42:24.356228+0000","last_undegraded":"2023-09-23T22:42:24.356228+0000","last_fullsized":"2023-09-23T22:42:24.356228+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3308,"log_dups_size":0,"ondisk_log_size":3308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:12.457010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216793088,"num_objects":3308,"num_object_clones":0,"num_object_copies":9924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3308,"num_write_kb":211712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":146,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.153756+0000","last_change":"2023-09-23T22:40:50.916941+0000","last_active":"2023-09-23T22:42:20.153756+0000","last_peered":"2023-09-23T22:42:20.153756+0000","last_clean":"2023-09-23T22:42:20.153756+0000","last_became_active":"2023-09-23T22:40:50.916668+0000","last_became_peered":"2023-09-23T22:40:50.916668+0000","last_unstale":"2023-09-23T22:42:20.153756+0000","last_undegraded":"2023-09-23T22:42:20.153756+0000","last_fullsized":"2023-09-23T22:42:20.153756+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:57.299908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.154828+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:42:20.154828+0000","last_peered":"2023-09-23T22:42:20.154828+0000","last_clean":"2023-09-23T22:42:20.154828+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:42:20.154828+0000","last_undegraded":"2023-09-23T22:42:20.154828+0000","last_fullsized":"2023-09-23T22:42:20.154828+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":143,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.110868+0000","last_change":"2023-09-23T22:40:50.916178+0000","last_active":"2023-09-23T22:42:19.110868+0000","last_peered":"2023-09-23T22:42:19.110868+0000","last_clean":"2023-09-23T22:42:19.110868+0000","last_became_active":"2023-09-23T22:40:50.915857+0000","last_became_peered":"2023-09-23T22:40:50.915857+0000","last_unstale":"2023-09-23T22:42:19.110868+0000","last_undegraded":"2023-09-23T22:42:19.110868+0000","last_fullsized":"2023-09-23T22:42:19.110868+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:11.736643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"163'3278","reported_seq":3304,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:24.388880+0000","last_change":"2023-09-23T22:41:36.965968+0000","last_active":"2023-09-23T22:42:24.388880+0000","last_peered":"2023-09-23T22:42:24.388880+0000","last_clean":"2023-09-23T22:42:24.388880+0000","last_became_active":"2023-09-23T22:41:36.965582+0000","last_became_peered":"2023-09-23T22:41:36.965582+0000","last_unstale":"2023-09-23T22:42:24.388880+0000","last_undegraded":"2023-09-23T22:42:24.388880+0000","last_fullsized":"2023-09-23T22:42:24.388880+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3278,"log_dups_size":0,"ondisk_log_size":3278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:51.048347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214827008,"num_objects":3278,"num_object_clones":0,"num_object_copies":9834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3278,"num_write_kb":209792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":168,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T22:42:20.231791+0000","last_change":"2023-09-23T22:42:20.231791+0000","last_active":"2023-09-23T22:42:20.231791+0000","last_peered":"2023-09-23T22:42:20.231791+0000","last_clean":"2023-09-23T22:42:20.231791+0000","last_became_active":"2023-09-23T22:42:20.231410+0000","last_became_peered":"2023-09-23T22:42:20.231410+0000","last_unstale":"2023-09-23T22:42:20.231791+0000","last_undegraded":"2023-09-23T22:42:20.231791+0000","last_fullsized":"2023-09-23T22:42:20.231791+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":95,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.105190+0000","last_change":"2023-09-23T22:40:50.916564+0000","last_active":"2023-09-23T22:42:19.105190+0000","last_peered":"2023-09-23T22:42:19.105190+0000","last_clean":"2023-09-23T22:42:19.105190+0000","last_became_active":"2023-09-23T22:40:50.915883+0000","last_became_peered":"2023-09-23T22:40:50.915883+0000","last_unstale":"2023-09-23T22:42:19.105190+0000","last_undegraded":"2023-09-23T22:42:19.105190+0000","last_fullsized":"2023-09-23T22:42:19.105190+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:25.767798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.116841+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:42:19.116841+0000","last_peered":"2023-09-23T22:42:19.116841+0000","last_clean":"2023-09-23T22:42:19.116841+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:42:19.116841+0000","last_undegraded":"2023-09-23T22:42:19.116841+0000","last_fullsized":"2023-09-23T22:42:19.116841+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":75,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.104473+0000","last_change":"2023-09-23T22:40:50.909960+0000","last_active":"2023-09-23T22:42:19.104473+0000","last_peered":"2023-09-23T22:42:19.104473+0000","last_clean":"2023-09-23T22:42:19.104473+0000","last_became_active":"2023-09-23T22:40:50.909314+0000","last_became_peered":"2023-09-23T22:40:50.909314+0000","last_unstale":"2023-09-23T22:42:19.104473+0000","last_undegraded":"2023-09-23T22:42:19.104473+0000","last_fullsized":"2023-09-23T22:42:19.104473+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:41.246849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.118206+0000","last_change":"2023-09-23T22:41:10.929809+0000","last_active":"2023-09-23T22:42:19.118206+0000","last_peered":"2023-09-23T22:42:19.118206+0000","last_clean":"2023-09-23T22:42:19.118206+0000","last_became_active":"2023-09-23T22:41:10.929502+0000","last_became_peered":"2023-09-23T22:41:10.929502+0000","last_unstale":"2023-09-23T22:42:19.118206+0000","last_undegraded":"2023-09-23T22:42:19.118206+0000","last_fullsized":"2023-09-23T22:42:19.118206+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:44.238238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.137800+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:42:19.137800+0000","last_peered":"2023-09-23T22:42:19.137800+0000","last_clean":"2023-09-23T22:42:19.137800+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:42:19.137800+0000","last_undegraded":"2023-09-23T22:42:19.137800+0000","last_fullsized":"2023-09-23T22:42:19.137800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114933+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:42:19.114933+0000","last_peered":"2023-09-23T22:42:19.114933+0000","last_clean":"2023-09-23T22:42:19.114933+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:42:19.114933+0000","last_undegraded":"2023-09-23T22:42:19.114933+0000","last_fullsized":"2023-09-23T22:42:19.114933+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.138133+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:42:19.138133+0000","last_peered":"2023-09-23T22:42:19.138133+0000","last_clean":"2023-09-23T22:42:19.138133+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:42:19.138133+0000","last_undegraded":"2023-09-23T22:42:19.138133+0000","last_fullsized":"2023-09-23T22:42:19.138133+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114516+0000","last_change":"2023-09-23T22:41:49.032087+0000","last_active":"2023-09-23T22:42:19.114516+0000","last_peered":"2023-09-23T22:42:19.114516+0000","last_clean":"2023-09-23T22:42:19.114516+0000","last_became_active":"2023-09-23T22:41:49.031690+0000","last_became_peered":"2023-09-23T22:41:49.031690+0000","last_unstale":"2023-09-23T22:42:19.114516+0000","last_undegraded":"2023-09-23T22:42:19.114516+0000","last_fullsized":"2023-09-23T22:42:19.114516+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":153,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.112238+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:42:19.112238+0000","last_peered":"2023-09-23T22:42:19.112238+0000","last_clean":"2023-09-23T22:42:19.112238+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:42:19.112238+0000","last_undegraded":"2023-09-23T22:42:19.112238+0000","last_fullsized":"2023-09-23T22:42:19.112238+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":176,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.110365+0000","last_change":"2023-09-23T22:41:10.931122+0000","last_active":"2023-09-23T22:42:19.110365+0000","last_peered":"2023-09-23T22:42:19.110365+0000","last_clean":"2023-09-23T22:42:19.110365+0000","last_became_active":"2023-09-23T22:41:10.930811+0000","last_became_peered":"2023-09-23T22:41:10.930811+0000","last_unstale":"2023-09-23T22:42:19.110365+0000","last_undegraded":"2023-09-23T22:42:19.110365+0000","last_fullsized":"2023-09-23T22:42:19.110365+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:16.649049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.108442+0000","last_change":"2023-09-23T22:41:10.926522+0000","last_active":"2023-09-23T22:42:19.108442+0000","last_peered":"2023-09-23T22:42:19.108442+0000","last_clean":"2023-09-23T22:42:19.108442+0000","last_became_active":"2023-09-23T22:41:10.926169+0000","last_became_peered":"2023-09-23T22:41:10.926169+0000","last_unstale":"2023-09-23T22:42:19.108442+0000","last_undegraded":"2023-09-23T22:42:19.108442+0000","last_fullsized":"2023-09-23T22:42:19.108442+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:45.457705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T22:42:19.114427+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:42:19.114427+0000","last_peered":"2023-09-23T22:42:19.114427+0000","last_clean":"2023-09-23T22:42:19.114427+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:42:19.114427+0000","last_undegraded":"2023-09-23T22:42:19.114427+0000","last_fullsized":"2023-09-23T22:42:19.114427+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3481272320,"num_objects":53120,"num_object_clones":0,"num_object_copies":159360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":890,"num_objects_unfound":0,"num_objects_dirty":53120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":53120,"num_write_kb":3399680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1582,"num_bytes_recovered":103677952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10581835776,"data_stored":10581835776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53120,"ondisk_log_size":53120,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430393,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1378540,"kb_used_data":1064932,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92366612,"statfs":{"total":95995035648,"available":94583410688,"internally_reserved":0,"allocated":1090490368,"data_stored":1090007219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.258,"5min":2.258,"15min":2.258},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.5600000000000001,"5min":2.5600000000000001,"15min":2.5600000000000001},"last":0.224}]},{"osd":1,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":6.2210000000000001,"5min":2.423,"15min":1.2290000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":73.269000000000005,"5min":73.269000000000005,"15min":73.269000000000005},"last":8.2189999999999994},{"interface":"front","average":{"1min":6.21,"5min":2.4089999999999998,"15min":1.208},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":73.281000000000006,"5min":73.281000000000006,"15min":73.281000000000006},"last":8.2349999999999994}]},{"osd":2,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":2.0920000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.093,"5min":3.093,"15min":3.093},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.1970000000000001,"5min":3.1970000000000001,"15min":3.1970000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.64700000000000002,"15min":0.61699999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.3019999999999996,"5min":4.3019999999999996,"15min":4.3019999999999996},"last":2.1499999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:41:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.47399999999999998,"15min":0.498},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.294,"5min":2.294,"15min":5.3140000000000001},"last":2.1240000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.54300000000000004,"15min":0.51400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.4109999999999996,"5min":5.4109999999999996,"15min":5.4109999999999996},"last":2.1920000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:41:34 2023","interfaces":[{"interface":"back","average":{"1min":12.460000000000001,"5min":2.883,"15min":1.2989999999999999},"min":{"1min":0.30599999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":226.22200000000001,"5min":226.22200000000001,"15min":226.22200000000001},"last":1.1970000000000001},{"interface":"front","average":{"1min":12.574999999999999,"5min":2.956,"15min":1.3779999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":225.50899999999999,"5min":225.50899999999999,"15min":225.50899999999999},"last":1.423}]},{"osd":6,"last update":"Sat Sep 23 22:41:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":2.2149999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.59599999999999997,"15min":0.59399999999999997},"min":{"1min":0.373,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.266,"5min":2.266,"15min":2.266},"last":0.40699999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626621,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332192,"kb_used_data":1022680,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92412960,"statfs":{"total":95995035648,"available":94630871040,"internally_reserved":0,"allocated":1047224320,"data_stored":1046753313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":1.792},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.984,"5min":1.984,"15min":1.984},"last":0.73399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:42:17 2023","interfaces":[{"interface":"back","average":{"1min":8.6210000000000004,"5min":3.6850000000000001,"15min":1.569},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":153.78399999999999,"5min":153.78399999999999,"15min":153.78399999999999},"last":2.5880000000000001},{"interface":"front","average":{"1min":8.6170000000000009,"5min":3.6549999999999998,"15min":1.599},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":154.148,"5min":154.148,"15min":154.148},"last":2.2890000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.7389999999999999,"15min":2.7389999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":43.408000000000001,"5min":43.408000000000001,"15min":43.408000000000001},"last":5.6500000000000004},{"interface":"front","average":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":43.356000000000002,"5min":43.356000000000002,"15min":43.356000000000002},"last":1.907}]},{"osd":3,"last update":"Sat Sep 23 22:42:17 2023","interfaces":[{"interface":"back","average":{"1min":8.1059999999999999,"5min":2.254,"15min":1.1020000000000001},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":80.277000000000001,"5min":80.277000000000001,"15min":80.277000000000001},"last":4.2999999999999998},{"interface":"front","average":{"1min":8.2669999999999995,"5min":2.4039999999999999,"15min":1.21},"min":{"1min":0.38500000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":80.513999999999996,"5min":80.513999999999996,"15min":80.513999999999996},"last":5.2699999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49199999999999999,"15min":0.48099999999999998},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.47899999999999998,"15min":0.49199999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.31900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.31},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.51500000000000001,"15min":0.53500000000000003},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.39800000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.56799999999999995,"15min":0.56000000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.53900000000000003,"15min":0.56499999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.34699999999999998}]}]},{"osd":1,"up_from":84,"seq":360777252931,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527956,"kb_used_data":1456204,"kb_used_omap":9,"kb_used_meta":71734,"kb_avail":92217196,"statfs":{"total":95995035648,"available":94430408704,"internally_reserved":0,"allocated":1491152896,"data_stored":1490469092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":73456617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.8279999999999998,"5min":2.8279999999999998,"15min":2.8279999999999998},"last":0.5},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.9710000000000001,"5min":1.9710000000000001,"15min":1.9710000000000001},"last":0.443}]},{"osd":2,"last update":"Sat Sep 23 22:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.8019999999999996,"5min":4.8019999999999996,"15min":4.8019999999999996},"last":1.6200000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":6.4020000000000001},"last":1.659}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.52600000000000002,"15min":0.42999999999999999},"min":{"1min":0.33200000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.0470000000000002,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.5069999999999999,"5min":0.60599999999999998,"15min":0.46000000000000002},"min":{"1min":0.33900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":13.416,"5min":13.416,"15min":13.416},"last":0.63800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.47,"15min":0.91400000000000003},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.7640000000000002,"5min":69.344999999999999,"15min":69.344999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.41699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.8339999999999996,"5min":69.359999999999999,"15min":69.359999999999999},"last":1.6859999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0369999999999999,"15min":0.77000000000000002},"min":{"1min":0.46999999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0880000000000001,"5min":28.837,"15min":28.837},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.98099999999999998,"15min":0.72699999999999998},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.726,"5min":28.986999999999998,"15min":28.986999999999998},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.75,"15min":0.67400000000000004},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.8550000000000004,"5min":4.8550000000000004,"15min":4.8550000000000004},"last":1.7509999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.73299999999999998,"15min":0.66600000000000004},"min":{"1min":0.51300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.1509999999999998,"5min":3.1509999999999998,"15min":3.1509999999999998},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":2.6429999999999998,"15min":1.3069999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.7889999999999997,"5min":159.018,"15min":159.018},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.228,"5min":2.7149999999999999,"15min":1.347},"min":{"1min":0.499,"5min":0.37,"15min":0.37},"max":{"1min":4.8860000000000001,"5min":158.96000000000001,"15min":158.96000000000001},"last":1.728}]}]},{"osd":0,"up_from":126,"seq":541165879316,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1731972,"kb_used_data":1655612,"kb_used_omap":28,"kb_used_meta":76323,"kb_avail":92013180,"statfs":{"total":95995035648,"available":94221496320,"internally_reserved":0,"allocated":1695346688,"data_stored":1694610804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":78155600},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.754,"15min":0.754},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.3759999999999999,"5min":3.3759999999999999,"15min":3.3759999999999999},"last":6.6349999999999998},{"interface":"front","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.532,"5min":2.532,"15min":2.532},"last":2.4550000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":6.6840000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":105.818,"5min":105.818,"15min":105.818},"last":7.5549999999999997},{"interface":"front","average":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":105.77500000000001,"5min":105.77500000000001,"15min":105.77500000000001},"last":3.5649999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.3899999999999997,"5min":4.3899999999999997,"15min":4.3899999999999997},"last":7.3170000000000002},{"interface":"front","average":{"1min":1.258,"5min":1.258,"15min":1.258},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9939999999999998,"5min":4.9939999999999998,"15min":4.9939999999999998},"last":6.7190000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"last":3.036},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":3.4180000000000001,"5min":3.4180000000000001,"15min":3.4180000000000001},"last":2.516}]},{"osd":5,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":4.2549999999999999,"5min":4.2549999999999999,"15min":4.2549999999999999},"last":3.0070000000000001},{"interface":"front","average":{"1min":0.875,"5min":0.875,"15min":0.875},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":3.4489999999999998,"5min":3.4489999999999998,"15min":3.4489999999999998},"last":2.5720000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":3.476,"5min":3.476,"15min":3.476},"last":2.5409999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.6919999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":2.4780000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":2.411},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":4.3200000000000003,"5min":4.3200000000000003,"15min":4.3200000000000003},"last":3.069}]}]},{"osd":2,"up_from":131,"seq":562640715792,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1313316,"kb_used_data":1236892,"kb_used_omap":81,"kb_used_meta":76334,"kb_avail":92431836,"statfs":{"total":95995035648,"available":94650200064,"internally_reserved":0,"allocated":1266577408,"data_stored":1265870423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":78166234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":3.399,"15min":3.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":48.973999999999997,"5min":48.973999999999997,"15min":48.973999999999997},"last":0.79000000000000004},{"interface":"front","average":{"1min":4.6580000000000004,"5min":4.6580000000000004,"15min":4.6580000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":50.616999999999997,"5min":50.616999999999997,"15min":50.616999999999997},"last":2.645}]},{"osd":1,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":3.0819999999999999,"15min":3.0819999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":21.114000000000001,"5min":21.114000000000001,"15min":21.114000000000001},"last":2.6150000000000002},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":22.021000000000001,"5min":22.021000000000001,"15min":22.021000000000001},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.2509999999999999,"5min":3.2509999999999999,"15min":3.2509999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.0830000000000002,"5min":4.0830000000000002,"15min":4.0830000000000002},"last":0.53400000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.0069999999999997,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":6.8129999999999997,"5min":6.8129999999999997,"15min":6.8129999999999997},"last":2.581}]},{"osd":5,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":6.8719999999999999,"5min":6.8719999999999999,"15min":6.8719999999999999},"last":2.532},{"interface":"front","average":{"1min":0.752,"5min":0.752,"15min":0.752},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.1890000000000001,"5min":3.1890000000000001,"15min":3.1890000000000001},"last":3.5259999999999998},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.0579999999999998,"5min":4.0579999999999998,"15min":4.0579999999999998},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":4.0359999999999996,"5min":4.0359999999999996,"15min":4.0359999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.355,"5min":1.355,"15min":1.355},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":6.9100000000000001,"5min":6.9100000000000001,"15min":6.9100000000000001},"last":2.7290000000000001}]}]},{"osd":3,"up_from":94,"seq":403726925878,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1991484,"kb_used_data":1887732,"kb_used_omap":7,"kb_used_meta":103736,"kb_avail":91753668,"statfs":{"total":95995035648,"available":93955756032,"internally_reserved":0,"allocated":1933037568,"data_stored":1932366911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":106226280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":26.385000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6319999999999997,"5min":6.6319999999999997,"15min":6.6319999999999997},"last":26.190999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.42499999999999999,"15min":0.375},"min":{"1min":0.32600000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.254,"5min":3.254,"15min":3.254},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.272,"5min":0.55100000000000005,"15min":0.44500000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.843,"5min":6.843,"15min":6.843},"last":0.52000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.3550000000000004,"5min":4.3550000000000004,"15min":4.3550000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.911,"5min":3.911,"15min":3.911},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.627,"15min":0.624},"min":{"1min":0.43099999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.2250000000000001,"5min":3.2250000000000001,"15min":3.2250000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.077,"5min":0.69999999999999996,"15min":0.68799999999999994},"min":{"1min":0.377,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.0279999999999996,"5min":4.0279999999999996,"15min":4.0279999999999996},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":0.67600000000000005,"15min":0.66800000000000004},"min":{"1min":0.35099999999999998,"5min":0.312,"15min":0.312},"max":{"1min":3.9980000000000002,"5min":3.9980000000000002,"15min":3.9980000000000002},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.65300000000000002,"15min":0.61599999999999999},"min":{"1min":0.45400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.915,"5min":3.0739999999999998,"15min":3.0739999999999998},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.8620000000000001,"5min":3.8620000000000001,"15min":3.8620000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67400000000000004,"15min":0.66500000000000004},"min":{"1min":0.39200000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.179,"5min":3.1520000000000001,"15min":3.1520000000000001},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.68700000000000006,"15min":0.66400000000000003},"min":{"1min":0.39500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.2569999999999999,"5min":3.1200000000000001,"15min":3.1200000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.441,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.1640000000000001,"5min":3.1640000000000001,"15min":3.1640000000000001},"last":0.61099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019077,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1362724,"kb_used_data":1065500,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92382428,"statfs":{"total":95995035648,"available":94599606272,"internally_reserved":0,"allocated":1091072000,"data_stored":1090596429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7690000000000001,"5min":2.7690000000000001,"15min":2.7690000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.96699999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.621,"15min":0.60799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4829999999999997,"5min":5.4829999999999997,"15min":5.4829999999999997},"last":1.5780000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.60299999999999998,"15min":0.57099999999999995},"min":{"1min":0.32500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.6070000000000002,"5min":5.6070000000000002,"15min":5.6070000000000002},"last":0.52500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:42:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.6890000000000001,"15min":2.6890000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":29.370999999999999,"5min":29.370999999999999,"15min":29.370999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.54,"5min":2.54,"15min":2.54},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":28.879000000000001,"5min":28.879000000000001,"15min":28.879000000000001},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.018,"15min":0.78200000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.921,"5min":44.994,"15min":44.994},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.05,"15min":0.74199999999999999},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.6819999999999999,"5min":44.765000000000001,"15min":44.765000000000001},"last":1.605}]},{"osd":5,"last update":"Sat Sep 23 22:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.93500000000000005,"15min":0.63100000000000001},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.431,"5min":43.189999999999998,"15min":43.189999999999998},"last":1.4950000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.94499999999999995,"15min":0.64400000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.1950000000000001,"5min":42.450000000000003,"15min":42.450000000000003},"last":0.29399999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.76900000000000002,"15min":0.60399999999999998},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81899999999999995,"5min":27.056000000000001,"15min":27.056000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.83399999999999996,"15min":0.64000000000000001},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.91800000000000004,"5min":26.940999999999999,"15min":26.940999999999999},"last":1.5449999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.629,"15min":0.57999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.1459999999999999,"5min":4.1459999999999999,"15min":4.1459999999999999},"last":1.6279999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55200000000000005,"15min":0.54700000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.48999999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822849,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141964,"kb_used_data":844740,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92603188,"statfs":{"total":95995035648,"available":94825664512,"internally_reserved":0,"allocated":865013760,"data_stored":864557879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":1.784,"5min":1.784,"15min":1.784},"last":1.968},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":0.59799999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":9.3829999999999991,"5min":2.3239999999999998,"15min":1.153},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":149.203,"5min":149.203,"15min":149.203},"last":1.093},{"interface":"front","average":{"1min":9.3109999999999999,"5min":2.2839999999999998,"15min":1.1160000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":149.554,"5min":149.554,"15min":149.554},"last":4.0940000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":9.1739999999999995,"5min":9.1739999999999995,"15min":9.1739999999999995},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":116.123,"5min":116.123,"15min":116.123},"last":3.9969999999999999},{"interface":"front","average":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":116.14100000000001,"5min":116.14100000000001,"15min":116.14100000000001},"last":2.069}]},{"osd":3,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.35899999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.089,"5min":2.286,"15min":2.286},"last":2.169},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.875,"5min":3.4740000000000002,"15min":3.4740000000000002},"last":2.2599999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.46200000000000002,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":4.0350000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.51600000000000001,"15min":0.5},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.3780000000000001,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":2.1669999999999998,"15min":1.0580000000000001},"min":{"1min":0.33100000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.349,"5min":140.07499999999999,"15min":140.07499999999999},"last":4.0730000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.1800000000000002,"15min":1.0840000000000001},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.319,"5min":139.739,"15min":139.739},"last":0.44900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.623,"15min":0.57699999999999996},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":2.0129999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.56000000000000005,"15min":0.56100000000000005},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":1.5980000000000001,"5min":1.5980000000000001,"15min":1.5980000000000001},"last":4.056}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1712128000,"data_stored":1712128000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1515913216,"data_stored":1515913216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1282473984,"data_stored":1282473984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1937768448,"data_stored":1937768448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1102577664,"data_stored":1102577664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":877592576,"data_stored":877592576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1046478848,"data_stored":1046478848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1106903040,"data_stored":1106903040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:42:31.148 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:42:31.211 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:31.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:31.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:31.341 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:31.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:31.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:31.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:31.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:31 smithi151 ceph-mon[159210]: pgmap v383: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 890/168300 objects misplaced (0.529%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T22:42:31.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:31 smithi151 ceph-mon[159210]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T22:42:31.488 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:31.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:31 smithi052 ceph-mon[187743]: pgmap v383: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 890/168300 objects misplaced (0.529%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T22:42:31.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:31 smithi052 ceph-mon[187743]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T22:42:31.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:31 smithi052 ceph-mon[180979]: pgmap v383: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 890/168300 objects misplaced (0.529%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T22:42:31.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:31 smithi052 ceph-mon[180979]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T22:42:31.496 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 44 16 59876 59860 84.9885 34.1875 0.00368375 0.0116083 2023-09-23T22:42:31.513 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:31.555 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:31.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:31.659 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:31.685 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:31.726 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:31.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:31.833 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:31.864 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:31.907 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:31.920 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:31.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:32.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:32.104 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:32.177 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:32 smithi151 ceph-mon[159210]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.177 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:32 smithi151 ceph-mon[159210]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.177 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:32 smithi151 ceph-mon[159210]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.177 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:32 smithi151 ceph-mon[159210]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T22:42:32.289 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:32.315 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:32.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:32.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:32.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[180979]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.321 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[180979]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[180979]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.322 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[180979]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T22:42:32.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[187743]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[187743]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[187743]: failed to encode map e165 with expected crc 2023-09-23T22:42:32.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:32 smithi052 ceph-mon[187743]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T22:42:32.420 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10170 6.1 6:8aa77601:::benchmark_data_smithi052_234456_object10169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:32.432 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 98.223639688000006, 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.264625532, 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10170 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199126+0000", 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199153+0000", 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7100000000000001e-05 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199170+0000", 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6382e-05 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199177+0000", 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7449999999999995e-06 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199217+0000", 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9523000000000001e-05 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199406+0000", 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018866 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199606+0000", 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000200278 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201347+0000", 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001740969 2023-09-23T22:42:32.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463581+0000", 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26223432499999999 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463651+0000", 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9529999999999993e-05 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463753+0000", 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000102389 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10180 6.1 6:8dfddb97:::benchmark_data_smithi052_234456_object10179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 98.223525909000003, 2023-09-23T22:42:32.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26492964299999999, 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10180 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199272+0000", 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1467999999999998e-05 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199288+0000", 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5582000000000001e-05 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199294+0000", 2023-09-23T22:42:32.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200146+0000", 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00085146099999999997 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200281+0000", 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013534300000000001 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200378+0000", 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6699000000000004e-05 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201357+0000", 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097968399999999998 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464013+0000", 2023-09-23T22:42:32.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26265580399999999 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464077+0000", 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4029999999999995e-05 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464171+0000", 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3728999999999997e-05 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.445 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10183 6.1 6:8243606a:::benchmark_data_smithi052_234456_object10182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 98.223466587000004, 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265208887, 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.446 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10183 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199300+0000", 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199322+0000", 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1863999999999999e-05 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199329+0000", 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4529999999999996e-06 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199335+0000", 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6749999999999999e-06 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200743+0000", 2023-09-23T22:42:32.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001407569 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200839+0000", 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5946999999999995e-05 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200957+0000", 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011885200000000001 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201525+0000", 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00056793000000000002 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464362+0000", 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26283688599999999 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464416+0000", 2023-09-23T22:42:32.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4191999999999999e-05 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464509+0000", 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3066000000000006e-05 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10194 6.1 6:818db385:::benchmark_data_smithi052_234456_object10193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:32.452 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.204091+0000", 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 98.218675684999994, 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.261536294, 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10194 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204091+0000", 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204091+0000", 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204090+0000", 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:32.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204114+0000", 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4051999999999999e-05 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204123+0000", 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6130000000000004e-06 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204130+0000", 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0480000000000001e-06 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204174+0000", 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4172000000000003e-05 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204288+0000", 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113712 2023-09-23T22:42:32.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204452+0000", 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163604 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.205205+0000", 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00075335599999999995 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.465471+0000", 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.260265737 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.465528+0000", 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7648000000000001e-05 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.465628+0000", 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9532999999999997e-05 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10221 6.1 6:8e2897f0:::benchmark_data_smithi052_234456_object10220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.217270+0000", 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 98.205497545, 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24949115699999999, 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.459 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10221 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217270+0000", 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217270+0000", 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217268+0000", 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217287+0000", 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8828999999999999e-05 2023-09-23T22:42:32.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217303+0000", 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5421999999999999e-05 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217310+0000", 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7859999999999997e-06 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217353+0000", 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2451999999999999e-05 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217499+0000", 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014595400000000001 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217626+0000", 2023-09-23T22:42:32.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000126971 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.218176+0000", 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00055033800000000004 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.466625+0000", 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248449319 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.466673+0000", 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7509999999999997e-05 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.466761+0000", 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7847000000000001e-05 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36160 6.4 6:2dcd31ff:::benchmark_data_smithi052_234456_object14016:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 85.574496741999994, 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32418800800000003, 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36160 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.466 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848269+0000", 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848277+0000", 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2050000000000002e-06 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848301+0000", 2023-09-23T22:42:32.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3801000000000001e-05 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848311+0000", 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0621e-05 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848367+0000", 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5881999999999998e-05 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848522+0000", 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000154544 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848651+0000", 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000129157 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850169+0000", 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015178349999999999 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850319+0000", 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015018400000000001 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172387+0000", 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32206747699999999 2023-09-23T22:42:32.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172420+0000", 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3751999999999998e-05 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172458+0000", 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8139000000000002e-05 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.472 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:38747 6.4 6:20877948:::benchmark_data_smithi052_234456_object16603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:08.547567+0000", 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 83.875200649999996, 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.198348212, 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 38747 2023-09-23T22:42:32.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547567+0000", 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547567+0000", 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547565+0000", 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547571+0000", 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5679999999999999e-06 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547586+0000", 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4633e-05 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547593+0000", 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8189999999999992e-06 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547642+0000", 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8548999999999999e-05 2023-09-23T22:42:32.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547743+0000", 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101178 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547824+0000", 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1098000000000002e-05 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.548983+0000", 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011587209999999999 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.549218+0000", 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023472200000000001 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.745831+0000", 2023-09-23T22:42:32.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19661288199999999 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.745877+0000", 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6539999999999998e-05 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.745915+0000", 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.756e-05 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.479 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:4838 7.b 7:da708ce4:::benchmark_data_smithi052_277165_object4837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:49.373150+0000", 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 43.049617413, 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.145418715, 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:32.480 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 4838 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373150+0000", 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373150+0000", 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373148+0000", 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373173+0000", 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4386999999999999e-05 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373185+0000", 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2911999999999999e-05 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373193+0000", 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373373+0000", 2023-09-23T22:42:32.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017994199999999999 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373513+0000", 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013928899999999999 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373656+0000", 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143292 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518396+0000", 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14473989200000001 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518408+0000", 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1838000000000001e-05 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518465+0000", 2023-09-23T22:42:32.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7352000000000002e-05 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518520+0000", 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5319999999999999e-05 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518568+0000", 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8055e-05 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.486 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:4839 7.b 7:d556b235:::benchmark_data_smithi052_277165_object4838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:49.373295+0000", 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 43.049472139999999, 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14542644099999999, 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:32.487 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 4839 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373295+0000", 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373295+0000", 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373294+0000", 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373316+0000", 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2101000000000001e-05 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373327+0000", 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1389000000000001e-05 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373335+0000", 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.25e-06 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374291+0000", 2023-09-23T22:42:32.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00095615000000000001 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374422+0000", 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013170900000000001 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374557+0000", 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000134559 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518416+0000", 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14385942600000001 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518616+0000", 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019989300000000001 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518646+0000", 2023-09-23T22:42:32.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0107000000000002e-05 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518691+0000", 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4511000000000001e-05 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518721+0000", 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0577000000000002e-05 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.493 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:4836 7.1 e154/146 7:8d95b6ae:::benchmark_data_smithi052_277165_object4835:head v 154'299, mlcod=154'293)", 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:49.373839+0000", 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 43.048928128999997, 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.144997289, 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373839+0000", 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373839+0000", 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373838+0000", 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373944+0000", 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010548 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373955+0000", 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0838e-05 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.373961+0000", 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3350000000000003e-06 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374375+0000", 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000414053 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374393+0000", 2023-09-23T22:42:32.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7391999999999999e-05 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518798+0000", 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.144405803 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518836+0000", 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7988000000000003e-05 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.498 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:4840 7.1 e154/146 7:8d01ace8:::benchmark_data_smithi052_277165_object4839:head v 154'300, mlcod=154'294)", 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:49.374706+0000", 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 43.048061130999997, 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14506904500000001, 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374706+0000", 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374706+0000", 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374705+0000", 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374736+0000", 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0519999999999999e-05 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374746+0000", 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0471e-05 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374753+0000", 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5289999999999997e-06 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374791+0000", 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8009999999999997e-05 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.374809+0000", 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7853e-05 2023-09-23T22:42:32.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.519736+0000", 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14492750600000001 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.519775+0000", 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8784e-05 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.503 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:4845 7.9 e154/146 7:96137274:::benchmark_data_smithi052_277165_object4844:head v 154'285, mlcod=154'283)", 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:49.376397+0000", 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 43.046370639999999, 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14551515300000001, 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.376397+0000", 2023-09-23T22:42:32.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.376397+0000", 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.376396+0000", 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.376427+0000", 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1251999999999998e-05 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518314+0000", 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.141886975 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518329+0000", 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5165e-05 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518757+0000", 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00042809 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.518774+0000", 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6453999999999999e-05 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.521863+0000", 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003089638 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:49.521912+0000", 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8310000000000003e-05 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10377 7.d e154/146 7:b0a9409a:::benchmark_data_smithi052_277165_object10376:head v 154'659, mlcod=154'657)", 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.075588+0000", 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.347179288, 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.227911682, 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075588+0000", 2023-09-23T22:42:32.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075588+0000", 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075587+0000", 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075629+0000", 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1495999999999997e-05 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075641+0000", 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2062000000000001e-05 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075648+0000", 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8820000000000001e-06 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075693+0000", 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4137000000000001e-05 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075725+0000", 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2147999999999998e-05 2023-09-23T22:42:32.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078485+0000", 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027598710000000001 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303500+0000", 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225014933 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10385 7.1 e154/146 7:87e353d9:::benchmark_data_smithi052_277165_object10384:head v 154'628, mlcod=154'626)", 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.347064715999998, 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22782445100000001, 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:42:32.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075735+0000", 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2697000000000002e-05 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075744+0000", 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6800000000000005e-06 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075750+0000", 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1940000000000003e-06 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075790+0000", 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9804e-05 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075807+0000", 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6789000000000001e-05 2023-09-23T22:42:32.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078501+0000", 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026934939999999998 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303527+0000", 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22502640500000001 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.518 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10380 7.2 e154/146 7:4efbeca9:::benchmark_data_smithi052_277165_object10379:head v 154'647, mlcod=154'645)", 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.076327+0000", 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.346440370000003, 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22720847199999999, 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076327+0000", 2023-09-23T22:42:32.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076327+0000", 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076326+0000", 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076358+0000", 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1412999999999999e-05 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076368+0000", 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0597e-05 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076375+0000", 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3940000000000001e-06 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076440+0000", 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5314999999999994e-05 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076460+0000", 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0482000000000001e-05 2023-09-23T22:42:32.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078511+0000", 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020503309999999999 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303535+0000", 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225024578 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.523 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10382 7.0 e154/146 7:08701358:::benchmark_data_smithi052_277165_object10381:head v 154'618, mlcod=154'615)", 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.077097+0000", 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.345670277000004, 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.227032811, 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077097+0000", 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077097+0000", 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077096+0000", 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077144+0000", 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7518999999999997e-05 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077155+0000", 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1168999999999999e-05 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077162+0000", 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8979999999999999e-06 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077339+0000", 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017684600000000001 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077357+0000", 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8813000000000001e-05 2023-09-23T22:42:32.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304094+0000", 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.226736363 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304130+0000", 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5899000000000003e-05 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.528 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10384 7.0 e154/146 7:090486be:::benchmark_data_smithi052_277165_object10383:head v 154'619, mlcod=154'615)", 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.345589707999999, 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.226981144, 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077224+0000", 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6554000000000002e-05 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077234+0000", 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.006e-05 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077240+0000", 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8549999999999998e-06 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077811+0000", 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00057189800000000005 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077830+0000", 2023-09-23T22:42:32.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8470000000000001e-05 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304136+0000", 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22630588600000001 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304159+0000", 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2762999999999999e-05 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.533 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10390 7.0 e154/146 7:01932cc7:::benchmark_data_smithi052_277165_object10389:head v 154'620, mlcod=154'615)", 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.345522389000003, 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22694154499999999, 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077277+0000", 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2830000000000002e-05 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077284+0000", 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3439999999999997e-06 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077288+0000", 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3889999999999998e-06 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078158+0000", 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00087037699999999998 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078175+0000", 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6781000000000001e-05 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304161+0000", 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225985827 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304186+0000", 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5267000000000001e-05 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10393 7.9 e154/146 7:914dcfe4:::benchmark_data_smithi052_277165_object10392:head v 154'624, mlcod=154'622)", 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.079905+0000", 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.342861845999998, 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225263939, 2023-09-23T22:42:32.539 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079905+0000", 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079905+0000", 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079904+0000", 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079930+0000", 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5576999999999999e-05 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303519+0000", 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22358936300000001 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303527+0000", 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8699999999999992e-06 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303571+0000", 2023-09-23T22:42:32.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3480999999999999e-05 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303593+0000", 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2467999999999999e-05 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.305128+0000", 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001534934 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.305169+0000", 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1085999999999999e-05 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10403 7.0 e154/146 7:08ef69f1:::benchmark_data_smithi052_277165_object10402:head v 154'621, mlcod=154'616)", 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.082968+0000", 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 40.339798706000003, 2023-09-23T22:42:32.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22345253500000001, 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.082968+0000", 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:32.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.082968+0000", 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.082967+0000", 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.083095+0000", 2023-09-23T22:42:32.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012813500000000001 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303474+0000", 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.220378193 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303485+0000", 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1378e-05 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303534+0000", 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9326000000000002e-05 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303557+0000", 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3101000000000001e-05 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306380+0000", 2023-09-23T22:42:32.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028225289999999998 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306421+0000", 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0988999999999997e-05 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:32.549 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:42:32.550 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 45 16 60641 60625 84.1626 47.8125 0.0113255 0.0118747 2023-09-23T22:42:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:32.551 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:32.551 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:32.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:32.652 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:32.736 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:33.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:42:33.350 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[180979]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.351 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[180979]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.351 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[180979]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.351 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[180979]: pgmap v386: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8.8 MiB/s, 141 objects/s recovering 2023-09-23T22:42:33.351 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:33.351 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[187743]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.352 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[187743]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.352 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[187743]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.352 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:33 smithi052 ceph-mon[187743]: pgmap v386: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8.8 MiB/s, 141 objects/s recovering 2023-09-23T22:42:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:33.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:33.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:33 smithi151 ceph-mon[159210]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:33 smithi151 ceph-mon[159210]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:33 smithi151 ceph-mon[159210]: failed to encode map e166 with expected crc 2023-09-23T22:42:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:33 smithi151 ceph-mon[159210]: pgmap v386: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8.8 MiB/s, 141 objects/s recovering 2023-09-23T22:42:33.455 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:33.497 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 46 16 61449 61433 83.4309 50.5 0.0106979 0.01198 2023-09-23T22:42:33.526 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:33.620 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:42:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:42:33] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.33.4" 2023-09-23T22:42:34.497 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 47 16 62441 62425 82.9749 62 0.0024889 0.0120351 2023-09-23T22:42:34.768 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:34.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:34.878 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:34.908 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:42:34.912 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:34.967 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:34.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:35.073 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:35.210 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:35.361 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:35 smithi052 ceph-mon[180979]: pgmap v387: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:42:35.361 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:35.363 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:35 smithi052 ceph-mon[187743]: pgmap v387: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:42:35.364 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:35.427 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:35.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:35.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:35 smithi151 ceph-mon[159210]: pgmap v387: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:42:35.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:35.497 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 48 16 63242 63226 82.2893 50.0625 0.0212141 0.0121471 2023-09-23T22:42:35.539 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:35.576 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:35.629 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:35.638 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:35.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:35.743 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:35.844 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:36.014 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:36.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:36.126 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:36.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:36.245 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:42:36.286 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/766879163' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:36.286 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:36.286 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/766879163' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:36.286 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:36.410 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/766879163' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:36.497 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:42:36.499 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 49 16 64153 64137 81.7673 56.9375 0.0108769 0.0122241 2023-09-23T22:42:36.529 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:42:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: pgmap v388: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:42:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:42:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:37.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T22:42:37.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: failed to encode map e167 with expected crc 2023-09-23T22:42:37.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:37 smithi151 ceph-mon[159210]: failed to encode map e167 with expected crc 2023-09-23T22:42:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: pgmap v388: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:42:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:42:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T22:42:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: failed to encode map e167 with expected crc 2023-09-23T22:42:37.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[180979]: failed to encode map e167 with expected crc 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: pgmap v388: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: failed to encode map e167 with expected crc 2023-09-23T22:42:37.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:37 smithi052 ceph-mon[187743]: failed to encode map e167 with expected crc 2023-09-23T22:42:37.499 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 50 16 65221 65205 81.4669 66.75 0.0128951 0.0122694 2023-09-23T22:42:37.998 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:38.105 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:38.135 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:38.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:38 smithi052 ceph-mon[180979]: from='client.54844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:38.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:38 smithi052 ceph-mon[187743]: from='client.54844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:38.500 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 51 16 66307 66291 81.2004 67.875 0.0153124 0.0123107 2023-09-23T22:42:38.665 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:38.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:38 smithi151 ceph-mon[159210]: from='client.54844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:38.772 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:38.806 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:38.856 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:38.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:38.870 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":389,"stamp":"2023-09-23T22:42:36.210434+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4084799392,"num_objects":62344,"num_object_clones":0,"num_object_copies":187032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62344,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":62580,"num_write_kb":3990925,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2480,"num_bytes_recovered":162004992,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62514,"ondisk_log_size":62514,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13624260,"kb_used_data":12034116,"kb_used_omap":126,"kb_used_meta":1589953,"kb_avail":736336956,"statfs":{"total":767960285184,"available":754009042944,"internally_reserved":0,"allocated":12322934784,"data_stored":12318214816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1628112507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,1,4,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":409206784,"num_objects":6244,"num_object_clones":0,"num_object_copies":18732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-890,"num_objects_unfound":0,"num_objects_dirty":6244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6244,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":890,"num_bytes_recovered":58327040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.185475"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":163,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746076+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:42:30.746076+0000","last_peered":"2023-09-23T22:42:30.746076+0000","last_clean":"2023-09-23T22:42:30.746076+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:42:30.746076+0000","last_undegraded":"2023-09-23T22:42:30.746076+0000","last_fullsized":"2023-09-23T22:42:30.746076+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":135,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745805+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:42:30.745805+0000","last_peered":"2023-09-23T22:42:30.745805+0000","last_clean":"2023-09-23T22:42:30.745805+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:42:30.745805+0000","last_undegraded":"2023-09-23T22:42:30.745805+0000","last_fullsized":"2023-09-23T22:42:30.745805+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":121,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752106+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:42:30.752106+0000","last_peered":"2023-09-23T22:42:30.752106+0000","last_clean":"2023-09-23T22:42:30.752106+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:42:30.752106+0000","last_undegraded":"2023-09-23T22:42:30.752106+0000","last_fullsized":"2023-09-23T22:42:30.752106+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":104,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.758166+0000","last_change":"2023-09-23T22:40:50.920332+0000","last_active":"2023-09-23T22:42:30.758166+0000","last_peered":"2023-09-23T22:42:30.758166+0000","last_clean":"2023-09-23T22:42:30.758166+0000","last_became_active":"2023-09-23T22:40:50.919888+0000","last_became_peered":"2023-09-23T22:40:50.919888+0000","last_unstale":"2023-09-23T22:42:30.758166+0000","last_undegraded":"2023-09-23T22:42:30.758166+0000","last_fullsized":"2023-09-23T22:42:30.758166+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:27:24.536141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":179,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745561+0000","last_change":"2023-09-23T22:40:50.916258+0000","last_active":"2023-09-23T22:42:30.745561+0000","last_peered":"2023-09-23T22:42:30.745561+0000","last_clean":"2023-09-23T22:42:30.745561+0000","last_became_active":"2023-09-23T22:40:50.915948+0000","last_became_peered":"2023-09-23T22:40:50.915948+0000","last_unstale":"2023-09-23T22:42:30.745561+0000","last_undegraded":"2023-09-23T22:42:30.745561+0000","last_fullsized":"2023-09-23T22:42:30.745561+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:32.210743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752220+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:42:30.752220+0000","last_peered":"2023-09-23T22:42:30.752220+0000","last_clean":"2023-09-23T22:42:30.752220+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:42:30.752220+0000","last_undegraded":"2023-09-23T22:42:30.752220+0000","last_fullsized":"2023-09-23T22:42:30.752220+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":102,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.761391+0000","last_change":"2023-09-23T22:40:50.926694+0000","last_active":"2023-09-23T22:42:30.761391+0000","last_peered":"2023-09-23T22:42:30.761391+0000","last_clean":"2023-09-23T22:42:30.761391+0000","last_became_active":"2023-09-23T22:40:50.926142+0000","last_became_peered":"2023-09-23T22:40:50.926142+0000","last_unstale":"2023-09-23T22:42:30.761391+0000","last_undegraded":"2023-09-23T22:42:30.761391+0000","last_fullsized":"2023-09-23T22:42:30.761391+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:53.874735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":162,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745591+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:42:30.745591+0000","last_peered":"2023-09-23T22:42:30.745591+0000","last_clean":"2023-09-23T22:42:30.745591+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:42:30.745591+0000","last_undegraded":"2023-09-23T22:42:30.745591+0000","last_fullsized":"2023-09-23T22:42:30.745591+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754907+0000","last_change":"2023-09-23T22:37:03.803929+0000","last_active":"2023-09-23T22:42:30.754907+0000","last_peered":"2023-09-23T22:42:30.754907+0000","last_clean":"2023-09-23T22:42:30.754907+0000","last_became_active":"2023-09-23T22:37:03.803607+0000","last_became_peered":"2023-09-23T22:37:03.803607+0000","last_unstale":"2023-09-23T22:42:30.754907+0000","last_undegraded":"2023-09-23T22:42:30.754907+0000","last_fullsized":"2023-09-23T22:42:30.754907+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":93,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745997+0000","last_change":"2023-09-23T22:38:06.151474+0000","last_active":"2023-09-23T22:42:30.745997+0000","last_peered":"2023-09-23T22:42:30.745997+0000","last_clean":"2023-09-23T22:42:30.745997+0000","last_became_active":"2023-09-23T22:38:06.151201+0000","last_became_peered":"2023-09-23T22:38:06.151201+0000","last_unstale":"2023-09-23T22:42:30.745997+0000","last_undegraded":"2023-09-23T22:42:30.745997+0000","last_fullsized":"2023-09-23T22:42:30.745997+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:40.404756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":135,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744702+0000","last_change":"2023-09-23T22:38:06.157765+0000","last_active":"2023-09-23T22:42:30.744702+0000","last_peered":"2023-09-23T22:42:30.744702+0000","last_clean":"2023-09-23T22:42:30.744702+0000","last_became_active":"2023-09-23T22:38:06.157289+0000","last_became_peered":"2023-09-23T22:38:06.157289+0000","last_unstale":"2023-09-23T22:42:30.744702+0000","last_undegraded":"2023-09-23T22:42:30.744702+0000","last_fullsized":"2023-09-23T22:42:30.744702+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":133,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.755535+0000","last_change":"2023-09-23T22:41:10.927399+0000","last_active":"2023-09-23T22:42:30.755535+0000","last_peered":"2023-09-23T22:42:30.755535+0000","last_clean":"2023-09-23T22:42:30.755535+0000","last_became_active":"2023-09-23T22:41:10.926366+0000","last_became_peered":"2023-09-23T22:41:10.926366+0000","last_unstale":"2023-09-23T22:42:30.755535+0000","last_undegraded":"2023-09-23T22:42:30.755535+0000","last_fullsized":"2023-09-23T22:42:30.755535+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:54.104169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":139,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746415+0000","last_change":"2023-09-23T22:28:42.665559+0000","last_active":"2023-09-23T22:42:30.746415+0000","last_peered":"2023-09-23T22:42:30.746415+0000","last_clean":"2023-09-23T22:42:30.746415+0000","last_became_active":"2023-09-23T22:28:42.665275+0000","last_became_peered":"2023-09-23T22:28:42.665275+0000","last_unstale":"2023-09-23T22:42:30.746415+0000","last_undegraded":"2023-09-23T22:42:30.746415+0000","last_fullsized":"2023-09-23T22:42:30.746415+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746258+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:42:30.746258+0000","last_peered":"2023-09-23T22:42:30.746258+0000","last_clean":"2023-09-23T22:42:30.746258+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:42:30.746258+0000","last_undegraded":"2023-09-23T22:42:30.746258+0000","last_fullsized":"2023-09-23T22:42:30.746258+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":149,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744515+0000","last_change":"2023-09-23T22:41:10.925900+0000","last_active":"2023-09-23T22:42:30.744515+0000","last_peered":"2023-09-23T22:42:30.744515+0000","last_clean":"2023-09-23T22:42:30.744515+0000","last_became_active":"2023-09-23T22:41:10.925510+0000","last_became_peered":"2023-09-23T22:41:10.925510+0000","last_unstale":"2023-09-23T22:42:30.744515+0000","last_undegraded":"2023-09-23T22:42:30.744515+0000","last_fullsized":"2023-09-23T22:42:30.744515+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":155,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746936+0000","last_change":"2023-09-23T22:40:15.114667+0000","last_active":"2023-09-23T22:42:30.746936+0000","last_peered":"2023-09-23T22:42:30.746936+0000","last_clean":"2023-09-23T22:42:30.746936+0000","last_became_active":"2023-09-23T22:40:15.114241+0000","last_became_peered":"2023-09-23T22:40:15.114241+0000","last_unstale":"2023-09-23T22:42:30.746936+0000","last_undegraded":"2023-09-23T22:42:30.746936+0000","last_fullsized":"2023-09-23T22:42:30.746936+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:13.807087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":102,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.758077+0000","last_change":"2023-09-23T22:40:50.920182+0000","last_active":"2023-09-23T22:42:30.758077+0000","last_peered":"2023-09-23T22:42:30.758077+0000","last_clean":"2023-09-23T22:42:30.758077+0000","last_became_active":"2023-09-23T22:40:50.919699+0000","last_became_peered":"2023-09-23T22:40:50.919699+0000","last_unstale":"2023-09-23T22:42:30.758077+0000","last_undegraded":"2023-09-23T22:42:30.758077+0000","last_fullsized":"2023-09-23T22:42:30.758077+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:56.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754681+0000","last_change":"2023-09-23T22:38:06.157553+0000","last_active":"2023-09-23T22:42:30.754681+0000","last_peered":"2023-09-23T22:42:30.754681+0000","last_clean":"2023-09-23T22:42:30.754681+0000","last_became_active":"2023-09-23T22:38:06.157268+0000","last_became_peered":"2023-09-23T22:38:06.157268+0000","last_unstale":"2023-09-23T22:42:30.754681+0000","last_undegraded":"2023-09-23T22:42:30.754681+0000","last_fullsized":"2023-09-23T22:42:30.754681+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":117,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744229+0000","last_change":"2023-09-23T22:28:42.670329+0000","last_active":"2023-09-23T22:42:30.744229+0000","last_peered":"2023-09-23T22:42:30.744229+0000","last_clean":"2023-09-23T22:42:30.744229+0000","last_became_active":"2023-09-23T22:28:42.670062+0000","last_became_peered":"2023-09-23T22:28:42.670062+0000","last_unstale":"2023-09-23T22:42:30.744229+0000","last_undegraded":"2023-09-23T22:42:30.744229+0000","last_fullsized":"2023-09-23T22:42:30.744229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.758101+0000","last_change":"2023-09-23T22:40:50.920158+0000","last_active":"2023-09-23T22:42:30.758101+0000","last_peered":"2023-09-23T22:42:30.758101+0000","last_clean":"2023-09-23T22:42:30.758101+0000","last_became_active":"2023-09-23T22:40:50.919445+0000","last_became_peered":"2023-09-23T22:40:50.919445+0000","last_unstale":"2023-09-23T22:42:30.758101+0000","last_undegraded":"2023-09-23T22:42:30.758101+0000","last_fullsized":"2023-09-23T22:42:30.758101+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:33.679699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":195,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.749378+0000","last_change":"2023-09-23T22:42:00.067286+0000","last_active":"2023-09-23T22:42:30.749378+0000","last_peered":"2023-09-23T22:42:30.749378+0000","last_clean":"2023-09-23T22:42:30.749378+0000","last_became_active":"2023-09-23T22:41:59.891905+0000","last_became_peered":"2023-09-23T22:41:59.891905+0000","last_unstale":"2023-09-23T22:42:30.749378+0000","last_undegraded":"2023-09-23T22:42:30.749378+0000","last_fullsized":"2023-09-23T22:42:30.749378+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":135,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746120+0000","last_change":"2023-09-23T22:38:06.154965+0000","last_active":"2023-09-23T22:42:30.746120+0000","last_peered":"2023-09-23T22:42:30.746120+0000","last_clean":"2023-09-23T22:42:30.746120+0000","last_became_active":"2023-09-23T22:38:06.154525+0000","last_became_peered":"2023-09-23T22:38:06.154525+0000","last_unstale":"2023-09-23T22:42:30.746120+0000","last_undegraded":"2023-09-23T22:42:30.746120+0000","last_fullsized":"2023-09-23T22:42:30.746120+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:04.854740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"166'3790","reported_seq":3819,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:31.828813+0000","last_change":"2023-09-23T22:41:36.969870+0000","last_active":"2023-09-23T22:42:31.828813+0000","last_peered":"2023-09-23T22:42:31.828813+0000","last_clean":"2023-09-23T22:42:31.828813+0000","last_became_active":"2023-09-23T22:41:36.969535+0000","last_became_peered":"2023-09-23T22:41:36.969535+0000","last_unstale":"2023-09-23T22:42:31.828813+0000","last_undegraded":"2023-09-23T22:42:31.828813+0000","last_fullsized":"2023-09-23T22:42:31.828813+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3790,"log_dups_size":0,"ondisk_log_size":3790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:24.978626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248381440,"num_objects":3790,"num_object_clones":0,"num_object_copies":11370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3790,"num_write_kb":242560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":148,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744925+0000","last_change":"2023-09-23T22:40:50.909706+0000","last_active":"2023-09-23T22:42:30.744925+0000","last_peered":"2023-09-23T22:42:30.744925+0000","last_clean":"2023-09-23T22:42:30.744925+0000","last_became_active":"2023-09-23T22:40:50.909320+0000","last_became_peered":"2023-09-23T22:40:50.909320+0000","last_unstale":"2023-09-23T22:42:30.744925+0000","last_undegraded":"2023-09-23T22:42:30.744925+0000","last_fullsized":"2023-09-23T22:42:30.744925+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":116,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754185+0000","last_change":"2023-09-23T22:28:43.839792+0000","last_active":"2023-09-23T22:42:30.754185+0000","last_peered":"2023-09-23T22:42:30.754185+0000","last_clean":"2023-09-23T22:42:30.754185+0000","last_became_active":"2023-09-23T22:28:43.839460+0000","last_became_peered":"2023-09-23T22:28:43.839460+0000","last_unstale":"2023-09-23T22:42:30.754185+0000","last_undegraded":"2023-09-23T22:42:30.754185+0000","last_fullsized":"2023-09-23T22:42:30.754185+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"166'3871","reported_seq":3900,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:34.378765+0000","last_change":"2023-09-23T22:41:36.967245+0000","last_active":"2023-09-23T22:42:34.378765+0000","last_peered":"2023-09-23T22:42:34.378765+0000","last_clean":"2023-09-23T22:42:34.378765+0000","last_became_active":"2023-09-23T22:41:36.966938+0000","last_became_peered":"2023-09-23T22:41:36.966938+0000","last_unstale":"2023-09-23T22:42:34.378765+0000","last_undegraded":"2023-09-23T22:42:34.378765+0000","last_fullsized":"2023-09-23T22:42:34.378765+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3871,"log_dups_size":0,"ondisk_log_size":3871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:31.285489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253689856,"num_objects":3871,"num_object_clones":0,"num_object_copies":11613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3871,"num_write_kb":247744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":134,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744808+0000","last_change":"2023-09-23T22:37:03.808244+0000","last_active":"2023-09-23T22:42:30.744808+0000","last_peered":"2023-09-23T22:42:30.744808+0000","last_clean":"2023-09-23T22:42:30.744808+0000","last_became_active":"2023-09-23T22:37:03.807828+0000","last_became_peered":"2023-09-23T22:37:03.807828+0000","last_unstale":"2023-09-23T22:42:30.744808+0000","last_undegraded":"2023-09-23T22:42:30.744808+0000","last_fullsized":"2023-09-23T22:42:30.744808+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":146,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746229+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:42:30.746229+0000","last_peered":"2023-09-23T22:42:30.746229+0000","last_clean":"2023-09-23T22:42:30.746229+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:42:30.746229+0000","last_undegraded":"2023-09-23T22:42:30.746229+0000","last_fullsized":"2023-09-23T22:42:30.746229+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"166'3868","reported_seq":3897,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:33.911123+0000","last_change":"2023-09-23T22:41:36.969607+0000","last_active":"2023-09-23T22:42:33.911123+0000","last_peered":"2023-09-23T22:42:33.911123+0000","last_clean":"2023-09-23T22:42:33.911123+0000","last_became_active":"2023-09-23T22:41:36.969311+0000","last_became_peered":"2023-09-23T22:41:36.969311+0000","last_unstale":"2023-09-23T22:42:33.911123+0000","last_undegraded":"2023-09-23T22:42:33.911123+0000","last_fullsized":"2023-09-23T22:42:33.911123+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3868,"log_dups_size":0,"ondisk_log_size":3868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:27.916919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253493248,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3868,"num_write_kb":247552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":118,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744806+0000","last_change":"2023-09-23T22:28:42.667869+0000","last_active":"2023-09-23T22:42:30.744806+0000","last_peered":"2023-09-23T22:42:30.744806+0000","last_clean":"2023-09-23T22:42:30.744806+0000","last_became_active":"2023-09-23T22:28:42.667520+0000","last_became_peered":"2023-09-23T22:28:42.667520+0000","last_unstale":"2023-09-23T22:42:30.744806+0000","last_undegraded":"2023-09-23T22:42:30.744806+0000","last_fullsized":"2023-09-23T22:42:30.744806+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.749606+0000","last_change":"2023-09-23T22:37:03.806890+0000","last_active":"2023-09-23T22:42:30.749606+0000","last_peered":"2023-09-23T22:42:30.749606+0000","last_clean":"2023-09-23T22:42:30.749606+0000","last_became_active":"2023-09-23T22:37:03.806498+0000","last_became_peered":"2023-09-23T22:37:03.806498+0000","last_unstale":"2023-09-23T22:42:30.749606+0000","last_undegraded":"2023-09-23T22:42:30.749606+0000","last_fullsized":"2023-09-23T22:42:30.749606+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"166'3864","reported_seq":3893,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:33.937144+0000","last_change":"2023-09-23T22:41:36.966000+0000","last_active":"2023-09-23T22:42:33.937144+0000","last_peered":"2023-09-23T22:42:33.937144+0000","last_clean":"2023-09-23T22:42:33.937144+0000","last_became_active":"2023-09-23T22:41:36.965691+0000","last_became_peered":"2023-09-23T22:41:36.965691+0000","last_unstale":"2023-09-23T22:42:33.937144+0000","last_undegraded":"2023-09-23T22:42:33.937144+0000","last_fullsized":"2023-09-23T22:42:33.937144+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3864,"log_dups_size":0,"ondisk_log_size":3864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:59.028709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253231104,"num_objects":3864,"num_object_clones":0,"num_object_copies":11592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3864,"num_write_kb":247296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754441+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:42:30.754441+0000","last_peered":"2023-09-23T22:42:30.754441+0000","last_clean":"2023-09-23T22:42:30.754441+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:42:30.754441+0000","last_undegraded":"2023-09-23T22:42:30.754441+0000","last_fullsized":"2023-09-23T22:42:30.754441+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":95,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.756941+0000","last_change":"2023-09-23T22:40:50.917176+0000","last_active":"2023-09-23T22:42:30.756941+0000","last_peered":"2023-09-23T22:42:30.756941+0000","last_clean":"2023-09-23T22:42:30.756941+0000","last_became_active":"2023-09-23T22:40:50.915836+0000","last_became_peered":"2023-09-23T22:40:50.915836+0000","last_unstale":"2023-09-23T22:42:30.756941+0000","last_undegraded":"2023-09-23T22:42:30.756941+0000","last_fullsized":"2023-09-23T22:42:30.756941+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:37.211725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"166'3783","reported_seq":3811,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:33.941799+0000","last_change":"2023-09-23T22:41:36.966198+0000","last_active":"2023-09-23T22:42:33.941799+0000","last_peered":"2023-09-23T22:42:33.941799+0000","last_clean":"2023-09-23T22:42:33.941799+0000","last_became_active":"2023-09-23T22:41:36.965865+0000","last_became_peered":"2023-09-23T22:41:36.965865+0000","last_unstale":"2023-09-23T22:42:33.941799+0000","last_undegraded":"2023-09-23T22:42:33.941799+0000","last_fullsized":"2023-09-23T22:42:33.941799+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3783,"log_dups_size":0,"ondisk_log_size":3783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:03.020464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247922688,"num_objects":3783,"num_object_clones":0,"num_object_copies":11349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3783,"num_write_kb":242112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":149,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:31.747825+0000","last_change":"2023-09-23T22:40:50.914941+0000","last_active":"2023-09-23T22:42:31.747825+0000","last_peered":"2023-09-23T22:42:31.747825+0000","last_clean":"2023-09-23T22:42:31.747825+0000","last_became_active":"2023-09-23T22:40:50.914602+0000","last_became_peered":"2023-09-23T22:40:50.914602+0000","last_unstale":"2023-09-23T22:42:31.747825+0000","last_undegraded":"2023-09-23T22:42:31.747825+0000","last_fullsized":"2023-09-23T22:42:31.747825+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:04.998198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746585+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:42:30.746585+0000","last_peered":"2023-09-23T22:42:30.746585+0000","last_clean":"2023-09-23T22:42:30.746585+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:42:30.746585+0000","last_undegraded":"2023-09-23T22:42:30.746585+0000","last_fullsized":"2023-09-23T22:42:30.746585+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"166'3826","reported_seq":3855,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:33.926662+0000","last_change":"2023-09-23T22:41:36.972030+0000","last_active":"2023-09-23T22:42:33.926662+0000","last_peered":"2023-09-23T22:42:33.926662+0000","last_clean":"2023-09-23T22:42:33.926662+0000","last_became_active":"2023-09-23T22:41:36.971693+0000","last_became_peered":"2023-09-23T22:41:36.971693+0000","last_unstale":"2023-09-23T22:42:33.926662+0000","last_undegraded":"2023-09-23T22:42:33.926662+0000","last_fullsized":"2023-09-23T22:42:33.926662+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3826,"log_dups_size":0,"ondisk_log_size":3826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:42:52.762839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250740736,"num_objects":3826,"num_object_clones":0,"num_object_copies":11478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3826,"num_write_kb":244864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":150,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.749743+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:42:30.749743+0000","last_peered":"2023-09-23T22:42:30.749743+0000","last_clean":"2023-09-23T22:42:30.749743+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:42:30.749743+0000","last_undegraded":"2023-09-23T22:42:30.749743+0000","last_fullsized":"2023-09-23T22:42:30.749743+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746448+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:42:30.746448+0000","last_peered":"2023-09-23T22:42:30.746448+0000","last_clean":"2023-09-23T22:42:30.746448+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:42:30.746448+0000","last_undegraded":"2023-09-23T22:42:30.746448+0000","last_fullsized":"2023-09-23T22:42:30.746448+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"166'3953","reported_seq":3980,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:35.397983+0000","last_change":"2023-09-23T22:41:36.971145+0000","last_active":"2023-09-23T22:42:35.397983+0000","last_peered":"2023-09-23T22:42:35.397983+0000","last_clean":"2023-09-23T22:42:35.397983+0000","last_became_active":"2023-09-23T22:41:36.970815+0000","last_became_peered":"2023-09-23T22:41:36.970815+0000","last_unstale":"2023-09-23T22:42:35.397983+0000","last_undegraded":"2023-09-23T22:42:35.397983+0000","last_fullsized":"2023-09-23T22:42:35.397983+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3953,"log_dups_size":0,"ondisk_log_size":3953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259063808,"num_objects":3953,"num_object_clones":0,"num_object_copies":11859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3953,"num_write_kb":252992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746044+0000","last_change":"2023-09-23T22:41:10.925985+0000","last_active":"2023-09-23T22:42:30.746044+0000","last_peered":"2023-09-23T22:42:30.746044+0000","last_clean":"2023-09-23T22:42:30.746044+0000","last_became_active":"2023-09-23T22:41:10.925598+0000","last_became_peered":"2023-09-23T22:41:10.925598+0000","last_unstale":"2023-09-23T22:42:30.746044+0000","last_undegraded":"2023-09-23T22:42:30.746044+0000","last_fullsized":"2023-09-23T22:42:30.746044+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":149,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746529+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:42:30.746529+0000","last_peered":"2023-09-23T22:42:30.746529+0000","last_clean":"2023-09-23T22:42:30.746529+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:42:30.746529+0000","last_undegraded":"2023-09-23T22:42:30.746529+0000","last_fullsized":"2023-09-23T22:42:30.746529+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"166'3922","reported_seq":3950,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:35.401791+0000","last_change":"2023-09-23T22:41:36.970313+0000","last_active":"2023-09-23T22:42:35.401791+0000","last_peered":"2023-09-23T22:42:35.401791+0000","last_clean":"2023-09-23T22:42:35.401791+0000","last_became_active":"2023-09-23T22:41:36.969917+0000","last_became_peered":"2023-09-23T22:41:36.969917+0000","last_unstale":"2023-09-23T22:42:35.401791+0000","last_undegraded":"2023-09-23T22:42:35.401791+0000","last_fullsized":"2023-09-23T22:42:35.401791+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3922,"log_dups_size":0,"ondisk_log_size":3922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257032192,"num_objects":3922,"num_object_clones":0,"num_object_copies":11766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3922,"num_write_kb":251008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":137,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745472+0000","last_change":"2023-09-23T22:38:06.155404+0000","last_active":"2023-09-23T22:42:30.745472+0000","last_peered":"2023-09-23T22:42:30.745472+0000","last_clean":"2023-09-23T22:42:30.745472+0000","last_became_active":"2023-09-23T22:38:06.154959+0000","last_became_peered":"2023-09-23T22:38:06.154959+0000","last_unstale":"2023-09-23T22:42:30.745472+0000","last_undegraded":"2023-09-23T22:42:30.745472+0000","last_fullsized":"2023-09-23T22:42:30.745472+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:57.496049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":89,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.758042+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:42:30.758042+0000","last_peered":"2023-09-23T22:42:30.758042+0000","last_clean":"2023-09-23T22:42:30.758042+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:42:30.758042+0000","last_undegraded":"2023-09-23T22:42:30.758042+0000","last_fullsized":"2023-09-23T22:42:30.758042+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"166'4035","reported_seq":9026,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:35.385114+0000","last_change":"2023-09-23T22:42:31.753433+0000","last_active":"2023-09-23T22:42:35.385114+0000","last_peered":"2023-09-23T22:42:35.385114+0000","last_clean":"2023-09-23T22:42:35.385114+0000","last_became_active":"2023-09-23T22:42:31.748584+0000","last_became_peered":"2023-09-23T22:42:31.748584+0000","last_unstale":"2023-09-23T22:42:35.385114+0000","last_undegraded":"2023-09-23T22:42:35.385114+0000","last_fullsized":"2023-09-23T22:42:35.385114+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4035,"log_dups_size":0,"ondisk_log_size":4035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264437760,"num_objects":4035,"num_object_clones":0,"num_object_copies":12105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4035,"num_write_kb":258240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":162004992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":161,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752075+0000","last_change":"2023-09-23T22:40:50.918358+0000","last_active":"2023-09-23T22:42:30.752075+0000","last_peered":"2023-09-23T22:42:30.752075+0000","last_clean":"2023-09-23T22:42:30.752075+0000","last_became_active":"2023-09-23T22:40:50.918016+0000","last_became_peered":"2023-09-23T22:40:50.918016+0000","last_unstale":"2023-09-23T22:42:30.752075+0000","last_undegraded":"2023-09-23T22:42:30.752075+0000","last_fullsized":"2023-09-23T22:42:30.752075+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:14.593638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":162,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754777+0000","last_change":"2023-09-23T22:41:10.929497+0000","last_active":"2023-09-23T22:42:30.754777+0000","last_peered":"2023-09-23T22:42:30.754777+0000","last_clean":"2023-09-23T22:42:30.754777+0000","last_became_active":"2023-09-23T22:41:10.929128+0000","last_became_peered":"2023-09-23T22:41:10.929128+0000","last_unstale":"2023-09-23T22:42:30.754777+0000","last_undegraded":"2023-09-23T22:42:30.754777+0000","last_fullsized":"2023-09-23T22:42:30.754777+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":106,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.755382+0000","last_change":"2023-09-23T22:40:14.755608+0000","last_active":"2023-09-23T22:42:30.755382+0000","last_peered":"2023-09-23T22:42:30.755382+0000","last_clean":"2023-09-23T22:42:30.755382+0000","last_became_active":"2023-09-23T22:40:14.755335+0000","last_became_peered":"2023-09-23T22:40:14.755335+0000","last_unstale":"2023-09-23T22:42:30.755382+0000","last_undegraded":"2023-09-23T22:42:30.755382+0000","last_fullsized":"2023-09-23T22:42:30.755382+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:31.014369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"166'3992","reported_seq":4020,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:35.420823+0000","last_change":"2023-09-23T22:41:36.972228+0000","last_active":"2023-09-23T22:42:35.420823+0000","last_peered":"2023-09-23T22:42:35.420823+0000","last_clean":"2023-09-23T22:42:35.420823+0000","last_became_active":"2023-09-23T22:41:36.971904+0000","last_became_peered":"2023-09-23T22:41:36.971904+0000","last_unstale":"2023-09-23T22:42:35.420823+0000","last_undegraded":"2023-09-23T22:42:35.420823+0000","last_fullsized":"2023-09-23T22:42:35.420823+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3992,"log_dups_size":0,"ondisk_log_size":3992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3992,"num_write_kb":255488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":136,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754599+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:42:30.754599+0000","last_peered":"2023-09-23T22:42:30.754599+0000","last_clean":"2023-09-23T22:42:30.754599+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:42:30.754599+0000","last_undegraded":"2023-09-23T22:42:30.754599+0000","last_fullsized":"2023-09-23T22:42:30.754599+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":147,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746360+0000","last_change":"2023-09-23T22:40:50.916179+0000","last_active":"2023-09-23T22:42:30.746360+0000","last_peered":"2023-09-23T22:42:30.746360+0000","last_clean":"2023-09-23T22:42:30.746360+0000","last_became_active":"2023-09-23T22:40:50.915799+0000","last_became_peered":"2023-09-23T22:40:50.915799+0000","last_unstale":"2023-09-23T22:42:30.746360+0000","last_undegraded":"2023-09-23T22:42:30.746360+0000","last_fullsized":"2023-09-23T22:42:30.746360+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":158,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754314+0000","last_change":"2023-09-23T22:41:10.929964+0000","last_active":"2023-09-23T22:42:30.754314+0000","last_peered":"2023-09-23T22:42:30.754314+0000","last_clean":"2023-09-23T22:42:30.754314+0000","last_became_active":"2023-09-23T22:41:10.929689+0000","last_became_peered":"2023-09-23T22:41:10.929689+0000","last_unstale":"2023-09-23T22:42:30.754314+0000","last_undegraded":"2023-09-23T22:42:30.754314+0000","last_fullsized":"2023-09-23T22:42:30.754314+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"166'3773","reported_seq":3802,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:33.896635+0000","last_change":"2023-09-23T22:41:36.968996+0000","last_active":"2023-09-23T22:42:33.896635+0000","last_peered":"2023-09-23T22:42:33.896635+0000","last_clean":"2023-09-23T22:42:33.896635+0000","last_became_active":"2023-09-23T22:41:36.968694+0000","last_became_peered":"2023-09-23T22:41:36.968694+0000","last_unstale":"2023-09-23T22:42:33.896635+0000","last_undegraded":"2023-09-23T22:42:33.896635+0000","last_fullsized":"2023-09-23T22:42:33.896635+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3773,"log_dups_size":0,"ondisk_log_size":3773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:14.636304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247267328,"num_objects":3773,"num_object_clones":0,"num_object_copies":11319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3773,"num_write_kb":241472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":138,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.757815+0000","last_change":"2023-09-23T22:41:10.929848+0000","last_active":"2023-09-23T22:42:30.757815+0000","last_peered":"2023-09-23T22:42:30.757815+0000","last_clean":"2023-09-23T22:42:30.757815+0000","last_became_active":"2023-09-23T22:41:10.929552+0000","last_became_peered":"2023-09-23T22:41:10.929552+0000","last_unstale":"2023-09-23T22:42:30.757815+0000","last_undegraded":"2023-09-23T22:42:30.757815+0000","last_fullsized":"2023-09-23T22:42:30.757815+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:09.783138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752765+0000","last_change":"2023-09-23T22:41:10.927089+0000","last_active":"2023-09-23T22:42:30.752765+0000","last_peered":"2023-09-23T22:42:30.752765+0000","last_clean":"2023-09-23T22:42:30.752765+0000","last_became_active":"2023-09-23T22:41:10.926767+0000","last_became_peered":"2023-09-23T22:41:10.926767+0000","last_unstale":"2023-09-23T22:42:30.752765+0000","last_undegraded":"2023-09-23T22:42:30.752765+0000","last_fullsized":"2023-09-23T22:42:30.752765+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:03.097039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":157,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.751072+0000","last_change":"2023-09-23T22:40:51.909811+0000","last_active":"2023-09-23T22:42:30.751072+0000","last_peered":"2023-09-23T22:42:30.751072+0000","last_clean":"2023-09-23T22:42:30.751072+0000","last_became_active":"2023-09-23T22:40:51.909464+0000","last_became_peered":"2023-09-23T22:40:51.909464+0000","last_unstale":"2023-09-23T22:42:30.751072+0000","last_undegraded":"2023-09-23T22:42:30.751072+0000","last_fullsized":"2023-09-23T22:42:30.751072+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.517012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"166'3901","reported_seq":3930,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:33.914121+0000","last_change":"2023-09-23T22:41:36.963752+0000","last_active":"2023-09-23T22:42:33.914121+0000","last_peered":"2023-09-23T22:42:33.914121+0000","last_clean":"2023-09-23T22:42:33.914121+0000","last_became_active":"2023-09-23T22:41:36.963461+0000","last_became_peered":"2023-09-23T22:41:36.963461+0000","last_unstale":"2023-09-23T22:42:33.914121+0000","last_undegraded":"2023-09-23T22:42:33.914121+0000","last_fullsized":"2023-09-23T22:42:33.914121+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3901,"log_dups_size":0,"ondisk_log_size":3901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:02.507496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255655936,"num_objects":3901,"num_object_clones":0,"num_object_copies":11703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3901,"num_write_kb":249664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":151,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754338+0000","last_change":"2023-09-23T22:41:10.925966+0000","last_active":"2023-09-23T22:42:30.754338+0000","last_peered":"2023-09-23T22:42:30.754338+0000","last_clean":"2023-09-23T22:42:30.754338+0000","last_became_active":"2023-09-23T22:41:10.925674+0000","last_became_peered":"2023-09-23T22:41:10.925674+0000","last_unstale":"2023-09-23T22:42:30.754338+0000","last_undegraded":"2023-09-23T22:42:30.754338+0000","last_fullsized":"2023-09-23T22:42:30.754338+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":383,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744213+0000","last_change":"2023-09-23T22:40:50.904717+0000","last_active":"2023-09-23T22:42:30.744213+0000","last_peered":"2023-09-23T22:42:30.744213+0000","last_clean":"2023-09-23T22:42:30.744213+0000","last_became_active":"2023-09-23T22:40:50.904259+0000","last_became_peered":"2023-09-23T22:40:50.904259+0000","last_unstale":"2023-09-23T22:42:30.744213+0000","last_undegraded":"2023-09-23T22:42:30.744213+0000","last_fullsized":"2023-09-23T22:42:30.744213+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745685+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:42:30.745685+0000","last_peered":"2023-09-23T22:42:30.745685+0000","last_clean":"2023-09-23T22:42:30.745685+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:42:30.745685+0000","last_undegraded":"2023-09-23T22:42:30.745685+0000","last_fullsized":"2023-09-23T22:42:30.745685+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":116,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752304+0000","last_change":"2023-09-23T22:28:43.841186+0000","last_active":"2023-09-23T22:42:30.752304+0000","last_peered":"2023-09-23T22:42:30.752304+0000","last_clean":"2023-09-23T22:42:30.752304+0000","last_became_active":"2023-09-23T22:28:43.840901+0000","last_became_peered":"2023-09-23T22:28:43.840901+0000","last_unstale":"2023-09-23T22:42:30.752304+0000","last_undegraded":"2023-09-23T22:42:30.752304+0000","last_fullsized":"2023-09-23T22:42:30.752304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"166'4053","reported_seq":4081,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:35.418216+0000","last_change":"2023-09-23T22:41:36.966223+0000","last_active":"2023-09-23T22:42:35.418216+0000","last_peered":"2023-09-23T22:42:35.418216+0000","last_clean":"2023-09-23T22:42:35.418216+0000","last_became_active":"2023-09-23T22:41:36.965848+0000","last_became_peered":"2023-09-23T22:41:36.965848+0000","last_unstale":"2023-09-23T22:42:35.418216+0000","last_undegraded":"2023-09-23T22:42:35.418216+0000","last_fullsized":"2023-09-23T22:42:35.418216+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4053,"log_dups_size":0,"ondisk_log_size":4053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265617408,"num_objects":4053,"num_object_clones":0,"num_object_copies":12159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4053,"num_write_kb":259392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":166,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744111+0000","last_change":"2023-09-23T22:40:50.912406+0000","last_active":"2023-09-23T22:42:30.744111+0000","last_peered":"2023-09-23T22:42:30.744111+0000","last_clean":"2023-09-23T22:42:30.744111+0000","last_became_active":"2023-09-23T22:40:50.912085+0000","last_became_peered":"2023-09-23T22:40:50.912085+0000","last_unstale":"2023-09-23T22:42:30.744111+0000","last_undegraded":"2023-09-23T22:42:30.744111+0000","last_fullsized":"2023-09-23T22:42:30.744111+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.757722+0000","last_change":"2023-09-23T22:41:49.035892+0000","last_active":"2023-09-23T22:42:30.757722+0000","last_peered":"2023-09-23T22:42:30.757722+0000","last_clean":"2023-09-23T22:42:30.757722+0000","last_became_active":"2023-09-23T22:41:49.035318+0000","last_became_peered":"2023-09-23T22:41:49.035318+0000","last_unstale":"2023-09-23T22:42:30.757722+0000","last_undegraded":"2023-09-23T22:42:30.757722+0000","last_fullsized":"2023-09-23T22:42:30.757722+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:06.155982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":120,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.758422+0000","last_change":"2023-09-23T22:41:10.928059+0000","last_active":"2023-09-23T22:42:30.758422+0000","last_peered":"2023-09-23T22:42:30.758422+0000","last_clean":"2023-09-23T22:42:30.758422+0000","last_became_active":"2023-09-23T22:41:10.927749+0000","last_became_peered":"2023-09-23T22:41:10.927749+0000","last_unstale":"2023-09-23T22:42:30.758422+0000","last_undegraded":"2023-09-23T22:42:30.758422+0000","last_fullsized":"2023-09-23T22:42:30.758422+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:53.307028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"166'3913","reported_seq":3941,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:35.559797+0000","last_change":"2023-09-23T22:41:36.969378+0000","last_active":"2023-09-23T22:42:35.559797+0000","last_peered":"2023-09-23T22:42:35.559797+0000","last_clean":"2023-09-23T22:42:35.559797+0000","last_became_active":"2023-09-23T22:41:36.969153+0000","last_became_peered":"2023-09-23T22:41:36.969153+0000","last_unstale":"2023-09-23T22:42:35.559797+0000","last_undegraded":"2023-09-23T22:42:35.559797+0000","last_fullsized":"2023-09-23T22:42:35.559797+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3913,"log_dups_size":0,"ondisk_log_size":3913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256442368,"num_objects":3913,"num_object_clones":0,"num_object_copies":11739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3913,"num_write_kb":250432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":151,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.744714+0000","last_change":"2023-09-23T22:40:50.904566+0000","last_active":"2023-09-23T22:42:30.744714+0000","last_peered":"2023-09-23T22:42:30.744714+0000","last_clean":"2023-09-23T22:42:30.744714+0000","last_became_active":"2023-09-23T22:40:50.904230+0000","last_became_peered":"2023-09-23T22:40:50.904230+0000","last_unstale":"2023-09-23T22:42:30.744714+0000","last_undegraded":"2023-09-23T22:42:30.744714+0000","last_fullsized":"2023-09-23T22:42:30.744714+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752395+0000","last_change":"2023-09-23T22:41:10.930375+0000","last_active":"2023-09-23T22:42:30.752395+0000","last_peered":"2023-09-23T22:42:30.752395+0000","last_clean":"2023-09-23T22:42:30.752395+0000","last_became_active":"2023-09-23T22:41:10.930050+0000","last_became_peered":"2023-09-23T22:41:10.930050+0000","last_unstale":"2023-09-23T22:42:30.752395+0000","last_undegraded":"2023-09-23T22:42:30.752395+0000","last_fullsized":"2023-09-23T22:42:30.752395+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:29.593309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.756462+0000","last_change":"2023-09-23T22:40:50.915078+0000","last_active":"2023-09-23T22:42:30.756462+0000","last_peered":"2023-09-23T22:42:30.756462+0000","last_clean":"2023-09-23T22:42:30.756462+0000","last_became_active":"2023-09-23T22:40:50.914759+0000","last_became_peered":"2023-09-23T22:40:50.914759+0000","last_unstale":"2023-09-23T22:42:30.756462+0000","last_undegraded":"2023-09-23T22:42:30.756462+0000","last_fullsized":"2023-09-23T22:42:30.756462+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:58.569943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"166'3921","reported_seq":3949,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:34.390559+0000","last_change":"2023-09-23T22:41:36.966638+0000","last_active":"2023-09-23T22:42:34.390559+0000","last_peered":"2023-09-23T22:42:34.390559+0000","last_clean":"2023-09-23T22:42:34.390559+0000","last_became_active":"2023-09-23T22:41:36.966433+0000","last_became_peered":"2023-09-23T22:41:36.966433+0000","last_unstale":"2023-09-23T22:42:34.390559+0000","last_undegraded":"2023-09-23T22:42:34.390559+0000","last_fullsized":"2023-09-23T22:42:34.390559+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3921,"log_dups_size":0,"ondisk_log_size":3921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:12.457010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256966656,"num_objects":3921,"num_object_clones":0,"num_object_copies":11763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3921,"num_write_kb":250944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":149,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746172+0000","last_change":"2023-09-23T22:40:50.916941+0000","last_active":"2023-09-23T22:42:30.746172+0000","last_peered":"2023-09-23T22:42:30.746172+0000","last_clean":"2023-09-23T22:42:30.746172+0000","last_became_active":"2023-09-23T22:40:50.916668+0000","last_became_peered":"2023-09-23T22:40:50.916668+0000","last_unstale":"2023-09-23T22:42:30.746172+0000","last_undegraded":"2023-09-23T22:42:30.746172+0000","last_fullsized":"2023-09-23T22:42:30.746172+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:57.299908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.750658+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:42:30.750658+0000","last_peered":"2023-09-23T22:42:30.750658+0000","last_clean":"2023-09-23T22:42:30.750658+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:42:30.750658+0000","last_undegraded":"2023-09-23T22:42:30.750658+0000","last_fullsized":"2023-09-23T22:42:30.750658+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":146,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745464+0000","last_change":"2023-09-23T22:40:50.916178+0000","last_active":"2023-09-23T22:42:30.745464+0000","last_peered":"2023-09-23T22:42:30.745464+0000","last_clean":"2023-09-23T22:42:30.745464+0000","last_became_active":"2023-09-23T22:40:50.915857+0000","last_became_peered":"2023-09-23T22:40:50.915857+0000","last_unstale":"2023-09-23T22:42:30.745464+0000","last_undegraded":"2023-09-23T22:42:30.745464+0000","last_fullsized":"2023-09-23T22:42:30.745464+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:11.736643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"166'3855","reported_seq":3884,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T22:42:34.395438+0000","last_change":"2023-09-23T22:41:36.965968+0000","last_active":"2023-09-23T22:42:34.395438+0000","last_peered":"2023-09-23T22:42:34.395438+0000","last_clean":"2023-09-23T22:42:34.395438+0000","last_became_active":"2023-09-23T22:41:36.965582+0000","last_became_peered":"2023-09-23T22:41:36.965582+0000","last_unstale":"2023-09-23T22:42:34.395438+0000","last_undegraded":"2023-09-23T22:42:34.395438+0000","last_fullsized":"2023-09-23T22:42:34.395438+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":3855,"log_dups_size":0,"ondisk_log_size":3855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:51.048347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252641280,"num_objects":3855,"num_object_clones":0,"num_object_copies":11565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3855,"num_write_kb":246720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":171,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.752135+0000","last_change":"2023-09-23T22:42:20.231791+0000","last_active":"2023-09-23T22:42:30.752135+0000","last_peered":"2023-09-23T22:42:30.752135+0000","last_clean":"2023-09-23T22:42:30.752135+0000","last_became_active":"2023-09-23T22:42:20.231410+0000","last_became_peered":"2023-09-23T22:42:20.231410+0000","last_unstale":"2023-09-23T22:42:30.752135+0000","last_undegraded":"2023-09-23T22:42:30.752135+0000","last_fullsized":"2023-09-23T22:42:30.752135+0000","mapping_epoch":162,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":98,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.758244+0000","last_change":"2023-09-23T22:40:50.916564+0000","last_active":"2023-09-23T22:42:30.758244+0000","last_peered":"2023-09-23T22:42:30.758244+0000","last_clean":"2023-09-23T22:42:30.758244+0000","last_became_active":"2023-09-23T22:40:50.915883+0000","last_became_peered":"2023-09-23T22:40:50.915883+0000","last_unstale":"2023-09-23T22:42:30.758244+0000","last_undegraded":"2023-09-23T22:42:30.758244+0000","last_fullsized":"2023-09-23T22:42:30.758244+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:25.767798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":116,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746127+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:42:30.746127+0000","last_peered":"2023-09-23T22:42:30.746127+0000","last_clean":"2023-09-23T22:42:30.746127+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:42:30.746127+0000","last_undegraded":"2023-09-23T22:42:30.746127+0000","last_fullsized":"2023-09-23T22:42:30.746127+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":78,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.757028+0000","last_change":"2023-09-23T22:40:50.909960+0000","last_active":"2023-09-23T22:42:30.757028+0000","last_peered":"2023-09-23T22:42:30.757028+0000","last_clean":"2023-09-23T22:42:30.757028+0000","last_became_active":"2023-09-23T22:40:50.909314+0000","last_became_peered":"2023-09-23T22:40:50.909314+0000","last_unstale":"2023-09-23T22:42:30.757028+0000","last_undegraded":"2023-09-23T22:42:30.757028+0000","last_fullsized":"2023-09-23T22:42:30.757028+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:41.246849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":129,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.757900+0000","last_change":"2023-09-23T22:41:10.929809+0000","last_active":"2023-09-23T22:42:30.757900+0000","last_peered":"2023-09-23T22:42:30.757900+0000","last_clean":"2023-09-23T22:42:30.757900+0000","last_became_active":"2023-09-23T22:41:10.929502+0000","last_became_peered":"2023-09-23T22:41:10.929502+0000","last_unstale":"2023-09-23T22:42:30.757900+0000","last_undegraded":"2023-09-23T22:42:30.757900+0000","last_fullsized":"2023-09-23T22:42:30.757900+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:44.238238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":134,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754523+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:42:30.754523+0000","last_peered":"2023-09-23T22:42:30.754523+0000","last_clean":"2023-09-23T22:42:30.754523+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:42:30.754523+0000","last_undegraded":"2023-09-23T22:42:30.754523+0000","last_fullsized":"2023-09-23T22:42:30.754523+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":165,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746357+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:42:30.746357+0000","last_peered":"2023-09-23T22:42:30.746357+0000","last_clean":"2023-09-23T22:42:30.746357+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:42:30.746357+0000","last_undegraded":"2023-09-23T22:42:30.746357+0000","last_fullsized":"2023-09-23T22:42:30.746357+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":119,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.754895+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:42:30.754895+0000","last_peered":"2023-09-23T22:42:30.754895+0000","last_clean":"2023-09-23T22:42:30.754895+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:42:30.754895+0000","last_undegraded":"2023-09-23T22:42:30.754895+0000","last_fullsized":"2023-09-23T22:42:30.754895+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746079+0000","last_change":"2023-09-23T22:41:49.032087+0000","last_active":"2023-09-23T22:42:30.746079+0000","last_peered":"2023-09-23T22:42:30.746079+0000","last_clean":"2023-09-23T22:42:30.746079+0000","last_became_active":"2023-09-23T22:41:49.031690+0000","last_became_peered":"2023-09-23T22:41:49.031690+0000","last_unstale":"2023-09-23T22:42:30.746079+0000","last_undegraded":"2023-09-23T22:42:30.746079+0000","last_fullsized":"2023-09-23T22:42:30.746079+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":156,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.746106+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:42:30.746106+0000","last_peered":"2023-09-23T22:42:30.746106+0000","last_clean":"2023-09-23T22:42:30.746106+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:42:30.746106+0000","last_undegraded":"2023-09-23T22:42:30.746106+0000","last_fullsized":"2023-09-23T22:42:30.746106+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":179,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745939+0000","last_change":"2023-09-23T22:41:10.931122+0000","last_active":"2023-09-23T22:42:30.745939+0000","last_peered":"2023-09-23T22:42:30.745939+0000","last_clean":"2023-09-23T22:42:30.745939+0000","last_became_active":"2023-09-23T22:41:10.930811+0000","last_became_peered":"2023-09-23T22:41:10.930811+0000","last_unstale":"2023-09-23T22:42:30.745939+0000","last_undegraded":"2023-09-23T22:42:30.745939+0000","last_fullsized":"2023-09-23T22:42:30.745939+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:16.649049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":108,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.761420+0000","last_change":"2023-09-23T22:41:10.926522+0000","last_active":"2023-09-23T22:42:30.761420+0000","last_peered":"2023-09-23T22:42:30.761420+0000","last_clean":"2023-09-23T22:42:30.761420+0000","last_became_active":"2023-09-23T22:41:10.926169+0000","last_became_peered":"2023-09-23T22:41:10.926169+0000","last_unstale":"2023-09-23T22:42:30.761420+0000","last_undegraded":"2023-09-23T22:42:30.761420+0000","last_fullsized":"2023-09-23T22:42:30.761420+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:45.457705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":131,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T22:42:30.745971+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:42:30.745971+0000","last_peered":"2023-09-23T22:42:30.745971+0000","last_clean":"2023-09-23T22:42:30.745971+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:42:30.745971+0000","last_undegraded":"2023-09-23T22:42:30.745971+0000","last_fullsized":"2023-09-23T22:42:30.745971+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4084203520,"num_objects":62320,"num_object_clones":0,"num_object_copies":186960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62320,"num_write_kb":3988480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":162004992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12397641728,"data_stored":12397641728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62320,"ondisk_log_size":62320,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430395,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1560260,"kb_used_data":1254844,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92184892,"statfs":{"total":95995035648,"available":94397329408,"internally_reserved":0,"allocated":1284960256,"data_stored":1284476152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.258,"5min":2.258,"15min":2.258},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.5600000000000001,"5min":2.5600000000000001,"15min":2.5600000000000001},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":6.2210000000000001,"5min":2.423,"15min":1.2290000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":73.269000000000005,"5min":73.269000000000005,"15min":73.269000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":6.21,"5min":2.4089999999999998,"15min":1.208},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":73.281000000000006,"5min":73.281000000000006,"15min":73.281000000000006},"last":0.439}]},{"osd":2,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.093,"5min":3.093,"15min":3.093},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.1970000000000001,"5min":3.1970000000000001,"15min":3.1970000000000001},"last":1.639},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.64700000000000002,"15min":0.61699999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.3019999999999996,"5min":4.3019999999999996,"15min":4.3019999999999996},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":0.84499999999999997,"15min":0.623},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":28.213000000000001,"5min":28.213000000000001,"15min":28.213000000000001},"last":0.432},{"interface":"front","average":{"1min":2.3439999999999999,"5min":0.91000000000000003,"15min":0.63800000000000001},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":27.933,"5min":27.933,"15min":27.933},"last":0.498}]},{"osd":5,"last update":"Sat Sep 23 22:42:34 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":2.956,"15min":1.323},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":2.3769999999999998,"5min":226.22200000000001,"15min":226.22200000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":2.9740000000000002,"15min":1.3859999999999999},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.425,"5min":225.50899999999999,"15min":225.50899999999999},"last":0.57999999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:42:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2149999999999999,"5min":2.2149999999999999,"15min":2.2149999999999999},"last":0.871},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.64000000000000001,"15min":0.61199999999999999},"min":{"1min":0.40600000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.0870000000000002,"5min":2.266,"15min":2.266},"last":0.52500000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626623,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1566904,"kb_used_data":1261488,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92178248,"statfs":{"total":95995035648,"available":94390525952,"internally_reserved":0,"allocated":1291763712,"data_stored":1291291750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.984,"5min":1.984,"15min":1.984},"last":0.45400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:42:17 2023","interfaces":[{"interface":"back","average":{"1min":8.6210000000000004,"5min":3.6850000000000001,"15min":1.569},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":153.78399999999999,"5min":153.78399999999999,"15min":153.78399999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":8.6170000000000009,"5min":3.6549999999999998,"15min":1.599},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":154.148,"5min":154.148,"15min":154.148},"last":0.51600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.7389999999999999,"15min":2.7389999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":43.408000000000001,"5min":43.408000000000001,"15min":43.408000000000001},"last":59.281999999999996},{"interface":"front","average":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":43.356000000000002,"5min":43.356000000000002,"15min":43.356000000000002},"last":59.149999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:42:17 2023","interfaces":[{"interface":"back","average":{"1min":8.1059999999999999,"5min":2.254,"15min":1.1020000000000001},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":80.277000000000001,"5min":80.277000000000001,"15min":80.277000000000001},"last":0.442},{"interface":"front","average":{"1min":8.2669999999999995,"5min":2.4039999999999999,"15min":1.21},"min":{"1min":0.38500000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":80.513999999999996,"5min":80.513999999999996,"15min":80.513999999999996},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 22:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49199999999999999,"15min":0.48099999999999998},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.47899999999999998,"15min":0.49199999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.31900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.51500000000000001,"15min":0.53500000000000003},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.56799999999999995,"15min":0.56000000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.5},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.53900000000000003,"15min":0.56499999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.50600000000000001}]}]},{"osd":1,"up_from":84,"seq":360777252932,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1662428,"kb_used_data":1586580,"kb_used_omap":9,"kb_used_meta":75830,"kb_avail":92082724,"statfs":{"total":95995035648,"available":94292709376,"internally_reserved":0,"allocated":1624657920,"data_stored":1623973788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":77650921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.8279999999999998,"5min":2.8279999999999998,"15min":2.8279999999999998},"last":83.670000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.9710000000000001,"5min":1.9710000000000001,"15min":1.9710000000000001},"last":84}]},{"osd":2,"last update":"Sat Sep 23 22:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.8019999999999996,"5min":4.8019999999999996,"15min":4.8019999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":6.4020000000000001},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.52600000000000002,"15min":0.42999999999999999},"min":{"1min":0.33200000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.0470000000000002,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":1.036},{"interface":"front","average":{"1min":1.5069999999999999,"5min":0.60599999999999998,"15min":0.46000000000000002},"min":{"1min":0.33900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":13.416,"5min":13.416,"15min":13.416},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.47,"15min":0.91400000000000003},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.7640000000000002,"5min":69.344999999999999,"15min":69.344999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.41699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.8339999999999996,"5min":69.359999999999999,"15min":69.359999999999999},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0369999999999999,"15min":0.77000000000000002},"min":{"1min":0.46999999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0880000000000001,"5min":28.837,"15min":28.837},"last":1.073},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.98099999999999998,"15min":0.72699999999999998},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.726,"5min":28.986999999999998,"15min":28.986999999999998},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.75,"15min":0.67400000000000004},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.8550000000000004,"5min":4.8550000000000004,"15min":4.8550000000000004},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.73299999999999998,"15min":0.66600000000000004},"min":{"1min":0.51300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.1509999999999998,"5min":3.1509999999999998,"15min":3.1509999999999998},"last":1.107}]},{"osd":7,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":2.6429999999999998,"15min":1.3069999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.7889999999999997,"5min":159.018,"15min":159.018},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.228,"5min":2.7149999999999999,"15min":1.347},"min":{"1min":0.499,"5min":0.37,"15min":0.37},"max":{"1min":4.8860000000000001,"5min":158.96000000000001,"15min":158.96000000000001},"last":0.70999999999999996}]}]},{"osd":0,"up_from":126,"seq":541165879318,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2060584,"kb_used_data":1966176,"kb_used_omap":28,"kb_used_meta":94371,"kb_avail":91684568,"statfs":{"total":95995035648,"available":93884997632,"internally_reserved":0,"allocated":2013364224,"data_stored":2012616127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":96636752},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.754,"15min":0.754},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.3759999999999999,"5min":3.3759999999999999,"15min":3.3759999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.532,"5min":2.532,"15min":2.532},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":6.6840000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":105.818,"5min":105.818,"15min":105.818},"last":0.30299999999999999},{"interface":"front","average":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":105.77500000000001,"5min":105.77500000000001,"15min":105.77500000000001},"last":0.78200000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.3899999999999997,"5min":4.3899999999999997,"15min":4.3899999999999997},"last":2.1429999999999998},{"interface":"front","average":{"1min":1.258,"5min":1.258,"15min":1.258},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9939999999999998,"5min":4.9939999999999998,"15min":4.9939999999999998},"last":0.36399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":3.4180000000000001,"5min":3.4180000000000001,"15min":3.4180000000000001},"last":0.61199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":4.2549999999999999,"5min":4.2549999999999999,"15min":4.2549999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.875,"5min":0.875,"15min":0.875},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":3.4489999999999998,"5min":3.4489999999999998,"15min":3.4489999999999998},"last":0.751}]},{"osd":6,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":3.476,"5min":3.476,"15min":3.476},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.6919999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.5}]},{"osd":7,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":4.3200000000000003,"5min":4.3200000000000003,"15min":4.3200000000000003},"last":0.41999999999999998}]}]},{"osd":2,"up_from":131,"seq":562640715794,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1563464,"kb_used_data":1473088,"kb_used_omap":81,"kb_used_meta":90286,"kb_avail":92181688,"statfs":{"total":95995035648,"available":94394048512,"internally_reserved":0,"allocated":1508442112,"data_stored":1507722914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":92453082},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":3.399,"15min":3.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":48.973999999999997,"5min":48.973999999999997,"15min":48.973999999999997},"last":0.79900000000000004},{"interface":"front","average":{"1min":4.6580000000000004,"5min":4.6580000000000004,"15min":4.6580000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":50.616999999999997,"5min":50.616999999999997,"15min":50.616999999999997},"last":3.29}]},{"osd":1,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":3.0819999999999999,"15min":3.0819999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":21.114000000000001,"5min":21.114000000000001,"15min":21.114000000000001},"last":3.4399999999999999},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":22.021000000000001,"5min":22.021000000000001,"15min":22.021000000000001},"last":3.254}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.2509999999999999,"5min":3.2509999999999999,"15min":3.2509999999999999},"last":8.7390000000000008},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.0830000000000002,"5min":4.0830000000000002,"15min":4.0830000000000002},"last":1.1120000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.0069999999999997,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.83499999999999996},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":6.8129999999999997,"5min":6.8129999999999997,"15min":6.8129999999999997},"last":3.3410000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":6.8719999999999999,"5min":6.8719999999999999,"15min":6.8719999999999999},"last":3.379},{"interface":"front","average":{"1min":0.752,"5min":0.752,"15min":0.752},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":3.3159999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.1890000000000001,"5min":3.1890000000000001,"15min":3.1890000000000001},"last":3.359},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.0579999999999998,"5min":4.0579999999999998,"15min":4.0579999999999998},"last":0.86099999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":4.0359999999999996,"5min":4.0359999999999996,"15min":4.0359999999999996},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.355,"5min":1.355,"15min":1.355},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":6.9100000000000001,"5min":6.9100000000000001,"15min":6.9100000000000001},"last":3.411}]}]},{"osd":3,"up_from":94,"seq":403726925880,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2334876,"kb_used_data":2231124,"kb_used_omap":7,"kb_used_meta":103736,"kb_avail":91410276,"statfs":{"total":95995035648,"available":93604122624,"internally_reserved":0,"allocated":2284670976,"data_stored":2283991863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":106226280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":0.308},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6319999999999997,"5min":6.6319999999999997,"15min":6.6319999999999997},"last":4.1470000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.42499999999999999,"15min":0.375},"min":{"1min":0.32600000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.254,"5min":3.254,"15min":3.254},"last":3.262},{"interface":"front","average":{"1min":1.272,"5min":0.55100000000000005,"15min":0.44500000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.843,"5min":6.843,"15min":6.843},"last":0.28699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.3550000000000004,"5min":4.3550000000000004,"15min":4.3550000000000004},"last":41.558},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.911,"5min":3.911,"15min":3.911},"last":42.088999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.627,"15min":0.624},"min":{"1min":0.43099999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.2250000000000001,"5min":3.2250000000000001,"15min":3.2250000000000001},"last":2.3929999999999998},{"interface":"front","average":{"1min":1.077,"5min":0.69999999999999996,"15min":0.68799999999999994},"min":{"1min":0.377,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.0279999999999996,"5min":4.0279999999999996,"15min":4.0279999999999996},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":0.67600000000000005,"15min":0.66800000000000004},"min":{"1min":0.35099999999999998,"5min":0.312,"15min":0.312},"max":{"1min":3.9980000000000002,"5min":3.9980000000000002,"15min":3.9980000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.65300000000000002,"15min":0.61599999999999999},"min":{"1min":0.45400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.915,"5min":3.0739999999999998,"15min":3.0739999999999998},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.8620000000000001,"5min":3.8620000000000001,"15min":3.8620000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67400000000000004,"15min":0.66500000000000004},"min":{"1min":0.39200000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.179,"5min":3.1520000000000001,"15min":3.1520000000000001},"last":0.58599999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.68700000000000006,"15min":0.66400000000000003},"min":{"1min":0.39500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.2569999999999999,"5min":3.1200000000000001,"15min":3.1200000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.441,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.1640000000000001,"5min":3.1640000000000001,"15min":3.1640000000000001},"last":2.431}]}]},{"osd":4,"up_from":30,"seq":128849019079,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1566204,"kb_used_data":1256692,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92178948,"statfs":{"total":95995035648,"available":94391242752,"internally_reserved":0,"allocated":1286852608,"data_stored":1286376082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7690000000000001,"5min":2.7690000000000001,"15min":2.7690000000000001},"last":3.6040000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.42799999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.621,"15min":0.60799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4829999999999997,"5min":5.4829999999999997,"15min":5.4829999999999997},"last":0.379},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.60299999999999998,"15min":0.57099999999999995},"min":{"1min":0.32500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.6070000000000002,"5min":5.6070000000000002,"15min":5.6070000000000002},"last":1.4910000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:42:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.6890000000000001,"15min":2.6890000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":29.370999999999999,"5min":29.370999999999999,"15min":29.370999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.54,"5min":2.54,"15min":2.54},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":28.879000000000001,"5min":28.879000000000001,"15min":28.879000000000001},"last":1.77}]},{"osd":3,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.018,"15min":0.78200000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.921,"5min":44.994,"15min":44.994},"last":2.3639999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.05,"15min":0.74199999999999999},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.6819999999999999,"5min":44.765000000000001,"15min":44.765000000000001},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.93500000000000005,"15min":0.63100000000000001},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.431,"5min":43.189999999999998,"15min":43.189999999999998},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.94499999999999995,"15min":0.64400000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.1950000000000001,"5min":42.450000000000003,"15min":42.450000000000003},"last":0.28699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.76900000000000002,"15min":0.60399999999999998},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81899999999999995,"5min":27.056000000000001,"15min":27.056000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.83399999999999996,"15min":0.64000000000000001},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.91800000000000004,"5min":26.940999999999999,"15min":26.940999999999999},"last":0.307}]},{"osd":7,"last update":"Sat Sep 23 22:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.629,"15min":0.57999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.1459999999999999,"5min":4.1459999999999999,"15min":4.1459999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55200000000000005,"15min":0.54700000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.33400000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822851,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309540,"kb_used_data":1004124,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92435612,"statfs":{"total":95995035648,"available":94654066688,"internally_reserved":0,"allocated":1028222976,"data_stored":1027766140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":1.784,"5min":1.784,"15min":1.784},"last":3.1970000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":1.589}]},{"osd":1,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":9.3829999999999991,"5min":2.3239999999999998,"15min":1.153},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":149.203,"5min":149.203,"15min":149.203},"last":0.36299999999999999},{"interface":"front","average":{"1min":9.3109999999999999,"5min":2.2839999999999998,"15min":1.1160000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":149.554,"5min":149.554,"15min":149.554},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":9.1739999999999995,"5min":9.1739999999999995,"15min":9.1739999999999995},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":116.123,"5min":116.123,"15min":116.123},"last":1.5580000000000001},{"interface":"front","average":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":116.14100000000001,"5min":116.14100000000001,"15min":116.14100000000001},"last":1.6990000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.35899999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.089,"5min":2.286,"15min":2.286},"last":1.9399999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.875,"5min":3.4740000000000002,"15min":3.4740000000000002},"last":1.804}]},{"osd":4,"last update":"Sat Sep 23 22:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.46200000000000002,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.51600000000000001,"15min":0.5},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.3780000000000001,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":2.1669999999999998,"15min":1.0580000000000001},"min":{"1min":0.33100000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.349,"5min":140.07499999999999,"15min":140.07499999999999},"last":0.439},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.1800000000000002,"15min":1.0840000000000001},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.319,"5min":139.739,"15min":139.739},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.623,"15min":0.57699999999999996},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.56000000000000005,"15min":0.56100000000000005},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":1.5980000000000001,"5min":1.5980000000000001,"15min":1.5980000000000001},"last":0.47599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2025717760,"data_stored":2025717760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1624834048,"data_stored":1624834048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1520697344,"data_stored":1520697344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2297888768,"data_stored":2297888768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1293418496,"data_stored":1293418496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1034813440,"data_stored":1034813440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1304166400,"data_stored":1304166400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1296105472,"data_stored":1296105472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:42:38.876 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:42:38.876 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:42:38.876 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:42:38.876 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T22:42:38.962 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:39.030 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:39.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:39 smithi052 ceph-mon[180979]: pgmap v390: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 38 MiB/s wr, 611 op/s 2023-09-23T22:42:39.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:39 smithi052 ceph-mon[187743]: pgmap v390: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 38 MiB/s wr, 611 op/s 2023-09-23T22:42:39.396 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:39.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:39.512 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 52 16 67313 67297 80.8406 62.875 0.0142654 0.012364 2023-09-23T22:42:39.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:39.597 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:39.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:39 smithi151 ceph-mon[159210]: pgmap v390: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 38 MiB/s wr, 611 op/s 2023-09-23T22:42:39.720 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:39.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:39.823 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:39.912 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:40.315 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:40.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:40.412 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:42:40.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:40.453 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:42:40.455 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:40.476 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:40.490 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:42:40.506 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 53 16 68385 68369 80.5774 67 0.0222184 0.0124049 2023-09-23T22:42:40.533 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:42:40.537 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:40.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:40.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:40.578 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:42:40.623 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:42:40.644 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:40.657 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:42:40.677 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:40.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:42:40.692 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.693 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:42:40.733 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:40.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:40.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:40.838 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:40.871 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:40.925 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:42:40.926 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:40.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:40.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:40 smithi052 ceph-mon[180979]: pgmap v391: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T22:42:40.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/141503879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:40.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:40 smithi052 ceph-mon[187743]: pgmap v391: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T22:42:40.976 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/141503879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:41.032 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:41.053 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:41.053 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:41.063 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:40 smithi151 ceph-mon[159210]: pgmap v391: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T22:42:41.063 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/141503879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:41.066 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.119 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:41.225 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:41.262 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.319 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:41.426 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:41.460 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.509 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 54 16 69381 69365 80.2333 62.25 0.00885516 0.0124582 2023-09-23T22:42:41.529 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:41.637 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:41.670 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.713 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:41.818 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:41.854 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.914 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:42.022 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:42.059 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:42.119 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:42.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:42.227 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:42.257 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:42.509 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 55 16 70307 70291 79.8269 57.875 0.0190815 0.0125217 2023-09-23T22:42:42.654 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:42.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:42.765 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:42.847 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:42.992 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:42.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:43.095 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:43.126 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:43.182 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:43.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:43.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:43.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:43 smithi052 ceph-mon[180979]: pgmap v392: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2023-09-23T22:42:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:43 smithi052 ceph-mon[187743]: pgmap v392: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2023-09-23T22:42:43.287 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:43.334 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:43.400 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:43.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:43.405 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:43 smithi151 ceph-mon[159210]: pgmap v392: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2023-09-23T22:42:43.506 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:43.511 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 56 16 71337 71321 79.5511 64.375 0.0176832 0.012565 2023-09-23T22:42:43.583 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:43.658 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:42:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:42:43] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-09-23T22:42:43.688 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:42:43.806 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:43.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:43.922 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:44.021 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:44.097 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:44.149 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:44.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:44.261 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:44.353 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:44.509 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 57 16 72214 72198 79.1172 54.8125 0.0186173 0.0126338 2023-09-23T22:42:44.597 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:44.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:44.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:44.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:42:44.797 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:44.936 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:44 smithi052 conmon[180954]: 2023-09-23T22:42:44.617+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e167 definitely_dead 0 2023-09-23T22:42:44.991 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:44.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:45.099 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:45.148 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:45.151 INFO:teuthology.orchestra.run.smithi052.stderr:marked out osd.7. 2023-09-23T22:42:45.195 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:45.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[187743]: pgmap v393: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2023-09-23T22:42:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3681257752' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:42:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[187743]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:42:45.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[180979]: pgmap v393: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2023-09-23T22:42:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3681257752' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:42:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[180979]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:42:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:45.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:45.330 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:45 smithi151 ceph-mon[159210]: pgmap v393: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2023-09-23T22:42:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3681257752' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:42:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:45 smithi151 ceph-mon[159210]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:42:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:45.510 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 58 16 72839 72823 78.4269 39.0625 0.00265078 0.0126704 2023-09-23T22:42:45.876 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:45.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:45.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:46.007 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[180979]: failed to encode map e168 with expected crc 2023-09-23T22:42:46.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3681257752' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:42:46.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[180979]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T22:42:46.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[180979]: failed to encode map e168 with expected crc 2023-09-23T22:42:46.157 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:46.252 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:46.262 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:46.299 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.349 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:46.365 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:46.429 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[187743]: failed to encode map e168 with expected crc 2023-09-23T22:42:46.429 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3681257752' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:42:46.429 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[187743]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T22:42:46.429 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:46 smithi052 ceph-mon[187743]: failed to encode map e168 with expected crc 2023-09-23T22:42:46.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:46 smithi151 ceph-mon[159210]: failed to encode map e168 with expected crc 2023-09-23T22:42:46.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3681257752' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:42:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:46 smithi151 ceph-mon[159210]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T22:42:46.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:46 smithi151 ceph-mon[159210]: failed to encode map e168 with expected crc 2023-09-23T22:42:46.454 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:46.478 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.510 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 59 16 72839 72823 77.0983 0 - 0.0126704 2023-09-23T22:42:46.520 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:46.624 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:46.650 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.690 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:46.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:46.795 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:46.825 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.932 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:46.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:47.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:47.065 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:47.107 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:47.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:47.212 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:47.234 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:42:47.238 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:47.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: pgmap v395: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T22:42:47.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T22:42:47.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3352705235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: pgmap v395: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3352705235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:47.290 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:47.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:47.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:47.395 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:47.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: pgmap v395: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T22:42:47.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T22:42:47.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T22:42:47.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: failed to encode map e169 with expected crc 2023-09-23T22:42:47.423 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3352705235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:47.425 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:47.480 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:47.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:47.510 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:42:47.510345+0000 min lat: 0.00113345 max lat: 2.5358 avg lat: 0.0129123 2023-09-23T22:42:47.510 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:42:47.510 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 60 16 72896 72880 75.8733 1.78125 0.00482087 0.0129123 2023-09-23T22:42:47.586 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:47.619 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:47.673 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:47.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:47.780 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:47.811 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:47.863 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:47.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:47.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:47.968 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:48.002 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:48.054 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:48.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:48.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:48.160 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:48.189 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:48.242 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:48.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:48.347 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:48.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:48.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[187743]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3352705235' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:48.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[187743]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T22:42:48.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[187743]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[187743]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:48.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[180979]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3352705235' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:48.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[180979]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T22:42:48.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[180979]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:48 smithi052 ceph-mon[180979]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:48 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:48.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:48 smithi151 ceph-mon[159210]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3352705235' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:48.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:48 smithi151 ceph-mon[159210]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T22:42:48.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:48 smithi151 ceph-mon[159210]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:48 smithi151 ceph-mon[159210]: failed to encode map e170 with expected crc 2023-09-23T22:42:48.455 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:48.514 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 61 16 73579 73563 75.3282 42.6875 0.0308131 0.0132694 2023-09-23T22:42:48.992 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:49.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:49.101 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:49.174 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:49.294 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:49.300 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:49 smithi052 ceph-mon[187743]: pgmap v398: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s 2023-09-23T22:42:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:49.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:49.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:49 smithi052 ceph-mon[180979]: pgmap v398: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s 2023-09-23T22:42:49.403 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:49 smithi151 ceph-mon[159210]: pgmap v398: 89 pgs: 15 peering, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s 2023-09-23T22:42:49.530 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:49.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 62 16 74364 74348 74.8952 49.0625 0.0354898 0.0133449 2023-09-23T22:42:49.848 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:49.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:49.855 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:49.855 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:49.916 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:42:49.957 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:49.969 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:42:49.985 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.034 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:50.053 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:42:50.101 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:42:50.139 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:50.148 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:42:50.170 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:50.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:42:50.340 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:50.436 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.519 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 63 16 75096 75080 74.4326 45.75 0.0180723 0.0134293 2023-09-23T22:42:50.561 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:50.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:50.676 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:50.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:50.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1286480664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:50 smithi151 ceph-mon[159210]: pgmap v399: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 15 peering, 69 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s; 18332/220851 objects degraded (8.301%); 14 B/s, 0 objects/s recovering 2023-09-23T22:42:50.705 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1286480664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[187743]: pgmap v399: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 15 peering, 69 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s; 18332/220851 objects degraded (8.301%); 14 B/s, 0 objects/s recovering 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1286480664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:50.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:50 smithi052 ceph-mon[180979]: pgmap v399: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 15 peering, 69 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s; 18332/220851 objects degraded (8.301%); 14 B/s, 0 objects/s recovering 2023-09-23T22:42:50.766 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:50.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:50.873 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:50.908 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.968 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:51.076 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:51.109 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.120 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:42:51.174 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:51.180 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:42:51.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:42:51.181 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-23T22:42:51.181 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:42:51.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:51.282 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:51.316 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.369 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:51.477 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:51.508 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.522 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 64 16 75787 75771 73.9446 43.1875 0.0138061 0.0135186 2023-09-23T22:42:51.565 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:51.570 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:51 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 18332/220851 objects degraded (8.301%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:42:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:51.672 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:51.689 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:42:51.701 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.748 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:51.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:51 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 18332/220851 objects degraded (8.301%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:42:51.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:51 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 18332/220851 objects degraded (8.301%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:42:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:51.856 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:51.969 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.065 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:52.114 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:52.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:52.157 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:52.230 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:52.316 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.519 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 65 16 76431 76415 73.4265 40.25 0.0315752 0.0136129 2023-09-23T22:42:52.594 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:52.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:52.703 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:42:52.735 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.773 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:52 smithi052 ceph-mon[180979]: pgmap v400: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 586 op/s; 18332/223560 objects degraded (8.200%); 4352/223560 objects misplaced (1.947%); 1.2 MiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T22:42:52.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:52 smithi052 ceph-mon[187743]: pgmap v400: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 586 op/s; 18332/223560 objects degraded (8.200%); 4352/223560 objects misplaced (1.947%); 1.2 MiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T22:42:52.785 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:42:52.793 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:52 smithi151 ceph-mon[159210]: pgmap v400: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 586 op/s; 18332/223560 objects degraded (8.200%); 4352/223560 objects misplaced (1.947%); 1.2 MiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T22:42:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:52.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:42:52.895 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:52.929 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.999 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:53.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:53.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:53.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:53.140 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:53.201 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:53.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:53.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:53.393 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:53.521 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 66 16 77169 77153 73.0109 46.125 0.0619594 0.0136838 2023-09-23T22:42:53.719 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:42:53.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:53 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 5 pgs peering) 2023-09-23T22:42:53.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:42:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:42:53] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.33.4" 2023-09-23T22:42:53.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:53 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 5 pgs peering) 2023-09-23T22:42:53.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:53 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 5 pgs peering) 2023-09-23T22:42:54.072 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:54.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:54.189 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:54.220 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:54.273 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:54.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:54.379 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:54.476 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:54.521 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 67 16 77852 77836 72.5585 42.6875 0.021156 0.0137758 2023-09-23T22:42:54.726 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:54.731 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:54.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:54 smithi052 ceph-mon[180979]: pgmap v401: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 18332/228219 objects degraded (8.033%); 4352/228219 objects misplaced (1.907%); 1.0 MiB/s, 1 keys/s, 17 objects/s recovering 2023-09-23T22:42:54.733 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:54 smithi052 ceph-mon[187743]: pgmap v401: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 18332/228219 objects degraded (8.033%); 4352/228219 objects misplaced (1.907%); 1.0 MiB/s, 1 keys/s, 17 objects/s recovering 2023-09-23T22:42:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:54.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:54.852 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:54.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:54 smithi151 ceph-mon[159210]: pgmap v401: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 18332/228219 objects degraded (8.033%); 4352/228219 objects misplaced (1.907%); 1.0 MiB/s, 1 keys/s, 17 objects/s recovering 2023-09-23T22:42:54.967 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:55.164 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:55.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:55.286 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:55.320 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:55.374 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:55.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:55.407 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:55.480 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:55.510 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:55.521 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 68 16 78513 78497 72.0992 41.3125 0.00781031 0.0138641 2023-09-23T22:42:55.572 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:55.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:55.684 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:55.715 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:42:55.717 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:55.718 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:42:55.762 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:55.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:55.878 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:55.966 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:56.153 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:56 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/233367 objects degraded (7.855%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:42:56.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:56 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/233367 objects degraded (7.855%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:42:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:56 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/233367 objects degraded (7.855%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:42:56.522 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 69 16 79202 79186 71.6785 43.0625 0.0128059 0.013946 2023-09-23T22:42:56.529 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:56.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:56.634 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:56.668 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:57.035 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:57.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:57.143 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:57.165 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:57 smithi052 ceph-mon[180979]: from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:57.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:57 smithi052 ceph-mon[180979]: pgmap v402: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 599 op/s; 18332/233367 objects degraded (7.855%); 4141/233367 objects misplaced (1.774%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2023-09-23T22:42:57.166 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/968941186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.175 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:57.179 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:42:57.238 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:57.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:57.268 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":401,"stamp":"2023-09-23T22:42:54.036597+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4984543136,"num_objects":76073,"num_object_clones":0,"num_object_copies":228219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18332,"num_objects_misplaced":4352,"num_objects_unfound":0,"num_objects_dirty":76073,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":76309,"num_write_kb":4869581,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2623,"num_bytes_recovered":170595158,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76243,"ondisk_log_size":76243,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":258,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14476136,"kb_used_data":13175792,"kb_used_omap":126,"kb_used_meta":1300161,"kb_avail":641739928,"statfs":{"total":671965249536,"available":657141686272,"internally_reserved":0,"allocated":13492011008,"data_stored":13487711461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1331365499},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":276889600,"num_objects":4225,"num_object_clones":0,"num_object_copies":12675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18332,"num_objects_misplaced":4352,"num_objects_unfound":0,"num_objects_dirty":4225,"num_whiteouts":0,"num_read":-14,"num_read_kb":-12,"num_write":4225,"num_write_kb":270400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":8590166,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003941"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":167,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.137035+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:42:46.137035+0000","last_peered":"2023-09-23T22:42:46.137035+0000","last_clean":"2023-09-23T22:42:46.137035+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:42:46.137035+0000","last_undegraded":"2023-09-23T22:42:46.137035+0000","last_fullsized":"2023-09-23T22:42:46.137035+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":139,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.133365+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:42:46.133365+0000","last_peered":"2023-09-23T22:42:46.133365+0000","last_clean":"2023-09-23T22:42:46.133365+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:42:46.133365+0000","last_undegraded":"2023-09-23T22:42:46.133365+0000","last_fullsized":"2023-09-23T22:42:46.133365+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.126888+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:42:46.126888+0000","last_peered":"2023-09-23T22:42:46.126888+0000","last_clean":"2023-09-23T22:42:46.126888+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:42:46.126888+0000","last_undegraded":"2023-09-23T22:42:46.126888+0000","last_fullsized":"2023-09-23T22:42:46.126888+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":118,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:47.131835+0000","last_change":"2023-09-23T22:42:46.137037+0000","last_active":"2023-09-23T22:42:47.131835+0000","last_peered":"2023-09-23T22:42:47.131835+0000","last_clean":"2023-09-23T22:42:47.131835+0000","last_became_active":"2023-09-23T22:42:46.136711+0000","last_became_peered":"2023-09-23T22:42:46.136711+0000","last_unstale":"2023-09-23T22:42:47.131835+0000","last_undegraded":"2023-09-23T22:42:47.131835+0000","last_fullsized":"2023-09-23T22:42:47.131835+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:32.549377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":183,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.133799+0000","last_change":"2023-09-23T22:40:50.916258+0000","last_active":"2023-09-23T22:42:46.133799+0000","last_peered":"2023-09-23T22:42:46.133799+0000","last_clean":"2023-09-23T22:42:46.133799+0000","last_became_active":"2023-09-23T22:40:50.915948+0000","last_became_peered":"2023-09-23T22:40:50.915948+0000","last_unstale":"2023-09-23T22:42:46.133799+0000","last_undegraded":"2023-09-23T22:42:46.133799+0000","last_fullsized":"2023-09-23T22:42:46.133799+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:32.210743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.136241+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:42:46.136241+0000","last_peered":"2023-09-23T22:42:46.136241+0000","last_clean":"2023-09-23T22:42:46.136241+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:42:46.136241+0000","last_undegraded":"2023-09-23T22:42:46.136241+0000","last_fullsized":"2023-09-23T22:42:46.136241+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":106,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.128377+0000","last_change":"2023-09-23T22:40:50.926694+0000","last_active":"2023-09-23T22:42:46.128377+0000","last_peered":"2023-09-23T22:42:46.128377+0000","last_clean":"2023-09-23T22:42:46.128377+0000","last_became_active":"2023-09-23T22:40:50.926142+0000","last_became_peered":"2023-09-23T22:40:50.926142+0000","last_unstale":"2023-09-23T22:42:46.128377+0000","last_undegraded":"2023-09-23T22:42:46.128377+0000","last_fullsized":"2023-09-23T22:42:46.128377+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:53.874735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":167,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.133880+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:42:46.133880+0000","last_peered":"2023-09-23T22:42:46.133880+0000","last_clean":"2023-09-23T22:42:46.133880+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:42:46.133880+0000","last_undegraded":"2023-09-23T22:42:46.133880+0000","last_fullsized":"2023-09-23T22:42:46.133880+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":145,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.149540+0000","last_change":"2023-09-23T22:42:46.148114+0000","last_active":"2023-09-23T22:42:46.149540+0000","last_peered":"2023-09-23T22:42:46.149540+0000","last_clean":"2023-09-23T22:42:46.149540+0000","last_became_active":"2023-09-23T22:42:46.147722+0000","last_became_peered":"2023-09-23T22:42:46.147722+0000","last_unstale":"2023-09-23T22:42:46.149540+0000","last_undegraded":"2023-09-23T22:42:46.149540+0000","last_fullsized":"2023-09-23T22:42:46.149540+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":107,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.142285+0000","last_change":"2023-09-23T22:42:46.141366+0000","last_active":"2023-09-23T22:42:46.142285+0000","last_peered":"2023-09-23T22:42:46.142285+0000","last_clean":"2023-09-23T22:42:46.142285+0000","last_became_active":"2023-09-23T22:42:46.141088+0000","last_became_peered":"2023-09-23T22:42:46.141088+0000","last_unstale":"2023-09-23T22:42:46.142285+0000","last_undegraded":"2023-09-23T22:42:46.142285+0000","last_fullsized":"2023-09-23T22:42:46.142285+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.050826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":30,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:47.234244+0000","last_change":"2023-09-23T22:42:46.154236+0000","last_active":"2023-09-23T22:42:47.234244+0000","last_peered":"2023-09-23T22:42:47.234244+0000","last_clean":"2023-09-23T22:42:47.234244+0000","last_became_active":"2023-09-23T22:42:46.142775+0000","last_became_peered":"2023-09-23T22:42:46.142775+0000","last_unstale":"2023-09-23T22:42:47.234244+0000","last_undegraded":"2023-09-23T22:42:47.234244+0000","last_fullsized":"2023-09-23T22:42:47.234244+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:42.587667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.127301+0000","last_change":"2023-09-23T22:41:10.927399+0000","last_active":"2023-09-23T22:42:46.127301+0000","last_peered":"2023-09-23T22:42:46.127301+0000","last_clean":"2023-09-23T22:42:46.127301+0000","last_became_active":"2023-09-23T22:41:10.926366+0000","last_became_peered":"2023-09-23T22:41:10.926366+0000","last_unstale":"2023-09-23T22:42:46.127301+0000","last_undegraded":"2023-09-23T22:42:46.127301+0000","last_fullsized":"2023-09-23T22:42:46.127301+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:54.104169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":162,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.372632+0000","last_change":"2023-09-23T22:42:46.372286+0000","last_active":"2023-09-23T22:42:46.372632+0000","last_peered":"2023-09-23T22:42:46.372632+0000","last_clean":"2023-09-23T22:42:46.372632+0000","last_became_active":"2023-09-23T22:42:46.142963+0000","last_became_peered":"2023-09-23T22:42:46.142963+0000","last_unstale":"2023-09-23T22:42:46.372632+0000","last_undegraded":"2023-09-23T22:42:46.372632+0000","last_fullsized":"2023-09-23T22:42:46.372632+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.137719+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:42:46.137719+0000","last_peered":"2023-09-23T22:42:46.137719+0000","last_clean":"2023-09-23T22:42:46.137719+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:42:46.137719+0000","last_undegraded":"2023-09-23T22:42:46.137719+0000","last_fullsized":"2023-09-23T22:42:46.137719+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":30,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.199906+0000","last_change":"2023-09-23T22:42:46.199571+0000","last_active":"2023-09-23T22:42:46.199906+0000","last_peered":"2023-09-23T22:42:46.199906+0000","last_clean":"2023-09-23T22:42:46.199906+0000","last_became_active":"2023-09-23T22:42:46.141688+0000","last_became_peered":"2023-09-23T22:42:46.141688+0000","last_unstale":"2023-09-23T22:42:46.199906+0000","last_undegraded":"2023-09-23T22:42:46.199906+0000","last_fullsized":"2023-09-23T22:42:46.199906+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":169,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.139372+0000","last_change":"2023-09-23T22:42:46.138336+0000","last_active":"2023-09-23T22:42:46.139372+0000","last_peered":"2023-09-23T22:42:46.139372+0000","last_clean":"2023-09-23T22:42:46.139372+0000","last_became_active":"2023-09-23T22:42:46.137970+0000","last_became_peered":"2023-09-23T22:42:46.137970+0000","last_unstale":"2023-09-23T22:42:46.139372+0000","last_undegraded":"2023-09-23T22:42:46.139372+0000","last_fullsized":"2023-09-23T22:42:46.139372+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:54.943531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":115,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.141036+0000","last_change":"2023-09-23T22:42:46.137453+0000","last_active":"2023-09-23T22:42:46.141036+0000","last_peered":"2023-09-23T22:42:46.141036+0000","last_clean":"2023-09-23T22:42:46.141036+0000","last_became_active":"2023-09-23T22:42:46.137109+0000","last_became_peered":"2023-09-23T22:42:46.137109+0000","last_unstale":"2023-09-23T22:42:46.141036+0000","last_undegraded":"2023-09-23T22:42:46.141036+0000","last_fullsized":"2023-09-23T22:42:46.141036+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:57.553888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":144,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.149507+0000","last_change":"2023-09-23T22:42:46.148342+0000","last_active":"2023-09-23T22:42:46.149507+0000","last_peered":"2023-09-23T22:42:46.149507+0000","last_clean":"2023-09-23T22:42:46.149507+0000","last_became_active":"2023-09-23T22:42:46.147557+0000","last_became_peered":"2023-09-23T22:42:46.147557+0000","last_unstale":"2023-09-23T22:42:46.149507+0000","last_undegraded":"2023-09-23T22:42:46.149507+0000","last_fullsized":"2023-09-23T22:42:46.149507+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.151473+0000","last_change":"2023-09-23T22:42:46.146964+0000","last_active":"2023-09-23T22:42:46.151473+0000","last_peered":"2023-09-23T22:42:46.151473+0000","last_clean":"2023-09-23T22:42:46.151473+0000","last_became_active":"2023-09-23T22:42:46.146664+0000","last_became_peered":"2023-09-23T22:42:46.146664+0000","last_unstale":"2023-09-23T22:42:46.151473+0000","last_undegraded":"2023-09-23T22:42:46.151473+0000","last_fullsized":"2023-09-23T22:42:46.151473+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.138987+0000","last_change":"2023-09-23T22:42:46.137804+0000","last_active":"2023-09-23T22:42:46.138987+0000","last_peered":"2023-09-23T22:42:46.138987+0000","last_clean":"2023-09-23T22:42:46.138987+0000","last_became_active":"2023-09-23T22:42:46.137385+0000","last_became_peered":"2023-09-23T22:42:46.137385+0000","last_unstale":"2023-09-23T22:42:46.138987+0000","last_undegraded":"2023-09-23T22:42:46.138987+0000","last_fullsized":"2023-09-23T22:42:46.138987+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:32.576919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":199,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.134047+0000","last_change":"2023-09-23T22:42:00.067286+0000","last_active":"2023-09-23T22:42:46.134047+0000","last_peered":"2023-09-23T22:42:46.134047+0000","last_clean":"2023-09-23T22:42:46.134047+0000","last_became_active":"2023-09-23T22:41:59.891905+0000","last_became_peered":"2023-09-23T22:41:59.891905+0000","last_unstale":"2023-09-23T22:42:46.134047+0000","last_undegraded":"2023-09-23T22:42:46.134047+0000","last_fullsized":"2023-09-23T22:42:46.134047+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":148,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.139762+0000","last_change":"2023-09-23T22:42:46.138756+0000","last_active":"2023-09-23T22:42:46.139762+0000","last_peered":"2023-09-23T22:42:46.139762+0000","last_clean":"2023-09-23T22:42:46.139762+0000","last_became_active":"2023-09-23T22:42:46.138157+0000","last_became_peered":"2023-09-23T22:42:46.138157+0000","last_unstale":"2023-09-23T22:42:46.139762+0000","last_undegraded":"2023-09-23T22:42:46.139762+0000","last_fullsized":"2023-09-23T22:42:46.139762+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.885251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"170'4827","reported_seq":4860,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:51.846617+0000","last_change":"2023-09-23T22:41:36.969870+0000","last_active":"2023-09-23T22:42:51.846617+0000","last_peered":"2023-09-23T22:42:51.846617+0000","last_clean":"2023-09-23T22:42:51.846617+0000","last_became_active":"2023-09-23T22:41:36.969535+0000","last_became_peered":"2023-09-23T22:41:36.969535+0000","last_unstale":"2023-09-23T22:42:51.846617+0000","last_undegraded":"2023-09-23T22:42:51.846617+0000","last_fullsized":"2023-09-23T22:42:51.846617+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4827,"log_dups_size":0,"ondisk_log_size":4827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:24.978626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4827,"num_write_kb":308928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":28,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.462716+0000","last_change":"2023-09-23T22:42:46.462464+0000","last_active":"2023-09-23T22:42:46.462716+0000","last_peered":"2023-09-23T22:42:46.462716+0000","last_clean":"2023-09-23T22:42:46.462716+0000","last_became_active":"2023-09-23T22:42:46.144318+0000","last_became_peered":"2023-09-23T22:42:46.144318+0000","last_unstale":"2023-09-23T22:42:46.462716+0000","last_undegraded":"2023-09-23T22:42:46.462716+0000","last_fullsized":"2023-09-23T22:42:46.462716+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.149862+0000","last_change":"2023-09-23T22:42:46.149210+0000","last_active":"2023-09-23T22:42:46.149862+0000","last_peered":"2023-09-23T22:42:46.149862+0000","last_clean":"2023-09-23T22:42:46.149862+0000","last_became_active":"2023-09-23T22:42:46.148684+0000","last_became_peered":"2023-09-23T22:42:46.148684+0000","last_unstale":"2023-09-23T22:42:46.149862+0000","last_undegraded":"2023-09-23T22:42:46.149862+0000","last_fullsized":"2023-09-23T22:42:46.149862+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"170'4628","reported_seq":4661,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:49.347721+0000","last_change":"2023-09-23T22:41:36.967245+0000","last_active":"2023-09-23T22:42:49.347721+0000","last_peered":"2023-09-23T22:42:49.347721+0000","last_clean":"2023-09-23T22:42:49.347721+0000","last_became_active":"2023-09-23T22:41:36.966938+0000","last_became_peered":"2023-09-23T22:41:36.966938+0000","last_unstale":"2023-09-23T22:42:49.347721+0000","last_undegraded":"2023-09-23T22:42:49.347721+0000","last_fullsized":"2023-09-23T22:42:49.347721+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4628,"log_dups_size":0,"ondisk_log_size":4628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:31.285489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303300608,"num_objects":4628,"num_object_clones":0,"num_object_copies":13884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4628,"num_write_kb":296192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":28,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.418933+0000","last_change":"2023-09-23T22:42:46.418675+0000","last_active":"2023-09-23T22:42:46.418933+0000","last_peered":"2023-09-23T22:42:46.418933+0000","last_clean":"2023-09-23T22:42:46.418933+0000","last_became_active":"2023-09-23T22:42:46.144268+0000","last_became_peered":"2023-09-23T22:42:46.144268+0000","last_unstale":"2023-09-23T22:42:46.418933+0000","last_undegraded":"2023-09-23T22:42:46.418933+0000","last_fullsized":"2023-09-23T22:42:46.418933+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":150,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.141631+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:42:46.141631+0000","last_peered":"2023-09-23T22:42:46.141631+0000","last_clean":"2023-09-23T22:42:46.141631+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:42:46.141631+0000","last_undegraded":"2023-09-23T22:42:46.141631+0000","last_fullsized":"2023-09-23T22:42:46.141631+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"170'4816","reported_seq":4849,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:53.919584+0000","last_change":"2023-09-23T22:41:36.969607+0000","last_active":"2023-09-23T22:42:53.919584+0000","last_peered":"2023-09-23T22:42:53.919584+0000","last_clean":"2023-09-23T22:42:53.919584+0000","last_became_active":"2023-09-23T22:41:36.969311+0000","last_became_peered":"2023-09-23T22:41:36.969311+0000","last_unstale":"2023-09-23T22:42:53.919584+0000","last_undegraded":"2023-09-23T22:42:53.919584+0000","last_fullsized":"2023-09-23T22:42:53.919584+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4816,"log_dups_size":0,"ondisk_log_size":4816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:27.916919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315621376,"num_objects":4816,"num_object_clones":0,"num_object_copies":14448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4816,"num_write_kb":308224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":28,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.194834+0000","last_change":"2023-09-23T22:42:46.194517+0000","last_active":"2023-09-23T22:42:46.194834+0000","last_peered":"2023-09-23T22:42:46.194834+0000","last_clean":"2023-09-23T22:42:46.194834+0000","last_became_active":"2023-09-23T22:42:46.147763+0000","last_became_peered":"2023-09-23T22:42:46.147763+0000","last_unstale":"2023-09-23T22:42:46.194834+0000","last_undegraded":"2023-09-23T22:42:46.194834+0000","last_fullsized":"2023-09-23T22:42:46.194834+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.150721+0000","last_change":"2023-09-23T22:42:46.148855+0000","last_active":"2023-09-23T22:42:46.150721+0000","last_peered":"2023-09-23T22:42:46.150721+0000","last_clean":"2023-09-23T22:42:46.150721+0000","last_became_active":"2023-09-23T22:42:46.148479+0000","last_became_peered":"2023-09-23T22:42:46.148479+0000","last_unstale":"2023-09-23T22:42:46.150721+0000","last_undegraded":"2023-09-23T22:42:46.150721+0000","last_fullsized":"2023-09-23T22:42:46.150721+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"170'4862","reported_seq":4903,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:42:53.805032+0000","last_change":"2023-09-23T22:42:47.456571+0000","last_active":"2023-09-23T22:42:53.805032+0000","last_peered":"2023-09-23T22:42:53.805032+0000","last_clean":"2023-09-23T22:42:45.119339+0000","last_became_active":"2023-09-23T22:42:47.446567+0000","last_became_peered":"2023-09-23T22:42:47.446567+0000","last_unstale":"2023-09-23T22:42:53.805032+0000","last_undegraded":"2023-09-23T22:42:47.267547+0000","last_fullsized":"2023-09-23T22:42:47.133015+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4862,"log_dups_size":0,"ondisk_log_size":4862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:42:43.312921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318636032,"num_objects":4862,"num_object_clones":0,"num_object_copies":14586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4862,"num_write_kb":311168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4","objects":298},{"shards":"2,4,7","objects":4564}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.136984+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:42:46.136984+0000","last_peered":"2023-09-23T22:42:46.136984+0000","last_clean":"2023-09-23T22:42:46.136984+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:42:46.136984+0000","last_undegraded":"2023-09-23T22:42:46.136984+0000","last_fullsized":"2023-09-23T22:42:46.136984+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.128388+0000","last_change":"2023-09-23T22:40:50.917176+0000","last_active":"2023-09-23T22:42:46.128388+0000","last_peered":"2023-09-23T22:42:46.128388+0000","last_clean":"2023-09-23T22:42:46.128388+0000","last_became_active":"2023-09-23T22:40:50.915836+0000","last_became_peered":"2023-09-23T22:40:50.915836+0000","last_unstale":"2023-09-23T22:42:46.128388+0000","last_undegraded":"2023-09-23T22:42:46.128388+0000","last_fullsized":"2023-09-23T22:42:46.128388+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:37.211725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"170'4803","reported_seq":4842,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:42:53.916979+0000","last_change":"2023-09-23T22:42:47.392197+0000","last_active":"2023-09-23T22:42:53.916979+0000","last_peered":"2023-09-23T22:42:53.916979+0000","last_clean":"2023-09-23T22:42:45.086533+0000","last_became_active":"2023-09-23T22:42:47.381283+0000","last_became_peered":"2023-09-23T22:42:47.381283+0000","last_unstale":"2023-09-23T22:42:53.916979+0000","last_undegraded":"2023-09-23T22:42:47.267761+0000","last_fullsized":"2023-09-23T22:42:47.235668+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:00.393536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4803,"num_write_kb":307392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":305},{"shards":"0,3,7","objects":4498}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":153,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.132645+0000","last_change":"2023-09-23T22:40:50.914941+0000","last_active":"2023-09-23T22:42:46.132645+0000","last_peered":"2023-09-23T22:42:46.132645+0000","last_clean":"2023-09-23T22:42:46.132645+0000","last_became_active":"2023-09-23T22:40:50.914602+0000","last_became_peered":"2023-09-23T22:40:50.914602+0000","last_unstale":"2023-09-23T22:42:46.132645+0000","last_undegraded":"2023-09-23T22:42:46.132645+0000","last_fullsized":"2023-09-23T22:42:46.132645+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:04.998198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.138454+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:42:46.138454+0000","last_peered":"2023-09-23T22:42:46.138454+0000","last_clean":"2023-09-23T22:42:46.138454+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:42:46.138454+0000","last_undegraded":"2023-09-23T22:42:46.138454+0000","last_fullsized":"2023-09-23T22:42:46.138454+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"170'4867","reported_seq":4900,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:53.910743+0000","last_change":"2023-09-23T22:41:36.972030+0000","last_active":"2023-09-23T22:42:53.910743+0000","last_peered":"2023-09-23T22:42:53.910743+0000","last_clean":"2023-09-23T22:42:53.910743+0000","last_became_active":"2023-09-23T22:41:36.971693+0000","last_became_peered":"2023-09-23T22:41:36.971693+0000","last_unstale":"2023-09-23T22:42:53.910743+0000","last_undegraded":"2023-09-23T22:42:53.910743+0000","last_fullsized":"2023-09-23T22:42:53.910743+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4867,"log_dups_size":0,"ondisk_log_size":4867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:42:52.762839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4867,"num_write_kb":311488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.135170+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:42:46.135170+0000","last_peered":"2023-09-23T22:42:46.135170+0000","last_clean":"2023-09-23T22:42:46.135170+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:42:46.135170+0000","last_undegraded":"2023-09-23T22:42:46.135170+0000","last_fullsized":"2023-09-23T22:42:46.135170+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":165,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.138297+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:42:46.138297+0000","last_peered":"2023-09-23T22:42:46.138297+0000","last_clean":"2023-09-23T22:42:46.138297+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:42:46.138297+0000","last_undegraded":"2023-09-23T22:42:46.138297+0000","last_fullsized":"2023-09-23T22:42:46.138297+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"170'4653","reported_seq":4685,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:50.408648+0000","last_change":"2023-09-23T22:41:36.971145+0000","last_active":"2023-09-23T22:42:50.408648+0000","last_peered":"2023-09-23T22:42:50.408648+0000","last_clean":"2023-09-23T22:42:50.408648+0000","last_became_active":"2023-09-23T22:41:36.970815+0000","last_became_peered":"2023-09-23T22:41:36.970815+0000","last_unstale":"2023-09-23T22:42:50.408648+0000","last_undegraded":"2023-09-23T22:42:50.408648+0000","last_fullsized":"2023-09-23T22:42:50.408648+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4653,"log_dups_size":0,"ondisk_log_size":4653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304939008,"num_objects":4653,"num_object_clones":0,"num_object_copies":13959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4653,"num_write_kb":297792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":160,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.151518+0000","last_change":"2023-09-23T22:42:46.150139+0000","last_active":"2023-09-23T22:42:46.151518+0000","last_peered":"2023-09-23T22:42:46.151518+0000","last_clean":"2023-09-23T22:42:46.151518+0000","last_became_active":"2023-09-23T22:42:46.147558+0000","last_became_peered":"2023-09-23T22:42:46.147558+0000","last_unstale":"2023-09-23T22:42:46.151518+0000","last_undegraded":"2023-09-23T22:42:46.151518+0000","last_fullsized":"2023-09-23T22:42:46.151518+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":153,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.138884+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:42:46.138884+0000","last_peered":"2023-09-23T22:42:46.138884+0000","last_clean":"2023-09-23T22:42:46.138884+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:42:46.138884+0000","last_undegraded":"2023-09-23T22:42:46.138884+0000","last_fullsized":"2023-09-23T22:42:46.138884+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"170'4607","reported_seq":4907,"reported_epoch":170,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:42:50.395098+0000","last_change":"2023-09-23T22:42:47.394572+0000","last_active":"2023-09-23T22:42:50.395098+0000","last_peered":"2023-09-23T22:42:50.395098+0000","last_clean":"2023-09-23T22:42:45.106840+0000","last_became_active":"2023-09-23T22:42:47.366488+0000","last_became_peered":"2023-09-23T22:42:47.366488+0000","last_unstale":"2023-09-23T22:42:50.395098+0000","last_undegraded":"2023-09-23T22:42:50.395098+0000","last_fullsized":"2023-09-23T22:42:47.135341+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4607,"log_dups_size":0,"ondisk_log_size":4607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4352,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4607,"num_write_kb":294848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":8585216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":255},{"shards":"3,6,7","objects":4352}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":150,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.147355+0000","last_change":"2023-09-23T22:42:46.146993+0000","last_active":"2023-09-23T22:42:46.147355+0000","last_peered":"2023-09-23T22:42:46.147355+0000","last_clean":"2023-09-23T22:42:46.147355+0000","last_became_active":"2023-09-23T22:42:46.146662+0000","last_became_peered":"2023-09-23T22:42:46.146662+0000","last_unstale":"2023-09-23T22:42:46.147355+0000","last_undegraded":"2023-09-23T22:42:46.147355+0000","last_fullsized":"2023-09-23T22:42:46.147355+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:37.078736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":93,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.136213+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:42:46.136213+0000","last_peered":"2023-09-23T22:42:46.136213+0000","last_clean":"2023-09-23T22:42:46.136213+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:42:46.136213+0000","last_undegraded":"2023-09-23T22:42:46.136213+0000","last_fullsized":"2023-09-23T22:42:46.136213+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"170'4789","reported_seq":9784,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:50.409046+0000","last_change":"2023-09-23T22:42:31.753433+0000","last_active":"2023-09-23T22:42:50.409046+0000","last_peered":"2023-09-23T22:42:50.409046+0000","last_clean":"2023-09-23T22:42:50.409046+0000","last_became_active":"2023-09-23T22:42:31.748584+0000","last_became_peered":"2023-09-23T22:42:31.748584+0000","last_unstale":"2023-09-23T22:42:50.409046+0000","last_undegraded":"2023-09-23T22:42:50.409046+0000","last_fullsized":"2023-09-23T22:42:50.409046+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4789,"log_dups_size":0,"ondisk_log_size":4789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313851904,"num_objects":4789,"num_object_clones":0,"num_object_copies":14367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4789,"num_write_kb":306496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":162004992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":174,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.140570+0000","last_change":"2023-09-23T22:42:46.138650+0000","last_active":"2023-09-23T22:42:46.140570+0000","last_peered":"2023-09-23T22:42:46.140570+0000","last_clean":"2023-09-23T22:42:46.140570+0000","last_became_active":"2023-09-23T22:42:46.138351+0000","last_became_peered":"2023-09-23T22:42:46.138351+0000","last_unstale":"2023-09-23T22:42:46.140570+0000","last_undegraded":"2023-09-23T22:42:46.140570+0000","last_fullsized":"2023-09-23T22:42:46.140570+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:19:30.008132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":166,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.137749+0000","last_change":"2023-09-23T22:41:10.929497+0000","last_active":"2023-09-23T22:42:46.137749+0000","last_peered":"2023-09-23T22:42:46.137749+0000","last_clean":"2023-09-23T22:42:46.137749+0000","last_became_active":"2023-09-23T22:41:10.929128+0000","last_became_peered":"2023-09-23T22:41:10.929128+0000","last_unstale":"2023-09-23T22:42:46.137749+0000","last_undegraded":"2023-09-23T22:42:46.137749+0000","last_fullsized":"2023-09-23T22:42:46.137749+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.127161+0000","last_change":"2023-09-23T22:40:14.755608+0000","last_active":"2023-09-23T22:42:46.127161+0000","last_peered":"2023-09-23T22:42:46.127161+0000","last_clean":"2023-09-23T22:42:46.127161+0000","last_became_active":"2023-09-23T22:40:14.755335+0000","last_became_peered":"2023-09-23T22:40:14.755335+0000","last_unstale":"2023-09-23T22:42:46.127161+0000","last_undegraded":"2023-09-23T22:42:46.127161+0000","last_fullsized":"2023-09-23T22:42:46.127161+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:31.014369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"170'4705","reported_seq":4737,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:50.354569+0000","last_change":"2023-09-23T22:41:36.972228+0000","last_active":"2023-09-23T22:42:50.354569+0000","last_peered":"2023-09-23T22:42:50.354569+0000","last_clean":"2023-09-23T22:42:50.354569+0000","last_became_active":"2023-09-23T22:41:36.971904+0000","last_became_peered":"2023-09-23T22:41:36.971904+0000","last_unstale":"2023-09-23T22:42:50.354569+0000","last_undegraded":"2023-09-23T22:42:50.354569+0000","last_fullsized":"2023-09-23T22:42:50.354569+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308346880,"num_objects":4705,"num_object_clones":0,"num_object_copies":14115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":301120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":140,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.136887+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:42:46.136887+0000","last_peered":"2023-09-23T22:42:46.136887+0000","last_clean":"2023-09-23T22:42:46.136887+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:42:46.136887+0000","last_undegraded":"2023-09-23T22:42:46.136887+0000","last_fullsized":"2023-09-23T22:42:46.136887+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.138101+0000","last_change":"2023-09-23T22:40:50.916179+0000","last_active":"2023-09-23T22:42:46.138101+0000","last_peered":"2023-09-23T22:42:46.138101+0000","last_clean":"2023-09-23T22:42:46.138101+0000","last_became_active":"2023-09-23T22:40:50.915799+0000","last_became_peered":"2023-09-23T22:40:50.915799+0000","last_unstale":"2023-09-23T22:42:46.138101+0000","last_undegraded":"2023-09-23T22:42:46.138101+0000","last_fullsized":"2023-09-23T22:42:46.138101+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":162,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.136810+0000","last_change":"2023-09-23T22:41:10.929964+0000","last_active":"2023-09-23T22:42:46.136810+0000","last_peered":"2023-09-23T22:42:46.136810+0000","last_clean":"2023-09-23T22:42:46.136810+0000","last_became_active":"2023-09-23T22:41:10.929689+0000","last_became_peered":"2023-09-23T22:41:10.929689+0000","last_unstale":"2023-09-23T22:42:46.136810+0000","last_undegraded":"2023-09-23T22:42:46.136810+0000","last_fullsized":"2023-09-23T22:42:46.136810+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"170'4696","reported_seq":4729,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:53.959998+0000","last_change":"2023-09-23T22:41:36.968996+0000","last_active":"2023-09-23T22:42:53.959998+0000","last_peered":"2023-09-23T22:42:53.959998+0000","last_clean":"2023-09-23T22:42:53.959998+0000","last_became_active":"2023-09-23T22:41:36.968694+0000","last_became_peered":"2023-09-23T22:41:36.968694+0000","last_unstale":"2023-09-23T22:42:53.959998+0000","last_undegraded":"2023-09-23T22:42:53.959998+0000","last_fullsized":"2023-09-23T22:42:53.959998+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4696,"log_dups_size":0,"ondisk_log_size":4696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:14.636304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307757056,"num_objects":4696,"num_object_clones":0,"num_object_copies":14088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4696,"num_write_kb":300544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":142,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.133212+0000","last_change":"2023-09-23T22:41:10.929848+0000","last_active":"2023-09-23T22:42:46.133212+0000","last_peered":"2023-09-23T22:42:46.133212+0000","last_clean":"2023-09-23T22:42:46.133212+0000","last_became_active":"2023-09-23T22:41:10.929552+0000","last_became_peered":"2023-09-23T22:41:10.929552+0000","last_unstale":"2023-09-23T22:42:46.133212+0000","last_undegraded":"2023-09-23T22:42:46.133212+0000","last_fullsized":"2023-09-23T22:42:46.133212+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:09.783138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.127469+0000","last_change":"2023-09-23T22:41:10.927089+0000","last_active":"2023-09-23T22:42:46.127469+0000","last_peered":"2023-09-23T22:42:46.127469+0000","last_clean":"2023-09-23T22:42:46.127469+0000","last_became_active":"2023-09-23T22:41:10.926767+0000","last_became_peered":"2023-09-23T22:41:10.926767+0000","last_unstale":"2023-09-23T22:42:46.127469+0000","last_undegraded":"2023-09-23T22:42:46.127469+0000","last_fullsized":"2023-09-23T22:42:46.127469+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:03.097039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":161,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.128714+0000","last_change":"2023-09-23T22:40:51.909811+0000","last_active":"2023-09-23T22:42:46.128714+0000","last_peered":"2023-09-23T22:42:46.128714+0000","last_clean":"2023-09-23T22:42:46.128714+0000","last_became_active":"2023-09-23T22:40:51.909464+0000","last_became_peered":"2023-09-23T22:40:51.909464+0000","last_unstale":"2023-09-23T22:42:46.128714+0000","last_undegraded":"2023-09-23T22:42:46.128714+0000","last_fullsized":"2023-09-23T22:42:46.128714+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.517012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"170'4873","reported_seq":4911,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:42:53.888290+0000","last_change":"2023-09-23T22:42:47.382192+0000","last_active":"2023-09-23T22:42:53.888290+0000","last_peered":"2023-09-23T22:42:53.888290+0000","last_clean":"2023-09-23T22:42:45.097581+0000","last_became_active":"2023-09-23T22:42:47.373309+0000","last_became_peered":"2023-09-23T22:42:47.373309+0000","last_unstale":"2023-09-23T22:42:53.888290+0000","last_undegraded":"2023-09-23T22:42:47.264608+0000","last_fullsized":"2023-09-23T22:42:47.235147+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:54.196518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319356928,"num_objects":4873,"num_object_clones":0,"num_object_copies":14619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4601,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4873,"num_write_kb":311872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":272},{"shards":"0,3,7","objects":4601}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":164,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.142770+0000","last_change":"2023-09-23T22:42:46.141504+0000","last_active":"2023-09-23T22:42:46.142770+0000","last_peered":"2023-09-23T22:42:46.142770+0000","last_clean":"2023-09-23T22:42:46.142770+0000","last_became_active":"2023-09-23T22:42:46.141114+0000","last_became_peered":"2023-09-23T22:42:46.141114+0000","last_unstale":"2023-09-23T22:42:46.142770+0000","last_undegraded":"2023-09-23T22:42:46.142770+0000","last_fullsized":"2023-09-23T22:42:46.142770+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":273,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:42:47.405659+0000","last_change":"2023-09-23T22:42:47.405659+0000","last_active":"2023-09-23T22:42:47.405659+0000","last_peered":"2023-09-23T22:42:47.405659+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T22:42:47.405313+0000","last_became_peered":"2023-09-23T22:42:47.405313+0000","last_unstale":"2023-09-23T22:42:47.405659+0000","last_undegraded":"2023-09-23T22:42:47.234750+0000","last_fullsized":"2023-09-23T22:42:47.131409+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:43.228214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":98,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.132675+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:42:46.132675+0000","last_peered":"2023-09-23T22:42:46.132675+0000","last_clean":"2023-09-23T22:42:46.132675+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:42:46.132675+0000","last_undegraded":"2023-09-23T22:42:46.132675+0000","last_fullsized":"2023-09-23T22:42:46.132675+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.149805+0000","last_change":"2023-09-23T22:42:46.145584+0000","last_active":"2023-09-23T22:42:46.149805+0000","last_peered":"2023-09-23T22:42:46.149805+0000","last_clean":"2023-09-23T22:42:46.149805+0000","last_became_active":"2023-09-23T22:42:46.145197+0000","last_became_peered":"2023-09-23T22:42:46.145197+0000","last_unstale":"2023-09-23T22:42:46.149805+0000","last_undegraded":"2023-09-23T22:42:46.149805+0000","last_fullsized":"2023-09-23T22:42:46.149805+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"170'4977","reported_seq":5016,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:42:53.917401+0000","last_change":"2023-09-23T22:42:47.541787+0000","last_active":"2023-09-23T22:42:53.917401+0000","last_peered":"2023-09-23T22:42:53.917401+0000","last_clean":"2023-09-23T22:42:45.107206+0000","last_became_active":"2023-09-23T22:42:47.532032+0000","last_became_peered":"2023-09-23T22:42:47.532032+0000","last_unstale":"2023-09-23T22:42:53.917401+0000","last_undegraded":"2023-09-23T22:42:47.267628+0000","last_fullsized":"2023-09-23T22:42:47.132930+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4977,"log_dups_size":0,"ondisk_log_size":4977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.268054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326172672,"num_objects":4977,"num_object_clones":0,"num_object_copies":14931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4667,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4977,"num_write_kb":318528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":310},{"shards":"0,2,7","objects":4667}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.852819+0000","last_change":"2023-09-23T22:42:46.139194+0000","last_active":"2023-09-23T22:42:46.852819+0000","last_peered":"2023-09-23T22:42:46.852819+0000","last_clean":"2023-09-23T22:42:46.852819+0000","last_became_active":"2023-09-23T22:42:46.138865+0000","last_became_peered":"2023-09-23T22:42:46.138865+0000","last_unstale":"2023-09-23T22:42:46.852819+0000","last_undegraded":"2023-09-23T22:42:46.852819+0000","last_fullsized":"2023-09-23T22:42:46.852819+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:44.691520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":116,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:47.131421+0000","last_change":"2023-09-23T22:41:49.035892+0000","last_active":"2023-09-23T22:42:47.131421+0000","last_peered":"2023-09-23T22:42:47.131421+0000","last_clean":"2023-09-23T22:42:47.131421+0000","last_became_active":"2023-09-23T22:41:49.035318+0000","last_became_peered":"2023-09-23T22:41:49.035318+0000","last_unstale":"2023-09-23T22:42:47.131421+0000","last_undegraded":"2023-09-23T22:42:47.131421+0000","last_fullsized":"2023-09-23T22:42:47.131421+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:06.155982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.126851+0000","last_change":"2023-09-23T22:41:10.928059+0000","last_active":"2023-09-23T22:42:46.126851+0000","last_peered":"2023-09-23T22:42:46.126851+0000","last_clean":"2023-09-23T22:42:46.126851+0000","last_became_active":"2023-09-23T22:41:10.927749+0000","last_became_peered":"2023-09-23T22:41:10.927749+0000","last_unstale":"2023-09-23T22:42:46.126851+0000","last_undegraded":"2023-09-23T22:42:46.126851+0000","last_fullsized":"2023-09-23T22:42:46.126851+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:53.307028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"170'4693","reported_seq":4725,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:50.549981+0000","last_change":"2023-09-23T22:41:36.969378+0000","last_active":"2023-09-23T22:42:50.549981+0000","last_peered":"2023-09-23T22:42:50.549981+0000","last_clean":"2023-09-23T22:42:50.549981+0000","last_became_active":"2023-09-23T22:41:36.969153+0000","last_became_peered":"2023-09-23T22:41:36.969153+0000","last_unstale":"2023-09-23T22:42:50.549981+0000","last_undegraded":"2023-09-23T22:42:50.549981+0000","last_fullsized":"2023-09-23T22:42:50.549981+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307560448,"num_objects":4693,"num_object_clones":0,"num_object_copies":14079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4693,"num_write_kb":300352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":116,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:47.234085+0000","last_change":"2023-09-23T22:42:46.142541+0000","last_active":"2023-09-23T22:42:47.234085+0000","last_peered":"2023-09-23T22:42:47.234085+0000","last_clean":"2023-09-23T22:42:47.234085+0000","last_became_active":"2023-09-23T22:42:46.141740+0000","last_became_peered":"2023-09-23T22:42:46.141740+0000","last_unstale":"2023-09-23T22:42:47.234085+0000","last_undegraded":"2023-09-23T22:42:47.234085+0000","last_fullsized":"2023-09-23T22:42:47.234085+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:17.675577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":140,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.127549+0000","last_change":"2023-09-23T22:41:10.930375+0000","last_active":"2023-09-23T22:42:46.127549+0000","last_peered":"2023-09-23T22:42:46.127549+0000","last_clean":"2023-09-23T22:42:46.127549+0000","last_became_active":"2023-09-23T22:41:10.930050+0000","last_became_peered":"2023-09-23T22:41:10.930050+0000","last_unstale":"2023-09-23T22:42:46.127549+0000","last_undegraded":"2023-09-23T22:42:46.127549+0000","last_fullsized":"2023-09-23T22:42:46.127549+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:29.593309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":89,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:47.131641+0000","last_change":"2023-09-23T22:42:46.147970+0000","last_active":"2023-09-23T22:42:47.131641+0000","last_peered":"2023-09-23T22:42:47.131641+0000","last_clean":"2023-09-23T22:42:47.131641+0000","last_became_active":"2023-09-23T22:42:46.147624+0000","last_became_peered":"2023-09-23T22:42:46.147624+0000","last_unstale":"2023-09-23T22:42:47.131641+0000","last_undegraded":"2023-09-23T22:42:47.131641+0000","last_fullsized":"2023-09-23T22:42:47.131641+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:07.438720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"170'4658","reported_seq":4691,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:49.382121+0000","last_change":"2023-09-23T22:41:36.966638+0000","last_active":"2023-09-23T22:42:49.382121+0000","last_peered":"2023-09-23T22:42:49.382121+0000","last_clean":"2023-09-23T22:42:49.382121+0000","last_became_active":"2023-09-23T22:41:36.966433+0000","last_became_peered":"2023-09-23T22:41:36.966433+0000","last_unstale":"2023-09-23T22:42:49.382121+0000","last_undegraded":"2023-09-23T22:42:49.382121+0000","last_fullsized":"2023-09-23T22:42:49.382121+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4658,"log_dups_size":0,"ondisk_log_size":4658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:12.457010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305266688,"num_objects":4658,"num_object_clones":0,"num_object_copies":13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4658,"num_write_kb":298112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":166,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.150738+0000","last_change":"2023-09-23T22:42:46.150394+0000","last_active":"2023-09-23T22:42:46.150738+0000","last_peered":"2023-09-23T22:42:46.150738+0000","last_clean":"2023-09-23T22:42:46.150738+0000","last_became_active":"2023-09-23T22:42:46.142608+0000","last_became_peered":"2023-09-23T22:42:46.142608+0000","last_unstale":"2023-09-23T22:42:46.150738+0000","last_undegraded":"2023-09-23T22:42:46.150738+0000","last_fullsized":"2023-09-23T22:42:46.150738+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:57.622299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":109,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.127098+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:42:46.127098+0000","last_peered":"2023-09-23T22:42:46.127098+0000","last_clean":"2023-09-23T22:42:46.127098+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:42:46.127098+0000","last_undegraded":"2023-09-23T22:42:46.127098+0000","last_fullsized":"2023-09-23T22:42:46.127098+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":159,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.140658+0000","last_change":"2023-09-23T22:42:46.139166+0000","last_active":"2023-09-23T22:42:46.140658+0000","last_peered":"2023-09-23T22:42:46.140658+0000","last_clean":"2023-09-23T22:42:46.140658+0000","last_became_active":"2023-09-23T22:42:46.138846+0000","last_became_peered":"2023-09-23T22:42:46.138846+0000","last_unstale":"2023-09-23T22:42:46.140658+0000","last_undegraded":"2023-09-23T22:42:46.140658+0000","last_fullsized":"2023-09-23T22:42:46.140658+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:58.876511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"170'4595","reported_seq":4628,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:49.385840+0000","last_change":"2023-09-23T22:41:36.965968+0000","last_active":"2023-09-23T22:42:49.385840+0000","last_peered":"2023-09-23T22:42:49.385840+0000","last_clean":"2023-09-23T22:42:49.385840+0000","last_became_active":"2023-09-23T22:41:36.965582+0000","last_became_peered":"2023-09-23T22:41:36.965582+0000","last_unstale":"2023-09-23T22:42:49.385840+0000","last_undegraded":"2023-09-23T22:42:49.385840+0000","last_fullsized":"2023-09-23T22:42:49.385840+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":4595,"log_dups_size":0,"ondisk_log_size":4595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:51.048347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301137920,"num_objects":4595,"num_object_clones":0,"num_object_copies":13785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4595,"num_write_kb":294080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":184,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.151514+0000","last_change":"2023-09-23T22:42:46.148623+0000","last_active":"2023-09-23T22:42:46.151514+0000","last_peered":"2023-09-23T22:42:46.151514+0000","last_clean":"2023-09-23T22:42:46.151514+0000","last_became_active":"2023-09-23T22:42:46.148308+0000","last_became_peered":"2023-09-23T22:42:46.148308+0000","last_unstale":"2023-09-23T22:42:46.151514+0000","last_undegraded":"2023-09-23T22:42:46.151514+0000","last_fullsized":"2023-09-23T22:42:46.151514+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":102,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.126944+0000","last_change":"2023-09-23T22:40:50.916564+0000","last_active":"2023-09-23T22:42:46.126944+0000","last_peered":"2023-09-23T22:42:46.126944+0000","last_clean":"2023-09-23T22:42:46.126944+0000","last_became_active":"2023-09-23T22:40:50.915883+0000","last_became_peered":"2023-09-23T22:40:50.915883+0000","last_unstale":"2023-09-23T22:42:46.126944+0000","last_undegraded":"2023-09-23T22:42:46.126944+0000","last_fullsized":"2023-09-23T22:42:46.126944+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:25.767798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":120,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.141451+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:42:46.141451+0000","last_peered":"2023-09-23T22:42:46.141451+0000","last_clean":"2023-09-23T22:42:46.141451+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:42:46.141451+0000","last_undegraded":"2023-09-23T22:42:46.141451+0000","last_fullsized":"2023-09-23T22:42:46.141451+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":91,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.145429+0000","last_change":"2023-09-23T22:42:46.142270+0000","last_active":"2023-09-23T22:42:46.145429+0000","last_peered":"2023-09-23T22:42:46.145429+0000","last_clean":"2023-09-23T22:42:46.145429+0000","last_became_active":"2023-09-23T22:42:46.141711+0000","last_became_peered":"2023-09-23T22:42:46.141711+0000","last_unstale":"2023-09-23T22:42:46.145429+0000","last_undegraded":"2023-09-23T22:42:46.145429+0000","last_fullsized":"2023-09-23T22:42:46.145429+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:41.719643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.134190+0000","last_change":"2023-09-23T22:41:10.929809+0000","last_active":"2023-09-23T22:42:46.134190+0000","last_peered":"2023-09-23T22:42:46.134190+0000","last_clean":"2023-09-23T22:42:46.134190+0000","last_became_active":"2023-09-23T22:41:10.929502+0000","last_became_peered":"2023-09-23T22:41:10.929502+0000","last_unstale":"2023-09-23T22:42:46.134190+0000","last_undegraded":"2023-09-23T22:42:46.134190+0000","last_fullsized":"2023-09-23T22:42:46.134190+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:44.238238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":138,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.137095+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:42:46.137095+0000","last_peered":"2023-09-23T22:42:46.137095+0000","last_clean":"2023-09-23T22:42:46.137095+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:42:46.137095+0000","last_undegraded":"2023-09-23T22:42:46.137095+0000","last_fullsized":"2023-09-23T22:42:46.137095+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":169,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.138221+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:42:46.138221+0000","last_peered":"2023-09-23T22:42:46.138221+0000","last_clean":"2023-09-23T22:42:46.138221+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:42:46.138221+0000","last_undegraded":"2023-09-23T22:42:46.138221+0000","last_fullsized":"2023-09-23T22:42:46.138221+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":123,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.137830+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:42:46.137830+0000","last_peered":"2023-09-23T22:42:46.137830+0000","last_clean":"2023-09-23T22:42:46.137830+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:42:46.137830+0000","last_undegraded":"2023-09-23T22:42:46.137830+0000","last_fullsized":"2023-09-23T22:42:46.137830+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":26,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.137127+0000","last_change":"2023-09-23T22:41:49.032087+0000","last_active":"2023-09-23T22:42:46.137127+0000","last_peered":"2023-09-23T22:42:46.137127+0000","last_clean":"2023-09-23T22:42:46.137127+0000","last_became_active":"2023-09-23T22:41:49.031690+0000","last_became_peered":"2023-09-23T22:41:49.031690+0000","last_unstale":"2023-09-23T22:42:46.137127+0000","last_undegraded":"2023-09-23T22:42:46.137127+0000","last_fullsized":"2023-09-23T22:42:46.137127+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":160,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.133427+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:42:46.133427+0000","last_peered":"2023-09-23T22:42:46.133427+0000","last_clean":"2023-09-23T22:42:46.133427+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:42:46.133427+0000","last_undegraded":"2023-09-23T22:42:46.133427+0000","last_fullsized":"2023-09-23T22:42:46.133427+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":183,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.131552+0000","last_change":"2023-09-23T22:41:10.931122+0000","last_active":"2023-09-23T22:42:46.131552+0000","last_peered":"2023-09-23T22:42:46.131552+0000","last_clean":"2023-09-23T22:42:46.131552+0000","last_became_active":"2023-09-23T22:41:10.930811+0000","last_became_peered":"2023-09-23T22:41:10.930811+0000","last_unstale":"2023-09-23T22:42:46.131552+0000","last_undegraded":"2023-09-23T22:42:46.131552+0000","last_fullsized":"2023-09-23T22:42:46.131552+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:16.649049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":113,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-09-23T22:42:47.235638+0000","last_change":"2023-09-23T22:41:10.926522+0000","last_active":"2023-09-23T22:42:47.235638+0000","last_peered":"2023-09-23T22:42:47.235638+0000","last_clean":"2023-09-23T22:42:47.235638+0000","last_became_active":"2023-09-23T22:41:10.926169+0000","last_became_peered":"2023-09-23T22:41:10.926169+0000","last_unstale":"2023-09-23T22:42:47.235638+0000","last_undegraded":"2023-09-23T22:42:47.235638+0000","last_fullsized":"2023-09-23T22:42:47.235638+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:45.457705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-09-23T22:42:46.139660+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:42:46.139660+0000","last_peered":"2023-09-23T22:42:46.139660+0000","last_clean":"2023-09-23T22:42:46.139660+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:42:46.139660+0000","last_undegraded":"2023-09-23T22:42:46.139660+0000","last_fullsized":"2023-09-23T22:42:46.139660+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4983947264,"num_objects":76049,"num_object_clones":0,"num_object_copies":228147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18330,"num_objects_misplaced":4352,"num_objects_unfound":0,"num_objects_dirty":76049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76049,"num_write_kb":4867136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2603,"num_bytes_recovered":170590208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15046017024,"data_stored":15046017024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76049,"ondisk_log_size":76049,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430398,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626626,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1809756,"kb_used_data":1504340,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91935396,"statfs":{"total":95995035648,"available":94141845504,"internally_reserved":0,"allocated":1540444160,"data_stored":1539967172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.984,"5min":1.984,"15min":1.984},"last":0.27900000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:42:17 2023","interfaces":[{"interface":"back","average":{"1min":8.6210000000000004,"5min":3.6850000000000001,"15min":1.569},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":153.78399999999999,"5min":153.78399999999999,"15min":153.78399999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":8.6170000000000009,"5min":3.6549999999999998,"15min":1.599},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":154.148,"5min":154.148,"15min":154.148},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.7389999999999999,"15min":2.7389999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":43.408000000000001,"5min":43.408000000000001,"15min":43.408000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":43.356000000000002,"5min":43.356000000000002,"15min":43.356000000000002},"last":2.6819999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:42:17 2023","interfaces":[{"interface":"back","average":{"1min":8.1059999999999999,"5min":2.254,"15min":1.1020000000000001},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":80.277000000000001,"5min":80.277000000000001,"15min":80.277000000000001},"last":4.5430000000000001},{"interface":"front","average":{"1min":8.2669999999999995,"5min":2.4039999999999999,"15min":1.21},"min":{"1min":0.38500000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":80.513999999999996,"5min":80.513999999999996,"15min":80.513999999999996},"last":2.48}]},{"osd":4,"last update":"Sat Sep 23 22:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49199999999999999,"15min":0.48099999999999998},"min":{"1min":0.27700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.47899999999999998,"15min":0.49199999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.31900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.317},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.51500000000000001,"15min":0.53500000000000003},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:42:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.0600000000000001,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.52100000000000002,"15min":0.55900000000000005},"min":{"1min":0.32500000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.93899999999999995,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.40200000000000002}]}]},{"osd":1,"up_from":84,"seq":360777252936,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2013588,"kb_used_data":1931980,"kb_used_omap":9,"kb_used_meta":81590,"kb_avail":91731564,"statfs":{"total":95995035648,"available":93933121536,"internally_reserved":0,"allocated":1978347520,"data_stored":1977646989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":83549161},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.8279999999999998,"5min":2.8279999999999998,"15min":2.8279999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.9710000000000001,"5min":1.9710000000000001,"15min":1.9710000000000001},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.8019999999999996,"5min":4.8019999999999996,"15min":4.8019999999999996},"last":7.1239999999999997},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.4020000000000001,"5min":6.4020000000000001,"15min":6.4020000000000001},"last":7.0839999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.52600000000000002,"15min":0.42999999999999999},"min":{"1min":0.33200000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.0470000000000002,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":1.641},{"interface":"front","average":{"1min":1.5069999999999999,"5min":0.60599999999999998,"15min":0.46000000000000002},"min":{"1min":0.33900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":13.416,"5min":13.416,"15min":13.416},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.47,"15min":0.91400000000000003},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.7640000000000002,"5min":69.344999999999999,"15min":69.344999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.41699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.8339999999999996,"5min":69.359999999999999,"15min":69.359999999999999},"last":1.0669999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0369999999999999,"15min":0.77000000000000002},"min":{"1min":0.46999999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0880000000000001,"5min":28.837,"15min":28.837},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.98099999999999998,"15min":0.72699999999999998},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.726,"5min":28.986999999999998,"15min":28.986999999999998},"last":0.36099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.75,"15min":0.67400000000000004},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.8550000000000004,"5min":4.8550000000000004,"15min":4.8550000000000004},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.73299999999999998,"15min":0.66600000000000004},"min":{"1min":0.51300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.1509999999999998,"5min":3.1509999999999998,"15min":3.1509999999999998},"last":0.40300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":2.6429999999999998,"15min":1.3069999999999999},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.7889999999999997,"5min":159.018,"15min":159.018},"last":0.502},{"interface":"front","average":{"1min":1.228,"5min":2.7149999999999999,"15min":1.347},"min":{"1min":0.499,"5min":0.37,"15min":0.37},"max":{"1min":4.8860000000000001,"5min":158.96000000000001,"15min":158.96000000000001},"last":1.097}]}]},{"osd":0,"up_from":126,"seq":541165879322,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2551188,"kb_used_data":2469068,"kb_used_omap":28,"kb_used_meta":82083,"kb_avail":91193964,"statfs":{"total":95995035648,"available":93382619136,"internally_reserved":0,"allocated":2528325632,"data_stored":2527565289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":84053840},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:42:52 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":0.93500000000000005,"15min":0.81399999999999995},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.0419999999999998,"5min":8.0419999999999998,"15min":8.0419999999999998},"last":8.0419999999999998},{"interface":"front","average":{"1min":1.377,"5min":0.877,"15min":0.79400000000000004},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":3.9340000000000002,"5min":3.9340000000000002,"15min":3.9340000000000002},"last":3.9340000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":6.6840000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":105.818,"5min":105.818,"15min":105.818},"last":17.077999999999999},{"interface":"front","average":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":105.77500000000001,"5min":105.77500000000001,"15min":105.77500000000001},"last":15.882999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:42:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":1.4299999999999999,"15min":1.264},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":6.1130000000000004},{"interface":"front","average":{"1min":2.379,"5min":1.482,"15min":1.3320000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.695,"5min":10.695,"15min":10.695},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:42:52 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.0509999999999999,"15min":0.91700000000000004},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.0140000000000002,"5min":6.0140000000000002,"15min":6.0140000000000002},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.478,"5min":0.97099999999999997,"15min":0.88700000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.1999999999999993,"5min":8.1999999999999993,"15min":8.1999999999999993},"last":8.1999999999999993}]},{"osd":5,"last update":"Sat Sep 23 22:42:52 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.004,"15min":0.89400000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.0990000000000002,"5min":6.0990000000000002,"15min":6.0990000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.597,"5min":1.0189999999999999,"15min":0.92300000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.1690000000000005,"5min":8.1690000000000005,"15min":8.1690000000000005},"last":8.1690000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:42:52 2023","interfaces":[{"interface":"back","average":{"1min":2.016,"5min":1.1120000000000001,"15min":0.96099999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":11.161,"5min":11.161,"15min":11.161},"last":8.2360000000000007},{"interface":"front","average":{"1min":1.427,"5min":0.872,"15min":0.78000000000000003},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":3.8969999999999998,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":3.5870000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:42:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":0.85999999999999999,"15min":0.80300000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":3.7989999999999999,"5min":3.7989999999999999,"15min":3.7989999999999999},"last":3.6280000000000001},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.113,"15min":0.98899999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":6.069,"5min":6.069,"15min":6.069},"last":0.67100000000000004}]}]},{"osd":2,"up_from":131,"seq":562640715798,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989248,"kb_used_data":1894776,"kb_used_omap":81,"kb_used_meta":94382,"kb_avail":91755904,"statfs":{"total":95995035648,"available":93958045696,"internally_reserved":0,"allocated":1940250624,"data_stored":1939511867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":96647386},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":3.399,"15min":3.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":48.973999999999997,"5min":48.973999999999997,"15min":48.973999999999997},"last":0.19600000000000001},{"interface":"front","average":{"1min":4.6580000000000004,"5min":4.6580000000000004,"15min":4.6580000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":50.616999999999997,"5min":50.616999999999997,"15min":50.616999999999997},"last":14.519}]},{"osd":1,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":3.0819999999999999,"15min":3.0819999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":21.114000000000001,"5min":21.114000000000001,"15min":21.114000000000001},"last":14.656000000000001},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":22.021000000000001,"5min":22.021000000000001,"15min":22.021000000000001},"last":9.1809999999999992}]},{"osd":3,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.2509999999999999,"5min":3.2509999999999999,"15min":3.2509999999999999},"last":9.6189999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.0830000000000002,"5min":4.0830000000000002,"15min":4.0830000000000002},"last":0.37}]},{"osd":4,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.0069999999999997,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":6.8129999999999997,"5min":6.8129999999999997,"15min":6.8129999999999997},"last":14.566000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":6.8719999999999999,"5min":6.8719999999999999,"15min":6.8719999999999999},"last":14.599},{"interface":"front","average":{"1min":0.752,"5min":0.752,"15min":0.752},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":8.9550000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.1890000000000001,"5min":3.1890000000000001,"15min":3.1890000000000001},"last":8.9870000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.0579999999999998,"5min":4.0579999999999998,"15min":4.0579999999999998},"last":0.85499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":4.0359999999999996,"5min":4.0359999999999996,"15min":4.0359999999999996},"last":0.88100000000000001},{"interface":"front","average":{"1min":1.355,"5min":1.355,"15min":1.355},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":6.9100000000000001,"5min":6.9100000000000001,"15min":6.9100000000000001},"last":14.631}]}]},{"osd":3,"up_from":94,"seq":403726925883,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2793604,"kb_used_data":2667708,"kb_used_omap":7,"kb_used_meta":125880,"kb_avail":90951548,"statfs":{"total":95995035648,"available":93134385152,"internally_reserved":0,"allocated":2731732992,"data_stored":2731045354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":128901736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":13.525},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6319999999999997,"5min":6.6319999999999997,"15min":6.6319999999999997},"last":11.321}]},{"osd":1,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.42499999999999999,"15min":0.375},"min":{"1min":0.32600000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.254,"5min":3.254,"15min":3.254},"last":11.269},{"interface":"front","average":{"1min":1.272,"5min":0.55100000000000005,"15min":0.44500000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.843,"5min":6.843,"15min":6.843},"last":7.0609999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.3550000000000004,"5min":4.3550000000000004,"15min":4.3550000000000004},"last":3.423},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.911,"5min":3.911,"15min":3.911},"last":13.231}]},{"osd":4,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.627,"15min":0.624},"min":{"1min":0.43099999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.2250000000000001,"5min":3.2250000000000001,"15min":3.2250000000000001},"last":2.9169999999999998},{"interface":"front","average":{"1min":1.077,"5min":0.69999999999999996,"15min":0.68799999999999994},"min":{"1min":0.377,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.0279999999999996,"5min":4.0279999999999996,"15min":4.0279999999999996},"last":3.323}]},{"osd":5,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":0.67600000000000005,"15min":0.66800000000000004},"min":{"1min":0.35099999999999998,"5min":0.312,"15min":0.312},"max":{"1min":3.9980000000000002,"5min":3.9980000000000002,"15min":3.9980000000000002},"last":3.395},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.65300000000000002,"15min":0.61599999999999999},"min":{"1min":0.45400000000000001,"5min":0.308,"15min":0.308},"max":{"1min":1.915,"5min":3.0739999999999998,"15min":3.0739999999999998},"last":4.1840000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.8620000000000001,"5min":3.8620000000000001,"15min":3.8620000000000001},"last":3.3679999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67400000000000004,"15min":0.66500000000000004},"min":{"1min":0.39200000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.179,"5min":3.1520000000000001,"15min":3.1520000000000001},"last":4.1159999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.68700000000000006,"15min":0.66400000000000003},"min":{"1min":0.39500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.2569999999999999,"5min":3.1200000000000001,"15min":3.1200000000000001},"last":4.1520000000000001},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.441,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.1640000000000001,"5min":3.1640000000000001,"15min":3.1640000000000001},"last":2.952}]}]},{"osd":4,"up_from":30,"seq":128849019082,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804060,"kb_used_data":1498644,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91941092,"statfs":{"total":95995035648,"available":94147678208,"internally_reserved":0,"allocated":1534611456,"data_stored":1534133966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7690000000000001,"5min":2.7690000000000001,"15min":2.7690000000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":3.1269999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.621,"15min":0.60799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4829999999999997,"5min":5.4829999999999997,"15min":5.4829999999999997},"last":0.998},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.60299999999999998,"15min":0.57099999999999995},"min":{"1min":0.32500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.6070000000000002,"5min":5.6070000000000002,"15min":5.6070000000000002},"last":1.752}]},{"osd":2,"last update":"Sat Sep 23 22:42:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.6890000000000001,"15min":2.6890000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":29.370999999999999,"5min":29.370999999999999,"15min":29.370999999999999},"last":1.3280000000000001},{"interface":"front","average":{"1min":2.54,"5min":2.54,"15min":2.54},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":28.879000000000001,"5min":28.879000000000001,"15min":28.879000000000001},"last":0.63}]},{"osd":3,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.018,"15min":0.78200000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.921,"5min":44.994,"15min":44.994},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.05,"15min":0.74199999999999999},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.6819999999999999,"5min":44.765000000000001,"15min":44.765000000000001},"last":1.3839999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.93500000000000005,"15min":0.63100000000000001},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.431,"5min":43.189999999999998,"15min":43.189999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.94499999999999995,"15min":0.64400000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.1950000000000001,"5min":42.450000000000003,"15min":42.450000000000003},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.76900000000000002,"15min":0.60399999999999998},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81899999999999995,"5min":27.056000000000001,"15min":27.056000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.83399999999999996,"15min":0.64000000000000001},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.91800000000000004,"5min":26.940999999999999,"15min":26.940999999999999},"last":0.73299999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:42:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.65000000000000002,"15min":0.59199999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.5049999999999999,"5min":4.1459999999999999,"15min":4.1459999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.879,"5min":0.60999999999999999,"15min":0.57299999999999995},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9409999999999998,"5min":5.9409999999999998,"15min":5.9409999999999998},"last":0.45500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822854,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514692,"kb_used_data":1209276,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92230460,"statfs":{"total":95995035648,"available":94443991040,"internally_reserved":0,"allocated":1238298624,"data_stored":1237840824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":1.784,"5min":1.784,"15min":1.784},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":0.51800000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":9.3829999999999991,"5min":2.3239999999999998,"15min":1.153},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":149.203,"5min":149.203,"15min":149.203},"last":0.60199999999999998},{"interface":"front","average":{"1min":9.3109999999999999,"5min":2.2839999999999998,"15min":1.1160000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":149.554,"5min":149.554,"15min":149.554},"last":0.47999999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":9.1739999999999995,"5min":9.1739999999999995,"15min":9.1739999999999995},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":116.123,"5min":116.123,"15min":116.123},"last":0.46300000000000002},{"interface":"front","average":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":116.14100000000001,"5min":116.14100000000001,"15min":116.14100000000001},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.35899999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.089,"5min":2.286,"15min":2.286},"last":0.621},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.875,"5min":3.4740000000000002,"15min":3.4740000000000002},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.46200000000000002,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.378},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.51600000000000001,"15min":0.5},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.3780000000000001,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.30199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":2.1669999999999998,"15min":1.0580000000000001},"min":{"1min":0.33100000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.349,"5min":140.07499999999999,"15min":140.07499999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.1800000000000002,"15min":1.0840000000000001},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.319,"5min":139.739,"15min":139.739},"last":0.35199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.68799999999999994,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.1680000000000001,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.60499999999999998,"15min":0.58199999999999996},"min":{"1min":0.39500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.056,"5min":4.056,"15min":4.056},"last":0.42199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2536898560,"data_stored":2536898560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1992949760,"data_stored":1992949760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1949761536,"data_stored":1949761536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2742616064,"data_stored":2742616064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1544093696,"data_stored":1544093696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1243480064,"data_stored":1243480064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1541144576,"data_stored":1541144576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1495072768,"data_stored":1495072768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:42:57.277 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-09-23T22:42:57.343 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:57.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:57 smithi052 ceph-mon[187743]: from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:57.433 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:57 smithi052 ceph-mon[187743]: pgmap v402: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 599 op/s; 18332/233367 objects degraded (7.855%); 4141/233367 objects misplaced (1.774%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2023-09-23T22:42:57.433 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/968941186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.438 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:57.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:57 smithi151 ceph-mon[159210]: from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:42:57.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:57 smithi151 ceph-mon[159210]: pgmap v402: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 599 op/s; 18332/233367 objects degraded (7.855%); 4141/233367 objects misplaced (1.774%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2023-09-23T22:42:57.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/968941186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.523 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 70 16 79913 79897 71.2878 44.4375 0.0148806 0.0140218 2023-09-23T22:42:58.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/968941186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:58 smithi151 ceph-mon[159210]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/968941186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T22:42:58.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[180979]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.502 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/968941186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:58 smithi052 ceph-mon[187743]: failed to encode map e171 with expected crc 2023-09-23T22:42:58.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 71 16 80522 80506 70.819 38.0625 0.02458 0.0141149 2023-09-23T22:42:58.535 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:58.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:58.653 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:58.679 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:58.731 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:58.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34125.0:18 2.1 2:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.901972+0000", 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 128.741017145, 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.012967792, 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.817 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34125", 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3448800119", 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 18 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901972+0000", 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.818 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901972+0000", 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901970+0000", 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.819 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901977+0000", 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8569999999999997e-06 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901993+0000", 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6793999999999999e-05 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.902005+0000", 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1314e-05 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.902075+0000", 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0760999999999996e-05 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.902090+0000", 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4866e-05 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.914824+0000", 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0127341780000001 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.914939+0000", 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000114917 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.822 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:1181 6.b 6:d6b823aa:::benchmark_data_smithi052_234456_object1180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:51.312240+0000", 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 127.33074854100001, 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16517362199999999, 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1181 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.823 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312240+0000", 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312240+0000", 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312238+0000", 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312306+0000", 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7294999999999999e-05 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312324+0000", 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8998999999999999e-05 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312335+0000", 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0859999999999999e-05 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312524+0000", 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018910199999999999 2023-09-23T22:42:58.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312742+0000", 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021746800000000001 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312885+0000", 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014277300000000001 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.314104+0000", 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012193950000000001 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.316077+0000", 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001973027 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477297+0000", 2023-09-23T22:42:58.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16121960799999999 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477371+0000", 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4326999999999995e-05 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477414+0000", 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2735000000000003e-05 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.829 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:1189 6.b 6:d19c247e:::benchmark_data_smithi052_234456_object1188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:51.312418+0000", 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 127.330570743, 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16550995099999999, 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.830 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1189 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312418+0000", 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312418+0000", 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312417+0000", 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312444+0000", 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6220999999999998e-05 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312451+0000", 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5639999999999999e-06 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312457+0000", 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9399999999999999e-06 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.313211+0000", 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00075388099999999997 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.313332+0000", 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012050899999999999 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.313478+0000", 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014599599999999999 2023-09-23T22:42:58.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.314278+0000", 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00080022700000000003 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.316774+0000", 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024959959999999999 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477842+0000", 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.161068145 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477892+0000", 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9786000000000001e-05 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477928+0000", 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6229000000000002e-05 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:1191 6.6 6:684d433b:::benchmark_data_smithi052_234456_object1190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:51.312463+0000", 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 127.330525873, 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16480969100000001, 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.837 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1191 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312463+0000", 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312463+0000", 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312462+0000", 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312486+0000", 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3618999999999999e-05 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312493+0000", 2023-09-23T22:42:58.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9430000000000003e-06 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312498+0000", 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4600000000000002e-06 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312549+0000", 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0775000000000001e-05 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312705+0000", 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155592 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.312860+0000", 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155749 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.314044+0000", 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00118311 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.314196+0000", 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000152058 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477179+0000", 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.162983552 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477233+0000", 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4045000000000002e-05 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.477273+0000", 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9328e-05 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:1195 6.b 6:d80ebdb4:::benchmark_data_smithi052_234456_object1194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:51.315228+0000", 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 127.327760467, 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.163833165, 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 1195 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.845 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315228+0000", 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315228+0000", 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315227+0000", 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315243+0000", 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.541e-05 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315253+0000", 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.046e-05 2023-09-23T22:42:58.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315259+0000", 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3020000000000002e-06 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315323+0000", 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4511999999999995e-05 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315463+0000", 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139927 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.315615+0000", 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000151571 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.316303+0000", 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00068849999999999998 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.317405+0000", 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001101735 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.478971+0000", 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.161565616 2023-09-23T22:42:58.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.479018+0000", 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7141000000000001e-05 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:51.479061+0000", 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3742999999999998e-05 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.851 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10176 6.c 6:30687e13:::benchmark_data_smithi052_234456_object10175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199321+0000", 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 124.443667914, 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26354179999999999, 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10176 2023-09-23T22:42:58.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199321+0000", 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199321+0000", 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199319+0000", 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199341+0000", 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1223999999999999e-05 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199356+0000", 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5281000000000002e-05 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199365+0000", 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6680000000000007e-06 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199418+0000", 2023-09-23T22:42:58.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2932000000000001e-05 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199553+0000", 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000135623 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199695+0000", 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000142006 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201004+0000", 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013087070000000001 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.202654+0000", 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016499799999999999 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.462740+0000", 2023-09-23T22:42:58.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26008568700000001 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.462811+0000", 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1581000000000005e-05 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.462863+0000", 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.147e-05 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.858 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10208 6.c 6:38aea037:::benchmark_data_smithi052_234456_object10207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.210457+0000", 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 124.43253171800001, 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25350228899999999, 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.859 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10208 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210457+0000", 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210457+0000", 2023-09-23T22:42:58.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210456+0000", 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210477+0000", 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1296000000000001e-05 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210494+0000", 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6942000000000001e-05 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210502+0000", 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6820000000000003e-06 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210554+0000", 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2348000000000001e-05 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210709+0000", 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015471800000000001 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.210862+0000", 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000152564 2023-09-23T22:42:58.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.211459+0000", 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059781400000000001 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.212186+0000", 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00072627100000000001 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463846+0000", 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25165979900000002 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463914+0000", 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8257999999999996e-05 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463959+0000", 2023-09-23T22:42:58.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5501999999999999e-05 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36140 6.c 6:354f3782:::benchmark_data_smithi052_234456_object13996:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:06.835092+0000", 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 111.807896695, 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.33536728199999999, 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36140 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835092+0000", 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835092+0000", 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835091+0000", 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835096+0000", 2023-09-23T22:42:58.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.989e-06 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835108+0000", 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2334000000000001e-05 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835116+0000", 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7880000000000007e-06 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835168+0000", 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2649999999999999e-05 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835274+0000", 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010595900000000001 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.835371+0000", 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7146999999999997e-05 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.836770+0000", 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013990299999999999 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.837681+0000", 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00091053399999999995 2023-09-23T22:42:58.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.170351+0000", 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.33267043099999999 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.170410+0000", 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8650000000000003e-05 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.170459+0000", 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9274000000000003e-05 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36151 6.c 6:368aa1e9:::benchmark_data_smithi052_234456_object14007:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:06.841374+0000", 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 111.801614853, 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.330113822, 2023-09-23T22:42:58.873 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36151 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841374+0000", 2023-09-23T22:42:58.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841374+0000", 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841373+0000", 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841379+0000", 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8749999999999997e-06 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841392+0000", 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3919e-05 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841400+0000", 2023-09-23T22:42:58.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9529999999999999e-06 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841448+0000", 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7160000000000002e-05 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841538+0000", 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0049000000000005e-05 2023-09-23T22:42:58.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.841603+0000", 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5659000000000003e-05 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.842589+0000", 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0009857729999999999 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.842833+0000", 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000244033 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.171402+0000", 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32856925799999998 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.171448+0000", 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5362999999999999e-05 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.171488+0000", 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9950000000000002e-05 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.880 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36161 6.c 6:3a6067b9:::benchmark_data_smithi052_234456_object14017:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:06.848529+0000", 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 111.79445997400001, 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32596319699999998, 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36161 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848529+0000", 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848529+0000", 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848528+0000", 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848533+0000", 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0250000000000002e-06 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848547+0000", 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3967e-05 2023-09-23T22:42:58.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848557+0000", 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0484e-05 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848606+0000", 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8405999999999997e-05 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848727+0000", 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012107 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848845+0000", 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118541 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.849905+0000", 2023-09-23T22:42:58.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001059213 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850309+0000", 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040432 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.174416+0000", 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.324106852 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.174457+0000", 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1149000000000002e-05 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.174492+0000", 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5015000000000003e-05 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:38725 6.c 6:3e1f87d2:::benchmark_data_smithi052_234456_object16581:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:08.537160+0000", 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.10582886500001, 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20707393900000001, 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 38725 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.888 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537160+0000", 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537160+0000", 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537158+0000", 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537164+0000", 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5799999999999999e-06 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537179+0000", 2023-09-23T22:42:58.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4626e-05 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537186+0000", 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0779999999999998e-06 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537228+0000", 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2534999999999998e-05 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537358+0000", 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012927300000000001 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.537476+0000", 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011846899999999999 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.540168+0000", 2023-09-23T22:42:58.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026917930000000001 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.542434+0000", 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002266278 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.744144+0000", 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20170975999999999 2023-09-23T22:42:58.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.744192+0000", 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8310000000000003e-05 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.744234+0000", 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1603000000000001e-05 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.894 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:38734 6.c 6:39c0debf:::benchmark_data_smithi052_234456_object16590:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:08.541002+0000", 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.101986526, 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20394494799999999, 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.895 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 38734 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541002+0000", 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541002+0000", 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541001+0000", 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541007+0000", 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6170000000000003e-06 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541020+0000", 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2989999999999999e-05 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541027+0000", 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4010000000000002e-06 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541096+0000", 2023-09-23T22:42:58.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8977e-05 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541232+0000", 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013664800000000001 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.541355+0000", 2023-09-23T22:42:58.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012287600000000001 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.542494+0000", 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001139022 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.543132+0000", 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00063766699999999996 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.744872+0000", 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20174035200000001 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.744914+0000", 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1433000000000001e-05 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.744947+0000", 2023-09-23T22:42:58.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3534000000000001e-05 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40792 6.f 6:f7ead2d1:::benchmark_data_smithi052_234456_object18648:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e132)", 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:10.858925+0000", 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 107.78406403699999, 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.228704185, 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:42:58.902 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40792 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858925+0000", 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858925+0000", 2023-09-23T22:42:58.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858923+0000", 2023-09-23T22:42:58.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858931+0000", 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3750000000000003e-06 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858945+0000", 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4104000000000001e-05 2023-09-23T22:42:58.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858956+0000", 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0489000000000001e-05 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859059+0000", 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010275199999999999 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:42:58.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859069+0000", 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8160000000000005e-06 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.926942+0000", 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067873389000000006 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.926954+0000", 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2214e-05 2023-09-23T22:42:58.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.048290+0000", 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.121336195 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.048442+0000", 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000151454 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.087582+0000", 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.039140212000000001 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.087629+0000", 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6866999999999997e-05 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40806 6.f 6:fdb4d017:::benchmark_data_smithi052_234456_object18662:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e132)", 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:10.858975+0000", 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 107.78401405300001, 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.225864842, 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.911 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40806 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858975+0000", 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858975+0000", 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858974+0000", 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858978+0000", 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8039999999999999e-06 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858985+0000", 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4719999999999998e-06 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858991+0000", 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1040000000000003e-06 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859133+0000", 2023-09-23T22:42:58.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014187499999999999 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859140+0000", 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4139999999999999e-06 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.926978+0000", 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067838120000000002 2023-09-23T22:42:58.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.926986+0000", 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2849999999999995e-06 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.048862+0000", 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1218754719999999 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.049017+0000", 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155284 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.049190+0000", 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017290099999999999 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.052716+0000", 2023-09-23T22:42:58.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035265660000000001 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.059082+0000", 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063652969999999998 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.084441+0000", 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.02535946 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.084504+0000", 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2503999999999997e-05 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.084840+0000", 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033587599999999998 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40815 6.f 6:fca3f2d1:::benchmark_data_smithi052_234456_object18671:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e132)", 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:10.858997+0000", 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 107.783991433, 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.226141189, 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.920 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40815 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858997+0000", 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858997+0000", 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858997+0000", 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859000+0000", 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8459999999999999e-06 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859006+0000", 2023-09-23T22:42:58.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2450000000000003e-06 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859011+0000", 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5729999999999998e-06 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859164+0000", 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015270400000000001 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859169+0000", 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3959999999999997e-06 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927009+0000", 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067839231999999999 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927017+0000", 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.191e-06 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.050767+0000", 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.12375051 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.050936+0000", 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168288 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.051103+0000", 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016761900000000001 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.056800+0000", 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056970930000000003 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.059090+0000", 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022900009999999998 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.084892+0000", 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.025801991 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.084941+0000", 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8610999999999999e-05 2023-09-23T22:42:58.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085138+0000", 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000197505 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40816 6.f 6:fac562f2:::benchmark_data_smithi052_234456_object18672:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e132)", 2023-09-23T22:42:58.928 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:10.859037+0000", 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 107.783952149, 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.226353601, 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40816 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859037+0000", 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859037+0000", 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859036+0000", 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859039+0000", 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9749999999999999e-06 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859046+0000", 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8170000000000001e-06 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859065+0000", 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9086999999999999e-05 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859194+0000", 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012935600000000001 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:42:58.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859200+0000", 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1989999999999997e-06 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927039+0000", 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067838945999999997 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927047+0000", 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1359999999999997e-06 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.052741+0000", 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1256940529999999 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.052913+0000", 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017258200000000001 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.053118+0000", 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020461000000000001 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.056843+0000", 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037246760000000001 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.059098+0000", 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002255436 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085189+0000", 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.026090665999999998 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085234+0000", 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5439999999999999e-05 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085390+0000", 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015604400000000001 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40818 6.f 6:f38a98c2:::benchmark_data_smithi052_234456_object18674:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e132)", 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:10.859071+0000", 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 107.783917503, 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.226621341, 2023-09-23T22:42:58.937 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40818 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859071+0000", 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859071+0000", 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859071+0000", 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859074+0000", 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9340000000000002e-06 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859080+0000", 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6769999999999998e-06 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859085+0000", 2023-09-23T22:42:58.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0799999999999996e-06 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859224+0000", 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013813700000000001 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859230+0000", 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1229999999999997e-06 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927151+0000", 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067921370999999994 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927161+0000", 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5049999999999996e-06 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.054842+0000", 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1276811689999999 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.055106+0000", 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00026423899999999999 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.055274+0000", 2023-09-23T22:42:58.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016821499999999999 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.056883+0000", 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016086500000000001 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.059105+0000", 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022225349999999999 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085459+0000", 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.026353386999999999 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085511+0000", 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2658000000000003e-05 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085693+0000", 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018107299999999999 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40826 6.f 6:ffde327c:::benchmark_data_smithi052_234456_object18682:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e132)", 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:10.859090+0000", 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 107.783898865, 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.226883081, 2023-09-23T22:42:58.946 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40826 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859090+0000", 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859090+0000", 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859089+0000", 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859092+0000", 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5900000000000002e-06 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859098+0000", 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0970000000000002e-06 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859103+0000", 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2739999999999998e-06 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859308+0000", 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020415 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.859314+0000", 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4640000000000003e-06 2023-09-23T22:42:58.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927210+0000", 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.067895580999999997 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.927218+0000", 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6929999999999996e-06 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.057125+0000", 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.129907129 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.057328+0000", 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020236000000000001 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:42:58.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.057493+0000", 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000165167 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.059248+0000", 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017549690000000001 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.060577+0000", 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001328982 2023-09-23T22:42:58.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085747+0000", 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.025170251000000001 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085794+0000", 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7209999999999997e-05 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.085973+0000", 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000178572 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:10380 7.2 7:4efbeca9:::benchmark_data_smithi052_277165_object10379:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.074693+0000", 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 66.568295749000001, 2023-09-23T22:42:58.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.229292887, 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10380 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074693+0000", 2023-09-23T22:42:58.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074693+0000", 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074692+0000", 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074718+0000", 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5425000000000001e-05 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074727+0000", 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3009999999999995e-06 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074731+0000", 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3259999999999997e-06 2023-09-23T22:42:58.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074776+0000", 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4462999999999998e-05 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.074946+0000", 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000170514 2023-09-23T22:42:58.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075139+0000", 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000192547 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077285+0000", 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021460239999999998 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078061+0000", 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00077635099999999999 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303881+0000", 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225819886 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303940+0000", 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8613000000000003e-05 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303986+0000", 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5957000000000001e-05 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:10411 7.2 7:482ad6f1:::benchmark_data_smithi052_277165_object10410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:42:58.962 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.086038+0000", 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 66.556951115000004, 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22020151900000001, 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10411 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086038+0000", 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086038+0000", 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086036+0000", 2023-09-23T22:42:58.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086071+0000", 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4589999999999999e-05 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086082+0000", 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1048e-05 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086090+0000", 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9799999999999998e-06 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086137+0000", 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6665999999999997e-05 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086287+0000", 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015026800000000001 2023-09-23T22:42:58.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.086451+0000", 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016439199999999999 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.087290+0000", 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083841000000000004 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.087576+0000", 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00028666899999999998 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306155+0000", 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.218578469 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306205+0000", 2023-09-23T22:42:58.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0164999999999998e-05 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306239+0000", 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4072999999999997e-05 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:42:58.969 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:42:58.970 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:59.066 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:59.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:42:59 smithi151 ceph-mon[159210]: pgmap v404: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 18332/234024 objects degraded (7.833%); 4141/234024 objects misplaced (1.769%); 2.1 MiB/s, 0 keys/s, 35 objects/s recovering 2023-09-23T22:42:59.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:42:59 smithi052 ceph-mon[180979]: pgmap v404: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 18332/234024 objects degraded (7.833%); 4141/234024 objects misplaced (1.769%); 2.1 MiB/s, 0 keys/s, 35 objects/s recovering 2023-09-23T22:42:59.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:42:59 smithi052 ceph-mon[187743]: pgmap v404: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 18332/234024 objects degraded (7.833%); 4141/234024 objects misplaced (1.769%); 2.1 MiB/s, 0 keys/s, 35 objects/s recovering 2023-09-23T22:42:59.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 72 16 81119 81103 70.3538 37.3125 0.0141424 0.0142084 2023-09-23T22:42:59.551 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:59.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:59.660 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:59.727 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:59.823 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:42:59.890 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:59.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:59.965 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:00.001 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:00.037 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:00.108 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:00.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:00.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:00.215 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:00.295 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:00.299 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:00.437 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:00.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:00.504 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:00.525 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 73 16 81824 81808 69.9938 44.0625 0.0135476 0.0142805 2023-09-23T22:43:00.537 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:43:00.545 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:00.555 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:00.587 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:00.603 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:00.639 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:00.650 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:00.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:00.687 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:00.722 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:00 smithi052 ceph-mon[187743]: pgmap v405: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 18332/240114 objects degraded (7.635%); 4141/240114 objects misplaced (1.725%); 2.1 MiB/s, 0 keys/s, 35 objects/s recovering 2023-09-23T22:43:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/307864442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:00 smithi052 ceph-mon[180979]: pgmap v405: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 18332/240114 objects degraded (7.635%); 4141/240114 objects misplaced (1.725%); 2.1 MiB/s, 0 keys/s, 35 objects/s recovering 2023-09-23T22:43:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/307864442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:00.755 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:00.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:00.791 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:00.791 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:00 smithi151 ceph-mon[159210]: pgmap v405: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 40 MiB/s wr, 641 op/s; 18332/240114 objects degraded (7.635%); 4141/240114 objects misplaced (1.725%); 2.1 MiB/s, 0 keys/s, 35 objects/s recovering 2023-09-23T22:43:00.791 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/307864442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:00.800 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:00.826 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:00.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.858 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.858 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:43:00.904 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:00.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:01.010 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:01.083 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:01.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 74 16 82406 82390 69.5383 36.375 0.0095 0.0143722 2023-09-23T22:43:01.835 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:01.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:01.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:01.970 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:02.324 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:02.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:02.398 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:02 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/243990 objects degraded (7.513%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:02.400 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:02 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/243990 objects degraded (7.513%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:02.430 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:02.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:02 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/243990 objects degraded (7.513%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:02.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 75 16 83044 83028 69.143 39.875 0.0189626 0.0144567 2023-09-23T22:43:02.531 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:02.630 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:02.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:02.746 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:02.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:02.870 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:02.956 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:02.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:03.066 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:03.098 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:03.154 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:03.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:03 smithi052 ceph-mon[187743]: pgmap v406: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 18332/243990 objects degraded (7.513%); 3924/243990 objects misplaced (1.608%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:43:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:03 smithi052 ceph-mon[180979]: pgmap v406: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 18332/243990 objects degraded (7.513%); 3924/243990 objects misplaced (1.608%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:43:03.258 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:03.367 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:03 smithi151 ceph-mon[159210]: pgmap v406: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 18332/243990 objects degraded (7.513%); 3924/243990 objects misplaced (1.608%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:43:03.472 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:03.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:03.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 76 16 83700 83684 68.7729 41 0.0103402 0.0145352 2023-09-23T22:43:03.555 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:43:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:43:03] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.33.4" 2023-09-23T22:43:03.580 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:03.648 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:03.776 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:03.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:03.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:03.888 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:03.919 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:03.971 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:03.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:04.075 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:04.143 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:04.223 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 7 pgp_num to 16 2023-09-23T22:43:04.438 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:04.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2644523244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:43:04.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:04 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:43:04.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2644523244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:43:04.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:04 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:43:04.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2644523244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:43:04.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:04 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:43:04.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 77 16 84424 84408 68.4675 45.25 0.0161875 0.0145984 2023-09-23T22:43:04.567 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:04.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:04.680 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:04.802 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: pgmap v407: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 18332/248154 objects degraded (7.387%); 3924/248154 objects misplaced (1.581%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:05.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:05.256 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:05.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:05.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:05.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:05.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: pgmap v407: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 18332/248154 objects degraded (7.387%); 3924/248154 objects misplaced (1.581%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:05.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:05.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 78 16 85049 85033 68.0907 39.0625 0.0149447 0.0146804 2023-09-23T22:43:05.604 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:05.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:05.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: pgmap v407: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 18332/248154 objects degraded (7.387%); 3924/248154 objects misplaced (1.581%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: failed to encode map e172 with expected crc 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:05.719 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:05.754 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:05.811 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:05.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:05.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:05.826 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:05.919 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:05.948 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:06.164 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:06.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:06.268 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:06.310 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:06.412 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:06.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:06.516 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:06.529 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 79 16 85705 85689 67.7465 41 0.0286017 0.0147538 2023-09-23T22:43:06.554 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:06.676 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:06.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:06.770 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:06.770 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:06.780 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:06.892 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:07.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[180979]: pgmap v409: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 18332/252993 objects degraded (7.246%); 3705/252993 objects misplaced (1.464%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:43:07.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:43:07.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1133220477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:07.228 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:07.247 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:43:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[187743]: pgmap v409: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 18332/252993 objects degraded (7.246%); 3705/252993 objects misplaced (1.464%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:43:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:43:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1133220477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:07.528 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:43:07.529130+0000 min lat: 0.00113345 max lat: 2.5358 avg lat: 0.0148335 2023-09-23T22:43:07.529 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:43:07.529 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 80 16 86329 86313 67.3873 39 0.0107031 0.0148335 2023-09-23T22:43:07.531 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:07.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:07.643 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:07.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:07 smithi151 ceph-mon[159210]: pgmap v409: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 18332/252993 objects degraded (7.246%); 3705/252993 objects misplaced (1.464%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:43:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:43:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1133220477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:07.726 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:07.935 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:43:08.275 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:08.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:08.386 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:08.386 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:08.386 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T22:43:08.386 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[180979]: pgmap v411: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 18332/253593 objects degraded (7.229%); 3705/253593 objects misplaced (1.461%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:43:08.387 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:08.388 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:08.388 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:08.388 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T22:43:08.388 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.389 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.389 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.389 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.389 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.389 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:08 smithi052 ceph-mon[187743]: pgmap v411: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 18332/253593 objects degraded (7.229%); 3705/253593 objects misplaced (1.461%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:43:08.464 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:08.465 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:08.532 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 81 16 86928 86912 67.0154 37.4375 0.0397464 0.0149157 2023-09-23T22:43:08.591 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:08.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:08.689 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:08.689 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:08.689 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T22:43:08.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: failed to encode map e173 with expected crc 2023-09-23T22:43:08.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:08 smithi151 ceph-mon[159210]: pgmap v411: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 18332/253593 objects degraded (7.229%); 3705/253593 objects misplaced (1.461%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:43:08.699 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:08.740 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:08.948 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:08.994 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:09.020 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:09.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:09.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:09.039 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:09.072 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:09.123 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:09.125 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:09.165 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:09.167 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:09.217 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:09.257 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:09.273 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.273 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.274 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.274 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.274 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.274 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.274 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.274 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:43:09.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:09 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/253593 objects degraded (7.229%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:09.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1863627659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:09.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:09 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/253593 objects degraded (7.229%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:09.518 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1863627659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:09.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:09 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/253593 objects degraded (7.229%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:09.518 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1863627659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:09.532 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 82 16 87590 87574 66.7023 41.375 0.0281372 0.0149855 2023-09-23T22:43:09.995 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:10.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:10.102 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:10.166 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.281 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:10.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:10.418 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:43:10.432 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.444 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:43:10.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:43:10.444 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:43:10.444 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:43:10.444 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:43:10.444 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:43:10.539 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 83 16 88238 88222 66.3867 40.5 0.0453301 0.0150557 2023-09-23T22:43:10.558 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:10.664 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:10.683 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:10.683 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:43:10.683 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:43:10.683 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:43:10.683 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:10.694 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.755 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:10.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:10 smithi052 ceph-mon[180979]: pgmap v412: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 18332/259641 objects degraded (7.061%); 3705/259641 objects misplaced (1.427%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:43:10.774 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:10 smithi052 ceph-mon[187743]: pgmap v412: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 18332/259641 objects degraded (7.061%); 3705/259641 objects misplaced (1.427%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:43:10.859 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:10.895 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:10 smithi151 ceph-mon[159210]: pgmap v412: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 18332/259641 objects degraded (7.061%); 3705/259641 objects misplaced (1.427%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:43:10.899 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:11.327 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:11.405 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:11.532 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 84 16 88809 88793 66.0214 35.6875 0.0117925 0.0151375 2023-09-23T22:43:11.714 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:11.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:11.820 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:11.852 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:12.002 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:12.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:12.064 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:43:12.106 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:12.153 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:12.244 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:12.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:12.359 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:12.412 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:12.486 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:12.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:12.533 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 85 15 89482 89467 65.7405 42.125 0.0160178 0.0152065 2023-09-23T22:43:12.591 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:12.630 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:12.708 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:43:12.724 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:12.724 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:12.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:12.828 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:12.860 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.077 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:13.181 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:13.276 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:13 smithi151 ceph-mon[159210]: pgmap v413: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s; 18332/263166 objects degraded (6.966%); 3471/263166 objects misplaced (1.319%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:43:13.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:13 smithi052 ceph-mon[180979]: pgmap v413: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s; 18332/263166 objects degraded (6.966%); 3471/263166 objects misplaced (1.319%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:43:13.501 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:43:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:43:13] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-09-23T22:43:13.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:13 smithi052 ceph-mon[187743]: pgmap v413: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s; 18332/263166 objects degraded (6.966%); 3471/263166 objects misplaced (1.319%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:43:13.533 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 86 16 90140 90124 65.4531 41.0625 0.0235169 0.0152714 2023-09-23T22:43:13.769 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:13.883 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:13.916 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:14.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:14 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/267405 objects degraded (6.856%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:14.311 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:14.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:14.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:14.417 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:14.448 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:14 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/267405 objects degraded (6.856%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:14.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:14 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/267405 objects degraded (6.856%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:14.532 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:14.539 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 87 16 90801 90785 65.1718 41.3125 0.0241372 0.0153369 2023-09-23T22:43:14.755 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:43:14.756 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:43:14.756 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:43:14.805 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:14.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:14.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:14.912 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:14.912 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:43:14.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:43:14.912 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:43:14.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40802 6.e 6:7b05ccce:::benchmark_data_smithi052_234456_object18658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 125.046633411, 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533, 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40802 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862704+0000", 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0949999999999996e-06 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862712+0000", 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2490000000000001e-05 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862725+0000", 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.041e-06 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862732+0000", 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036689000000000001 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866401+0000", 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0216e-05 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866411+0000", 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99412088600000004 2023-09-23T22:43:14.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860532+0000", 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3456e-05 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860545+0000", 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2107602909999999 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071306+0000", 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014411400000000001 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071450+0000", 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017796199999999999 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:43:14.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071628+0000", 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036812979999999999 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075309+0000", 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001368813 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076678+0000", 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064926749999999998 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083170+0000", 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9931999999999994e-05 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083240+0000", 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034446200000000002 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083585+0000", 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533 2023-09-23T22:43:14.920 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40804 6.e 6:7187807f:::benchmark_data_smithi052_234456_object18660:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 125.043887842, 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001, 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:14.921 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40804 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:43:14.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865450+0000", 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4629999999999998e-06 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865457+0000", 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1907e-05 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865469+0000", 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1515000000000001e-05 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865480+0000", 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096401899999999999 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866444+0000", 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8879999999999997e-06 2023-09-23T22:43:14.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866452+0000", 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413119599999999 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860583+0000", 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7849999999999997e-06 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860592+0000", 2023-09-23T22:43:14.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.212291711 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072884+0000", 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011587900000000001 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073000+0000", 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073105+0000", 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002208621 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075313+0000", 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020589470000000002 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.077372+0000", 2023-09-23T22:43:14.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062710200000000004 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083643+0000", 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9950000000000002e-05 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083683+0000", 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000162177 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083845+0000", 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40805 6.e 6:798273ba:::benchmark_data_smithi052_234456_object18661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 125.043850902, 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999, 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.929 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40805 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865488+0000", 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6610000000000002e-06 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865492+0000", 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7939999999999997e-06 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865499+0000", 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7509999999999999e-06 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865504+0000", 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097241299999999999 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866476+0000", 2023-09-23T22:43:14.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6120000000000001e-06 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866484+0000", 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413304199999997 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860617+0000", 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3080000000000002e-06 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860624+0000", 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.213177682 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073802+0000", 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8152000000000005e-05 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073900+0000", 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106062 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.074006+0000", 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022883959999999998 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076294+0000", 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002385603 2023-09-23T22:43:14.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.078680+0000", 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052328410000000002 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083913+0000", 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2397000000000005e-05 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083985+0000", 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026911300000000002 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.084254+0000", 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40809 6.2 6:400fe117:::benchmark_data_smithi052_234456_object18665:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 125.042971581, 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002, 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40809 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.938 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866367+0000", 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8289999999999996e-06 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866373+0000", 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1122e-05 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866384+0000", 2023-09-23T22:43:14.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8669999999999996e-06 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866391+0000", 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000202492 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866594+0000", 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7580000000000004e-06 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866603+0000", 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.994511967 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861115+0000", 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6169999999999996e-06 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.943 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861125+0000", 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.209770244 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.070895+0000", 2023-09-23T22:43:14.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013600900000000001 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071031+0000", 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121377 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071153+0000", 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016472430000000001 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072800+0000", 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000377547 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073177+0000", 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0096328860000000002 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082810+0000", 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6335999999999996e-05 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082877+0000", 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000259743 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083136+0000", 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.947 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39271 7.1 7:803b19ab:::benchmark_data_smithi052_277165_object39270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.153749915000006, 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002, 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:14.948 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39271 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:14.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755588+0000", 2023-09-23T22:43:14.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100054 2023-09-23T22:43:14.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755689+0000", 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755701+0000", 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3199999999999999e-06 2023-09-23T22:43:14.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755707+0000", 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9419000000000002e-05 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755756+0000", 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5290000000000008e-06 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755765+0000", 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99249923500000004 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748264+0000", 2023-09-23T22:43:14.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6199999999999994e-06 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748273+0000", 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.538374364 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286648+0000", 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000173475 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286821+0000", 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024065800000000001 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.287062+0000", 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003676038 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290738+0000", 2023-09-23T22:43:14.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3826999999999998e-05 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290792+0000", 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2153e-05 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290804+0000", 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5701000000000002e-05 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290850+0000", 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093742900000000004 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291787+0000", 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.957 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39291 7.1 7:8a0225f8:::benchmark_data_smithi052_277165_object39290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.139041141000007, 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001, 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39291 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770297+0000", 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5182e-05 2023-09-23T22:43:14.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770362+0000", 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498999999999998e-05 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770378+0000", 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0129999999999996e-06 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770385+0000", 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2045000000000007e-05 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770447+0000", 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1433999999999999e-05 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770458+0000", 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97785107400000004 2023-09-23T22:43:14.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748309+0000", 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8459999999999996e-06 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748315+0000", 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5410234169999999 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289338+0000", 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117113 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289455+0000", 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000150991 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289606+0000", 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023599210000000001 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291966+0000", 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4410000000000004e-06 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291974+0000", 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8967999999999994e-05 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292043+0000", 2023-09-23T22:43:14.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7804000000000003e-05 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292111+0000", 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00075040600000000001 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292861+0000", 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.965 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39293 7.1 7:8d7e8869:::benchmark_data_smithi052_277165_object39292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.135622996999999, 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908, 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:14.966 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39293 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:14.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773716+0000", 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2105000000000003e-05 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773778+0000", 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2449000000000001e-05 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773790+0000", 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6099999999999997e-06 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773796+0000", 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9072000000000001e-05 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773835+0000", 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T22:43:14.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773843+0000", 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97449834000000002 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748341+0000", 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5279999999999997e-06 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748346+0000", 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.544637525 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292983+0000", 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016144799999999999 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293145+0000", 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015888799999999999 2023-09-23T22:43:14.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293304+0000", 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039379849999999997 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297242+0000", 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6013000000000001e-05 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297288+0000", 2023-09-23T22:43:14.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003665958 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300954+0000", 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7888999999999999e-05 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.301032+0000", 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012321579999999999 2023-09-23T22:43:14.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302264+0000", 2023-09-23T22:43:14.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908 2023-09-23T22:43:14.974 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.974 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.975 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.975 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39318 7.1 7:887a21fd:::benchmark_data_smithi052_277165_object39317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:14.975 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.114089372999999, 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997, 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39318 2023-09-23T22:43:14.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.977 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:43:14.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.977 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.978 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:43:14.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:14.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795249+0000", 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1755000000000002e-05 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795301+0000", 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3233e-05 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795314+0000", 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1940000000000001e-06 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795321+0000", 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1218999999999999e-05 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795372+0000", 2023-09-23T22:43:14.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0147e-05 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795383+0000", 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.95298535699999998 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748368+0000", 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2440000000000004e-06 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748372+0000", 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5462721529999999 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294644+0000", 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112796 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294757+0000", 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018578900000000001 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294943+0000", 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023897250000000001 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297333+0000", 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297340+0000", 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050090029999999997 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302349+0000", 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3208999999999999e-05 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302392+0000", 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00080982299999999999 2023-09-23T22:43:14.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303202+0000", 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39334 7.1 7:83eed48f:::benchmark_data_smithi052_277165_object39333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.102929786000004, 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001, 2023-09-23T22:43:14.985 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39334 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806408+0000", 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8466e-05 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806457+0000", 2023-09-23T22:43:14.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3062e-05 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806470+0000", 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2000000000000002e-06 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806475+0000", 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9246999999999998e-05 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806524+0000", 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0499999999999997e-06 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806533+0000", 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94188278000000003 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748416+0000", 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.52e-06 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748423+0000", 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5475971120000001 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296021+0000", 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015821300000000001 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296179+0000", 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013869899999999999 2023-09-23T22:43:14.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296317+0000", 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045890180000000003 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300906+0000", 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023854169999999999 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303292+0000", 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028062899999999999 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306098+0000", 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2596999999999999e-05 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306141+0000", 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068390599999999997 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306825+0000", 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39346 7.1 7:8a73189a:::benchmark_data_smithi052_277165_object39345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.097974078999997, 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001, 2023-09-23T22:43:14.993 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39346 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811364+0000", 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012578900000000001 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811490+0000", 2023-09-23T22:43:14.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8567999999999999e-05 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811508+0000", 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9329999999999996e-06 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811518+0000", 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8912000000000001e-05 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811577+0000", 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1018e-05 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811588+0000", 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93686738199999997 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748455+0000", 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0320000000000003e-06 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748460+0000", 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5488962179999999 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297357+0000", 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010246 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297459+0000", 2023-09-23T22:43:14.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012592199999999999 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297585+0000", 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004696787 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302282+0000", 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001097813 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:14.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303380+0000", 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034811529999999999 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306861+0000", 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9079000000000001e-05 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306920+0000", 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074140899999999995 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.307661+0000", 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39360 7.1 7:8f91ddcd:::benchmark_data_smithi052_277165_object39359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.086581683999995, 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001, 2023-09-23T22:43:15.001 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39360 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822756+0000", 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0686999999999998e-05 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822807+0000", 2023-09-23T22:43:15.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4600000000000001e-05 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822821+0000", 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2920000000000001e-06 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822828+0000", 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2878000000000003e-05 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822901+0000", 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9390000000000003e-06 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822909+0000", 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92559491500000002 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748504+0000", 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4299999999999997e-06 2023-09-23T22:43:15.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748510+0000", 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498821280000001 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298392+0000", 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1963e-05 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298484+0000", 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011828300000000001 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298602+0000", 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036917579999999998 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302294+0000", 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011534850000000001 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303447+0000", 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049678559999999997 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308415+0000", 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0241000000000001e-05 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308476+0000", 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00084742900000000002 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309323+0000", 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39366 7.1 7:86dcbc64:::benchmark_data_smithi052_277165_object39365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.009 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.082952487, 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001, 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39366 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826386+0000", 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4935999999999997e-05 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826441+0000", 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2864000000000001e-05 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826454+0000", 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7969999999999997e-06 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826460+0000", 2023-09-23T22:43:15.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5853000000000003e-05 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826505+0000", 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0890000000000006e-06 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826514+0000", 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92203426200000005 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748549+0000", 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9800000000000003e-06 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748555+0000", 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.55081775 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299372+0000", 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155606 2023-09-23T22:43:15.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299528+0000", 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021651799999999999 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299745+0000", 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035645 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303309+0000", 2023-09-23T22:43:15.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021267000000000001 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303522+0000", 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0058789619999999997 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309401+0000", 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0636999999999995e-05 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309471+0000", 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00099162699999999992 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310463+0000", 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.017 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39373 7.1 7:85f7eb6c:::benchmark_data_smithi052_277165_object39372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.079874966000006, 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812, 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39373 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829464+0000", 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1137000000000002e-05 2023-09-23T22:43:15.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829515+0000", 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3482000000000001e-05 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829528+0000", 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.959e-06 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829534+0000", 2023-09-23T22:43:15.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5408999999999999e-05 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829570+0000", 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2763e-05 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829582+0000", 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.91900465799999997 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748587+0000", 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7450000000000003e-06 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748592+0000", 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.554972391 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303564+0000", 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012435699999999999 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303689+0000", 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000131853 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303820+0000", 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004451455 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308272+0000", 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0795000000000003e-05 2023-09-23T22:43:15.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308333+0000", 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022271669999999999 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310560+0000", 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7315000000000001e-05 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310637+0000", 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083538299999999998 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311473+0000", 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.025 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39414 7.1 7:8075013a:::benchmark_data_smithi052_277165_object39413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.047714291000005, 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001, 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39414 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861624+0000", 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4787000000000002e-05 2023-09-23T22:43:15.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861678+0000", 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3568e-05 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861692+0000", 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1750000000000002e-06 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861698+0000", 2023-09-23T22:43:15.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9217e-05 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861747+0000", 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0080000000000002e-06 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861755+0000", 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88686667100000005 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748622+0000", 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0063e-05 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748632+0000", 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.562879379 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311512+0000", 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018041500000000001 2023-09-23T22:43:15.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311692+0000", 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020370699999999999 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311896+0000", 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077409310000000004 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319637+0000", 2023-09-23T22:43:15.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1062000000000006e-05 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319698+0000", 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103898 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319802+0000", 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7586e-05 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319869+0000", 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017220569999999999 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321591+0000", 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39416 7.1 7:83076ecc:::benchmark_data_smithi052_277165_object39415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.033 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.046325027999998, 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074, 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39416 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863014+0000", 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106535 2023-09-23T22:43:15.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863120+0000", 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1805e-05 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863132+0000", 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4430000000000002e-06 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863138+0000", 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0098999999999997e-05 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863178+0000", 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0545e-05 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863188+0000", 2023-09-23T22:43:15.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88547513300000003 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748663+0000", 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.507e-06 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748669+0000", 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5646240410000001 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313293+0000", 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110319 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313403+0000", 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196856 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313600+0000", 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060436789999999997 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319644+0000", 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011535 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319759+0000", 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019084060000000001 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321668+0000", 2023-09-23T22:43:15.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2410999999999996e-05 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321730+0000", 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011838980000000001 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322914+0000", 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39440 7.1 7:846ccd4b:::benchmark_data_smithi052_277165_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.025849722000004, 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002, 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39440 2023-09-23T22:43:15.042 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883489+0000", 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0798999999999997e-05 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883569+0000", 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5083999999999999e-05 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883594+0000", 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1099999999999999e-06 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883601+0000", 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5002e-05 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883646+0000", 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5539999999999998e-06 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883654+0000", 2023-09-23T22:43:15.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86504454099999994 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748699+0000", 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0530000000000003e-06 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748705+0000", 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.566234793 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.314939+0000", 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014568900000000001 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315085+0000", 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019602700000000001 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315281+0000", 2023-09-23T22:43:15.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004426915 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319708+0000", 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032665569999999998 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322975+0000", 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5554e-05 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323070+0000", 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1078999999999998e-05 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323111+0000", 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095274200000000002 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324064+0000", 2023-09-23T22:43:15.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39442 7.1 7:821661b1:::benchmark_data_smithi052_277165_object39441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.022812125000002, 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999, 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.052 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39442 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:43:15.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999895 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886517+0000", 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0640000000000002e-05 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886578+0000", 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4535999999999999e-05 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886592+0000", 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.024e-06 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886599+0000", 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5583e-05 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886655+0000", 2023-09-23T22:43:15.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9129999999999992e-06 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886664+0000", 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86206608799999995 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748730+0000", 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6200000000000001e-06 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748737+0000", 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.567891876 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316628+0000", 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014458900000000001 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316773+0000", 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000198579 2023-09-23T22:43:15.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316972+0000", 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027449810000000001 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319717+0000", 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033197000000000001 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323036+0000", 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001091297 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324128+0000", 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0636999999999997e-05 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324168+0000", 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071410600000000005 2023-09-23T22:43:15.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324882+0000", 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39446 7.1 7:8475cdd4:::benchmark_data_smithi052_277165_object39445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:43:15.060 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.019532897999994, 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001, 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39446 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:43:15.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889806+0000", 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7126000000000002e-05 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889853+0000", 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2593e-05 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889865+0000", 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889871+0000", 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7049999999999998e-05 2023-09-23T22:43:15.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889918+0000", 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5479999999999998e-06 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889924+0000", 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.858839715 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748764+0000", 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5000000000000001e-06 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748768+0000", 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5693916369999998 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318160+0000", 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016406600000000001 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318324+0000", 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000199247 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318523+0000", 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044604579999999996 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322984+0000", 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019935700000000001 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324977+0000", 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4416e-05 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325052+0000", 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9657000000000003e-05 2023-09-23T22:43:15.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325092+0000", 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00089603700000000003 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325988+0000", 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39458 7.1 7:846e660e:::benchmark_data_smithi052_277165_object39457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 64.003456229999998, 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002, 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39458 2023-09-23T22:43:15.069 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905882+0000", 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7948000000000002e-05 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905940+0000", 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1592e-05 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905952+0000", 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4750000000000001e-06 2023-09-23T22:43:15.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905957+0000", 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2416000000000001e-05 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906010+0000", 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4849999999999992e-06 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906018+0000", 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.84277395099999997 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748792+0000", 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748796+0000", 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5772432570000001 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326040+0000", 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015276800000000001 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326193+0000", 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018607500000000001 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326379+0000", 2023-09-23T22:43:15.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038150480000000001 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.330194+0000", 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001090159 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331284+0000", 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0065274460000000001 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337811+0000", 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1335999999999999e-05 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337853+0000", 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014175889999999999 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339270+0000", 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002 2023-09-23T22:43:15.076 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39462 7.1 7:8cf1a796:::benchmark_data_smithi052_277165_object39461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 63.992895517999997, 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001, 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:15.077 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39462 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:15.078 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916441+0000", 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5293000000000001e-05 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:15.079 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916497+0000", 2023-09-23T22:43:15.080 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3080000000000002e-05 2023-09-23T22:43:15.080 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.080 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.080 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:15.080 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916530+0000", 2023-09-23T22:43:15.080 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4319e-05 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916544+0000", 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018227799999999999 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.081 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916726+0000", 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3705999999999999e-05 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916760+0000", 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.83205244499999997 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748813+0000", 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5040000000000004e-06 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748817+0000", 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.578915206 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327732+0000", 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115865 2023-09-23T22:43:15.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327848+0000", 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000123075 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327971+0000", 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033208170000000002 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331292+0000", 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0080585329999999997 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339351+0000", 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1529e-05 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:15.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339412+0000", 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7031000000000002e-05 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339479+0000", 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010848730000000001 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.340564+0000", 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001 2023-09-23T22:43:15.086 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.087 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.087 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.087 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:15.087 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:15.087 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:15.194 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:15.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:15 smithi151 ceph-mon[159210]: pgmap v414: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 18332/267405 objects degraded (6.856%); 3471/267405 objects misplaced (1.298%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:43:15.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:15 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:15 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:15.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:15 smithi052 ceph-mon[187743]: pgmap v414: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 18332/267405 objects degraded (6.856%); 3471/267405 objects misplaced (1.298%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:43:15.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:15 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:15.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:15 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:15.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:15 smithi052 ceph-mon[180979]: pgmap v414: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 18332/267405 objects degraded (6.856%); 3471/267405 objects misplaced (1.298%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:43:15.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:15 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:15.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:15 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:15.541 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 88 16 91328 91312 64.8037 32.9375 0.0173923 0.0154261 2023-09-23T22:43:15.783 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:43:16.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:16.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:16.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:16.183 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:16.231 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:16.256 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:43:16.281 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:16.298 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:16.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:16.410 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:16.466 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2673940143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:16.466 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:16.466 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2673940143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:16.466 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:16.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2673940143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:16.497 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:16.503 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:16.542 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 89 16 92020 92004 64.5615 43.25 0.0202849 0.0154835 2023-09-23T22:43:16.746 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:16.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:16.859 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:16.906 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:16.948 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:16.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:16.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:17.052 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:17.102 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[180979]: pgmap v415: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 18332/271974 objects degraded (6.740%); 3263/271974 objects misplaced (1.200%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[180979]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[180979]: failed to encode map e174 with expected crc 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[180979]: failed to encode map e174 with expected crc 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[187743]: pgmap v415: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 18332/271974 objects degraded (6.740%); 3263/271974 objects misplaced (1.200%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[187743]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[187743]: failed to encode map e174 with expected crc 2023-09-23T22:43:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:17 smithi052 ceph-mon[187743]: failed to encode map e174 with expected crc 2023-09-23T22:43:17.542 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total time run: 90.0108 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total writes made: 92489 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Write size: 65536 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object size: 65536 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Bandwidth (MB/sec): 64.2208 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Bandwidth: 31.7639 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max bandwidth (MB/sec): 177.062 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average IOPS: 1027 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev IOPS: 508.233 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max IOPS: 2833 2023-09-23T22:43:17.543 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min IOPS: 0 2023-09-23T22:43:17.544 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average Latency(s): 0.0155671 2023-09-23T22:43:17.544 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Latency(s): 0.0486018 2023-09-23T22:43:17.544 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max latency(s): 2.5358 2023-09-23T22:43:17.544 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min latency(s): 0.00113345 2023-09-23T22:43:17.547 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:43:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:17 smithi151 ceph-mon[159210]: pgmap v415: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 18332/271974 objects degraded (6.740%); 3263/271974 objects misplaced (1.200%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:17 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:17 smithi151 ceph-mon[159210]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T22:43:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:17 smithi151 ceph-mon[159210]: failed to encode map e174 with expected crc 2023-09-23T22:43:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:17 smithi151 ceph-mon[159210]: failed to encode map e174 with expected crc 2023-09-23T22:43:17.814 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:17.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:17.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:17.924 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:18.012 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:18.313 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:18.313 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:43:18.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.313 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.35001.0:93019 7.e e174/146 7:7cb450aa:::benchmark_data_smithi052_277165_object527:head v 174'5829, mlcod=174'5827)", 2023-09-23T22:43:18.313 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:18.308610+0000", 2023-09-23T22:43:18.313 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.002349084, 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002378776, 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308610+0000", 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:18.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308610+0000", 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308609+0000", 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0451e-05 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308620+0000", 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3939e-05 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308634+0000", 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6259999999999997e-06 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308640+0000", 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8254999999999999e-05 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:18.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308679+0000", 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6538e-05 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:18.308695+0000", 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4785999999999998e-05 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 1 2023-09-23T22:43:18.317 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:18.325 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:18.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:18.436 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:18.471 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:18.528 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:18.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:18.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:18.694 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:18.721 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:18.757 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:18.813 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:18.846 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:18.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:18.876 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:18.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:18.952 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:18.967 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:18.985 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:19.031 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:19.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.062 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.231 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:19.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:19.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:19.302 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:43:19.352 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:19.397 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:19.471 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:19.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:19 smithi151 ceph-mon[159210]: pgmap v417: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 18332/272613 objects degraded (6.725%); 3263/272613 objects misplaced (1.197%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:19.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3346225125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:19.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:19.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:19.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:19.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[180979]: pgmap v417: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 18332/272613 objects degraded (6.725%); 3263/272613 objects misplaced (1.197%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:19.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3346225125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:19.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:19.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:19.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[187743]: pgmap v417: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 18332/272613 objects degraded (6.725%); 3263/272613 objects misplaced (1.197%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:19.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3346225125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:19.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:19.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:19.580 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:19.626 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:19.754 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:19.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:19.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:19.782 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:19.865 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:19.957 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:19.996 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:43:20.182 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:20.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:20.294 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:20.388 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:20.526 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:20.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:20.650 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:20.679 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:20.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:20.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/272613 objects degraded (6.725%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:20.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: pgmap v418: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 619 op/s; 18332/274341 objects degraded (6.682%); 3263/274341 objects misplaced (1.189%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:20.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:43:20.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:43:20.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:20.684 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:20.739 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:20.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/272613 objects degraded (6.725%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: pgmap v418: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 619 op/s; 18332/274341 objects degraded (6.682%); 3263/274341 objects misplaced (1.189%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:43:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:43:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:20.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:20.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:20.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/272613 objects degraded (6.725%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:20.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: pgmap v418: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 35 MiB/s wr, 619 op/s; 18332/274341 objects degraded (6.682%); 3263/274341 objects misplaced (1.189%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:20.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:43:20.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:43:20.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:20.802 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:20.844 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:20.875 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:20.934 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:20.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:21.037 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:21.151 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:21.215 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:43:21.215 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:43:21.215 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.8283965666961534 2023-09-23T22:43:21.215 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 0.8283965666961534 2023-09-23T22:43:21.350 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:21.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:21.468 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:21.517 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:21.628 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:21.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:21.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:21.715 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:21.716 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:21.733 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:21.771 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:21.945 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:21.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:22.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:22.159 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:22.228 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:22.368 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:22.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:22.496 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:22.535 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:22.592 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:22.689 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:43:22.715 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:22.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:22.820 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:22.853 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:22.935 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:22.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:23.046 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:23.088 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.146 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:23.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:23.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:23.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:23 smithi151 ceph-mon[159210]: pgmap v419: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 622 op/s; 18332/273231 objects degraded (6.709%); 3028/273231 objects misplaced (1.108%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:23.209 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:23.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:23 smithi052 ceph-mon[187743]: pgmap v419: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 622 op/s; 18332/273231 objects degraded (6.709%); 3028/273231 objects misplaced (1.108%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:23.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:23 smithi052 ceph-mon[180979]: pgmap v419: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 622 op/s; 18332/273231 objects degraded (6.709%); 3028/273231 objects misplaced (1.108%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:23.259 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:23.383 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.512 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:23.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:23.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:23.623 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:23.724 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:43:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:43:23] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.33.4" 2023-09-23T22:43:23.728 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.862 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:23.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:23.968 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:24.002 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.069 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:24.173 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:24.178 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:43:24.199 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.241 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:24.267 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:24.345 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:24.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3541383799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:24.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:24.378 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3541383799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:24.423 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:24.430 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3541383799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:24.430 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:24.674 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:24.782 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:24.872 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:25.085 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:25.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:25.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:25.142 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: pgmap v420: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 21 MiB/s wr, 624 op/s; 18332/268935 objects degraded (6.817%); 3028/268935 objects misplaced (1.126%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:25.161 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:25.161 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:25.183 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: pgmap v420: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 21 MiB/s wr, 624 op/s; 18332/268935 objects degraded (6.817%); 3028/268935 objects misplaced (1.126%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:25.183 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:25.184 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:43:25.184 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.184 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T22:43:25.184 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.184 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.184 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/268935 objects degraded (6.817%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:25.193 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:25.240 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:25.351 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:25.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:25.455 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:25.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: pgmap v420: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 21 MiB/s wr, 624 op/s; 18332/268935 objects degraded (6.817%); 3028/268935 objects misplaced (1.126%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:43:25.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:25.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:43:25.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T22:43:25.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/268935 objects degraded (6.817%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: failed to encode map e175 with expected crc 2023-09-23T22:43:25.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/268935 objects degraded (6.817%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:25.565 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:25.783 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:25.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:25.899 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:25.929 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:26.021 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:26.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:26.074 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:43:26.125 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:26.168 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:26.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2407618767' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]: dispatch 2023-09-23T22:43:26.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]: dispatch 2023-09-23T22:43:26.201 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 0.828397 (d411) 2023-09-23T22:43:26.236 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:26.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:26.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:26.342 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:26.385 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:26.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2407618767' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]: dispatch 2023-09-23T22:43:26.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]: dispatch 2023-09-23T22:43:26.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2407618767' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]: dispatch 2023-09-23T22:43:26.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]: dispatch 2023-09-23T22:43:26.506 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:26.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:26.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:26.537 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:26.614 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:26.653 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:26.696 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:26.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:26.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:26.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:26.837 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.090 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:43:27.149 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:27.176 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[180979]: pgmap v422: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 104 B/s rd, 12 MiB/s wr, 646 op/s; 18332/264042 objects degraded (6.943%); 2808/264042 objects misplaced (1.063%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:43:27.176 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[180979]: failed to encode map e176 with expected crc 2023-09-23T22:43:27.176 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]': finished 2023-09-23T22:43:27.176 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[180979]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T22:43:27.177 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[180979]: failed to encode map e176 with expected crc 2023-09-23T22:43:27.177 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[187743]: pgmap v422: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 104 B/s rd, 12 MiB/s wr, 646 op/s; 18332/264042 objects degraded (6.943%); 2808/264042 objects misplaced (1.063%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:43:27.177 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[187743]: failed to encode map e176 with expected crc 2023-09-23T22:43:27.177 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]': finished 2023-09-23T22:43:27.177 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[187743]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T22:43:27.177 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:27 smithi052 ceph-mon[187743]: failed to encode map e176 with expected crc 2023-09-23T22:43:27.253 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:27.403 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:27 smithi151 ceph-mon[159210]: pgmap v422: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 104 B/s rd, 12 MiB/s wr, 646 op/s; 18332/264042 objects degraded (6.943%); 2808/264042 objects misplaced (1.063%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:43:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:27 smithi151 ceph-mon[159210]: failed to encode map e176 with expected crc 2023-09-23T22:43:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.8283965666961534}]': finished 2023-09-23T22:43:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:27 smithi151 ceph-mon[159210]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T22:43:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:27 smithi151 ceph-mon[159210]: failed to encode map e176 with expected crc 2023-09-23T22:43:27.505 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:27.613 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:27.645 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.775 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:27.817 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:27.855 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:27.873 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:27.905 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:27.907 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:27.958 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:28.004 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:28.016 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:28.041 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:28.049 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:28.089 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.111 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:43:28.161 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:28.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:28.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:28.269 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:28.347 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:28.366 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:28.366 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.35001.0:99345 7.b e177/146 7:d5c64f68:::benchmark_data_smithi052_277165_object6853:head v 177'6354, mlcod=177'6352)", 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:28.359504+0000", 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0047868729999999996, 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048184209999999998, 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.367 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359504+0000", 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359504+0000", 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:28.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359503+0000", 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2737e-05 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359515+0000", 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2965999999999999e-05 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359528+0000", 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4529999999999999e-06 2023-09-23T22:43:28.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359535+0000", 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0951e-05 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359596+0000", 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7425e-05 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.370 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.359613+0000", 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000109632 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.35001.0:99355 7.b e177/146 7:d49d3390:::benchmark_data_smithi052_277165_object6863:head v 177'6355, mlcod=177'6352)", 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:28.361005+0000", 2023-09-23T22:43:28.371 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.00328547, 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003396212, 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.361005+0000", 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:28.372 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.361005+0000", 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.361004+0000", 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0979999999999994e-06 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.361012+0000", 2023-09-23T22:43:28.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2972e-05 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.361025+0000", 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1539999999999999e-06 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.361031+0000", 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001414946 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.362446+0000", 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2174000000000001e-05 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.362468+0000", 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001463556 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.375 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.35001.0:99356 7.1 e177/165)", 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:28.363608+0000", 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.00068249900000000004, 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00086028199999999997, 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "queued for pg", 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.363608+0000", 2023-09-23T22:43:28.376 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.363608+0000", 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.363607+0000", 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6600000000000007e-06 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.363616+0000", 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6402e-05 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.363633+0000", 2023-09-23T22:43:28.378 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7059999999999998e-06 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:28.363640+0000", 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1686000000000001e-05 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 3 2023-09-23T22:43:28.379 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:28.413 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[180979]: osdmap e177: 8 total, 8 up, 7 in 2023-09-23T22:43:28.413 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[180979]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.413 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[180979]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[180979]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[180979]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3212262949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[187743]: osdmap e177: 8 total, 8 up, 7 in 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[187743]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[187743]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[187743]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.414 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[187743]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.415 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3212262949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:28.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:28 smithi151 ceph-mon[159210]: osdmap e177: 8 total, 8 up, 7 in 2023-09-23T22:43:28.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:28 smithi151 ceph-mon[159210]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:28 smithi151 ceph-mon[159210]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:28 smithi151 ceph-mon[159210]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:28 smithi151 ceph-mon[159210]: failed to encode map e177 with expected crc 2023-09-23T22:43:28.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3212262949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:29.254 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:43:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:29 smithi052 ceph-mon[187743]: failed to encode map e177 with expected crc 2023-09-23T22:43:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:29 smithi052 ceph-mon[187743]: pgmap v425: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 18332/263436 objects degraded (6.959%); 2808/263436 objects misplaced (1.066%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:43:29.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:29 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:43:29.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:29 smithi052 ceph-mon[180979]: failed to encode map e177 with expected crc 2023-09-23T22:43:29.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:29 smithi052 ceph-mon[180979]: pgmap v425: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 18332/263436 objects degraded (6.959%); 2808/263436 objects misplaced (1.066%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:43:29.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:29 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:43:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:29 smithi151 ceph-mon[159210]: failed to encode map e177 with expected crc 2023-09-23T22:43:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:29 smithi151 ceph-mon[159210]: pgmap v425: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 18332/263436 objects degraded (6.959%); 2808/263436 objects misplaced (1.066%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:43:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:29 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:43:29.718 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:29.756 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:29.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:29.868 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:29.898 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:29.956 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:29.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:30.061 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:30.155 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:30.349 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:30.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:30.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/263436 objects degraded (6.959%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:30.462 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:30.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/263436 objects degraded (6.959%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:30.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/263436 objects degraded (6.959%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:30.550 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:30.650 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:30.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:30.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:30.754 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:30.811 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:30.908 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:30.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:30.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:31.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:31.119 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:31.270 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:31.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:31.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:31.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:31 smithi151 ceph-mon[159210]: pgmap v426: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 18332/257811 objects degraded (7.111%); 2808/257811 objects misplaced (1.089%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:43:31.360 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4276995544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:31.375 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:31.431 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:31.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:31 smithi052 ceph-mon[187743]: pgmap v426: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 18332/257811 objects degraded (7.111%); 2808/257811 objects misplaced (1.089%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:43:31.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4276995544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:31.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:31 smithi052 ceph-mon[180979]: pgmap v426: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 18332/257811 objects degraded (7.111%); 2808/257811 objects misplaced (1.089%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:43:31.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4276995544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:31.598 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:31.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:31.656 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:31.656 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:43:31.656 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:43:31.656 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:43:31.656 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:31.705 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:31.762 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:43:31.800 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:31.854 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:43:32.123 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:32.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:32.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:32.179 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:43:32.179 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:43:32.179 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T22:43:32.180 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T22:43:32.237 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:32.359 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:32.359 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4276995544' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:32.359 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[187743]: osdmap e178: 8 total, 8 up, 7 in 2023-09-23T22:43:32.359 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[187743]: failed to encode map e178 with expected crc 2023-09-23T22:43:32.360 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:32.360 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4276995544' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:32.361 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[180979]: osdmap e178: 8 total, 8 up, 7 in 2023-09-23T22:43:32.361 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:32 smithi052 ceph-mon[180979]: failed to encode map e178 with expected crc 2023-09-23T22:43:32.383 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:32.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:32 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:32.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4276995544' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:32.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:32 smithi151 ceph-mon[159210]: osdmap e178: 8 total, 8 up, 7 in 2023-09-23T22:43:32.569 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:32 smithi151 ceph-mon[159210]: failed to encode map e178 with expected crc 2023-09-23T22:43:32.581 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:43:33.013 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:33.041 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:33.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:33.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:33.153 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:33.234 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:33.292 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:33.359 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[180979]: failed to encode map e178 with expected crc 2023-09-23T22:43:33.359 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[180979]: failed to encode map e178 with expected crc 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[180979]: pgmap v428: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 18332/254544 objects degraded (7.202%); 2569/254544 objects misplaced (1.009%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:43:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:43:33] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.33.4" 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[187743]: failed to encode map e178 with expected crc 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[187743]: failed to encode map e178 with expected crc 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[187743]: pgmap v428: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 18332/254544 objects degraded (7.202%); 2569/254544 objects misplaced (1.009%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:43:33.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:33 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:43:33.377 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:33.377 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:33.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:33.489 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:33.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:33 smithi151 ceph-mon[159210]: failed to encode map e178 with expected crc 2023-09-23T22:43:33.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:33 smithi151 ceph-mon[159210]: failed to encode map e178 with expected crc 2023-09-23T22:43:33.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:33 smithi151 ceph-mon[159210]: pgmap v428: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 18332/254544 objects degraded (7.202%); 2569/254544 objects misplaced (1.009%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:43:33.536 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:33 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:43:33.539 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:33.648 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:33.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:33.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:33.751 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:33.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:34.070 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:34.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:34.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:34.280 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10176 6.c e128/99 6:30687e13:::benchmark_data_smithi052_234456_object10175:head v 128'611, mlcod=128'609)", 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.65021951599999, 2023-09-23T22:43:34.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26259265100000001, 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:43:34.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200005+0000", 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4515999999999999e-05 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200049+0000", 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0868e-05 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200060+0000", 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7479999999999999e-06 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200069+0000", 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8922999999999999e-05 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200118+0000", 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1420000000000002e-05 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200139+0000", 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26240534999999998 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462545+0000", 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3596e-05 2023-09-23T22:43:34.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462598+0000", 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26259265100000001 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10178 6.3 e128/104 6:cbb8686c:::benchmark_data_smithi052_234456_object10177:head v 128'684, mlcod=128'682)", 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:43:34.859 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.649957607, 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26233074499999998, 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:43:34.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200267+0000", 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2943999999999999e-05 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200290+0000", 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0489000000000001e-05 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200300+0000", 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4090000000000002e-06 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200305+0000", 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000490297 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200796+0000", 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2308999999999999e-05 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200818+0000", 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085734699999999995 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201675+0000", 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26092291299999998 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462598+0000", 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26233074499999998 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10170 6.1 e128/104 6:8aa77601:::benchmark_data_smithi052_234456_object10169:head v 128'666, mlcod=128'664)", 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.649440669, 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26275960799999998, 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.865 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1285999999999997e-05 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200825+0000", 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6779999999999995e-06 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200835+0000", 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.976e-06 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200841+0000", 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0885999999999999e-05 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200902+0000", 2023-09-23T22:43:34.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1832999999999999e-05 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200923+0000", 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26257463399999997 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463498+0000", 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6022000000000001e-05 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463544+0000", 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26275960799999998 2023-09-23T22:43:34.871 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10180 6.1 e128/104 6:8dfddb97:::benchmark_data_smithi052_234456_object10179:head v 128'667, mlcod=128'664)", 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.64935303499999, 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26309316700000002, 2023-09-23T22:43:34.872 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9139e-05 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200901+0000", 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2439999999999996e-06 2023-09-23T22:43:34.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200908+0000", 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200913+0000", 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045587999999999998 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201369+0000", 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8995e-05 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201388+0000", 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26253594800000002 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463924+0000", 2023-09-23T22:43:34.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1671000000000001e-05 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463965+0000", 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26309316700000002 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10183 6.1 e128/104 6:8243606a:::benchmark_data_smithi052_234456_object10182:head v 128'668, mlcod=128'664)", 2023-09-23T22:43:34.877 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.64811648899999, 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261887543, 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202108+0000", 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0642000000000002e-05 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202148+0000", 2023-09-23T22:43:34.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26041415499999998 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462563+0000", 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6589999999999996e-06 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462571+0000", 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052397000000000001 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463095+0000", 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2568000000000001e-05 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463118+0000", 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085566799999999997 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463973+0000", 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2731999999999999e-05 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463996+0000", 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261887543 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.882 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10181 6.3 e128/104 6:c0620c37:::benchmark_data_smithi052_234456_object10180:head v 128'685, mlcod=128'682)", 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.647634004, 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26238228600000002, 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202590+0000", 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5381999999999999e-05 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202616+0000", 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25993996400000002 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462556+0000", 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0231000000000001e-05 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462566+0000", 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2098000000000001e-05 2023-09-23T22:43:34.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462628+0000", 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0216999999999999e-05 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462658+0000", 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002274767 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464933+0000", 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0459e-05 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464973+0000", 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26238228600000002 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10192 6.a e128/126 6:5a145b56:::benchmark_data_smithi052_234456_object10191:head v 128'636, mlcod=128'634)", 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.64470463399999, 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.258119451, 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205519+0000", 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8529000000000001e-05 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205548+0000", 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25701668599999999 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462565+0000", 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9289999999999998e-06 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462573+0000", 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4702000000000004e-05 2023-09-23T22:43:34.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462647+0000", 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1945000000000001e-05 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462669+0000", 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093275200000000002 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463602+0000", 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8049000000000001e-05 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463640+0000", 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.258119451 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41337 7.7 7:eb68b087:::benchmark_data_smithi052_277165_object41336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.665753+0000", 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.184471731000002, 2023-09-23T22:43:34.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.215328203, 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41337 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665753+0000", 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665753+0000", 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999881 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665742+0000", 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1890000000000003e-05 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665833+0000", 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1992e-05 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665875+0000", 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6199999999999999e-06 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665883+0000", 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5296999999999994e-05 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665958+0000", 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139579 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666098+0000", 2023-09-23T22:43:34.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016923599999999999 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666267+0000", 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018763129999999999 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.668144+0000", 2023-09-23T22:43:34.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0061320660000000003 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.674276+0000", 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20671217 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.880988+0000", 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1666999999999998e-05 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881039+0000", 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2194000000000003e-05 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881082+0000", 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.215328203 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.902 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41339 7.d 7:b0decece:::benchmark_data_smithi052_277165_object41338:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.665913+0000", 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.184312027999994, 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21514861900000001, 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41339 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.903 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665913+0000", 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665913+0000", 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665913+0000", 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1168999999999999e-05 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665944+0000", 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0739999999999993e-06 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665952+0000", 2023-09-23T22:43:34.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4439999999999999e-06 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665957+0000", 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1968999999999997e-05 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665999+0000", 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011969 2023-09-23T22:43:34.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666119+0000", 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000156405 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666275+0000", 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018460270000000001 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.668121+0000", 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035066260000000001 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.671628+0000", 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20933426299999999 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.880962+0000", 2023-09-23T22:43:34.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6971000000000001e-05 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881019+0000", 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2601000000000001e-05 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881062+0000", 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21514861900000001 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.909 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41351 7.7 7:eaec7e2f:::benchmark_data_smithi052_277165_object41350:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.672112+0000", 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 80.178112935000001, 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21458672200000001, 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.910 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41351 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672112+0000", 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672112+0000", 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672111+0000", 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000102829 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672214+0000", 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2830000000000001e-05 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672226+0000", 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4049999999999999e-06 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672232+0000", 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110771 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672343+0000", 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013482699999999999 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672477+0000", 2023-09-23T22:43:34.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000186019 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672663+0000", 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017299290000000001 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.674393+0000", 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017630099999999999 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.674570+0000", 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21203038799999999 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.886600+0000", 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8053000000000001e-05 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.886658+0000", 2023-09-23T22:43:34.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0855000000000001e-05 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.886699+0000", 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21458672200000001 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61219 7.d 7:b3f702b2:::benchmark_data_smithi052_277165_object61218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:43:34.917 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 61.781115171000003, 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23283454100000001, 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61219 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069109+0000", 2023-09-23T22:43:34.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8376000000000001e-05 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069137+0000", 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0046e-05 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069147+0000", 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3259999999999997e-06 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069151+0000", 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9551999999999997e-05 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069191+0000", 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112429 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069303+0000", 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114738 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069418+0000", 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003836182 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073254+0000", 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022611850000000002 2023-09-23T22:43:34.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075516+0000", 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226299794 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301815+0000", 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7392999999999995e-05 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301903+0000", 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1786000000000003e-05 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301945+0000", 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23283454100000001 2023-09-23T22:43:34.924 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61222 7.d 7:b9a46085:::benchmark_data_smithi052_277165_object61221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 61.781066179, 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23289720799999999, 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.925 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61222 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069158+0000", 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8464999999999998e-05 2023-09-23T22:43:34.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069197+0000", 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3069999999999999e-06 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069203+0000", 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0140000000000001e-06 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069210+0000", 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002195729 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071406+0000", 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010020499999999999 2023-09-23T22:43:34.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071506+0000", 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013088800000000001 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071637+0000", 2023-09-23T22:43:34.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018844230000000001 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073521+0000", 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020741359999999999 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075596+0000", 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226399296 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301995+0000", 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6539999999999999e-05 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.302031+0000", 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4808e-05 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.302056+0000", 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23289720799999999 2023-09-23T22:43:34.933 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61226 7.7 7:ebd42417:::benchmark_data_smithi052_277165_object61225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:43:34.934 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 61.774114310000002, 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.227997757, 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61226 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076110+0000", 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9333999999999999e-05 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076159+0000", 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1613e-05 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076171+0000", 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129999999999996e-06 2023-09-23T22:43:34.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076175+0000", 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6888000000000002e-05 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076222+0000", 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129424 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076352+0000", 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8393000000000005e-05 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076450+0000", 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023315779999999999 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.078782+0000", 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001019966 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.079801+0000", 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.224238032 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304040+0000", 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3377000000000002e-05 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304083+0000", 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5698e-05 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304109+0000", 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.227997757 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.941 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61248 7.d 7:bf1664de:::benchmark_data_smithi052_277165_object61247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.091179+0000", 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 61.759046149, 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.212800882, 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.942 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61248 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091179+0000", 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091179+0000", 2023-09-23T22:43:34.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091178+0000", 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1151e-05 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091219+0000", 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2448e-05 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091231+0000", 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0849999999999996e-06 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091236+0000", 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3915000000000002e-05 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091280+0000", 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010614 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091387+0000", 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000119046 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:43:34.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.091506+0000", 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016745180000000001 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.093180+0000", 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00041871900000000001 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.093599+0000", 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.210306086 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.303905+0000", 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5026e-05 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.303940+0000", 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9947000000000002e-05 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.303980+0000", 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.212800882 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:92229 7.c 7:3ccf5324:::benchmark_data_smithi052_277165_object92228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:16.887818+0000", 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.962406940000001, 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19015305099999999, 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.949 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 92229 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887818+0000", 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887818+0000", 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887817+0000", 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6644999999999997e-05 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887874+0000", 2023-09-23T22:43:34.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1189000000000001e-05 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887885+0000", 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7820000000000001e-06 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887890+0000", 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4323999999999998e-05 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.887934+0000", 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000166813 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.888101+0000", 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015709200000000001 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:43:34.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.888258+0000", 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027017640000000002 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.890960+0000", 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017050019999999999 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:16.892665+0000", 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.183642476 2023-09-23T22:43:34.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:17.076307+0000", 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2508000000000005e-05 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:17.076370+0000", 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016017029999999999 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:17.077971+0000", 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19015305099999999 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.956 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100763 7.c 7:3dd4fd54:::benchmark_data_smithi052_277165_object8271:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.555586+0000", 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.2946395580000001, 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20470687700000001, 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100763 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.957 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.555586+0000", 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.555586+0000", 2023-09-23T22:43:34.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.555585+0000", 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.237e-06 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.555590+0000", 2023-09-23T22:43:34.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9740000000000002e-06 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.555598+0000", 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.711e-06 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.555604+0000", 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00252549 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.558129+0000", 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000111469 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.558241+0000", 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000122199 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.558363+0000", 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002404982 2023-09-23T22:43:34.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.560768+0000", 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013442189 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574210+0000", 2023-09-23T22:43:34.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.184096125 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758306+0000", 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9367999999999998e-05 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758366+0000", 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00192687 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760292+0000", 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20470687700000001 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.965 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop_reply(client.35001.0:100763 7.c e177/169 ondisk, result = 0)", 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.574822+0000", 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.2754028330000002, 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.185482064, 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574822+0000", 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574822+0000", 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574822+0000", 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9729999999999999e-06 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574826+0000", 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6810000000000003e-06 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574832+0000", 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.591e-06 2023-09-23T22:43:34.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.574835+0000", 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18345240300000001 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758288+0000", 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9699999999999994e-06 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758297+0000", 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020069010000000002 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760304+0000", 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.185482064 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100776 7.c 7:350d58b3:::benchmark_data_smithi052_277165_object8284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.2735202000000001, 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999, 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100776 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576703+0000", 2023-09-23T22:43:34.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0799999999999995e-06 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576712+0000", 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2679e-05 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576725+0000", 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6329999999999999e-06 2023-09-23T22:43:34.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576731+0000", 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18361677800000001 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760348+0000", 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000140829 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760489+0000", 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016031800000000001 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760649+0000", 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010853742 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.771503+0000", 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023432639000000002 2023-09-23T22:43:34.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.794936+0000", 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035400660000000001 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798476+0000", 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5828999999999999e-05 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798511+0000", 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012960090000000001 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.799807+0000", 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.977 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100781 7.c 7:39965936:::benchmark_data_smithi052_277165_object8289:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.2662415559999998, 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999, 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.978 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100781 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:43:34.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583981+0000", 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.59e-05 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583996+0000", 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.174193922 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758190+0000", 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2031000000000001e-05 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758202+0000", 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049640919999999998 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763167+0000", 2023-09-23T22:43:34.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143024 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763310+0000", 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016401100000000001 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763474+0000", 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012004406 2023-09-23T22:43:34.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.775478+0000", 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019532911 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.795011+0000", 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072752770000000001 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802286+0000", 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0192000000000002e-05 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802346+0000", 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018569999999999999 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.804203+0000", 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101958 7.7 7:e819df4a:::benchmark_data_smithi052_277165_object9466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 2.3234920969999999, 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001, 2023-09-23T22:43:34.985 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101958 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:34.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526731+0000", 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.396e-06 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526739+0000", 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3361e-05 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526752+0000", 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2920000000000003e-06 2023-09-23T22:43:34.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526757+0000", 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1291000000000002e-05 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526799+0000", 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121802 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526920+0000", 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014018900000000001 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527061+0000", 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033778649999999999 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530439+0000", 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000232978 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530671+0000", 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21847482400000001 2023-09-23T22:43:34.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749146+0000", 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749200+0000", 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5618000000000003e-05 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:34.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749245+0000", 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:34.993 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:35.130 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:43:35.216 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:35.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:35.323 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:35.402 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[180979]: pgmap v429: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 18332/250416 objects degraded (7.321%); 2569/250416 objects misplaced (1.026%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:43:35.402 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:35.402 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:35.402 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/250416 objects degraded (7.321%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:35.402 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[187743]: pgmap v429: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 18332/250416 objects degraded (7.321%); 2569/250416 objects misplaced (1.026%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:43:35.402 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:35.403 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:35.403 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/250416 objects degraded (7.321%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:35.410 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:35.602 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:35.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:35 smithi151 ceph-mon[159210]: pgmap v429: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 18332/250416 objects degraded (7.321%); 2569/250416 objects misplaced (1.026%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:43:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:43:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/250416 objects degraded (7.321%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:35.719 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:35.829 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:35.841 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:43:36.150 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:36.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:36.255 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:36.281 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:36.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:36.431 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:36.490 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:36.498 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:36.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:36.536 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:36.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:36.601 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:36.643 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:36.696 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:36.730 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:36.769 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:36.790 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:36.830 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:36.848 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.848 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.848 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.848 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.849 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.849 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.849 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.849 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:43:36.849 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:36.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:36.953 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:36.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:36 smithi052 conmon[180954]: 2023-09-23T22:43:36.733+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e178 definitely_dead 0 2023-09-23T22:43:37.072 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:37.276 INFO:teuthology.orchestra.run.smithi052.stderr:marked in osd.7. 2023-09-23T22:43:37.304 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[187743]: pgmap v430: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 18332/245964 objects degraded (7.453%); 2318/245964 objects misplaced (0.942%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T22:43:37.304 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/183005462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:37.304 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4091819279' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:43:37.304 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:43:37.304 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[180979]: pgmap v430: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 18332/245964 objects degraded (7.453%); 2318/245964 objects misplaced (0.942%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T22:43:37.305 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/183005462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:37.305 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4091819279' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:43:37.305 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:43:37.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:37 smithi151 ceph-mon[159210]: pgmap v430: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 18332/245964 objects degraded (7.453%); 2318/245964 objects misplaced (0.942%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T22:43:37.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/183005462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:37.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4091819279' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:43:37.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:43:38.049 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:38.049 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:43:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T22:43:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: pgmap v432: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 18332/245505 objects degraded (7.467%); 2318/245505 objects misplaced (0.944%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T22:43:38.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:38 smithi151 ceph-mon[159210]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T22:43:38.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:43:38.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T22:43:38.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: pgmap v432: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 18332/245505 objects degraded (7.467%); 2318/245505 objects misplaced (0.944%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T22:43:38.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[187743]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T22:43:38.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:43:38.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T22:43:38.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: failed to encode map e179 with expected crc 2023-09-23T22:43:38.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: pgmap v432: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 18332/245505 objects degraded (7.467%); 2318/245505 objects misplaced (0.944%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T22:43:38.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:38 smithi052 ceph-mon[180979]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T22:43:38.793 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:43:39.200 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T22:43:39.245 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:39.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:39.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:39.365 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:39.367 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:39.435 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:39.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.500 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[180979]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:39 smithi052 ceph-mon[187743]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.547 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:39.553 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.554 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.555 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:39 smithi151 ceph-mon[159210]: failed to encode map e180 with expected crc 2023-09-23T22:43:39.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:39.656 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:39.732 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:39.877 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:39.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:39.994 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:40.082 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:40.203 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:40.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:40.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:40.313 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:40.407 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T22:43:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18332/245505 objects degraded (7.467%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[187743]: pgmap v435: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 8028/241053 objects degraded (3.330%); 2318/241053 objects misplaced (0.962%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18332/245505 objects degraded (7.467%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:40.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:40 smithi052 ceph-mon[180979]: pgmap v435: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 8028/241053 objects degraded (3.330%); 2318/241053 objects misplaced (0.962%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T22:43:40.496 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:40.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:40.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:40.625 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T22:43:40.625 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.625 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.625 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.625 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: failed to encode map e181 with expected crc 2023-09-23T22:43:40.626 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18332/245505 objects degraded (7.467%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:40.626 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:40 smithi151 ceph-mon[159210]: pgmap v435: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 8028/241053 objects degraded (3.330%); 2318/241053 objects misplaced (0.962%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T22:43:40.628 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:40.671 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:40.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:40.722 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:40.775 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:40.835 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:43:40.906 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:41.094 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:41.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:41.207 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:41.269 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:41.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3130495899' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:41.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:41 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:41.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3130495899' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:41.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:41 smithi151 ceph-mon[159210]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T22:43:41.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:41 smithi151 ceph-mon[159210]: failed to encode map e182 with expected crc 2023-09-23T22:43:41.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:41 smithi151 ceph-mon[159210]: failed to encode map e182 with expected crc 2023-09-23T22:43:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3130495899' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3130495899' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[180979]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T22:43:41.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[180979]: failed to encode map e182 with expected crc 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[180979]: failed to encode map e182 with expected crc 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3130495899' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3130495899' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[187743]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[187743]: failed to encode map e182 with expected crc 2023-09-23T22:43:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:41 smithi052 ceph-mon[187743]: failed to encode map e182 with expected crc 2023-09-23T22:43:41.577 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:41.577 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:41.870 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:41.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:41.980 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:42.033 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:42.106 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:42.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:42.134 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:42.214 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:42.255 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:42.330 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:42.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:42.433 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:42.499 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:42.610 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:42 smithi151 ceph-mon[159210]: failed to encode map e182 with expected crc 2023-09-23T22:43:42.610 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:42 smithi151 ceph-mon[159210]: failed to encode map e182 with expected crc 2023-09-23T22:43:42.610 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:42 smithi151 ceph-mon[159210]: pgmap v437: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8026/238053 objects degraded (3.372%); 1072/238053 objects misplaced (0.450%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:43:42.632 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:43:42.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:42 smithi052 ceph-mon[180979]: failed to encode map e182 with expected crc 2023-09-23T22:43:42.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:42 smithi052 ceph-mon[180979]: failed to encode map e182 with expected crc 2023-09-23T22:43:42.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:42 smithi052 ceph-mon[180979]: pgmap v437: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8026/238053 objects degraded (3.372%); 1072/238053 objects misplaced (0.450%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:43:42.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:42 smithi052 ceph-mon[187743]: failed to encode map e182 with expected crc 2023-09-23T22:43:42.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:42 smithi052 ceph-mon[187743]: failed to encode map e182 with expected crc 2023-09-23T22:43:42.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:42 smithi052 ceph-mon[187743]: pgmap v437: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8026/238053 objects degraded (3.372%); 1072/238053 objects misplaced (0.450%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:43:42.771 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:42.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:42.881 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:42.933 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:43.014 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:43.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:43.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:43.121 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:43.133 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:43:43.150 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:43.196 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:43.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:43.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:43.389 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:43.495 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:43:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:43:43] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T22:43:43.556 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:43.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:43.671 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:43.749 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:43.873 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:43.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:43.987 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:43:43.991 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:44.050 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:44.201 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:43:44.201 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:43:44.201 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T22:43:44.201 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:43:44.201 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:43:44.202 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T22:43:44.254 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:44.289 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:44.294 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:44.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:44.353 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:44.400 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:44.410 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:44.435 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:44.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:44.502 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:44.541 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:44.580 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.618 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:43:44.642 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:44.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:44.755 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:44.844 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:44.950 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:44.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:44.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:45.057 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:45.106 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:45.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:45 smithi151 ceph-mon[159210]: pgmap v438: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 8026/233538 objects degraded (3.437%); 1072/233538 objects misplaced (0.459%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:43:45.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3546227946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:45.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8026/233538 objects degraded (3.437%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1177 6.8 6:1fae9040:::benchmark_data_smithi052_234456_object1176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:43:45.176 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.85976976500001, 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16501295899999999, 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1177 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.177 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312080+0000", 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6277999999999999e-05 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312137+0000", 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9989999999999997e-06 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312146+0000", 2023-09-23T22:43:45.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6670000000000004e-06 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312150+0000", 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8374999999999998e-05 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312189+0000", 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8557e-05 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312267+0000", 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011574900000000001 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312383+0000", 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16315368599999999 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475537+0000", 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013937680000000001 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476930+0000", 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1847000000000006e-05 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476992+0000", 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010188600000000001 2023-09-23T22:43:45.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477094+0000", 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16501295899999999 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1180 6.3 6:c4de4eba:::benchmark_data_smithi052_234456_object1179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.85928782900001, 2023-09-23T22:43:45.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.165229389, 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1180 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:43:45.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312562+0000", 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000111852 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312674+0000", 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3390999999999999e-05 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312687+0000", 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312694+0000", 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9179999999999999e-05 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312753+0000", 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16226790299999999 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475021+0000", 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018092 2023-09-23T22:43:45.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475202+0000", 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020914169999999999 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477293+0000", 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035115500000000002 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477644+0000", 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2717999999999999e-05 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477697+0000", 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5310999999999996e-05 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477792+0000", 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.165229389 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.189 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1182 6.8 6:12f46353:::benchmark_data_smithi052_234456_object1181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.85913999300001, 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16475775100000001, 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.190 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1182 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:43:45.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312710+0000", 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5873e-05 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312746+0000", 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4910000000000005e-06 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312755+0000", 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8200000000000002e-06 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312760+0000", 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9837000000000002e-05 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312830+0000", 2023-09-23T22:43:45.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.162295933 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475126+0000", 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015127000000000001 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475277+0000", 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001836475 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477114+0000", 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000174838 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477289+0000", 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9021000000000003e-05 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477348+0000", 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120896 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477469+0000", 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16475775100000001 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.196 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1184 6.5 6:ad9e7e6c:::benchmark_data_smithi052_234456_object1183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.859076421, 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16539384100000001, 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1184 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:43:45.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6648000000000001e-05 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312861+0000", 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2448e-05 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312873+0000", 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7630000000000003e-06 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312880+0000", 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16255894300000001 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475439+0000", 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129219 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475568+0000", 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000164919 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475733+0000", 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022748180000000001 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478008+0000", 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478015+0000", 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4327999999999997e-05 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478089+0000", 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2700000000000001e-05 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478132+0000", 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6504999999999997e-05 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478168+0000", 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16539384100000001 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.203 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:1191 6.6 e127/99 6:684d433b:::benchmark_data_smithi052_234456_object1190:head v 127'68, mlcod=127'65)", 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.85865813000001, 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16367246499999999, 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313192+0000", 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8267000000000001e-05 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313270+0000", 2023-09-23T22:43:45.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16176015999999999 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475030+0000", 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7800000000000006e-06 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475039+0000", 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00056310300000000002 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475602+0000", 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0098e-05 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475622+0000", 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012034210000000001 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476826+0000", 2023-09-23T22:43:45.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9504e-05 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476865+0000", 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16367246499999999 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10178 6.3 6:cbb8686c:::benchmark_data_smithi052_234456_object10177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.208 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.97223640600001, 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26379366399999998, 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10178 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8393999999999994e-05 2023-09-23T22:43:45.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199702+0000", 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0538e-05 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199713+0000", 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5840000000000002e-06 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199718+0000", 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7531000000000001e-05 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199786+0000", 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012609700000000001 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199912+0000", 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024177399999999999 2023-09-23T22:43:45.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200154+0000", 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029848309999999999 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203139+0000", 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.260079748 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463218+0000", 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3586999999999999e-05 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463292+0000", 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116289 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463408+0000", 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26379366399999998 2023-09-23T22:43:45.214 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10181 6.3 6:c0620c37:::benchmark_data_smithi052_234456_object10180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.972033568, 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.265707154, 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.215 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10181 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8221e-05 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199845+0000", 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3889999999999999e-06 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199852+0000", 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9130000000000002e-06 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199857+0000", 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085247700000000003 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200710+0000", 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001666491 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202376+0000", 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124387 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202501+0000", 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00086751799999999996 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203368+0000", 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26193364000000002 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465302+0000", 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106265 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465408+0000", 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116496 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465524+0000", 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.265707154 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10191 6.5 6:a7cca2de:::benchmark_data_smithi052_234456_object10190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.221 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.967976838, 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26015834900000001, 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10191 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203873+0000", 2023-09-23T22:43:45.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010616300000000001 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203979+0000", 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2459999999999992e-06 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203989+0000", 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129999999999996e-06 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203993+0000", 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5849000000000002e-05 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204029+0000", 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105581 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204135+0000", 2023-09-23T22:43:45.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000160994 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204296+0000", 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00127407 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205570+0000", 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052499199999999995 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206095+0000", 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25784317800000001 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463938+0000", 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5868000000000001e-05 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463994+0000", 2023-09-23T22:43:45.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8639000000000001e-05 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464032+0000", 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26015834900000001 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10193 6.5 6:a177b7dd:::benchmark_data_smithi052_234456_object10192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.228 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.96761158499999, 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26049174200000003, 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10193 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6399999999999997e-05 2023-09-23T22:43:45.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204275+0000", 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1413000000000001e-05 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204286+0000", 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9870000000000002e-06 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204293+0000", 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054160700000000005 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204835+0000", 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013009600000000001 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204965+0000", 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016865000000000001 2023-09-23T22:43:45.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205134+0000", 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00065124099999999999 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205785+0000", 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037879500000000002 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206164+0000", 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25846449399999999 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464628+0000", 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9333000000000003e-05 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464688+0000", 2023-09-23T22:43:45.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3455999999999999e-05 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464731+0000", 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26049174200000003 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10206 6.3 6:cca2051f:::benchmark_data_smithi052_234456_object10205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:43:45.235 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.96277722600001, 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25668903900000001, 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10206 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:43:45.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209072+0000", 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1259000000000002e-05 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209154+0000", 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3302000000000001e-05 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209167+0000", 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3980000000000002e-06 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209174+0000", 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4868000000000003e-05 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209239+0000", 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018571700000000001 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209425+0000", 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018386100000000001 2023-09-23T22:43:45.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209609+0000", 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00066804500000000003 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.210277+0000", 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.255318247 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465595+0000", 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6978999999999998e-05 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465652+0000", 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110548 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465763+0000", 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25668903900000001 2023-09-23T22:43:45.241 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10215 6.3 6:c04bb586:::benchmark_data_smithi052_234456_object10214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.958543257, 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25266233500000002, 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.242 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10215 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:43:45.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213306+0000", 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3293000000000001e-05 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213340+0000", 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0866000000000001e-05 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213350+0000", 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2249999999999997e-06 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213357+0000", 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1418999999999997e-05 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213408+0000", 2023-09-23T22:43:45.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6898999999999995e-05 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213505+0000", 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132287 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213637+0000", 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00064820999999999998 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214285+0000", 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25152426300000003 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465810+0000", 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4280999999999997e-05 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465864+0000", 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105955 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465970+0000", 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25266233500000002 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10217 6.3 6:c781daf6:::benchmark_data_smithi052_234456_object10216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.248 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.95699453899999, 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25134704800000002, 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10217 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214855+0000", 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8257999999999994e-05 2023-09-23T22:43:45.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214944+0000", 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1997999999999999e-05 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214956+0000", 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1810000000000004e-06 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214963+0000", 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9869e-05 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215013+0000", 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015027000000000001 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215163+0000", 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021892400000000001 2023-09-23T22:43:45.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215382+0000", 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073708599999999997 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.216119+0000", 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.249917901 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466037+0000", 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8545999999999998e-05 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466095+0000", 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010809199999999999 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466203+0000", 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25134704800000002 2023-09-23T22:43:45.254 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10224 6.3 6:c3676ada:::benchmark_data_smithi052_234456_object10223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 170.95144703299999, 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.246054935, 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.255 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10224 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:43:45.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220402+0000", 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7369e-05 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220449+0000", 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6140000000000001e-05 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220466+0000", 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8979999999999996e-06 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220473+0000", 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2861000000000002e-05 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220516+0000", 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011724500000000001 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220634+0000", 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013401900000000001 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220768+0000", 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00059656599999999998 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.221364+0000", 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24489038599999999 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466255+0000", 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6962e-05 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466321+0000", 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013730400000000001 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466459+0000", 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.246054935 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40799 6.8 6:14756be4:::benchmark_data_smithi052_234456_object18655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:43:45.261 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 155.310281436, 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2385245110000001, 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40799 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861568+0000", 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9182999999999999e-05 2023-09-23T22:43:45.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861587+0000", 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3654e-05 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861600+0000", 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.588e-06 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861608+0000", 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114259 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861722+0000", 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.251e-06 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861731+0000", 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028264639999999999 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864558+0000", 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4350000000000003e-06 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864567+0000", 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99719855899999998 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861766+0000", 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9449999999999998e-05 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861785+0000", 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2282697419999999 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090055+0000", 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000194957 2023-09-23T22:43:45.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090250+0000", 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000107145 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090357+0000", 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030987689999999999 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.093456+0000", 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023293989999999998 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.095785+0000", 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038349410000000001 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.099620+0000", 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1438000000000004e-05 2023-09-23T22:43:45.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.099682+0000", 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00041221600000000003 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.100094+0000", 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2385245110000001 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40800 6.3 6:c0d13c02:::benchmark_data_smithi052_234456_object18656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 155.31021741000001, 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.236326756, 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40800 2023-09-23T22:43:45.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.693e-06 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861637+0000", 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6470000000000001e-06 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861643+0000", 2023-09-23T22:43:45.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.408e-06 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861648+0000", 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4624999999999998e-05 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861722+0000", 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8899999999999996e-06 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861731+0000", 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003144053 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864875+0000", 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6930000000000002e-06 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864885+0000", 2023-09-23T22:43:45.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99906347699999998 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.863948+0000", 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2204999999999999e-05 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.863961+0000", 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2265645000000001 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090525+0000", 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016863800000000001 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090694+0000", 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018472599999999999 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090878+0000", 2023-09-23T22:43:45.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018197459999999999 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.092698+0000", 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.679e-06 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.092708+0000", 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048923589999999998 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097600+0000", 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7243000000000007e-05 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097667+0000", 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00029281000000000001 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097960+0000", 2023-09-23T22:43:45.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.236326756 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:5893 7.5 7:a9ee901c:::benchmark_data_smithi052_277165_object5892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:49.844393+0000", 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 115.327458302, 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.14460056800000001, 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.280 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 5893 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844393+0000", 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844393+0000", 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844391+0000", 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6589000000000003e-05 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844447+0000", 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2269999999999999e-05 2023-09-23T22:43:45.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844470+0000", 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.482e-05 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844484+0000", 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7153000000000001e-05 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844532+0000", 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016836800000000001 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844700+0000", 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018661599999999999 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:45.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.844887+0000", 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012449379999999999 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.846131+0000", 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00069888700000000003 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.846830+0000", 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.142052664 2023-09-23T22:43:45.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.988883+0000", 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8085999999999999e-05 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.988941+0000", 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2043000000000003e-05 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:49.988993+0000", 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.14460056800000001 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41338 7.5 7:a210bb19:::benchmark_data_smithi052_277165_object41337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.665781+0000", 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 90.506069441999998, 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21540489700000001, 2023-09-23T22:43:45.287 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41338 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665781+0000", 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665781+0000", 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665780+0000", 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012445499999999999 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665905+0000", 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3737000000000001e-05 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665918+0000", 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5450000000000003e-06 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665924+0000", 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8445e-05 2023-09-23T22:43:45.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665982+0000", 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016478 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666147+0000", 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000182738 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:45.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666330+0000", 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002808867 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.669139+0000", 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005151704 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.674291+0000", 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20678310799999999 2023-09-23T22:43:45.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881074+0000", 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6613999999999997e-05 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881130+0000", 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6079000000000003e-05 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881186+0000", 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21540489700000001 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41350 7.5 7:ab47f499:::benchmark_data_smithi052_277165_object41349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.672107+0000", 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 90.499743484999996, 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21391692500000001, 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.294 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41350 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672107+0000", 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672107+0000", 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672106+0000", 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7646999999999995e-05 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672203+0000", 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2326e-05 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672216+0000", 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5760000000000002e-06 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672222+0000", 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0977999999999999e-05 2023-09-23T22:43:45.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672273+0000", 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016777 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672441+0000", 2023-09-23T22:43:45.299 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:45 smithi052 ceph-mon[187743]: pgmap v438: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 8026/233538 objects degraded (3.437%); 1072/233538 objects misplaced (0.459%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:43:45.299 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3546227946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:45.299 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8026/233538 objects degraded (3.437%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:45.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000201454 2023-09-23T22:43:45.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672642+0000", 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002397255 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.675040+0000", 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014347139999999999 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.676474+0000", 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20943123999999999 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.885906+0000", 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7896000000000002e-05 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.885973+0000", 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0927000000000002e-05 2023-09-23T22:43:45.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.886024+0000", 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21391692500000001 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61224 7.5 7:ad9959c8:::benchmark_data_smithi052_277165_object61223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.072846+0000", 2023-09-23T22:43:45.302 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 72.099004876999999, 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22912544700000001, 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61224 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072846+0000", 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072846+0000", 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072844+0000", 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4922000000000001e-05 2023-09-23T22:43:45.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072889+0000", 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5446000000000001e-05 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072904+0000", 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.409e-06 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072911+0000", 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6582000000000002e-05 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072957+0000", 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000147684 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073105+0000", 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000195095 2023-09-23T22:43:45.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073300+0000", 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020691300000000002 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075369+0000", 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00065103100000000005 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076020+0000", 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22586408299999999 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301884+0000", 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9163999999999999e-05 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301934+0000", 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7830999999999997e-05 2023-09-23T22:43:45.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301971+0000", 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22912544700000001 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61242 7.5 7:a582954c:::benchmark_data_smithi052_277165_object61241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.085367+0000", 2023-09-23T22:43:45.309 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 72.086483623999996, 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.219389263, 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61242 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:43:45.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085367+0000", 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085367+0000", 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085366+0000", 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3773999999999997e-05 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085420+0000", 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3003000000000001e-05 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085433+0000", 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8730000000000004e-06 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:43:45.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085439+0000", 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3528000000000001e-05 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085482+0000", 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012782900000000001 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085610+0000", 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000135647 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085746+0000", 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001705193 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.087451+0000", 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00082960900000000003 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.088280+0000", 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.216402284 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304683+0000", 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6258000000000003e-05 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304729+0000", 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7606e-05 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:43:45.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304757+0000", 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.219389263 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:43:45.316 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:43:45.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:45 smithi052 ceph-mon[180979]: pgmap v438: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 8026/233538 objects degraded (3.437%); 1072/233538 objects misplaced (0.459%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:43:45.317 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3546227946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:45.317 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8026/233538 objects degraded (3.437%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:45.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:45.317 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:45.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:45.418 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:45.495 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:45.778 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:45.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:45.795 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:45.886 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:45.923 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:45.991 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:46.000 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:46.111 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:46.153 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:46.285 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:46.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:46.296 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:43:46.393 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:46.508 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:46.764 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:46.874 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:46.902 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:46.948 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:47.056 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:47.096 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:47.149 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:47.152 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:47 smithi151 ceph-mon[159210]: pgmap v439: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 8026/228507 objects degraded (3.512%); 579/228507 objects misplaced (0.253%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T22:43:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:47.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:47.257 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:47.343 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:47 smithi052 ceph-mon[180979]: pgmap v439: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 8026/228507 objects degraded (3.512%); 579/228507 objects misplaced (0.253%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T22:43:47.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:47 smithi052 ceph-mon[187743]: pgmap v439: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 8026/228507 objects degraded (3.512%); 579/228507 objects misplaced (0.253%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T22:43:47.769 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:43:48.147 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:48.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:48.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:48.252 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:48.350 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:48.407 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:48.493 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:48.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:48.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:48.598 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:48.699 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:49.045 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:49.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:49.154 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:49.173 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:43:49.173 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:43:49.174 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:43:49.188 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:49.260 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:49.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:49.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:49.322 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:49.368 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:49.425 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:49 smithi052 ceph-mon[187743]: pgmap v440: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 8026/227892 objects degraded (3.522%); 579/227892 objects misplaced (0.254%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T22:43:49.425 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:49 smithi052 ceph-mon[180979]: pgmap v440: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 8026/227892 objects degraded (3.522%); 579/227892 objects misplaced (0.254%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T22:43:49.429 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:49 smithi151 ceph-mon[159210]: pgmap v440: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 8026/227892 objects degraded (3.522%); 579/227892 objects misplaced (0.254%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T22:43:50.178 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:43:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3640394167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:50 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8026/227892 objects degraded (3.522%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3640394167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:50.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:50.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8026/227892 objects degraded (3.522%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3640394167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:50.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8026/227892 objects degraded (3.522%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:50.660 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:50.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:50.770 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:50.801 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:51.170 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:51.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:51.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:51.280 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:51.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:51.355 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[187743]: pgmap v441: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8026/221556 objects degraded (3.623%); 579/221556 objects misplaced (0.261%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:43:51.355 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:51.355 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[187743]: failed to encode map e183 with expected crc 2023-09-23T22:43:51.355 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[187743]: failed to encode map e183 with expected crc 2023-09-23T22:43:51.356 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:51.356 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[187743]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T22:43:51.357 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[180979]: pgmap v441: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8026/221556 objects degraded (3.623%); 579/221556 objects misplaced (0.261%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:43:51.357 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:51.357 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[180979]: failed to encode map e183 with expected crc 2023-09-23T22:43:51.357 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[180979]: failed to encode map e183 with expected crc 2023-09-23T22:43:51.357 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:51.357 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:51 smithi052 ceph-mon[180979]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T22:43:51.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:51 smithi151 ceph-mon[159210]: pgmap v441: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8026/221556 objects degraded (3.623%); 579/221556 objects misplaced (0.261%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:43:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:51 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:51 smithi151 ceph-mon[159210]: failed to encode map e183 with expected crc 2023-09-23T22:43:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:51 smithi151 ceph-mon[159210]: failed to encode map e183 with expected crc 2023-09-23T22:43:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:51 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:51 smithi151 ceph-mon[159210]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T22:43:51.543 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:51.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:51.651 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:51.686 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:51.758 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:51.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:51.865 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:51.908 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:51.976 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:51.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:52.068 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:43:52.068 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:43:52.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.068 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113914 7.6 7:6f4122c2:::benchmark_data_smithi052_277165_object21422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.068 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.041721+0000", 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.022156528000000002, 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022188462999999999, 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113914 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041721+0000", 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041721+0000", 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041720+0000", 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041726+0000", 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7580000000000003e-06 2023-09-23T22:43:52.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041743+0000", 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6503000000000001e-05 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041753+0000", 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9159999999999996e-06 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041845+0000", 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2294999999999996e-05 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.041987+0000", 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014231199999999999 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.042132+0000", 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014494900000000001 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.046103+0000", 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039703330000000004 2023-09-23T22:43:52.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051748+0000", 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056451549999999998 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113899 7.6 7:657015e1:::benchmark_data_smithi052_277165_object21407:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.019522+0000", 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.044355657, 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.044485243000000001, 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.076 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113899 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.019522+0000", 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.019522+0000", 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.019521+0000", 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.019525+0000", 2023-09-23T22:43:52.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7570000000000002e-06 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.019534+0000", 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2720000000000003e-06 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.019541+0000", 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.027840+0000", 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0082993300000000006 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.027974+0000", 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013430899999999999 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:43:52.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.028105+0000", 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013042199999999999 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.040112+0000", 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01200729 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051491+0000", 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011379119 2023-09-23T22:43:52.081 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113921 7.6 7:67d079ff:::benchmark_data_smithi052_277165_object21429:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.051757+0000", 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.012119937000000001, 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012326089, 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.082 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113921 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051757+0000", 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051757+0000", 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051756+0000", 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051762+0000", 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5929999999999996e-06 2023-09-23T22:43:52.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051773+0000", 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0900000000000001e-05 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051780+0000", 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3420000000000002e-06 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051837+0000", 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7136000000000003e-05 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051996+0000", 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015887900000000001 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:43:52.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052365+0000", 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00036936599999999999 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.057111+0000", 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047457519999999998 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063692+0000", 2023-09-23T22:43:52.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065812659999999997 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113922 7.4 7:27f5b4f7:::benchmark_data_smithi052_277165_object21430:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.051793+0000", 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.012084477, 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012358305999999999, 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.088 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113922 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051793+0000", 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051793+0000", 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051793+0000", 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051795+0000", 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6280000000000001e-06 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051801+0000", 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1140000000000002e-06 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051806+0000", 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7439999999999998e-06 2023-09-23T22:43:52.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051847+0000", 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1041000000000002e-05 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051975+0000", 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012798 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052113+0000", 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013819 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.059573+0000", 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074599920000000004 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.059593+0000", 2023-09-23T22:43:52.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9349000000000001e-05 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113923 7.f 7:fb2267b9:::benchmark_data_smithi052_277165_object21431:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.052125+0000", 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.011752232, 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012100474999999999, 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.094 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113923 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052125+0000", 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052125+0000", 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052124+0000", 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052130+0000", 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8819999999999998e-06 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052142+0000", 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2197e-05 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052152+0000", 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4679999999999998e-06 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052226+0000", 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4544999999999999e-05 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052375+0000", 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148512 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.052505+0000", 2023-09-23T22:43:52.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012974899999999999 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.056388+0000", 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0038839149999999999 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113903 7.6 7:6eda4b8f:::benchmark_data_smithi052_277165_object21411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.099 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.020271+0000", 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.043606493000000003, 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.044000093999999997, 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113903 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.020271+0000", 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.020271+0000", 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.020270+0000", 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:52.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.020273+0000", 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8940000000000002e-06 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.020280+0000", 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4590000000000003e-06 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.020285+0000", 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1869999999999996e-06 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.032392+0000", 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012106703999999999 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.032547+0000", 2023-09-23T22:43:52.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015538600000000001 2023-09-23T22:43:52.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:43:52.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.032673+0000", 2023-09-23T22:43:52.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012567000000000001 2023-09-23T22:43:52.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.105 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.105 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.040121+0000", 2023-09-23T22:43:52.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074478499999999998 2023-09-23T22:43:52.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:52.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051525+0000", 2023-09-23T22:43:52.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011403961000000001 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:113925 7.e e183/146 7:78fd3c71:::benchmark_data_smithi052_277165_object21433:head v 183'7156, mlcod=183'7154)", 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.060034+0000", 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0038429919999999999, 2023-09-23T22:43:52.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042859329999999996, 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060034+0000", 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060034+0000", 2023-09-23T22:43:52.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060034+0000", 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060042+0000", 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2290000000000003e-06 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060089+0000", 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7465e-05 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060097+0000", 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7500000000000003e-06 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060137+0000", 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0367999999999997e-05 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.060162+0000", 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4479000000000001e-05 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113928 7.9 7:9ec20095:::benchmark_data_smithi052_277165_object21436:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.063495+0000", 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.000382295, 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00085861100000000001, 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113928 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063495+0000", 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063495+0000", 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063494+0000", 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063499+0000", 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7039999999999999e-06 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063520+0000", 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0310999999999999e-05 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063529+0000", 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1300000000000007e-06 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063679+0000", 2023-09-23T22:43:52.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014991700000000001 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063800+0000", 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121699 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063894+0000", 2023-09-23T22:43:52.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3257000000000005e-05 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.35001.0:113922 7.4 e183/146)", 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.063558+0000", 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.00031892500000000001, 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083358499999999997, 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:43:52.117 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063558+0000", 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063558+0000", 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063558+0000", 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063562+0000", 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.827e-06 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063570+0000", 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9980000000000003e-06 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063576+0000", 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8039999999999998e-06 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:113912 7.6 7:60ee5df5:::benchmark_data_smithi052_277165_object21420:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.034326+0000", 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.029551452999999998, 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.030094778999999999, 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 113912 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.121 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.034326+0000", 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.034326+0000", 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.034325+0000", 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.034331+0000", 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2079999999999997e-06 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.034342+0000", 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0975000000000001e-05 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.034349+0000", 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4490000000000002e-06 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.036475+0000", 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021255649999999998 2023-09-23T22:43:52.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.036663+0000", 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018872999999999999 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.036840+0000", 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000176119 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.040279+0000", 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034397500000000001 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.051555+0000", 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011275294 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:113926 7.e e183/146 7:7e2fcf3d:::benchmark_data_smithi052_277165_object21434:head v 183'7157, mlcod=183'7154)", 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.063887+0000", 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4294967295.99999, 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00058049399999999998, 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:43:52.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063887+0000", 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063887+0000", 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063887+0000", 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063895+0000", 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7619999999999995e-06 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:43:52.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063905+0000", 2023-09-23T22:43:52.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0001e-05 2023-09-23T22:43:52.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:43:52.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063911+0000", 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7139999999999996e-06 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063958+0000", 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6947999999999998e-05 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:43:52.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.063977+0000", 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9043e-05 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 11 2023-09-23T22:43:52.132 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:43:52.132 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:52.214 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:52.964 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:53.076 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:53.160 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:53 smithi052 ceph-mon[187743]: pgmap v443: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8026/217665 objects degraded (3.687%); 6.7 MiB/s, 116 objects/s recovering 2023-09-23T22:43:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:53 smithi052 ceph-mon[180979]: pgmap v443: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8026/217665 objects degraded (3.687%); 6.7 MiB/s, 116 objects/s recovering 2023-09-23T22:43:53.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:43:53.434 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:53.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:53.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:53 smithi151 ceph-mon[159210]: pgmap v443: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8026/217665 objects degraded (3.687%); 6.7 MiB/s, 116 objects/s recovering 2023-09-23T22:43:53.536 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:53.629 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:43:53.630 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:43:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:43:53] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-09-23T22:43:53.651 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:53.801 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:53.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:53.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:53.956 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.011 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:54.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:54.117 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:43:54.117 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:54.157 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:43:54.201 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:43:54.249 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.250 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:43:54.295 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:54.300 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:43:54.348 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:43:54.394 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:43:54.422 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:54.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:54.430 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:43:54.446 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.447 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:43:54.486 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/981992921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:54.489 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/981992921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:54.531 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:54.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/981992921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:54.566 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.619 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:54.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:54.726 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:54.781 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.836 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:54.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:54.940 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:54.966 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:55.009 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:55.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:55.114 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:55.143 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:55.191 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:55.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:55.294 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:55.343 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:55.407 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:55.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:55.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:55.445 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: pgmap v444: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5993/213249 objects degraded (2.810%); 1229/213249 objects misplaced (0.576%); 6.8 MiB/s, 117 objects/s recovering 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/213249 objects degraded (2.810%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: pgmap v444: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5993/213249 objects degraded (2.810%); 1229/213249 objects misplaced (0.576%); 6.8 MiB/s, 117 objects/s recovering 2023-09-23T22:43:55.446 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T22:43:55.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.447 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/213249 objects degraded (2.810%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:55.515 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:55.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: pgmap v444: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5993/213249 objects degraded (2.810%); 1229/213249 objects misplaced (0.576%); 6.8 MiB/s, 117 objects/s recovering 2023-09-23T22:43:55.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T22:43:55.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: failed to encode map e184 with expected crc 2023-09-23T22:43:55.559 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/213249 objects degraded (2.810%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:55.566 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:55.620 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:55.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:55.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:55.726 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:55.741 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:55.853 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:56.029 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:56.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:56.069 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:43:56.070 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:43:56.070 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T22:43:56.070 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:43:56.070 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:43:56.070 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T22:43:56.142 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:56.170 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:56.220 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:56.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:56.325 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:56.355 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:56.420 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:56.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:56.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:56.427 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:56 smithi151 ceph-mon[159210]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.427 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:56 smithi151 ceph-mon[159210]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T22:43:56.427 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:56 smithi151 ceph-mon[159210]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.427 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:56 smithi151 ceph-mon[159210]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.528 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:56.580 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[187743]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[187743]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T22:43:56.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[187743]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[187743]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[180979]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[180979]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T22:43:56.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[180979]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:56 smithi052 ceph-mon[180979]: failed to encode map e185 with expected crc 2023-09-23T22:43:56.594 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:56.939 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:56.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:57.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:57.146 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:57.348 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:57.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:57.457 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:57.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:57 smithi052 ceph-mon[187743]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 5993/208578 objects degraded (2.873%); 1229/208578 objects misplaced (0.589%); 6.2 MiB/s, 223 objects/s recovering 2023-09-23T22:43:57.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:57 smithi052 ceph-mon[180979]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 5993/208578 objects degraded (2.873%); 1229/208578 objects misplaced (0.589%); 6.2 MiB/s, 223 objects/s recovering 2023-09-23T22:43:57.523 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:57.573 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:57.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:57.676 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:57.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:57 smithi151 ceph-mon[159210]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 5993/208578 objects degraded (2.873%); 1229/208578 objects misplaced (0.589%); 6.2 MiB/s, 223 objects/s recovering 2023-09-23T22:43:57.710 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:57.787 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:57.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:57.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:57.850 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:57.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:57.996 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:58.104 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:58.176 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:58.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:58.286 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:58.344 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:58.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:58 smithi052 ceph-mon[187743]: pgmap v448: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5993/208014 objects degraded (2.881%); 1229/208014 objects misplaced (0.591%); 4.7 MiB/s, 169 objects/s recovering 2023-09-23T22:43:58.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:58 smithi052 ceph-mon[180979]: pgmap v448: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5993/208014 objects degraded (2.881%); 1229/208014 objects misplaced (0.591%); 4.7 MiB/s, 169 objects/s recovering 2023-09-23T22:43:58.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:58 smithi151 ceph-mon[159210]: pgmap v448: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5993/208014 objects degraded (2.881%); 1229/208014 objects misplaced (0.591%); 4.7 MiB/s, 169 objects/s recovering 2023-09-23T22:43:58.787 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:58.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:58.892 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:58.920 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:58.970 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:58.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:59.074 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:59.104 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:59.131 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:59.132 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:59.172 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:59.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:59.281 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:59.317 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:59.337 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:43:59.357 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2006268920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:59.357 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:43:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:59.359 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2006268920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:59.359 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:43:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:59.368 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:59.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2006268920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:43:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:59.475 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:59.553 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:00.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: failed to encode map e186 with expected crc 2023-09-23T22:44:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T22:44:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: failed to encode map e186 with expected crc 2023-09-23T22:44:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/208014 objects degraded (2.881%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:00 smithi151 ceph-mon[159210]: pgmap v450: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 5993/202860 objects degraded (2.954%); 892/202860 objects misplaced (0.440%); 3.5 MiB/s, 165 objects/s recovering 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: failed to encode map e186 with expected crc 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: failed to encode map e186 with expected crc 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/208014 objects degraded (2.881%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[187743]: pgmap v450: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 5993/202860 objects degraded (2.954%); 892/202860 objects misplaced (0.440%); 3.5 MiB/s, 165 objects/s recovering 2023-09-23T22:44:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:00.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: failed to encode map e186 with expected crc 2023-09-23T22:44:00.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:00.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T22:44:00.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: failed to encode map e186 with expected crc 2023-09-23T22:44:00.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/208014 objects degraded (2.881%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:00.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:00 smithi052 ceph-mon[180979]: pgmap v450: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 5993/202860 objects degraded (2.954%); 892/202860 objects misplaced (0.440%); 3.5 MiB/s, 165 objects/s recovering 2023-09-23T22:44:01.359 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:44:01.593 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:01.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:01.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:01.632 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:01.706 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:01.737 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:01.795 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:01.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:01.902 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:01.996 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:02.124 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:02.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:02.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:02.240 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:02.269 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:02.321 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:02.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:02.338 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:44:02.338 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:44:02.339 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:44:02.395 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:02.429 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:02.519 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:03 smithi151 ceph-mon[159210]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 5993/199512 objects degraded (3.004%); 892/199512 objects misplaced (0.447%); 2.7 MiB/s, 126 objects/s recovering 2023-09-23T22:44:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:03 smithi052 ceph-mon[180979]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 5993/199512 objects degraded (3.004%); 892/199512 objects misplaced (0.447%); 2.7 MiB/s, 126 objects/s recovering 2023-09-23T22:44:03.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:03 smithi052 ceph-mon[187743]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 5993/199512 objects degraded (3.004%); 892/199512 objects misplaced (0.447%); 2.7 MiB/s, 126 objects/s recovering 2023-09-23T22:44:03.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:44:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:44:03] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-09-23T22:44:03.677 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:44:03.769 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:03.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:03.876 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:03.910 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:03.974 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:03.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:04.086 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:04.120 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:04.715 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:04.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:04.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:04.821 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:04.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:05.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:05 smithi052 ceph-mon[180979]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 5993/197220 objects degraded (3.039%); 892/197220 objects misplaced (0.452%); 2.4 MiB/s, 112 objects/s recovering 2023-09-23T22:44:05.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:05.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/197220 objects degraded (3.039%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:05 smithi052 ceph-mon[187743]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 5993/197220 objects degraded (3.039%); 892/197220 objects misplaced (0.452%); 2.4 MiB/s, 112 objects/s recovering 2023-09-23T22:44:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:05.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/197220 objects degraded (3.039%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:05 smithi151 ceph-mon[159210]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 5993/197220 objects degraded (3.039%); 892/197220 objects misplaced (0.452%); 2.4 MiB/s, 112 objects/s recovering 2023-09-23T22:44:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:05.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/197220 objects degraded (3.039%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:05.499 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:44:05.499 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:44:05.499 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:44:05.554 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:05.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:05.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:05.608 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:44:05.608 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:44:05.608 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:44:05.615 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:44:05.633 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.635 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:44:05.660 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:05.693 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:05.756 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:05.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:05.861 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:05.964 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:06.100 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:06.186 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:06.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:06.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3347545524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:06.253 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3347545524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:06.302 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:06.338 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:06.397 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:06.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:06.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:06.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3347545524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:06.505 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:06.592 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:06.797 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:06.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:06.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:06.904 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:07.006 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:07.137 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:07.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:07.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:07.241 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:07.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:3 6.9 6:977e6172:::benchmark_data_smithi052_234456_object2:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.899530+0000", 2023-09-23T22:44:07.395 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48999954199999, 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.016604193, 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 3 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.396 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899530+0000", 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899530+0000", 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899527+0000", 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:44:07.397 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899606+0000", 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8311000000000003e-05 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899620+0000", 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3997e-05 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.398 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899633+0000", 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3023000000000001e-05 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921844+0000", 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022211709 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.399 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921861+0000", 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6385999999999999e-05 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.903617+0000", 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98175606299999996 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.903777+0000", 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015961799999999999 2023-09-23T22:44:07.400 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.903919+0000", 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000142418 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912920+0000", 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090012289999999995 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.401 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.915972+0000", 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030520180000000001 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916033+0000", 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0594999999999998e-05 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916134+0000", 2023-09-23T22:44:07.402 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000100994 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:4 6.9 6:966b311b:::benchmark_data_smithi052_234456_object3:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.899651+0000", 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.489878331, 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0166748489999999, 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.403 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 4 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899651+0000", 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.404 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899651+0000", 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899650+0000", 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.405 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899721+0000", 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0710999999999994e-05 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899731+0000", 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9599999999999995e-06 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.899740+0000", 2023-09-23T22:44:07.406 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0289999999999994e-06 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921883+0000", 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022143121000000002 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921888+0000", 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7880000000000003e-06 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.407 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904432+0000", 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98254400399999997 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904566+0000", 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013371999999999999 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.408 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904707+0000", 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014122700000000001 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912936+0000", 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0082280860000000008 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916177+0000", 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003241945 2023-09-23T22:44:07.409 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916244+0000", 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6903999999999995e-05 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916326+0000", 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1295e-05 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.410 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10 6.9 6:93a199a5:::benchmark_data_smithi052_234456_object9:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48951582199999, 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0164958529999999, 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.411 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.412 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900013+0000", 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900062+0000", 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9654000000000003e-05 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.413 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900074+0000", 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1392999999999999e-05 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900088+0000", 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4338999999999999e-05 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.414 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921910+0000", 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021822426999999998 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921916+0000", 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3680000000000001e-06 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.904977+0000", 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98306100500000004 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.415 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905090+0000", 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113242 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905219+0000", 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012862999999999999 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.416 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912941+0000", 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0077225569999999997 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916368+0000", 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003426751 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916415+0000", 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6869000000000001e-05 2023-09-23T22:44:07.417 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916509+0000", 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4359999999999998e-05 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:16 6.9 6:9e1b8017:::benchmark_data_smithi052_234456_object15:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.418 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900101+0000", 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48942850399999, 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0165771290000001, 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 16 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.419 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900101+0000", 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900101+0000", 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900100+0000", 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.420 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900145+0000", 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5096999999999999e-05 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900153+0000", 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7009999999999996e-06 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.421 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900158+0000", 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6999999999999999e-06 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921938+0000", 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021780167 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921943+0000", 2023-09-23T22:44:07.422 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5969999999999999e-06 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905456+0000", 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98351275599999999 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905557+0000", 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010088 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.423 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905686+0000", 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0001292 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912946+0000", 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072599969999999998 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.424 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916550+0000", 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036041070000000001 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916592+0000", 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1462999999999999e-05 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916678+0000", 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6054e-05 2023-09-23T22:44:07.425 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:22 6.0 6:02bff156:::benchmark_data_smithi052_234456_object21:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.489361836, 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.025782167, 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.426 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 22 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.427 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900167+0000", 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900206+0000", 2023-09-23T22:44:07.428 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8720000000000002e-05 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900210+0000", 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9269999999999996e-06 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900214+0000", 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8650000000000003e-06 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923636+0000", 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023421252 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923641+0000", 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5260000000000001e-06 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920207+0000", 2023-09-23T22:44:07.430 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.996566222 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920350+0000", 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014280299999999999 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920498+0000", 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014755399999999999 2023-09-23T22:44:07.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925223+0000", 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047253130000000001 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925814+0000", 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059094599999999996 2023-09-23T22:44:07.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925870+0000", 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6121e-05 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925949+0000", 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9439999999999998e-05 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.433 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:23 6.0 6:0341baf4:::benchmark_data_smithi052_234456_object22:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900224+0000", 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48930547099999, 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0258131610000001, 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.434 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 23 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900224+0000", 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900224+0000", 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900223+0000", 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900269+0000", 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5688e-05 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900275+0000", 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9619999999999998e-06 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900280+0000", 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.575e-06 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923664+0000", 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023383731000000001 2023-09-23T22:44:07.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923668+0000", 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.484e-06 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920864+0000", 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.997196254 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920973+0000", 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000108769 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921106+0000", 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013302600000000001 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.439 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925254+0000", 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041479220000000001 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925959+0000", 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00070547000000000003 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925993+0000", 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3612000000000003e-05 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926037+0000", 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4141999999999999e-05 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:24 6.0 6:0b99a54d:::benchmark_data_smithi052_234456_object23:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48923805699999, 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.026286555, 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 24 2023-09-23T22:44:07.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900291+0000", 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900328+0000", 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7772999999999999e-05 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900334+0000", 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2070000000000003e-06 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900337+0000", 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6550000000000002e-06 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923686+0000", 2023-09-23T22:44:07.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023348694 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923692+0000", 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.818e-06 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921474+0000", 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.997782736 2023-09-23T22:44:07.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921590+0000", 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011517 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921733+0000", 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143277 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925283+0000", 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035501679999999998 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926440+0000", 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011565950000000001 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926524+0000", 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4742999999999998e-05 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926578+0000", 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3273999999999998e-05 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.449 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:28 6.9 6:9a0dcc21:::benchmark_data_smithi052_234456_object27:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48918373199999, 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0164962040000001, 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 28 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900345+0000", 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900385+0000", 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0535999999999999e-05 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900391+0000", 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1619999999999999e-06 2023-09-23T22:44:07.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900394+0000", 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6370000000000001e-06 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921968+0000", 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021573569000000001 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921975+0000", 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6569999999999997e-06 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.905926+0000", 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98395083999999999 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906026+0000", 2023-09-23T22:44:07.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9716000000000005e-05 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906162+0000", 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000136558 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912952+0000", 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0067891499999999999 2023-09-23T22:44:07.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916720+0000", 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037686159999999998 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916761+0000", 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1146999999999998e-05 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.916842+0000", 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0220000000000001e-05 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:45 6.9 6:93672be9:::benchmark_data_smithi052_234456_object44:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900401+0000", 2023-09-23T22:44:07.457 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48912808, 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0188872680000001, 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 45 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900401+0000", 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900401+0000", 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900400+0000", 2023-09-23T22:44:07.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900442+0000", 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1789999999999998e-05 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900447+0000", 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8130000000000003e-06 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900456+0000", 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9109999999999999e-06 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.921997+0000", 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021541227 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922586+0000", 2023-09-23T22:44:07.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000589358 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906367+0000", 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98378031200000005 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906447+0000", 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0572e-05 2023-09-23T22:44:07.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906575+0000", 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000127472 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912957+0000", 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063821770000000002 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919146+0000", 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0061887330000000001 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919195+0000", 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9317000000000002e-05 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919288+0000", 2023-09-23T22:44:07.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3289999999999996e-05 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:72 6.0 6:029c92c4:::benchmark_data_smithi052_234456_object71:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.489065719, 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0262414900000001, 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.465 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900463+0000", 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900507+0000", 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4363000000000002e-05 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900515+0000", 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6760000000000007e-06 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900520+0000", 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6700000000000002e-06 2023-09-23T22:44:07.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923713+0000", 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023193394999999999 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923716+0000", 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.083e-06 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.921995+0000", 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99827871099999999 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922138+0000", 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014364600000000001 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922288+0000", 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014937699999999999 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.925313+0000", 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030248839999999998 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926586+0000", 2023-09-23T22:44:07.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012736679999999999 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926632+0000", 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6100999999999998e-05 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926705+0000", 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2418999999999999e-05 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.472 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:75 6.0 6:0ddd5ad2:::benchmark_data_smithi052_234456_object74:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.488998111, 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.027114453, 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.473 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900531+0000", 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900613+0000", 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2361999999999995e-05 2023-09-23T22:44:07.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900618+0000", 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5040000000000002e-06 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900622+0000", 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9149999999999998e-06 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923738+0000", 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023115244 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923741+0000", 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0400000000000001e-06 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922543+0000", 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.998801936 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922656+0000", 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011299 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.922796+0000", 2023-09-23T22:44:07.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014071299999999999 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926713+0000", 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039166870000000003 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927511+0000", 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079828500000000003 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927558+0000", 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6536000000000003e-05 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927645+0000", 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7645000000000005e-05 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:87 6.9 6:9eaa18f5:::benchmark_data_smithi052_234456_object86:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.488898654, 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0188088259999999, 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 87 2023-09-23T22:44:07.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900630+0000", 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900674+0000", 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3816000000000002e-05 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900681+0000", 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5780000000000001e-06 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900686+0000", 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2640000000000002e-06 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922616+0000", 2023-09-23T22:44:07.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021929976 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922623+0000", 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5390000000000002e-06 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906760+0000", 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.984136699 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906861+0000", 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010134200000000001 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.906990+0000", 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012864899999999999 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912962+0000", 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059724019999999999 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919330+0000", 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.006368243 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919371+0000", 2023-09-23T22:44:07.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0352000000000003e-05 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919439+0000", 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8473000000000007e-05 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.488 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:90 6.0 6:030f5829:::benchmark_data_smithi052_234456_object89:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900695+0000", 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.488834325, 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0271018009999999, 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 90 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900695+0000", 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900695+0000", 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900694+0000", 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900734+0000", 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9341999999999997e-05 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900738+0000", 2023-09-23T22:44:07.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7639999999999996e-06 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900742+0000", 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2909999999999999e-06 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923763+0000", 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023021491000000002 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923766+0000", 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0070000000000002e-06 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923164+0000", 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99939733799999997 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923299+0000", 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000135239 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923470+0000", 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017162899999999999 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926718+0000", 2023-09-23T22:44:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032476250000000001 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927680+0000", 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00096186999999999998 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927718+0000", 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7710000000000003e-05 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927797+0000", 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9010000000000004e-05 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:92 6.9 6:9807eb97:::benchmark_data_smithi052_234456_object91:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:44:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.488762681, 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.01884157, 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900766+0000", 2023-09-23T22:44:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900824+0000", 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8517999999999998e-05 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900831+0000", 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0739999999999998e-06 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900835+0000", 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3209999999999998e-06 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922653+0000", 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021818273999999999 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922660+0000", 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9940000000000003e-06 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907201+0000", 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98454060200000004 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907305+0000", 2023-09-23T22:44:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103848 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907388+0000", 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3669000000000001e-05 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912967+0000", 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055790249999999996 2023-09-23T22:44:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919483+0000", 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065161849999999999 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919528+0000", 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4042000000000003e-05 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919608+0000", 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0524000000000004e-05 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:93 6.0 6:0f027d47:::benchmark_data_smithi052_234456_object92:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e126)", 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900843+0000", 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48868608399999, 2023-09-23T22:44:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0271189679999999, 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 93 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900843+0000", 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900843+0000", 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900842+0000", 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900881+0000", 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8917e-05 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900903+0000", 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1769000000000001e-05 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900909+0000", 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0510000000000002e-06 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923785+0000", 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022875933000000001 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.923788+0000", 2023-09-23T22:44:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9119999999999998e-06 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923720+0000", 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99993241899999996 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.923845+0000", 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012472899999999999 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3", 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.924026+0000", 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000180886 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.926726+0000", 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026996870000000001 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927834+0000", 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011082279999999999 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927883+0000", 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9249999999999998e-05 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.927962+0000", 2023-09-23T22:44:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8947000000000002e-05 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:101 6.9 6:9ca0677b:::benchmark_data_smithi052_234456_object100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:49.900930+0000", 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 197.48859942000001, 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0190877739999999, 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900930+0000", 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900930+0000", 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900929+0000", 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.900998+0000", 2023-09-23T22:44:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8745000000000007e-05 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901031+0000", 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3470999999999999e-05 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.901062+0000", 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0997000000000002e-05 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922689+0000", 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021626510000000002 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:49.922695+0000", 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.913e-06 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907555+0000", 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98486037800000004 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907675+0000", 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119653 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 7", 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.907764+0000", 2023-09-23T22:44:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8918000000000005e-05 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.912972+0000", 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052083060000000002 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919650+0000", 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0066783859999999997 2023-09-23T22:44:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.919692+0000", 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1408000000000001e-05 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:50.920017+0000", 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032568300000000001 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40807 6.9 6:9345ac67:::benchmark_data_smithi052_234456_object18663:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861640+0000", 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 177.52788963200001, 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2337652800000001, 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40807 2023-09-23T22:44:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861640+0000", 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861640+0000", 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861637+0000", 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861648+0000", 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0575e-05 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861672+0000", 2023-09-23T22:44:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4164000000000001e-05 2023-09-23T22:44:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861684+0000", 2023-09-23T22:44:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2551999999999999e-05 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861747+0000", 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2843999999999997e-05 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861755+0000", 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2349999999999999e-06 2023-09-23T22:44:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.868387+0000", 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0066316300000000003 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.868400+0000", 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.29e-05 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858560+0000", 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99016036799999996 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.858575+0000", 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4430000000000001e-05 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.086138+0000", 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2275634580000001 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.086287+0000", 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014845 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.086427+0000", 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014055599999999999 2023-09-23T22:44:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.088326+0000", 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018989790000000001 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.094860+0000", 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065336420000000001 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.095008+0000", 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014823399999999999 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.095077+0000", 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8437999999999998e-05 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.095405+0000", 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032823699999999999 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40811 6.0 6:02d50e34:::benchmark_data_smithi052_234456_object18667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.877262+0000", 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 177.51226728099999, 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.226087551, 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40811 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877262+0000", 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877262+0000", 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877260+0000", 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877267+0000", 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1940000000000001e-06 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877280+0000", 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2565000000000001e-05 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877288+0000", 2023-09-23T22:44:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2060000000000007e-06 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877521+0000", 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000232687 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.877533+0000", 2023-09-23T22:44:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1844999999999999e-05 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.923517+0000", 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0459847 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.923529+0000", 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1555e-05 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.097078+0000", 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.173549138 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.097281+0000", 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000202773 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:44:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.097491+0000", 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000210132 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.101496+0000", 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040048030000000004 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.101536+0000", 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0089000000000002e-05 2023-09-23T22:44:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.103009+0000", 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014735950000000001 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.103064+0000", 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4772999999999999e-05 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:12.103349+0000", 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00028529100000000002 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:61218 7.4 7:2b5a9e4b:::benchmark_data_smithi052_277165_object61217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:33.069073+0000", 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 94.320456195000006, 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.234406752, 2023-09-23T22:44:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 61218 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069073+0000", 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069073+0000", 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069072+0000", 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069096+0000", 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3374999999999999e-05 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069105+0000", 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069112+0000", 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6520000000000006e-06 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069170+0000", 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7989000000000003e-05 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069309+0000", 2023-09-23T22:44:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013882200000000001 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.069416+0000", 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106835 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.071824+0000", 2023-09-23T22:44:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024078839999999999 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.072114+0000", 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029004299999999997 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.303425+0000", 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23131093699999999 2023-09-23T22:44:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.303452+0000", 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6669000000000002e-05 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.303480+0000", 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8161999999999999e-05 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:61232 7.4 7:285c487d:::benchmark_data_smithi052_277165_object61231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:33.076027+0000", 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 94.313501885999997, 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24089580699999999, 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 61232 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076027+0000", 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076027+0000", 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076026+0000", 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076048+0000", 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2599000000000002e-05 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076073+0000", 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4434e-05 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076080+0000", 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3089999999999998e-06 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.076121+0000", 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1247e-05 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.301292+0000", 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22517095100000001 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:44:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.301455+0000", 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016304199999999999 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.307792+0000", 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063368160000000003 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.307829+0000", 2023-09-23T22:44:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6993000000000003e-05 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.316826+0000", 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089964959999999997 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.316881+0000", 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5436000000000003e-05 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.316923+0000", 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2123000000000003e-05 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:07.556 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:44:07.557 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:07 smithi151 ceph-mon[159210]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5993/190902 objects degraded (3.139%); 530/190902 objects misplaced (0.278%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:07.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:07 smithi052 ceph-mon[180979]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5993/190902 objects degraded (3.139%); 530/190902 objects misplaced (0.278%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:07.557 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:07 smithi052 ceph-mon[187743]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5993/190902 objects degraded (3.139%); 530/190902 objects misplaced (0.278%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:08.057 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:08.172 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:08.200 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.256 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:08.369 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:08.395 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.437 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:08.545 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:08.588 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.661 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:08.776 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:08.808 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.861 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:44:08.876 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:08.890 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:44:08.982 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:09.021 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.077 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:09.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:09.203 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:09 smithi052 ceph-mon[180979]: pgmap v454: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 5993/190293 objects degraded (3.149%); 530/190293 objects misplaced (0.279%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:09.218 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.270 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:09.375 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:09.407 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.467 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:09.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:09 smithi052 ceph-mon[187743]: pgmap v454: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 5993/190293 objects degraded (3.149%); 530/190293 objects misplaced (0.279%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:09.470 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:09 smithi151 ceph-mon[159210]: pgmap v454: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 5993/190293 objects degraded (3.149%); 530/190293 objects misplaced (0.279%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:09.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:09.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:09.611 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.615 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:44:09.615 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:44:09.615 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-23T22:44:09.615 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:44:09.667 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:09.771 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:09.880 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.901 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:10.029 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:10.096 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:10.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:10.202 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:10.296 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:10.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/190293 objects degraded (3.149%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:10.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3097826675' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:10.410 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:10.411 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/190293 objects degraded (3.149%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:10.411 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3097826675' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:10.412 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:10.510 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:10.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:10.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:10.617 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:10.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/190293 objects degraded (3.149%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:10.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3097826675' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:10.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:10.727 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:10.844 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:44:11.354 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:11.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:11.363 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[187743]: pgmap v455: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5993/184173 objects degraded (3.254%); 200/184173 objects misplaced (0.109%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:44:11.363 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:11.364 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:11.364 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[187743]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T22:44:11.366 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[180979]: pgmap v455: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5993/184173 objects degraded (3.254%); 200/184173 objects misplaced (0.109%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:44:11.367 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:11.367 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:11.367 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:11 smithi052 ceph-mon[180979]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T22:44:11.463 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:11.487 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:11 smithi151 ceph-mon[159210]: pgmap v455: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5993/184173 objects degraded (3.254%); 200/184173 objects misplaced (0.109%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:44:11.487 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:11 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:11.487 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:11.487 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:11 smithi151 ceph-mon[159210]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T22:44:11.489 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:11.530 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:11.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:11.635 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:11.669 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:11.745 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:11.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:11.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:11.849 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:11.929 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:12.409 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:12.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:12.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:12.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[180979]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[180979]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[180979]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[180979]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[187743]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[187743]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[187743]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:12 smithi052 ceph-mon[187743]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.520 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:12.543 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:12 smithi151 ceph-mon[159210]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.543 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:12 smithi151 ceph-mon[159210]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.543 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:12 smithi151 ceph-mon[159210]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.544 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:12 smithi151 ceph-mon[159210]: failed to encode map e187 with expected crc 2023-09-23T22:44:12.545 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:12.589 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:12.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:12.695 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:12.730 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:12.789 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:12.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:12.896 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:12.927 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:13.003 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:13.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:13.110 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:13.204 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:13.222 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:13.347 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:13 smithi052 ceph-mon[187743]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 5993/180768 objects degraded (3.315%); 200/180768 objects misplaced (0.111%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:44:13.350 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:13 smithi052 ceph-mon[180979]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 5993/180768 objects degraded (3.315%); 200/180768 objects misplaced (0.111%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:44:13.350 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:44:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:44:13] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.33.4" 2023-09-23T22:44:13.365 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:13.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:13.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:13.469 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:13.586 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:13.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:13 smithi151 ceph-mon[159210]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 5993/180768 objects degraded (3.315%); 200/180768 objects misplaced (0.111%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:44:13.840 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:13.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:13.947 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:13.955 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:44:13.985 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:44:13.997 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:14.025 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:44:14.061 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:14.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:14.073 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:44:14.113 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:44:14.167 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:14.185 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:44:14.198 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:44:14.242 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:44:14.244 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:14.244 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:14.264 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4112801525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.271 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:44:14.297 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:14.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4112801525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:14.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4112801525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:14.868 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:14.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:14.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:14.977 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:15.046 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:15.173 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:44:15.234 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:15.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:15.349 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:15.483 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:15.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:15 smithi052 ceph-mon[187743]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5993/178500 objects degraded (3.357%); 200/178500 objects misplaced (0.112%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:44:15.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/178500 objects degraded (3.357%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:15.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:15 smithi052 ceph-mon[180979]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5993/178500 objects degraded (3.357%); 200/178500 objects misplaced (0.112%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:44:15.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/178500 objects degraded (3.357%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:15.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:15 smithi151 ceph-mon[159210]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5993/178500 objects degraded (3.357%); 200/178500 objects misplaced (0.112%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:44:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/178500 objects degraded (3.357%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:16.399 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:16.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:16.517 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:16.562 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:16.618 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:16.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:16.724 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:16.742 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:44:16.839 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:17.030 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:17.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:17.078 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:17.138 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:17.177 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:17.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:17.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:17.291 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:17 smithi052 ceph-mon[180979]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5993/172086 objects degraded (3.483%); 3.3 MiB/s, 73 objects/s recovering 2023-09-23T22:44:17.294 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:17 smithi052 ceph-mon[187743]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5993/172086 objects degraded (3.483%); 3.3 MiB/s, 73 objects/s recovering 2023-09-23T22:44:17.344 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:17.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:17.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:17.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:17 smithi151 ceph-mon[159210]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5993/172086 objects degraded (3.483%); 3.3 MiB/s, 73 objects/s recovering 2023-09-23T22:44:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:17.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:17.550 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:17.653 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:17.979 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:17.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:44:18.038 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:44:18.091 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:18.128 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:18.197 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:18.322 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:18.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:18.429 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:18.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:18 smithi052 ceph-mon[180979]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:18.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:18 smithi052 ceph-mon[180979]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5993/171492 objects degraded (3.495%); 3.3 MiB/s, 73 objects/s recovering 2023-09-23T22:44:18.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:18 smithi052 ceph-mon[187743]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:18.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:18 smithi052 ceph-mon[187743]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5993/171492 objects degraded (3.495%); 3.3 MiB/s, 73 objects/s recovering 2023-09-23T22:44:18.502 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:18.591 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:44:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:18 smithi151 ceph-mon[159210]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:18 smithi151 ceph-mon[159210]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 5993/171492 objects degraded (3.495%); 3.3 MiB/s, 73 objects/s recovering 2023-09-23T22:44:19.748 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":459,"stamp":"2023-09-23T22:44:16.088871+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3758233528,"num_objects":57362,"num_object_clones":0,"num_object_copies":172086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5993,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57362,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":127902,"num_write_kb":5921742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8177,"num_bytes_recovered":471083006,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125136,"ondisk_log_size":125136,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13253488,"kb_used_data":11505136,"kb_used_omap":126,"kb_used_meta":1747521,"kb_avail":736707728,"statfs":{"total":767960285184,"available":754388713472,"internally_reserved":0,"allocated":11781259264,"data_stored":11776300975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1789462139},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,3,6,8,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-411041792,"num_objects":-6272,"num_object_clones":0,"num_object_copies":-18816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-530,"num_objects_unfound":0,"num_objects_dirty":-6272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6272,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":740,"num_bytes_recovered":34734080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009001"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":184,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314892+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:43:59.314892+0000","last_peered":"2023-09-23T22:43:59.314892+0000","last_clean":"2023-09-23T22:43:59.314892+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:43:59.314892+0000","last_undegraded":"2023-09-23T22:43:59.314892+0000","last_fullsized":"2023-09-23T22:43:59.314892+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.319599+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:43:59.319599+0000","last_peered":"2023-09-23T22:43:59.319599+0000","last_clean":"2023-09-23T22:43:59.319599+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:43:59.319599+0000","last_undegraded":"2023-09-23T22:43:59.319599+0000","last_fullsized":"2023-09-23T22:43:59.319599+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":142,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.318907+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:43:59.318907+0000","last_peered":"2023-09-23T22:43:59.318907+0000","last_clean":"2023-09-23T22:43:59.318907+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:43:59.318907+0000","last_undegraded":"2023-09-23T22:43:59.318907+0000","last_fullsized":"2023-09-23T22:43:59.318907+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":144,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314864+0000","last_change":"2023-09-23T22:43:38.409378+0000","last_active":"2023-09-23T22:43:59.314864+0000","last_peered":"2023-09-23T22:43:59.314864+0000","last_clean":"2023-09-23T22:43:59.314864+0000","last_became_active":"2023-09-23T22:43:38.409027+0000","last_became_peered":"2023-09-23T22:43:38.409027+0000","last_unstale":"2023-09-23T22:43:59.314864+0000","last_undegraded":"2023-09-23T22:43:59.314864+0000","last_fullsized":"2023-09-23T22:43:59.314864+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:16.178843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":201,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.323102+0000","last_change":"2023-09-23T22:40:50.916258+0000","last_active":"2023-09-23T22:43:59.323102+0000","last_peered":"2023-09-23T22:43:59.323102+0000","last_clean":"2023-09-23T22:43:59.323102+0000","last_became_active":"2023-09-23T22:40:50.915948+0000","last_became_peered":"2023-09-23T22:40:50.915948+0000","last_unstale":"2023-09-23T22:43:59.323102+0000","last_undegraded":"2023-09-23T22:43:59.323102+0000","last_fullsized":"2023-09-23T22:43:59.323102+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:32.210743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.310200+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:43:59.310200+0000","last_peered":"2023-09-23T22:43:59.310200+0000","last_clean":"2023-09-23T22:43:59.310200+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:43:59.310200+0000","last_undegraded":"2023-09-23T22:43:59.310200+0000","last_fullsized":"2023-09-23T22:43:59.310200+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":124,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.318468+0000","last_change":"2023-09-23T22:40:50.926694+0000","last_active":"2023-09-23T22:43:59.318468+0000","last_peered":"2023-09-23T22:43:59.318468+0000","last_clean":"2023-09-23T22:43:59.318468+0000","last_became_active":"2023-09-23T22:40:50.926142+0000","last_became_peered":"2023-09-23T22:40:50.926142+0000","last_unstale":"2023-09-23T22:43:59.318468+0000","last_undegraded":"2023-09-23T22:43:59.318468+0000","last_fullsized":"2023-09-23T22:43:59.318468+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:53.874735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":184,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.323186+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:43:59.323186+0000","last_peered":"2023-09-23T22:43:59.323186+0000","last_clean":"2023-09-23T22:43:59.323186+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:43:59.323186+0000","last_undegraded":"2023-09-23T22:43:59.323186+0000","last_fullsized":"2023-09-23T22:43:59.323186+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":171,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.311151+0000","last_change":"2023-09-23T22:43:38.401745+0000","last_active":"2023-09-23T22:43:59.311151+0000","last_peered":"2023-09-23T22:43:59.311151+0000","last_clean":"2023-09-23T22:43:59.311151+0000","last_became_active":"2023-09-23T22:43:38.401387+0000","last_became_peered":"2023-09-23T22:43:38.401387+0000","last_unstale":"2023-09-23T22:43:59.311151+0000","last_undegraded":"2023-09-23T22:43:59.311151+0000","last_fullsized":"2023-09-23T22:43:59.311151+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":133,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.319516+0000","last_change":"2023-09-23T22:43:38.275026+0000","last_active":"2023-09-23T22:43:59.319516+0000","last_peered":"2023-09-23T22:43:59.319516+0000","last_clean":"2023-09-23T22:43:59.319516+0000","last_became_active":"2023-09-23T22:43:38.274558+0000","last_became_peered":"2023-09-23T22:43:38.274558+0000","last_unstale":"2023-09-23T22:43:59.319516+0000","last_undegraded":"2023-09-23T22:43:59.319516+0000","last_fullsized":"2023-09-23T22:43:59.319516+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:49.131212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":23,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309083+0000","last_change":"2023-09-23T22:43:38.491757+0000","last_active":"2023-09-23T22:43:59.309083+0000","last_peered":"2023-09-23T22:43:59.309083+0000","last_clean":"2023-09-23T22:43:59.309083+0000","last_became_active":"2023-09-23T22:43:38.402444+0000","last_became_peered":"2023-09-23T22:43:38.402444+0000","last_unstale":"2023-09-23T22:43:59.309083+0000","last_undegraded":"2023-09-23T22:43:59.309083+0000","last_fullsized":"2023-09-23T22:43:59.309083+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.910563+0000","last_change":"2023-09-23T22:41:10.927399+0000","last_active":"2023-09-23T22:44:01.910563+0000","last_peered":"2023-09-23T22:44:01.910563+0000","last_clean":"2023-09-23T22:44:01.910563+0000","last_became_active":"2023-09-23T22:41:10.926366+0000","last_became_peered":"2023-09-23T22:41:10.926366+0000","last_unstale":"2023-09-23T22:44:01.910563+0000","last_undegraded":"2023-09-23T22:44:01.910563+0000","last_fullsized":"2023-09-23T22:44:01.910563+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:54.104169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":198,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315330+0000","last_change":"2023-09-23T22:43:38.653479+0000","last_active":"2023-09-23T22:43:59.315330+0000","last_peered":"2023-09-23T22:43:59.315330+0000","last_clean":"2023-09-23T22:43:59.315330+0000","last_became_active":"2023-09-23T22:43:38.479484+0000","last_became_peered":"2023-09-23T22:43:38.479484+0000","last_unstale":"2023-09-23T22:43:59.315330+0000","last_undegraded":"2023-09-23T22:43:59.315330+0000","last_fullsized":"2023-09-23T22:43:59.315330+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314868+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:43:59.314868+0000","last_peered":"2023-09-23T22:43:59.314868+0000","last_clean":"2023-09-23T22:43:59.314868+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:43:59.314868+0000","last_undegraded":"2023-09-23T22:43:59.314868+0000","last_fullsized":"2023-09-23T22:43:59.314868+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":23,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.307281+0000","last_change":"2023-09-23T22:43:38.498107+0000","last_active":"2023-09-23T22:43:59.307281+0000","last_peered":"2023-09-23T22:43:59.307281+0000","last_clean":"2023-09-23T22:43:59.307281+0000","last_became_active":"2023-09-23T22:43:38.402493+0000","last_became_peered":"2023-09-23T22:43:38.402493+0000","last_unstale":"2023-09-23T22:43:59.307281+0000","last_undegraded":"2023-09-23T22:43:59.307281+0000","last_fullsized":"2023-09-23T22:43:59.307281+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.319945+0000","last_change":"2023-09-23T22:43:38.275350+0000","last_active":"2023-09-23T22:43:59.319945+0000","last_peered":"2023-09-23T22:43:59.319945+0000","last_clean":"2023-09-23T22:43:59.319945+0000","last_became_active":"2023-09-23T22:43:38.275063+0000","last_became_peered":"2023-09-23T22:43:38.275063+0000","last_unstale":"2023-09-23T22:43:59.319945+0000","last_undegraded":"2023-09-23T22:43:59.319945+0000","last_fullsized":"2023-09-23T22:43:59.319945+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:31.113686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":142,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.313122+0000","last_change":"2023-09-23T22:43:38.408783+0000","last_active":"2023-09-23T22:43:59.313122+0000","last_peered":"2023-09-23T22:43:59.313122+0000","last_clean":"2023-09-23T22:43:59.313122+0000","last_became_active":"2023-09-23T22:43:38.408462+0000","last_became_peered":"2023-09-23T22:43:38.408462+0000","last_unstale":"2023-09-23T22:43:59.313122+0000","last_undegraded":"2023-09-23T22:43:59.313122+0000","last_fullsized":"2023-09-23T22:43:59.313122+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:55.591282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309487+0000","last_change":"2023-09-23T22:43:38.401944+0000","last_active":"2023-09-23T22:43:59.309487+0000","last_peered":"2023-09-23T22:43:59.309487+0000","last_clean":"2023-09-23T22:43:59.309487+0000","last_became_active":"2023-09-23T22:43:38.401493+0000","last_became_peered":"2023-09-23T22:43:38.401493+0000","last_unstale":"2023-09-23T22:43:59.309487+0000","last_undegraded":"2023-09-23T22:43:59.309487+0000","last_fullsized":"2023-09-23T22:43:59.309487+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.306900+0000","last_change":"2023-09-23T22:43:38.395054+0000","last_active":"2023-09-23T22:43:59.306900+0000","last_peered":"2023-09-23T22:43:59.306900+0000","last_clean":"2023-09-23T22:43:59.306900+0000","last_became_active":"2023-09-23T22:43:38.394718+0000","last_became_peered":"2023-09-23T22:43:38.394718+0000","last_unstale":"2023-09-23T22:43:59.306900+0000","last_undegraded":"2023-09-23T22:43:59.306900+0000","last_fullsized":"2023-09-23T22:43:59.306900+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":143,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314927+0000","last_change":"2023-09-23T22:43:38.409062+0000","last_active":"2023-09-23T22:43:59.314927+0000","last_peered":"2023-09-23T22:43:59.314927+0000","last_clean":"2023-09-23T22:43:59.314927+0000","last_became_active":"2023-09-23T22:43:38.408560+0000","last_became_peered":"2023-09-23T22:43:38.408560+0000","last_unstale":"2023-09-23T22:43:59.314927+0000","last_undegraded":"2023-09-23T22:43:59.314927+0000","last_fullsized":"2023-09-23T22:43:59.314927+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:24.513511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":216,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.304662+0000","last_change":"2023-09-23T22:42:00.067286+0000","last_active":"2023-09-23T22:43:59.304662+0000","last_peered":"2023-09-23T22:43:59.304662+0000","last_clean":"2023-09-23T22:43:59.304662+0000","last_became_active":"2023-09-23T22:41:59.891905+0000","last_became_peered":"2023-09-23T22:41:59.891905+0000","last_unstale":"2023-09-23T22:43:59.304662+0000","last_undegraded":"2023-09-23T22:43:59.304662+0000","last_fullsized":"2023-09-23T22:43:59.304662+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":175,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.320127+0000","last_change":"2023-09-23T22:43:38.401471+0000","last_active":"2023-09-23T22:43:59.320127+0000","last_peered":"2023-09-23T22:43:59.320127+0000","last_clean":"2023-09-23T22:43:59.320127+0000","last_became_active":"2023-09-23T22:43:38.401122+0000","last_became_peered":"2023-09-23T22:43:38.401122+0000","last_unstale":"2023-09-23T22:43:59.320127+0000","last_undegraded":"2023-09-23T22:43:59.320127+0000","last_fullsized":"2023-09-23T22:43:59.320127+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:45.419642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"187'7959","reported_seq":8010,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:11.867573+0000","last_change":"2023-09-23T22:41:36.969870+0000","last_active":"2023-09-23T22:44:11.867573+0000","last_peered":"2023-09-23T22:44:11.867573+0000","last_clean":"2023-09-23T22:44:11.867573+0000","last_became_active":"2023-09-23T22:41:36.969535+0000","last_became_peered":"2023-09-23T22:41:36.969535+0000","last_unstale":"2023-09-23T22:44:11.867573+0000","last_undegraded":"2023-09-23T22:44:11.867573+0000","last_fullsized":"2023-09-23T22:44:11.867573+0000","mapping_epoch":146,"log_start":"154'500","ondisk_log_start":"154'500","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7459,"log_dups_size":0,"ondisk_log_size":7459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:24.978626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252641280,"num_objects":3855,"num_object_clones":0,"num_object_copies":11565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7959,"num_write_kb":378048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":21,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309545+0000","last_change":"2023-09-23T22:43:38.580186+0000","last_active":"2023-09-23T22:43:59.309545+0000","last_peered":"2023-09-23T22:43:59.309545+0000","last_clean":"2023-09-23T22:43:59.309545+0000","last_became_active":"2023-09-23T22:43:38.413778+0000","last_became_peered":"2023-09-23T22:43:38.413778+0000","last_unstale":"2023-09-23T22:43:59.309545+0000","last_undegraded":"2023-09-23T22:43:59.309545+0000","last_fullsized":"2023-09-23T22:43:59.309545+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309080+0000","last_change":"2023-09-23T22:43:38.402526+0000","last_active":"2023-09-23T22:43:59.309080+0000","last_peered":"2023-09-23T22:43:59.309080+0000","last_clean":"2023-09-23T22:43:59.309080+0000","last_became_active":"2023-09-23T22:43:38.402129+0000","last_became_peered":"2023-09-23T22:43:38.402129+0000","last_unstale":"2023-09-23T22:43:59.309080+0000","last_undegraded":"2023-09-23T22:43:59.309080+0000","last_fullsized":"2023-09-23T22:43:59.309080+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"187'7974","reported_seq":8023,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:14.570418+0000","last_change":"2023-09-23T22:41:36.967245+0000","last_active":"2023-09-23T22:44:14.570418+0000","last_peered":"2023-09-23T22:44:14.570418+0000","last_clean":"2023-09-23T22:44:14.570418+0000","last_became_active":"2023-09-23T22:41:36.966938+0000","last_became_peered":"2023-09-23T22:41:36.966938+0000","last_unstale":"2023-09-23T22:44:14.570418+0000","last_undegraded":"2023-09-23T22:44:14.570418+0000","last_fullsized":"2023-09-23T22:44:14.570418+0000","mapping_epoch":146,"log_start":"154'600","ondisk_log_start":"154'600","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:31.285489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229900288,"num_objects":3508,"num_object_clones":0,"num_object_copies":10524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7974,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":21,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.307434+0000","last_change":"2023-09-23T22:43:38.494018+0000","last_active":"2023-09-23T22:43:59.307434+0000","last_peered":"2023-09-23T22:43:59.307434+0000","last_clean":"2023-09-23T22:43:59.307434+0000","last_became_active":"2023-09-23T22:43:38.402480+0000","last_became_peered":"2023-09-23T22:43:38.402480+0000","last_unstale":"2023-09-23T22:43:59.307434+0000","last_undegraded":"2023-09-23T22:43:59.307434+0000","last_fullsized":"2023-09-23T22:43:59.307434+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":167,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315141+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:43:59.315141+0000","last_peered":"2023-09-23T22:43:59.315141+0000","last_clean":"2023-09-23T22:43:59.315141+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:43:59.315141+0000","last_undegraded":"2023-09-23T22:43:59.315141+0000","last_fullsized":"2023-09-23T22:43:59.315141+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"187'7923","reported_seq":7974,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:13.903879+0000","last_change":"2023-09-23T22:41:36.969607+0000","last_active":"2023-09-23T22:44:13.903879+0000","last_peered":"2023-09-23T22:44:13.903879+0000","last_clean":"2023-09-23T22:44:13.903879+0000","last_became_active":"2023-09-23T22:41:36.969311+0000","last_became_peered":"2023-09-23T22:41:36.969311+0000","last_unstale":"2023-09-23T22:44:13.903879+0000","last_undegraded":"2023-09-23T22:44:13.903879+0000","last_fullsized":"2023-09-23T22:44:13.903879+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7923,"log_dups_size":0,"ondisk_log_size":7923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:27.916919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232194048,"num_objects":3543,"num_object_clones":0,"num_object_copies":10629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7923,"num_write_kb":366912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":21,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309246+0000","last_change":"2023-09-23T22:43:38.477268+0000","last_active":"2023-09-23T22:43:59.309246+0000","last_peered":"2023-09-23T22:43:59.309246+0000","last_clean":"2023-09-23T22:43:59.309246+0000","last_became_active":"2023-09-23T22:43:38.395830+0000","last_became_peered":"2023-09-23T22:43:38.395830+0000","last_unstale":"2023-09-23T22:43:59.309246+0000","last_undegraded":"2023-09-23T22:43:59.309246+0000","last_fullsized":"2023-09-23T22:43:59.309246+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":170,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.304402+0000","last_change":"2023-09-23T22:43:38.396237+0000","last_active":"2023-09-23T22:43:59.304402+0000","last_peered":"2023-09-23T22:43:59.304402+0000","last_clean":"2023-09-23T22:43:59.304402+0000","last_became_active":"2023-09-23T22:43:38.395904+0000","last_became_peered":"2023-09-23T22:43:38.395904+0000","last_unstale":"2023-09-23T22:43:59.304402+0000","last_undegraded":"2023-09-23T22:43:59.304402+0000","last_fullsized":"2023-09-23T22:43:59.304402+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"187'8023","reported_seq":8092,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:44:14.126690+0000","last_change":"2023-09-23T22:43:38.495841+0000","last_active":"2023-09-23T22:44:14.126690+0000","last_peered":"2023-09-23T22:44:14.126690+0000","last_clean":"2023-09-23T22:42:45.119339+0000","last_became_active":"2023-09-23T22:43:38.485142+0000","last_became_peered":"2023-09-23T22:43:38.485142+0000","last_unstale":"2023-09-23T22:44:14.126690+0000","last_undegraded":"2023-09-23T22:43:38.271941+0000","last_fullsized":"2023-09-23T22:43:38.260546+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8023,"log_dups_size":0,"ondisk_log_size":8023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:23:41.274734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234684416,"num_objects":3581,"num_object_clones":0,"num_object_copies":10743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2005,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8023,"num_write_kb":371328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1238},{"shards":"2,4","objects":2343}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":153,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309297+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:43:59.309297+0000","last_peered":"2023-09-23T22:43:59.309297+0000","last_clean":"2023-09-23T22:43:59.309297+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:43:59.309297+0000","last_undegraded":"2023-09-23T22:43:59.309297+0000","last_fullsized":"2023-09-23T22:43:59.309297+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":117,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.313407+0000","last_change":"2023-09-23T22:40:50.917176+0000","last_active":"2023-09-23T22:43:59.313407+0000","last_peered":"2023-09-23T22:43:59.313407+0000","last_clean":"2023-09-23T22:43:59.313407+0000","last_became_active":"2023-09-23T22:40:50.915836+0000","last_became_peered":"2023-09-23T22:40:50.915836+0000","last_unstale":"2023-09-23T22:43:59.313407+0000","last_undegraded":"2023-09-23T22:43:59.313407+0000","last_fullsized":"2023-09-23T22:43:59.313407+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:37.211725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"187'7825","reported_seq":7891,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:44:13.986300+0000","last_change":"2023-09-23T22:43:38.563229+0000","last_active":"2023-09-23T22:44:13.986300+0000","last_peered":"2023-09-23T22:44:13.986300+0000","last_clean":"2023-09-23T22:42:45.086533+0000","last_became_active":"2023-09-23T22:43:38.491175+0000","last_became_peered":"2023-09-23T22:43:38.491175+0000","last_unstale":"2023-09-23T22:44:13.986300+0000","last_undegraded":"2023-09-23T22:43:38.272998+0000","last_fullsized":"2023-09-23T22:43:38.254963+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7825,"log_dups_size":0,"ondisk_log_size":7825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:20.109499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234553344,"num_objects":3579,"num_object_clones":0,"num_object_copies":10737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1973,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7825,"num_write_kb":364928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2375},{"shards":"0,3,4","objects":1204}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":170,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.319857+0000","last_change":"2023-09-23T22:40:50.914941+0000","last_active":"2023-09-23T22:43:59.319857+0000","last_peered":"2023-09-23T22:43:59.319857+0000","last_clean":"2023-09-23T22:43:59.319857+0000","last_became_active":"2023-09-23T22:40:50.914602+0000","last_became_peered":"2023-09-23T22:40:50.914602+0000","last_unstale":"2023-09-23T22:43:59.319857+0000","last_undegraded":"2023-09-23T22:43:59.319857+0000","last_fullsized":"2023-09-23T22:43:59.319857+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:04.998198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315222+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:43:59.315222+0000","last_peered":"2023-09-23T22:43:59.315222+0000","last_clean":"2023-09-23T22:43:59.315222+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:43:59.315222+0000","last_undegraded":"2023-09-23T22:43:59.315222+0000","last_fullsized":"2023-09-23T22:43:59.315222+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"187'7980","reported_seq":8031,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:14.229986+0000","last_change":"2023-09-23T22:41:36.972030+0000","last_active":"2023-09-23T22:44:14.229986+0000","last_peered":"2023-09-23T22:44:14.229986+0000","last_clean":"2023-09-23T22:44:14.229986+0000","last_became_active":"2023-09-23T22:41:36.971693+0000","last_became_peered":"2023-09-23T22:41:36.971693+0000","last_unstale":"2023-09-23T22:44:14.229986+0000","last_undegraded":"2023-09-23T22:44:14.229986+0000","last_fullsized":"2023-09-23T22:44:14.229986+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7980,"log_dups_size":0,"ondisk_log_size":7980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:42:52.762839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236584960,"num_objects":3610,"num_object_clones":0,"num_object_copies":10830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7980,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":171,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.304539+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:43:59.304539+0000","last_peered":"2023-09-23T22:43:59.304539+0000","last_clean":"2023-09-23T22:43:59.304539+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:43:59.304539+0000","last_undegraded":"2023-09-23T22:43:59.304539+0000","last_fullsized":"2023-09-23T22:43:59.304539+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":182,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315212+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:43:59.315212+0000","last_peered":"2023-09-23T22:43:59.315212+0000","last_clean":"2023-09-23T22:43:59.315212+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:43:59.315212+0000","last_undegraded":"2023-09-23T22:43:59.315212+0000","last_fullsized":"2023-09-23T22:43:59.315212+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"187'7970","reported_seq":8020,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:15.680416+0000","last_change":"2023-09-23T22:41:36.971145+0000","last_active":"2023-09-23T22:44:15.680416+0000","last_peered":"2023-09-23T22:44:15.680416+0000","last_clean":"2023-09-23T22:44:15.680416+0000","last_became_active":"2023-09-23T22:41:36.970815+0000","last_became_peered":"2023-09-23T22:41:36.970815+0000","last_unstale":"2023-09-23T22:44:15.680416+0000","last_undegraded":"2023-09-23T22:44:15.680416+0000","last_fullsized":"2023-09-23T22:44:15.680416+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230621208,"num_objects":3520,"num_object_clones":0,"num_object_copies":10560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3520,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7970,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":186,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314898+0000","last_change":"2023-09-23T22:43:38.403153+0000","last_active":"2023-09-23T22:43:59.314898+0000","last_peered":"2023-09-23T22:43:59.314898+0000","last_clean":"2023-09-23T22:43:59.314898+0000","last_became_active":"2023-09-23T22:43:38.402640+0000","last_became_peered":"2023-09-23T22:43:38.402640+0000","last_unstale":"2023-09-23T22:43:59.314898+0000","last_undegraded":"2023-09-23T22:43:59.314898+0000","last_fullsized":"2023-09-23T22:43:59.314898+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315672+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:43:59.315672+0000","last_peered":"2023-09-23T22:43:59.315672+0000","last_clean":"2023-09-23T22:43:59.315672+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:43:59.315672+0000","last_undegraded":"2023-09-23T22:43:59.315672+0000","last_fullsized":"2023-09-23T22:43:59.315672+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"187'7906","reported_seq":16427,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:15.638897+0000","last_change":"2023-09-23T22:43:55.249702+0000","last_active":"2023-09-23T22:44:15.638897+0000","last_peered":"2023-09-23T22:44:15.638897+0000","last_clean":"2023-09-23T22:44:15.638897+0000","last_became_active":"2023-09-23T22:43:55.236412+0000","last_became_peered":"2023-09-23T22:43:55.236412+0000","last_unstale":"2023-09-23T22:44:15.638897+0000","last_undegraded":"2023-09-23T22:44:15.638897+0000","last_fullsized":"2023-09-23T22:44:15.638897+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7906,"log_dups_size":0,"ondisk_log_size":7906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228589568,"num_objects":3488,"num_object_clones":0,"num_object_copies":10464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7906,"num_write_kb":364608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4223,"num_bytes_recovered":227672064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":176,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.322946+0000","last_change":"2023-09-23T22:43:38.401528+0000","last_active":"2023-09-23T22:43:59.322946+0000","last_peered":"2023-09-23T22:43:59.322946+0000","last_clean":"2023-09-23T22:43:59.322946+0000","last_became_active":"2023-09-23T22:43:38.401205+0000","last_became_peered":"2023-09-23T22:43:38.401205+0000","last_unstale":"2023-09-23T22:43:59.322946+0000","last_undegraded":"2023-09-23T22:43:59.322946+0000","last_fullsized":"2023-09-23T22:43:59.322946+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:19.740066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.317516+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:43:59.317516+0000","last_peered":"2023-09-23T22:43:59.317516+0000","last_clean":"2023-09-23T22:43:59.317516+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:43:59.317516+0000","last_undegraded":"2023-09-23T22:43:59.317516+0000","last_fullsized":"2023-09-23T22:43:59.317516+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"187'8133","reported_seq":13145,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:15.426270+0000","last_change":"2023-09-23T22:42:31.753433+0000","last_active":"2023-09-23T22:44:15.426270+0000","last_peered":"2023-09-23T22:44:15.426270+0000","last_clean":"2023-09-23T22:44:15.426270+0000","last_became_active":"2023-09-23T22:42:31.748584+0000","last_became_peered":"2023-09-23T22:42:31.748584+0000","last_unstale":"2023-09-23T22:44:15.426270+0000","last_undegraded":"2023-09-23T22:44:15.426270+0000","last_fullsized":"2023-09-23T22:44:15.426270+0000","mapping_epoch":165,"log_start":"154'400","ondisk_log_start":"154'400","created":146,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7733,"log_dups_size":0,"ondisk_log_size":7733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238878720,"num_objects":3645,"num_object_clones":0,"num_object_copies":10935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8133,"num_write_kb":376896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":162004992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":201,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.456156+0000","last_change":"2023-09-23T22:43:38.479448+0000","last_active":"2023-09-23T22:43:59.456156+0000","last_peered":"2023-09-23T22:43:59.456156+0000","last_clean":"2023-09-23T22:43:59.456156+0000","last_became_active":"2023-09-23T22:43:38.479188+0000","last_became_peered":"2023-09-23T22:43:38.479188+0000","last_unstale":"2023-09-23T22:43:59.456156+0000","last_undegraded":"2023-09-23T22:43:59.456156+0000","last_fullsized":"2023-09-23T22:43:59.456156+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:56.012373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309567+0000","last_change":"2023-09-23T22:41:10.929497+0000","last_active":"2023-09-23T22:43:59.309567+0000","last_peered":"2023-09-23T22:43:59.309567+0000","last_clean":"2023-09-23T22:43:59.309567+0000","last_became_active":"2023-09-23T22:41:10.929128+0000","last_became_peered":"2023-09-23T22:41:10.929128+0000","last_unstale":"2023-09-23T22:43:59.309567+0000","last_undegraded":"2023-09-23T22:43:59.309567+0000","last_fullsized":"2023-09-23T22:43:59.309567+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":128,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.456288+0000","last_change":"2023-09-23T22:40:14.755608+0000","last_active":"2023-09-23T22:43:59.456288+0000","last_peered":"2023-09-23T22:43:59.456288+0000","last_clean":"2023-09-23T22:43:59.456288+0000","last_became_active":"2023-09-23T22:40:14.755335+0000","last_became_peered":"2023-09-23T22:40:14.755335+0000","last_unstale":"2023-09-23T22:43:59.456288+0000","last_undegraded":"2023-09-23T22:43:59.456288+0000","last_fullsized":"2023-09-23T22:43:59.456288+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:31.014369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"187'8084","reported_seq":8133,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:15.656440+0000","last_change":"2023-09-23T22:41:36.972228+0000","last_active":"2023-09-23T22:44:15.656440+0000","last_peered":"2023-09-23T22:44:15.656440+0000","last_clean":"2023-09-23T22:44:15.656440+0000","last_became_active":"2023-09-23T22:41:36.971904+0000","last_became_peered":"2023-09-23T22:41:36.971904+0000","last_unstale":"2023-09-23T22:44:15.656440+0000","last_undegraded":"2023-09-23T22:44:15.656440+0000","last_fullsized":"2023-09-23T22:44:15.656440+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8084,"log_dups_size":0,"ondisk_log_size":8084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233963520,"num_objects":3570,"num_object_clones":0,"num_object_copies":10710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8084,"num_write_kb":372928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":157,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309829+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:43:59.309829+0000","last_peered":"2023-09-23T22:43:59.309829+0000","last_clean":"2023-09-23T22:43:59.309829+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:43:59.309829+0000","last_undegraded":"2023-09-23T22:43:59.309829+0000","last_fullsized":"2023-09-23T22:43:59.309829+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":168,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315010+0000","last_change":"2023-09-23T22:40:50.916179+0000","last_active":"2023-09-23T22:43:59.315010+0000","last_peered":"2023-09-23T22:43:59.315010+0000","last_clean":"2023-09-23T22:43:59.315010+0000","last_became_active":"2023-09-23T22:40:50.915799+0000","last_became_peered":"2023-09-23T22:40:50.915799+0000","last_unstale":"2023-09-23T22:43:59.315010+0000","last_undegraded":"2023-09-23T22:43:59.315010+0000","last_fullsized":"2023-09-23T22:43:59.315010+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":179,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309199+0000","last_change":"2023-09-23T22:41:10.929964+0000","last_active":"2023-09-23T22:43:59.309199+0000","last_peered":"2023-09-23T22:43:59.309199+0000","last_clean":"2023-09-23T22:43:59.309199+0000","last_became_active":"2023-09-23T22:41:10.929689+0000","last_became_peered":"2023-09-23T22:41:10.929689+0000","last_unstale":"2023-09-23T22:43:59.309199+0000","last_undegraded":"2023-09-23T22:43:59.309199+0000","last_fullsized":"2023-09-23T22:43:59.309199+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"187'7788","reported_seq":7839,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:14.006817+0000","last_change":"2023-09-23T22:41:36.968996+0000","last_active":"2023-09-23T22:44:14.006817+0000","last_peered":"2023-09-23T22:44:14.006817+0000","last_clean":"2023-09-23T22:44:14.006817+0000","last_became_active":"2023-09-23T22:41:36.968694+0000","last_became_peered":"2023-09-23T22:41:36.968694+0000","last_unstale":"2023-09-23T22:44:14.006817+0000","last_undegraded":"2023-09-23T22:44:14.006817+0000","last_fullsized":"2023-09-23T22:44:14.006817+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7788,"log_dups_size":0,"ondisk_log_size":7788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:14.636304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227672064,"num_objects":3474,"num_object_clones":0,"num_object_copies":10422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7788,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":160,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.317113+0000","last_change":"2023-09-23T22:41:10.929848+0000","last_active":"2023-09-23T22:43:59.317113+0000","last_peered":"2023-09-23T22:43:59.317113+0000","last_clean":"2023-09-23T22:43:59.317113+0000","last_became_active":"2023-09-23T22:41:10.929552+0000","last_became_peered":"2023-09-23T22:41:10.929552+0000","last_unstale":"2023-09-23T22:43:59.317113+0000","last_undegraded":"2023-09-23T22:43:59.317113+0000","last_fullsized":"2023-09-23T22:43:59.317113+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:09.783138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":158,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.318388+0000","last_change":"2023-09-23T22:41:10.927089+0000","last_active":"2023-09-23T22:43:59.318388+0000","last_peered":"2023-09-23T22:43:59.318388+0000","last_clean":"2023-09-23T22:43:59.318388+0000","last_became_active":"2023-09-23T22:41:10.926767+0000","last_became_peered":"2023-09-23T22:41:10.926767+0000","last_unstale":"2023-09-23T22:43:59.318388+0000","last_undegraded":"2023-09-23T22:43:59.318388+0000","last_fullsized":"2023-09-23T22:43:59.318388+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:03.097039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":179,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.459644+0000","last_change":"2023-09-23T22:40:51.909811+0000","last_active":"2023-09-23T22:43:59.459644+0000","last_peered":"2023-09-23T22:43:59.459644+0000","last_clean":"2023-09-23T22:43:59.459644+0000","last_became_active":"2023-09-23T22:40:51.909464+0000","last_became_peered":"2023-09-23T22:40:51.909464+0000","last_unstale":"2023-09-23T22:43:59.459644+0000","last_undegraded":"2023-09-23T22:43:59.459644+0000","last_fullsized":"2023-09-23T22:43:59.459644+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.517012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"187'8096","reported_seq":8163,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:44:13.956448+0000","last_change":"2023-09-23T22:43:38.558350+0000","last_active":"2023-09-23T22:44:13.956448+0000","last_peered":"2023-09-23T22:44:13.956448+0000","last_clean":"2023-09-23T22:42:45.097581+0000","last_became_active":"2023-09-23T22:43:38.491068+0000","last_became_peered":"2023-09-23T22:43:38.491068+0000","last_unstale":"2023-09-23T22:44:13.956448+0000","last_undegraded":"2023-09-23T22:43:38.275118+0000","last_fullsized":"2023-09-23T22:43:38.253783+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8096,"log_dups_size":0,"ondisk_log_size":8096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:11.723247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239730688,"num_objects":3658,"num_object_clones":0,"num_object_copies":10974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8096,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2382},{"shards":"0,3,5","objects":1276}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":190,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309983+0000","last_change":"2023-09-23T22:43:38.401731+0000","last_active":"2023-09-23T22:43:59.309983+0000","last_peered":"2023-09-23T22:43:59.309983+0000","last_clean":"2023-09-23T22:43:59.309983+0000","last_became_active":"2023-09-23T22:43:38.401376+0000","last_became_peered":"2023-09-23T22:43:38.401376+0000","last_unstale":"2023-09-23T22:43:59.309983+0000","last_undegraded":"2023-09-23T22:43:59.309983+0000","last_fullsized":"2023-09-23T22:43:59.309983+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":402,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.307200+0000","last_change":"2023-09-23T22:43:39.306382+0000","last_active":"2023-09-23T22:43:59.307200+0000","last_peered":"2023-09-23T22:43:59.307200+0000","last_clean":"2023-09-23T22:43:59.307200+0000","last_became_active":"2023-09-23T22:43:39.305922+0000","last_became_peered":"2023-09-23T22:43:39.305922+0000","last_unstale":"2023-09-23T22:43:59.307200+0000","last_undegraded":"2023-09-23T22:43:59.307200+0000","last_fullsized":"2023-09-23T22:43:59.307200+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":115,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.319492+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:43:59.319492+0000","last_peered":"2023-09-23T22:43:59.319492+0000","last_clean":"2023-09-23T22:43:59.319492+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:43:59.319492+0000","last_undegraded":"2023-09-23T22:43:59.319492+0000","last_fullsized":"2023-09-23T22:43:59.319492+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.310283+0000","last_change":"2023-09-23T22:43:38.404134+0000","last_active":"2023-09-23T22:43:59.310283+0000","last_peered":"2023-09-23T22:43:59.310283+0000","last_clean":"2023-09-23T22:43:59.310283+0000","last_became_active":"2023-09-23T22:43:38.403762+0000","last_became_peered":"2023-09-23T22:43:38.403762+0000","last_unstale":"2023-09-23T22:43:59.310283+0000","last_undegraded":"2023-09-23T22:43:59.310283+0000","last_fullsized":"2023-09-23T22:43:59.310283+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"187'8146","reported_seq":11115,"reported_epoch":187,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:44:14.238541+0000","last_change":"2023-09-23T22:43:53.823767+0000","last_active":"2023-09-23T22:44:14.238541+0000","last_peered":"2023-09-23T22:44:14.238541+0000","last_clean":"2023-09-23T22:42:45.107206+0000","last_became_active":"2023-09-23T22:43:38.481697+0000","last_became_peered":"2023-09-23T22:43:38.481697+0000","last_unstale":"2023-09-23T22:44:14.238541+0000","last_undegraded":"2023-09-23T22:44:14.238541+0000","last_fullsized":"2023-09-23T22:43:38.254661+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8146,"log_dups_size":0,"ondisk_log_size":8146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:02.871891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8146,"num_write_kb":378176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1452,"num_bytes_recovered":81395712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3672}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.307032+0000","last_change":"2023-09-23T22:43:38.395436+0000","last_active":"2023-09-23T22:43:59.307032+0000","last_peered":"2023-09-23T22:43:59.307032+0000","last_clean":"2023-09-23T22:43:59.307032+0000","last_became_active":"2023-09-23T22:43:38.395095+0000","last_became_peered":"2023-09-23T22:43:38.395095+0000","last_unstale":"2023-09-23T22:43:59.307032+0000","last_undegraded":"2023-09-23T22:43:59.307032+0000","last_fullsized":"2023-09-23T22:43:59.307032+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":133,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314096+0000","last_change":"2023-09-23T22:41:49.035892+0000","last_active":"2023-09-23T22:43:59.314096+0000","last_peered":"2023-09-23T22:43:59.314096+0000","last_clean":"2023-09-23T22:43:59.314096+0000","last_became_active":"2023-09-23T22:41:49.035318+0000","last_became_peered":"2023-09-23T22:41:49.035318+0000","last_unstale":"2023-09-23T22:43:59.314096+0000","last_undegraded":"2023-09-23T22:43:59.314096+0000","last_fullsized":"2023-09-23T22:43:59.314096+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:06.155982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":142,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314654+0000","last_change":"2023-09-23T22:41:10.928059+0000","last_active":"2023-09-23T22:43:59.314654+0000","last_peered":"2023-09-23T22:43:59.314654+0000","last_clean":"2023-09-23T22:43:59.314654+0000","last_became_active":"2023-09-23T22:41:10.927749+0000","last_became_peered":"2023-09-23T22:41:10.927749+0000","last_unstale":"2023-09-23T22:43:59.314654+0000","last_undegraded":"2023-09-23T22:43:59.314654+0000","last_fullsized":"2023-09-23T22:43:59.314654+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:53.307028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"187'7959","reported_seq":8008,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:15.481126+0000","last_change":"2023-09-23T22:41:36.969378+0000","last_active":"2023-09-23T22:44:15.481126+0000","last_peered":"2023-09-23T22:44:15.481126+0000","last_clean":"2023-09-23T22:44:15.481126+0000","last_became_active":"2023-09-23T22:41:36.969153+0000","last_became_peered":"2023-09-23T22:41:36.969153+0000","last_unstale":"2023-09-23T22:44:15.481126+0000","last_undegraded":"2023-09-23T22:44:15.481126+0000","last_fullsized":"2023-09-23T22:44:15.481126+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233766912,"num_objects":3567,"num_object_clones":0,"num_object_copies":10701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7959,"num_write_kb":368832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309163+0000","last_change":"2023-09-23T22:43:38.414528+0000","last_active":"2023-09-23T22:43:59.309163+0000","last_peered":"2023-09-23T22:43:59.309163+0000","last_clean":"2023-09-23T22:43:59.309163+0000","last_became_active":"2023-09-23T22:43:38.414174+0000","last_became_peered":"2023-09-23T22:43:38.414174+0000","last_unstale":"2023-09-23T22:43:59.309163+0000","last_undegraded":"2023-09-23T22:43:59.309163+0000","last_fullsized":"2023-09-23T22:43:59.309163+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":158,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.456455+0000","last_change":"2023-09-23T22:41:10.930375+0000","last_active":"2023-09-23T22:43:59.456455+0000","last_peered":"2023-09-23T22:43:59.456455+0000","last_clean":"2023-09-23T22:43:59.456455+0000","last_became_active":"2023-09-23T22:41:10.930050+0000","last_became_peered":"2023-09-23T22:41:10.930050+0000","last_unstale":"2023-09-23T22:43:59.456455+0000","last_undegraded":"2023-09-23T22:43:59.456455+0000","last_fullsized":"2023-09-23T22:43:59.456455+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:29.593309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":115,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.313258+0000","last_change":"2023-09-23T22:43:38.408959+0000","last_active":"2023-09-23T22:43:59.313258+0000","last_peered":"2023-09-23T22:43:59.313258+0000","last_clean":"2023-09-23T22:43:59.313258+0000","last_became_active":"2023-09-23T22:43:38.408608+0000","last_became_peered":"2023-09-23T22:43:38.408608+0000","last_unstale":"2023-09-23T22:43:59.313258+0000","last_undegraded":"2023-09-23T22:43:59.313258+0000","last_fullsized":"2023-09-23T22:43:59.313258+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:45.019126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"187'7947","reported_seq":7998,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:14.568967+0000","last_change":"2023-09-23T22:41:36.966638+0000","last_active":"2023-09-23T22:44:14.568967+0000","last_peered":"2023-09-23T22:44:14.568967+0000","last_clean":"2023-09-23T22:44:14.568967+0000","last_became_active":"2023-09-23T22:41:36.966433+0000","last_became_peered":"2023-09-23T22:41:36.966433+0000","last_unstale":"2023-09-23T22:44:14.568967+0000","last_undegraded":"2023-09-23T22:44:14.568967+0000","last_fullsized":"2023-09-23T22:44:14.568967+0000","mapping_epoch":146,"log_start":"154'600","ondisk_log_start":"154'600","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7347,"log_dups_size":0,"ondisk_log_size":7347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:12.457010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231931904,"num_objects":3539,"num_object_clones":0,"num_object_copies":10617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7947,"num_write_kb":367552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":196,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.320041+0000","last_change":"2023-09-23T22:43:38.411825+0000","last_active":"2023-09-23T22:43:59.320041+0000","last_peered":"2023-09-23T22:43:59.320041+0000","last_clean":"2023-09-23T22:43:59.320041+0000","last_became_active":"2023-09-23T22:43:38.401046+0000","last_became_peered":"2023-09-23T22:43:38.401046+0000","last_unstale":"2023-09-23T22:43:59.320041+0000","last_undegraded":"2023-09-23T22:43:59.320041+0000","last_fullsized":"2023-09-23T22:43:59.320041+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:09.059178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":127,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.456265+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:43:59.456265+0000","last_peered":"2023-09-23T22:43:59.456265+0000","last_clean":"2023-09-23T22:43:59.456265+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:43:59.456265+0000","last_undegraded":"2023-09-23T22:43:59.456265+0000","last_fullsized":"2023-09-23T22:43:59.456265+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":186,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.563462+0000","last_change":"2023-09-23T22:43:38.401501+0000","last_active":"2023-09-23T22:44:04.563462+0000","last_peered":"2023-09-23T22:44:04.563462+0000","last_clean":"2023-09-23T22:44:04.563462+0000","last_became_active":"2023-09-23T22:43:38.401083+0000","last_became_peered":"2023-09-23T22:43:38.401083+0000","last_unstale":"2023-09-23T22:44:04.563462+0000","last_undegraded":"2023-09-23T22:44:04.563462+0000","last_fullsized":"2023-09-23T22:44:04.563462+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:27.690217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"187'7929","reported_seq":7980,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:44:14.545352+0000","last_change":"2023-09-23T22:41:36.965968+0000","last_active":"2023-09-23T22:44:14.545352+0000","last_peered":"2023-09-23T22:44:14.545352+0000","last_clean":"2023-09-23T22:44:14.545352+0000","last_became_active":"2023-09-23T22:41:36.965582+0000","last_became_peered":"2023-09-23T22:41:36.965582+0000","last_unstale":"2023-09-23T22:44:14.545352+0000","last_undegraded":"2023-09-23T22:44:14.545352+0000","last_fullsized":"2023-09-23T22:44:14.545352+0000","mapping_epoch":146,"log_start":"154'600","ondisk_log_start":"154'600","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:51.048347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231276544,"num_objects":3529,"num_object_clones":0,"num_object_copies":10587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7929,"num_write_kb":366656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":210,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.310156+0000","last_change":"2023-09-23T22:43:38.396140+0000","last_active":"2023-09-23T22:43:59.310156+0000","last_peered":"2023-09-23T22:43:59.310156+0000","last_clean":"2023-09-23T22:43:59.310156+0000","last_became_active":"2023-09-23T22:43:38.395787+0000","last_became_peered":"2023-09-23T22:43:38.395787+0000","last_unstale":"2023-09-23T22:43:59.310156+0000","last_undegraded":"2023-09-23T22:43:59.310156+0000","last_fullsized":"2023-09-23T22:43:59.310156+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":120,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314741+0000","last_change":"2023-09-23T22:40:50.916564+0000","last_active":"2023-09-23T22:43:59.314741+0000","last_peered":"2023-09-23T22:43:59.314741+0000","last_clean":"2023-09-23T22:43:59.314741+0000","last_became_active":"2023-09-23T22:40:50.915883+0000","last_became_peered":"2023-09-23T22:40:50.915883+0000","last_unstale":"2023-09-23T22:43:59.314741+0000","last_undegraded":"2023-09-23T22:43:59.314741+0000","last_fullsized":"2023-09-23T22:43:59.314741+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:20:25.767798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314982+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:43:59.314982+0000","last_peered":"2023-09-23T22:43:59.314982+0000","last_clean":"2023-09-23T22:43:59.314982+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:43:59.314982+0000","last_undegraded":"2023-09-23T22:43:59.314982+0000","last_fullsized":"2023-09-23T22:43:59.314982+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":118,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.313494+0000","last_change":"2023-09-23T22:43:38.409027+0000","last_active":"2023-09-23T22:43:59.313494+0000","last_peered":"2023-09-23T22:43:59.313494+0000","last_clean":"2023-09-23T22:43:59.313494+0000","last_became_active":"2023-09-23T22:43:38.408636+0000","last_became_peered":"2023-09-23T22:43:38.408636+0000","last_unstale":"2023-09-23T22:43:59.313494+0000","last_undegraded":"2023-09-23T22:43:59.313494+0000","last_fullsized":"2023-09-23T22:43:59.313494+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:20.099948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":129,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.456372+0000","last_change":"2023-09-23T22:43:27.204192+0000","last_active":"2023-09-23T22:43:59.456372+0000","last_peered":"2023-09-23T22:43:59.456372+0000","last_clean":"2023-09-23T22:43:59.456372+0000","last_became_active":"2023-09-23T22:43:27.203807+0000","last_became_peered":"2023-09-23T22:43:27.203807+0000","last_unstale":"2023-09-23T22:43:59.456372+0000","last_undegraded":"2023-09-23T22:43:59.456372+0000","last_fullsized":"2023-09-23T22:43:59.456372+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:40.541572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309350+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:43:59.309350+0000","last_peered":"2023-09-23T22:43:59.309350+0000","last_clean":"2023-09-23T22:43:59.309350+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:43:59.309350+0000","last_undegraded":"2023-09-23T22:43:59.309350+0000","last_fullsized":"2023-09-23T22:43:59.309350+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":186,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.315093+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:43:59.315093+0000","last_peered":"2023-09-23T22:43:59.315093+0000","last_clean":"2023-09-23T22:43:59.315093+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:43:59.315093+0000","last_undegraded":"2023-09-23T22:43:59.315093+0000","last_fullsized":"2023-09-23T22:43:59.315093+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":140,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.309666+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:43:59.309666+0000","last_peered":"2023-09-23T22:43:59.309666+0000","last_clean":"2023-09-23T22:43:59.309666+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:43:59.309666+0000","last_undegraded":"2023-09-23T22:43:59.309666+0000","last_fullsized":"2023-09-23T22:43:59.309666+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314977+0000","last_change":"2023-09-23T22:41:49.032087+0000","last_active":"2023-09-23T22:43:59.314977+0000","last_peered":"2023-09-23T22:43:59.314977+0000","last_clean":"2023-09-23T22:43:59.314977+0000","last_became_active":"2023-09-23T22:41:49.031690+0000","last_became_peered":"2023-09-23T22:41:49.031690+0000","last_unstale":"2023-09-23T22:43:59.314977+0000","last_undegraded":"2023-09-23T22:43:59.314977+0000","last_fullsized":"2023-09-23T22:43:59.314977+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":178,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.563572+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:44:04.563572+0000","last_peered":"2023-09-23T22:44:04.563572+0000","last_clean":"2023-09-23T22:44:04.563572+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:44:04.563572+0000","last_undegraded":"2023-09-23T22:44:04.563572+0000","last_fullsized":"2023-09-23T22:44:04.563572+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":201,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.319436+0000","last_change":"2023-09-23T22:41:10.931122+0000","last_active":"2023-09-23T22:43:59.319436+0000","last_peered":"2023-09-23T22:43:59.319436+0000","last_clean":"2023-09-23T22:43:59.319436+0000","last_became_active":"2023-09-23T22:41:10.930811+0000","last_became_peered":"2023-09-23T22:41:10.930811+0000","last_unstale":"2023-09-23T22:43:59.319436+0000","last_undegraded":"2023-09-23T22:43:59.319436+0000","last_fullsized":"2023-09-23T22:43:59.319436+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:16.649049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":145,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.318545+0000","last_change":"2023-09-23T22:43:27.220868+0000","last_active":"2023-09-23T22:43:59.318545+0000","last_peered":"2023-09-23T22:43:59.318545+0000","last_clean":"2023-09-23T22:43:59.318545+0000","last_became_active":"2023-09-23T22:43:27.195700+0000","last_became_peered":"2023-09-23T22:43:27.195700+0000","last_unstale":"2023-09-23T22:43:59.318545+0000","last_undegraded":"2023-09-23T22:43:59.318545+0000","last_fullsized":"2023-09-23T22:43:59.318545+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:53.576662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:43:59.314655+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:43:59.314655+0000","last_peered":"2023-09-23T22:43:59.314655+0000","last_clean":"2023-09-23T22:43:59.314655+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:43:59.314655+0000","last_undegraded":"2023-09-23T22:43:59.314655+0000","last_fullsized":"2023-09-23T22:43:59.314655+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3757637656,"num_objects":57338,"num_object_clones":0,"num_object_copies":172014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5993,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57338,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127642,"num_write_kb":5919297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8150,"num_bytes_recovered":471072768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11708870656,"data_stored":11708858440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124942,"ondisk_log_size":124942,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10238,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430415,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389292,"kb_used_data":1087972,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92355860,"statfs":{"total":95995035648,"available":94572400640,"internally_reserved":0,"allocated":1114083328,"data_stored":1113595294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:58 2023","interfaces":[{"interface":"back","average":{"1min":4.891,"5min":1.6299999999999999,"15min":0.89800000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":82.403999999999996,"5min":82.403999999999996,"15min":82.403999999999996},"last":0.27200000000000002},{"interface":"front","average":{"1min":6.2400000000000002,"5min":2.1320000000000001,"15min":1.202},"min":{"1min":0.251,"5min":0.224,"15min":0.224},"max":{"1min":82.453999999999994,"5min":82.453999999999994,"15min":82.453999999999994},"last":2.109}]},{"osd":1,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":2.9700000000000002,"15min":1.409},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":11.180999999999999,"5min":73.269000000000005,"15min":73.269000000000005},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.0880000000000001,"5min":2.8780000000000001,"15min":1.361},"min":{"1min":0.35299999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.167999999999999,"5min":73.281000000000006,"15min":73.281000000000006},"last":0.42499999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":2.4350000000000001,"15min":1.282},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":24.039000000000001,"5min":92.488,"15min":92.488},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.226,"5min":2.669,"15min":1.4630000000000001},"min":{"1min":0.31,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":23.864999999999998,"5min":92.742999999999995,"15min":92.742999999999995},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":2.2629999999999999,"15min":1.127},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.308,"5min":85.956000000000003,"15min":85.956000000000003},"last":3.0110000000000001},{"interface":"front","average":{"1min":2.6200000000000001,"5min":2.399,"15min":1.2010000000000001},"min":{"1min":0.378,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.321,"5min":85.938000000000002,"15min":85.938000000000002},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.88400000000000001,"15min":0.63700000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.2269999999999999,"5min":28.213000000000001,"15min":28.213000000000001},"last":0.37},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.94799999999999995,"15min":0.65400000000000003},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4900000000000002,"5min":27.933,"15min":27.933},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":3.0099999999999998,"15min":1.343},"min":{"1min":0.33100000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":4.5039999999999996,"5min":226.22200000000001,"15min":226.22200000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.78900000000000003,"5min":3.0110000000000001,"15min":1.401},"min":{"1min":0.33400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.944,"5min":225.50899999999999,"15min":225.50899999999999},"last":0.71299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:43:38 2023","interfaces":[{"interface":"back","average":{"1min":6.0259999999999998,"5min":1.6359999999999999,"15min":0.91700000000000004},"min":{"1min":0.35199999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":114.63200000000001,"5min":114.63200000000001,"15min":114.63200000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":6.2999999999999998,"5min":1.7649999999999999,"15min":0.99399999999999999},"min":{"1min":0.35399999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":115.03100000000001,"5min":115.03100000000001,"15min":115.03100000000001},"last":0.68000000000000005}]}]},{"osd":6,"up_from":42,"seq":180388626643,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1458216,"kb_used_data":1152800,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92286936,"statfs":{"total":95995035648,"available":94501822464,"internally_reserved":0,"allocated":1180467200,"data_stored":1179983694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":1.7390000000000001,"15min":0.871},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":12.18,"5min":52.377000000000002,"15min":52.377000000000002},"last":6.8890000000000002},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.655,"15min":0.93799999999999994},"min":{"1min":0.25,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":10.314,"5min":51.970999999999997,"15min":51.970999999999997},"last":0.755}]},{"osd":1,"last update":"Sat Sep 23 22:43:20 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":4.157,"15min":1.7270000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":34.695999999999998,"5min":153.78399999999999,"15min":153.78399999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":3.101,"5min":4.1609999999999996,"15min":1.768},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":34.307000000000002,"5min":154.148,"15min":154.148},"last":7.1779999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:43:11 2023","interfaces":[{"interface":"back","average":{"1min":9.0280000000000005,"5min":3.996,"15min":3.1579999999999999},"min":{"1min":0.32500000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":100.785,"5min":100.785,"15min":100.785},"last":0.86799999999999999},{"interface":"front","average":{"1min":8.4049999999999994,"5min":3.8559999999999999,"15min":3.0979999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":100.77,"5min":100.77,"15min":100.77},"last":0.93700000000000006}]},{"osd":3,"last update":"Sat Sep 23 22:43:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3410000000000002,"5min":2.617,"15min":1.2230000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.343999999999999,"5min":80.277000000000001,"15min":80.277000000000001},"last":1.73},{"interface":"front","average":{"1min":2.1819999999999999,"5min":2.718,"15min":1.3149999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7530000000000001,"5min":80.513999999999996,"15min":80.513999999999996},"last":1.534}]},{"osd":4,"last update":"Sat Sep 23 22:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":0.68600000000000005,"15min":0.55500000000000005},"min":{"1min":0.27100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":18.085999999999999,"5min":18.085999999999999,"15min":18.085999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.387,"5min":0.65000000000000002,"15min":0.55600000000000005},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":18.597000000000001,"5min":18.597000000000001,"15min":18.597000000000001},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":0.70099999999999996,"15min":0.58399999999999996},"min":{"1min":0.31,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":18.972000000000001,"5min":18.972000000000001,"15min":18.972000000000001},"last":0.374},{"interface":"front","average":{"1min":1.605,"5min":0.71899999999999997,"15min":0.61199999999999999},"min":{"1min":0.32300000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":18.625,"5min":18.625,"15min":18.625},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:43:38 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.64500000000000002,"15min":0.59999999999999998},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":12.481999999999999,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.7789999999999999,"5min":0.76100000000000001,"15min":0.63900000000000001},"min":{"1min":0.376,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":16.661999999999999,"5min":16.661999999999999,"15min":16.661999999999999},"last":0.498}]}]},{"osd":1,"up_from":84,"seq":360777252952,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1599616,"kb_used_data":1495864,"kb_used_omap":9,"kb_used_meta":103734,"kb_avail":92145536,"statfs":{"total":95995035648,"available":94357028864,"internally_reserved":0,"allocated":1531764736,"data_stored":1531028169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":106224617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:59 2023","interfaces":[{"interface":"back","average":{"1min":5.2389999999999999,"5min":2.7890000000000001,"15min":1.4359999999999999},"min":{"1min":0.23499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":51.767000000000003,"5min":83.670000000000002,"15min":83.670000000000002},"last":6.1639999999999997},{"interface":"front","average":{"1min":5.2629999999999999,"5min":2.7549999999999999,"15min":1.2549999999999999},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":51.676000000000002,"5min":84,"15min":84},"last":7.4749999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":6.1289999999999996,"5min":2.0539999999999998,"15min":1.375},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.889000000000003,"5min":37.889000000000003,"15min":37.889000000000003},"last":0.36099999999999999},{"interface":"front","average":{"1min":5.4390000000000001,"5min":1.804,"15min":1.198},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":37.823,"5min":37.823,"15min":37.823},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":5.218,"5min":1.4930000000000001,"15min":0.752},"min":{"1min":0.23000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":50.033999999999999,"5min":50.033999999999999,"15min":50.033999999999999},"last":5.8799999999999999},{"interface":"front","average":{"1min":5.6749999999999998,"5min":1.663,"15min":0.81299999999999994},"min":{"1min":0.27400000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":52.491999999999997,"5min":52.491999999999997,"15min":52.491999999999997},"last":2.2839999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.571,"15min":0.94699999999999995},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.2949999999999999,"5min":69.344999999999999,"15min":69.344999999999999},"last":2.2290000000000001},{"interface":"front","average":{"1min":1.972,"5min":1.788,"15min":1.0369999999999999},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":7.7089999999999996,"5min":69.359999999999999,"15min":69.359999999999999},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":1.224,"15min":0.83199999999999996},"min":{"1min":0.33500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.532,"5min":28.837,"15min":28.837},"last":5.8120000000000003},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.1279999999999999,"15min":0.77600000000000002},"min":{"1min":0.36099999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.4020000000000001,"5min":28.986999999999998,"15min":28.986999999999998},"last":2.1960000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.0329999999999999,"15min":0.76900000000000002},"min":{"1min":0.36599999999999999,"5min":0.317,"15min":0.317},"max":{"1min":7.6710000000000003,"5min":7.6710000000000003,"15min":7.6710000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.369,"5min":0.88,"15min":0.71499999999999997},"min":{"1min":0.38900000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.5830000000000002,"5min":7.5830000000000002,"15min":7.5830000000000002},"last":5.8540000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":2.7890000000000001,"15min":1.3560000000000001},"min":{"1min":0.43099999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.323,"5min":159.018,"15min":159.018},"last":2.258},{"interface":"front","average":{"1min":2.4129999999999998,"5min":3.0649999999999999,"15min":1.464},"min":{"1min":0.50700000000000001,"5min":0.37,"15min":0.37},"max":{"1min":7.742,"5min":158.96000000000001,"15min":158.96000000000001},"last":0.436}]}]},{"osd":0,"up_from":126,"seq":541165879338,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1994636,"kb_used_data":1845956,"kb_used_omap":28,"kb_used_meta":148515,"kb_avail":91750516,"statfs":{"total":95995035648,"available":93952528384,"internally_reserved":0,"allocated":1890258944,"data_stored":1889451534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":152080208},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,5,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:43:52 2023","interfaces":[{"interface":"back","average":{"1min":5.9749999999999996,"5min":1.9790000000000001,"15min":1.1619999999999999},"min":{"1min":0.23200000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":22.991,"5min":22.991,"15min":22.991},"last":5.8230000000000004},{"interface":"front","average":{"1min":3.3260000000000001,"5min":1.3919999999999999,"15min":0.96599999999999997},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":17.792000000000002,"5min":17.792000000000002,"15min":17.792000000000002},"last":2.9940000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":8.7210000000000001,"5min":7.0910000000000002,"15min":6.819},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":101.43600000000001,"5min":105.818,"15min":105.818},"last":8.1280000000000001},{"interface":"front","average":{"1min":8.2850000000000001,"5min":6.5300000000000002,"15min":6.2380000000000004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":101.455,"5min":105.77500000000001,"15min":105.77500000000001},"last":2.2349999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:43:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":1.837,"15min":1.3999999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.952,"5min":9.952,"15min":9.952},"last":3.5499999999999998},{"interface":"front","average":{"1min":4.0599999999999996,"5min":2.0419999999999998,"15min":1.5189999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":17.030000000000001,"5min":17.030000000000001,"15min":17.030000000000001},"last":8.4380000000000006}]},{"osd":4,"last update":"Sat Sep 23 22:43:52 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.3380000000000001,"15min":1.012},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":8.8710000000000004,"5min":8.8710000000000004,"15min":8.8710000000000004},"last":3.6629999999999998},{"interface":"front","average":{"1min":3.2109999999999999,"5min":1.444,"15min":1.044},"min":{"1min":0.42799999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0030000000000001,"5min":9.0030000000000001,"15min":9.0030000000000001},"last":2.1899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:43:52 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":1.4119999999999999,"15min":1.03},"min":{"1min":0.372,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.853,"5min":11.853,"15min":11.853},"last":3.6899999999999999},{"interface":"front","average":{"1min":2.827,"5min":1.409,"15min":1.0529999999999999},"min":{"1min":0.39700000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.9469999999999992,"5min":8.9469999999999992,"15min":8.9469999999999992},"last":2.2120000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:43:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.5069999999999999,"15min":1.093},"min":{"1min":0.42299999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":9.0329999999999995,"5min":11.161,"15min":11.161},"last":2.1589999999999998},{"interface":"front","average":{"1min":2.4460000000000002,"5min":1.2150000000000001,"15min":0.89400000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":9.0679999999999996,"5min":9.0679999999999996,"15min":9.0679999999999996},"last":3.0249999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:43:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":1.1830000000000001,"15min":0.91100000000000003},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":9.0939999999999994,"5min":9.0939999999999994,"15min":9.0939999999999994},"last":3.056},{"interface":"front","average":{"1min":2.9900000000000002,"5min":1.5249999999999999,"15min":1.1259999999999999},"min":{"1min":0.46500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":10.547000000000001,"5min":10.547000000000001,"15min":10.547000000000001},"last":3.621}]}]},{"osd":2,"up_from":131,"seq":562640715814,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1676144,"kb_used_data":1557864,"kb_used_omap":81,"kb_used_meta":118190,"kb_avail":92069008,"statfs":{"total":95995035648,"available":94278664192,"internally_reserved":0,"allocated":1595252736,"data_stored":1594474277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":121026778},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":10.755000000000001,"5min":4.8700000000000001,"15min":3.8889999999999998},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":114.735,"5min":114.735,"15min":114.735},"last":1.905},{"interface":"front","average":{"1min":12.125999999999999,"5min":6.1509999999999998,"15min":5.1550000000000002},"min":{"1min":0.28399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":114.798,"5min":114.798,"15min":114.798},"last":0.88900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":3.0790000000000002,"15min":3.081},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":14.656000000000001,"5min":21.114000000000001,"15min":21.114000000000001},"last":7.7460000000000004},{"interface":"front","average":{"1min":2.1059999999999999,"5min":2.093,"15min":2.0910000000000002},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.7449999999999992,"5min":22.021000000000001,"15min":22.021000000000001},"last":2.2120000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":11.297000000000001,"5min":3.1179999999999999,"15min":1.7549999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":71.052000000000007,"5min":71.052000000000007,"15min":71.052000000000007},"last":2.355},{"interface":"front","average":{"1min":8.8870000000000005,"5min":2.6230000000000002,"15min":1.579},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":70.772999999999996,"5min":70.772999999999996,"15min":70.772999999999996},"last":1.968}]},{"osd":4,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.0529999999999999,"15min":0.95499999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.3179999999999996,"5min":7.3179999999999996,"15min":7.3179999999999996},"last":2.0569999999999999},{"interface":"front","average":{"1min":2.7599999999999998,"5min":1.6040000000000001,"15min":1.4119999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":14.566000000000001,"5min":14.566000000000001,"15min":14.566000000000001},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":1.556,"15min":1.421},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":14.599,"5min":14.599,"15min":14.599},"last":0.317},{"interface":"front","average":{"1min":1.974,"5min":0.996,"15min":0.83299999999999996},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.9550000000000001,"5min":8.9550000000000001,"15min":8.9550000000000001},"last":2.2480000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":1.0620000000000001,"15min":0.878},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.9870000000000001,"5min":8.9870000000000001,"15min":8.9870000000000001},"last":2.2770000000000001},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.077,"15min":0.95499999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":7.3810000000000002},"last":2.0310000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.1000000000000001,"15min":1.0149999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.3470000000000004,"5min":7.3470000000000004,"15min":7.3470000000000004},"last":2.0009999999999999},{"interface":"front","average":{"1min":2.9359999999999999,"5min":1.671,"15min":1.46},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":14.631,"5min":14.631,"15min":14.631},"last":0.371}]}]},{"osd":3,"up_from":94,"seq":403726925900,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2326896,"kb_used_data":2150440,"kb_used_omap":7,"kb_used_meta":175928,"kb_avail":91418256,"statfs":{"total":95995035648,"available":93612294144,"internally_reserved":0,"allocated":2202050560,"data_stored":2201327469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":180150888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":4.351,"5min":2.294,"15min":1.129},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":21.071999999999999,"5min":26.385000000000002,"15min":26.385000000000002},"last":12.996},{"interface":"front","average":{"1min":4.2069999999999999,"5min":2.1890000000000001,"15min":1.2370000000000001},"min":{"1min":0.26500000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.893999999999998,"5min":26.190999999999999,"15min":26.190999999999999},"last":6.0540000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1240000000000001,"5min":0.97999999999999998,"15min":0.56000000000000005},"min":{"1min":0.29799999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":11.269,"5min":11.269,"15min":11.269},"last":5.7080000000000002},{"interface":"front","average":{"1min":2.4140000000000001,"5min":0.95499999999999996,"15min":0.57999999999999996},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.712,"5min":10.712,"15min":10.712},"last":5.7569999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:43:11 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":1.728,"15min":1.341},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.558,"5min":41.558,"15min":41.558},"last":5.8419999999999996},{"interface":"front","average":{"1min":5.0019999999999998,"5min":1.655,"15min":1.097},"min":{"1min":0.378,"5min":0.309,"15min":0.309},"max":{"1min":42.088999999999999,"5min":42.088999999999999,"15min":42.088999999999999},"last":5.0499999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":0.95599999999999996,"15min":0.73399999999999999},"min":{"1min":0.40899999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.6310000000000002,"5min":6.6310000000000002,"15min":6.6310000000000002},"last":0.44},{"interface":"front","average":{"1min":1.8819999999999999,"5min":0.93899999999999995,"15min":0.76800000000000002},"min":{"1min":0.35399999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.04,"5min":5.04,"15min":5.04},"last":4.5380000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":0.94799999999999995,"15min":0.75900000000000001},"min":{"1min":0.35899999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.6909999999999998,"5min":5.6909999999999998,"15min":5.6909999999999998},"last":4.5880000000000001},{"interface":"front","average":{"1min":2.0409999999999999,"5min":0.94099999999999995,"15min":0.71199999999999997},"min":{"1min":0.33600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":7.0670000000000002,"5min":7.0670000000000002,"15min":7.0670000000000002},"last":4.976}]},{"osd":6,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":0.92800000000000005,"15min":0.76400000000000001},"min":{"1min":0.42799999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.1189999999999998,"5min":5.1189999999999998,"15min":5.1189999999999998},"last":4.6109999999999998},{"interface":"front","average":{"1min":2.851,"5min":1.1120000000000001,"15min":0.81100000000000005},"min":{"1min":0.37,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":10.755000000000001,"5min":10.755000000000001,"15min":10.755000000000001},"last":5.0140000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":1.0489999999999999,"15min":0.78500000000000003},"min":{"1min":0.40000000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.1399999999999997,"5min":7.1399999999999997,"15min":7.1399999999999997},"last":4.9329999999999998},{"interface":"front","average":{"1min":2.6970000000000001,"5min":1.0529999999999999,"15min":0.77000000000000002},"min":{"1min":0.441,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.0839999999999996,"5min":8.0839999999999996,"15min":8.0839999999999996},"last":4.5220000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019099,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507108,"kb_used_data":1209884,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92238044,"statfs":{"total":95995035648,"available":94451757056,"internally_reserved":0,"allocated":1238921216,"data_stored":1238440760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":0.79800000000000004,"15min":0.67900000000000005},"min":{"1min":0.24199999999999999,"5min":0.22,"15min":0.22},"max":{"1min":4.5140000000000002,"5min":4.5140000000000002,"15min":4.5140000000000002},"last":1.893},{"interface":"front","average":{"1min":1.944,"5min":0.98699999999999999,"15min":0.753},"min":{"1min":0.30299999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.5439999999999996,"5min":5.5439999999999996,"15min":5.5439999999999996},"last":3.3279999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":3.419,"5min":1.1839999999999999,"15min":0.79600000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":45.320999999999998,"5min":45.320999999999998,"15min":45.320999999999998},"last":2.464},{"interface":"front","average":{"1min":3.9180000000000001,"5min":1.2749999999999999,"15min":0.79500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":45.372999999999998,"5min":45.372999999999998,"15min":45.372999999999998},"last":1.4399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:44:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7770000000000001,"5min":2.9940000000000002,"15min":2.79},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":29.535,"5min":30.597000000000001,"15min":30.597000000000001},"last":3.1299999999999999},{"interface":"front","average":{"1min":3.4630000000000001,"5min":2.887,"15min":2.6549999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":26.719999999999999,"5min":30.117000000000001,"15min":30.117000000000001},"last":8.1150000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":1.4259999999999999,"15min":0.91800000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":23.318999999999999,"5min":44.994,"15min":44.994},"last":0.315},{"interface":"front","average":{"1min":2.5529999999999999,"5min":1.4430000000000001,"15min":0.873},"min":{"1min":0.371,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":23.469000000000001,"5min":44.765000000000001,"15min":44.765000000000001},"last":7.7380000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.0669999999999999,"15min":0.68300000000000005},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.2149999999999999,"5min":43.189999999999998,"15min":43.189999999999998},"last":2.3780000000000001},{"interface":"front","average":{"1min":0.996,"5min":1.0580000000000001,"15min":0.69299999999999995},"min":{"1min":0.30099999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.5839999999999996,"5min":42.450000000000003,"15min":42.450000000000003},"last":1.4670000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:43:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.82599999999999996,"15min":0.63200000000000001},"min":{"1min":0.32300000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.0110000000000001,"5min":27.056000000000001,"15min":27.056000000000001},"last":1.5489999999999999},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.89200000000000002,"15min":0.66900000000000004},"min":{"1min":0.307,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.4649999999999999,"5min":26.940999999999999,"15min":26.940999999999999},"last":2.411}]},{"osd":7,"last update":"Sat Sep 23 22:43:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.64900000000000002,"15min":0.59599999999999997},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.603,"5min":4.1459999999999999,"15min":4.1459999999999999},"last":2.4409999999999998},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.65400000000000003,"15min":0.59499999999999997},"min":{"1min":0.38800000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.129,"5min":5.9409999999999998,"15min":5.9409999999999998},"last":0.38800000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822871,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1301580,"kb_used_data":1004356,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92443572,"statfs":{"total":95995035648,"available":94662217728,"internally_reserved":0,"allocated":1028460544,"data_stored":1027999778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:53 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.974,"15min":1.0700000000000001},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.109,"5min":74.703999999999994,"15min":74.703999999999994},"last":45.420999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.6359999999999999,"15min":0.95599999999999996},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.6740000000000004,"5min":74.310000000000002,"15min":74.310000000000002},"last":45.353000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":2.7789999999999999,"15min":1.3049999999999999},"min":{"1min":0.318,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.388999999999999,"5min":149.203,"15min":149.203},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.528,"5min":2.6829999999999998,"15min":1.2490000000000001},"min":{"1min":0.28199999999999997,"5min":0.245,"15min":0.245},"max":{"1min":20.843,"5min":149.554,"15min":149.554},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":7.6909999999999998,"15min":8.6790000000000003},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.7859999999999996,"5min":116.123,"15min":116.123},"last":20.245999999999999},{"interface":"front","average":{"1min":1.3939999999999999,"5min":7.6180000000000003,"15min":8.6560000000000006},"min":{"1min":0.34499999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.633,"5min":116.14100000000001,"15min":116.14100000000001},"last":20.210000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":12.35,"5min":2.9180000000000001,"15min":1.3260000000000001},"min":{"1min":0.35699999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":121.94,"5min":121.94,"15min":121.94},"last":0.47799999999999998},{"interface":"front","average":{"1min":12.481999999999999,"5min":2.976,"15min":1.3999999999999999},"min":{"1min":0.36699999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":123.014,"5min":123.014,"15min":123.014},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.63300000000000001,"15min":0.53200000000000003},"min":{"1min":0.29699999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.165,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.57799999999999996,"15min":0.52400000000000002},"min":{"1min":0.309,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.0049999999999999,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":2.2309999999999999,"15min":1.0840000000000001},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.0730000000000004,"5min":140.07499999999999,"15min":140.07499999999999},"last":0.437},{"interface":"front","average":{"1min":0.75900000000000001,"5min":2.222,"15min":1.105},"min":{"1min":0.35199999999999998,"5min":0.254,"15min":0.254},"max":{"1min":2.7370000000000001,"5min":139.739,"15min":139.739},"last":0.38400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:43:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.70299999999999996,"15min":0.61299999999999999},"min":{"1min":0.35799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.774,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.67300000000000004,"15min":0.60799999999999998},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.311},"max":{"1min":3.1309999999999998,"5min":4.056,"15min":4.056},"last":0.40000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1883508736,"data_stored":1883504664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1527189504,"data_stored":1527185432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1585971200,"data_stored":1585971200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2188967936,"data_stored":2188967936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1230897152,"data_stored":1230897152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1020067840,"data_stored":1020067840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1173884928,"data_stored":1173880856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1098383360,"data_stored":1098383360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:44:19.755 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-09-23T22:44:19.819 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:19.820 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:19.907 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:19.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:19.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:20.017 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:20.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:20.354 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:20.459 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:20.485 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:20.527 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:20.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:20.672 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T22:44:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/171492 objects degraded (3.495%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[187743]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5993/165510 objects degraded (3.621%); 1.3 MiB/s, 104 objects/s recovering 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/171492 objects degraded (3.495%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:20.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:20 smithi052 ceph-mon[180979]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5993/165510 objects degraded (3.621%); 1.3 MiB/s, 104 objects/s recovering 2023-09-23T22:44:20.746 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:20.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T22:44:20.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: failed to encode map e188 with expected crc 2023-09-23T22:44:20.757 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/171492 objects degraded (3.495%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:20.758 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:20 smithi151 ceph-mon[159210]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 5993/165510 objects degraded (3.621%); 1.3 MiB/s, 104 objects/s recovering 2023-09-23T22:44:20.855 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:20.977 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:21.263 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:21.274 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:21.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:21.377 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:21.416 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:21.464 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:21.598 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T22:44:21.598 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.598 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/532218002' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:21.598 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:21.598 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.599 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:21.604 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:44:21.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T22:44:21.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/532218002' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:21.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:21.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:44:21.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:21 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:21.913 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:21.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:21.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:21.919 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T22:44:21.919 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.919 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/532218002' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:21.919 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:21.919 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: failed to encode map e189 with expected crc 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:21.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:21 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:22.020 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:22.133 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:22.856 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[187743]: failed to encode map e190 with expected crc 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[187743]: failed to encode map e190 with expected crc 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[187743]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[187743]: pgmap v465: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 5993/162417 objects degraded (3.690%); 0 B/s, 96 objects/s recovering 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[180979]: failed to encode map e190 with expected crc 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[180979]: failed to encode map e190 with expected crc 2023-09-23T22:44:22.857 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:44:22.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[180979]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T22:44:22.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:22 smithi052 ceph-mon[180979]: pgmap v465: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 5993/162417 objects degraded (3.690%); 0 B/s, 96 objects/s recovering 2023-09-23T22:44:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:22 smithi151 ceph-mon[159210]: failed to encode map e190 with expected crc 2023-09-23T22:44:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:22 smithi151 ceph-mon[159210]: failed to encode map e190 with expected crc 2023-09-23T22:44:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:22 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:44:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:22 smithi151 ceph-mon[159210]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T22:44:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:22 smithi151 ceph-mon[159210]: pgmap v465: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 5993/162417 objects degraded (3.690%); 0 B/s, 96 objects/s recovering 2023-09-23T22:44:23.102 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:23.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:23.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:23.211 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:23.307 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:23.394 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:44:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:44:23] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.33.4" 2023-09-23T22:44:23.431 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:23.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:23.539 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:23.631 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:23.794 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:23.794 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:23.801 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:23.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:23.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:23.960 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:24.108 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:24.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:24.224 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:24.319 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:24.804 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:24.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:24.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:24.911 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:24.974 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[187743]: pgmap v466: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 5993/162417 objects degraded (3.690%); 0 B/s, 96 objects/s recovering 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5993/162417 objects degraded (3.690%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[180979]: pgmap v466: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 5993/162417 objects degraded (3.690%); 0 B/s, 96 objects/s recovering 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:25.407 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5993/162417 objects degraded (3.690%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:25.408 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:25.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:44:25.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:25 smithi151 ceph-mon[159210]: pgmap v466: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 5993/162417 objects degraded (3.690%); 0 B/s, 96 objects/s recovering 2023-09-23T22:44:25.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5993/162417 objects degraded (3.690%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:25.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:44:25.542 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:25.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:25.660 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:25.868 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:25.995 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:26.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:26.061 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:44:26.106 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:26.121 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:44:26.140 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.165 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:44:26.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:26.231 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:44:26.275 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:44:26.310 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:26.321 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:44:26.342 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.375 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:44:26.390 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:26.395 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3659526187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:26.441 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:44:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.451 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.452 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.452 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:44:26.496 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:26.505 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3659526187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:26.506 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3659526187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:26.621 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.777 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:26.883 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:26.918 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.988 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:26.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:27.096 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:27.169 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:27.201 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 7 pgp_num to 16 2023-09-23T22:44:27.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:27 smithi052 ceph-mon[180979]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3978/154551 objects degraded (2.574%); 969/154551 objects misplaced (0.627%); 2.9 MiB/s, 50 objects/s recovering 2023-09-23T22:44:27.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1630631405' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:44:27.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:44:27.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:27 smithi052 ceph-mon[187743]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3978/154551 objects degraded (2.574%); 969/154551 objects misplaced (0.627%); 2.9 MiB/s, 50 objects/s recovering 2023-09-23T22:44:27.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1630631405' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:44:27.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:44:27.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:27 smithi151 ceph-mon[159210]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3978/154551 objects degraded (2.574%); 969/154551 objects misplaced (0.627%); 2.9 MiB/s, 50 objects/s recovering 2023-09-23T22:44:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1630631405' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:44:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:44:27.880 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:27.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:27.991 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:28.034 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:28.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[187743]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:44:28.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[187743]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T22:44:28.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[187743]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.229 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[187743]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.351 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[180979]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:44:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[180979]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T22:44:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[180979]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:28 smithi052 ceph-mon[180979]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.498 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:28.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:28.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:28.604 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:28.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:28 smithi151 ceph-mon[159210]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:28 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:44:28.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:28 smithi151 ceph-mon[159210]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T22:44:28.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:28 smithi151 ceph-mon[159210]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:28 smithi151 ceph-mon[159210]: failed to encode map e191 with expected crc 2023-09-23T22:44:28.635 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:28.684 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:28.700 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:28.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:28.805 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:28.927 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:29.051 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:29.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:29.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:29.160 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:29.312 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:29.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:29 smithi052 ceph-mon[187743]: pgmap v469: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 3978/153972 objects degraded (2.584%); 969/153972 objects misplaced (0.629%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:44:29.492 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:29 smithi052 ceph-mon[180979]: pgmap v469: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 3978/153972 objects degraded (2.584%); 969/153972 objects misplaced (0.629%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:44:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:29 smithi151 ceph-mon[159210]: pgmap v469: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 3978/153972 objects degraded (2.584%); 969/153972 objects misplaced (0.629%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:44:30.064 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:30.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:30.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:30.173 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:30.201 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:30.392 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:44:30.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3978/153972 objects degraded (2.584%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:30.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3978/153972 objects degraded (2.584%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3978/153972 objects degraded (2.584%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:30.904 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:30.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:30.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:31.014 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:31.046 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:31.107 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:31.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:31.214 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:31.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:31 smithi052 ceph-mon[187743]: pgmap v470: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3978/148998 objects degraded (2.670%); 680/148998 objects misplaced (0.456%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:31.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:31 smithi052 ceph-mon[180979]: pgmap v470: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3978/148998 objects degraded (2.670%); 680/148998 objects misplaced (0.456%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:31.246 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:31.274 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:31.308 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:31.312 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:31 smithi151 ceph-mon[159210]: pgmap v470: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3978/148998 objects degraded (2.670%); 680/148998 objects misplaced (0.456%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:44:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:31.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:31.413 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:31.446 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:31.514 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:31.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:31.622 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:31.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:31.723 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:31.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:31.829 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:31.857 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40792 6.f 6.8b4b57ef (undecoded) ondisk+retry+write+known_if_redirected e131)", 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:44:31.916 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 202.04385160499999, 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602794099999999, 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40792 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:44:31.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861820+0000", 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861828+0000", 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861841+0000", 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2709e-05 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861851+0000", 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861957+0000", 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106518 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861972+0000", 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5226e-05 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875022+0000", 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013050091999999999 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875034+0000", 2023-09-23T22:44:31.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1232e-05 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922087+0000", 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047052836 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922101+0000", 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4476e-05 2023-09-23T22:44:31.922 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40806 6.f 6.e80b2dbf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 202.04379250100001, 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602518750000001, 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.923 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40806 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861880+0000", 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861885+0000", 2023-09-23T22:44:31.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.561e-06 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862005+0000", 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120701 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862017+0000", 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:44:31.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862139+0000", 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012248000000000001 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862145+0000", 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881337+0000", 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019192607 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881351+0000", 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3879999999999999e-05 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922124+0000", 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0407723719999999 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922133+0000", 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40815 6.f 6.8b4fc53f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 202.02141136099999, 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037899828, 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.930 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40815 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884260+0000", 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884266+0000", 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6060000000000002e-06 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884276+0000", 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7180000000000008e-06 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884284+0000", 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6170000000000007e-06 2023-09-23T22:44:31.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884403+0000", 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011853099999999999 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884415+0000", 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1683e-05 2023-09-23T22:44:31.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922153+0000", 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037738136 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922162+0000", 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9279999999999999e-06 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40816 6.f 6.4f46a35f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 202.02137049500001, 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378988650000001, 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40816 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884302+0000", 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884309+0000", 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:44:31.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884317+0000", 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5660000000000007e-06 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884324+0000", 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0859999999999996e-06 2023-09-23T22:44:31.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884441+0000", 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011671 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884449+0000", 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.571e-06 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922191+0000", 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037741888 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922202+0000", 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.092e-05 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.941 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40818 6.f 6.431951cf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 202.01545317, 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.032011485, 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:31.942 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40818 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:44:31.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890219+0000", 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890224+0000", 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7840000000000003e-06 2023-09-23T22:44:31.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890235+0000", 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0899e-05 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890243+0000", 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.225e-06 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890301+0000", 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890312+0000", 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1235e-05 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922221+0000", 2023-09-23T22:44:31.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.031909277 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922232+0000", 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0465000000000001e-05 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40826 6.f 6.3e4c7bff (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:44:31.947 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 202.00326769099999, 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.019876078, 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40826 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902404+0000", 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902413+0000", 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5449999999999992e-06 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902432+0000", 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8488999999999999e-05 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902441+0000", 2023-09-23T22:44:31.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902502+0000", 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1064999999999999e-05 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902511+0000", 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7350000000000002e-06 2023-09-23T22:44:31.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922270+0000", 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0197598290000001 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922282+0000", 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.952 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:41355 7.e 7:753765c8:::benchmark_data_smithi052_277165_object41354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:14.675898+0000", 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 137.22977485000001, 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.209596168, 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.953 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 41355 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.675898+0000", 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.675898+0000", 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.675897+0000", 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.675932+0000", 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5324999999999998e-05 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.675944+0000", 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1951e-05 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.675953+0000", 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2849999999999995e-06 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.676012+0000", 2023-09-23T22:44:31.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9162e-05 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.676183+0000", 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000171409 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.676322+0000", 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013850000000000001 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.677845+0000", 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015236099999999999 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.677878+0000", 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2740999999999997e-05 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.885399+0000", 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20752073300000001 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.885452+0000", 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3127000000000001e-05 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.885495+0000", 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2589999999999997e-05 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.959 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:41360 7.e 7:70d31207:::benchmark_data_smithi052_277165_object41359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:14.678557+0000", 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 137.22711594800001, 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20987020000000001, 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.960 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 41360 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678557+0000", 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678557+0000", 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678556+0000", 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678576+0000", 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9445000000000001e-05 2023-09-23T22:44:31.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678590+0000", 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4113e-05 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678598+0000", 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1899999999999995e-06 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678656+0000", 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.766e-05 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678790+0000", 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013438600000000001 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.678936+0000", 2023-09-23T22:44:31.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014594599999999999 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.680505+0000", 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00156865 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.681974+0000", 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014696150000000001 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.888341+0000", 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.206366983 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.888387+0000", 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5120999999999997e-05 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:14.888427+0000", 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0936999999999998e-05 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:61238 7.e 7:7c256f0b:::benchmark_data_smithi052_277165_object61237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:44:31.967 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:33.081539+0000", 2023-09-23T22:44:31.968 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 118.824134362, 2023-09-23T22:44:31.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.228146231, 2023-09-23T22:44:31.968 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.968 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.968 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.968 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 61238 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081539+0000", 2023-09-23T22:44:31.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081539+0000", 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081538+0000", 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081562+0000", 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3954000000000001e-05 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081582+0000", 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0030999999999998e-05 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081589+0000", 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8140000000000001e-06 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081635+0000", 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5204999999999998e-05 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081764+0000", 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012969300000000001 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.081889+0000", 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012495900000000001 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.084334+0000", 2023-09-23T22:44:31.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024451429999999999 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.084353+0000", 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8131e-05 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.309587+0000", 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22523422400000001 2023-09-23T22:44:31.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.309640+0000", 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3319999999999998e-05 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:33.309685+0000", 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4997999999999998e-05 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:81038 7.8 7:13a2dcef:::benchmark_data_smithi052_277165_object81037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:59.284219+0000", 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 92.621454584000006, 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18565116400000001, 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 81038 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.976 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284219+0000", 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284219+0000", 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284217+0000", 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284240+0000", 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3422999999999999e-05 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284250+0000", 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5060000000000001e-06 2023-09-23T22:44:31.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284256+0000", 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.02e-06 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.286828+0000", 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025719800000000002 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:31.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.286975+0000", 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014717899999999999 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.287115+0000", 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013993799999999999 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.295503+0000", 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0083882249999999992 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.298088+0000", 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025842669999999999 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.467305+0000", 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16921724799999999 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.467363+0000", 2023-09-23T22:44:31.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8093000000000001e-05 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.469870+0000", 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025068820000000002 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:81039 7.8 7:1d7ea26b:::benchmark_data_smithi052_277165_object81038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:59.284269+0000", 2023-09-23T22:44:31.982 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 92.621403909999998, 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.1882365, 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 81039 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284269+0000", 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284269+0000", 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284269+0000", 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284288+0000", 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9249000000000002e-05 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284295+0000", 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6270000000000002e-06 2023-09-23T22:44:31.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.284300+0000", 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8459999999999999e-06 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.289884+0000", 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055845349999999998 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:31.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.290084+0000", 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020016200000000001 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.290259+0000", 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017523500000000001 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.295511+0000", 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052511459999999999 2023-09-23T22:44:31.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.298182+0000", 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026717199999999998 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.469952+0000", 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17176925400000001 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.470009+0000", 2023-09-23T22:44:31.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7710000000000001e-05 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.472506+0000", 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002496477 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:81043 7.8 7:1136dc21:::benchmark_data_smithi052_277165_object81042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-09-23T22:44:31.989 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:42:59.291192+0000", 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 92.614481545000004, 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18386490799999999, 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 81043 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.291192+0000", 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.291192+0000", 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.291190+0000", 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.291217+0000", 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7205000000000001e-05 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.291228+0000", 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0461e-05 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.291234+0000", 2023-09-23T22:44:31.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2140000000000001e-06 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.292959+0000", 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017246130000000001 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:31.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.293125+0000", 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016618899999999999 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.293270+0000", 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0001448 2023-09-23T22:44:31.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.295700+0000", 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024301420000000002 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.298233+0000", 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025336859999999998 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.472651+0000", 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17441747299999999 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.472709+0000", 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8229000000000001e-05 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:42:59.475057+0000", 2023-09-23T22:44:31.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023475150000000001 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:92224 7.e 7:7cc074f6:::benchmark_data_smithi052_277165_object92223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:16.881295+0000", 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 75.024378173000002, 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.201540412, 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:31.997 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92224 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881295+0000", 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:31.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881295+0000", 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881293+0000", 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:31.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881329+0000", 2023-09-23T22:44:32.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6409999999999999e-05 2023-09-23T22:44:32.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881343+0000", 2023-09-23T22:44:32.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.346e-05 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881351+0000", 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4659999999999999e-06 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881400+0000", 2023-09-23T22:44:32.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8748999999999997e-05 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881526+0000", 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000125967 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881675+0000", 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014883799999999999 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.886229+0000", 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0045543470000000003 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.886834+0000", 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00060536400000000005 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.082715+0000", 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19588014100000001 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.082780+0000", 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5339000000000006e-05 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.082835+0000", 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5478e-05 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:92227 7.5 7:aa914306:::benchmark_data_smithi052_277165_object92226:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:16.881370+0000", 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 75.024303525999997, 2023-09-23T22:44:32.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.197336294, 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92227 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881370+0000", 2023-09-23T22:44:32.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881370+0000", 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881369+0000", 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881392+0000", 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2504999999999999e-05 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881399+0000", 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2949999999999996e-06 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.881405+0000", 2023-09-23T22:44:32.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9399999999999999e-06 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.883363+0000", 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019582639999999999 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.883579+0000", 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021632800000000001 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.883756+0000", 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017634899999999999 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887077+0000", 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033213729999999999 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.075901+0000", 2023-09-23T22:44:32.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18882336 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.076278+0000", 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000377468 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.076333+0000", 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5136999999999997e-05 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.078706+0000", 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023727940000000001 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:92230 7.5 7:ac3ff760:::benchmark_data_smithi052_277165_object92229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:16.887734+0000", 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 75.017939530000007, 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19336736900000001, 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.012 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92230 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887734+0000", 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887734+0000", 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887732+0000", 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887757+0000", 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4825999999999999e-05 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887768+0000", 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1092e-05 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.887776+0000", 2023-09-23T22:44:32.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4209999999999998e-06 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.890336+0000", 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002560414 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.890517+0000", 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000181084 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.890699+0000", 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018143099999999999 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893541+0000", 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002842542 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.075932+0000", 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18239071400000001 2023-09-23T22:44:32.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.078759+0000", 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028268680000000002 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.078820+0000", 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0979e-05 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.081101+0000", 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022814189999999998 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:92229 7.c e174/169 7:3ccf5324:::benchmark_data_smithi052_277165_object92228:head v 174'5687, mlcod=154'1289)", 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:16.888238+0000", 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 75.017434835000003, 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18766786599999999, 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.019 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.888238+0000", 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.888238+0000", 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.888238+0000", 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.888347+0000", 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010901899999999999 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.888355+0000", 2023-09-23T22:44:32.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6980000000000004e-06 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.888360+0000", 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8330000000000001e-06 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.889257+0000", 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00089722900000000004 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.889283+0000", 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5771000000000001e-05 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.075875+0000", 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18659215900000001 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.075906+0000", 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0871000000000003e-05 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.024 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:92234 7.e 7:73302511:::benchmark_data_smithi052_277165_object92233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:16.893513+0000", 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 75.012160245000004, 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20031942999999999, 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92234 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.025 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893513+0000", 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893513+0000", 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893511+0000", 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893549+0000", 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8124000000000003e-05 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893561+0000", 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1829e-05 2023-09-23T22:44:32.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893569+0000", 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6580000000000002e-06 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.075822+0000", 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.182253146 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.076014+0000", 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000192346 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.076223+0000", 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000208579 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.082889+0000", 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0066666640000000001 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.090356+0000", 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074662670000000004 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.093758+0000", 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034020280000000001 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.093797+0000", 2023-09-23T22:44:32.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9869000000000001e-05 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.093832+0000", 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4895999999999999e-05 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:92235 7.e 7:71df773e:::benchmark_data_smithi052_277165_object92234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-09-23T22:44:32.031 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:16.893583+0000", 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 75.012090520000001, 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.200352433, 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 92235 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893583+0000", 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893583+0000", 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893582+0000", 2023-09-23T22:44:32.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893599+0000", 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6252999999999998e-05 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:16.893603+0000", 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7690000000000004e-06 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.034 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.075812+0000", 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.182208273 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.079337+0000", 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003525799 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.035 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.079505+0000", 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016794999999999999 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.079655+0000", 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014963800000000001 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.082940+0000", 2023-09-23T22:44:32.036 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003285278 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.090361+0000", 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074207739999999998 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.093871+0000", 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003509978 2023-09-23T22:44:32.037 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.093905+0000", 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4354999999999999e-05 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:17.093935+0000", 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9776000000000001e-05 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.038 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:101954 7.8 7:1f3c2d04:::benchmark_data_smithi052_277165_object9462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:32.520902+0000", 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 59.384770940000003, 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23800943099999999, 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101954 2023-09-23T22:44:32.039 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520902+0000", 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520902+0000", 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.040 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520900+0000", 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520908+0000", 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5850000000000002e-06 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520921+0000", 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3196e-05 2023-09-23T22:44:32.041 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520931+0000", 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5279999999999992e-06 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.520984+0000", 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3000000000000001e-05 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.042 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.521175+0000", 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019081999999999999 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.521321+0000", 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014635700000000001 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.656705+0000", 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.13538382099999999 2023-09-23T22:44:32.043 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.656753+0000", 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7877000000000003e-05 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.756467+0000", 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.099714338 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.044 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.756526+0000", 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9298e-05 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:32.758912+0000", 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023854150000000001 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.045 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:114040 7.5 7:a78c49b7:::benchmark_data_smithi052_277165_object21548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:52.198245+0000", 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 39.707428194000002, 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18495030500000001, 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114040 2023-09-23T22:44:32.046 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.198245+0000", 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.198245+0000", 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.047 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.198245+0000", 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.198247+0000", 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2520000000000002e-06 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.198253+0000", 2023-09-23T22:44:32.048 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6239999999999999e-06 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.198257+0000", 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7550000000000002e-06 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.205135+0000", 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.006877279 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.049 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.205277+0000", 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000142044 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.205409+0000", 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013284899999999999 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.210448+0000", 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050382279999999996 2023-09-23T22:44:32.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.212354+0000", 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019066930000000001 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.381904+0000", 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16954929399999999 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:32.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.381964+0000", 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0328000000000002e-05 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:52.383195+0000", 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001231298 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:32.052 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:44:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:32.053 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:32.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:32.154 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:32.264 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:33.319 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:44:33.355 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:44:33.377 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:33.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:33.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:33.485 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:33.529 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:33.634 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:44:33.634 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:44:33.634 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:44:33.635 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 1.000000 2023-09-23T22:44:33.635 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T22:44:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:33 smithi151 ceph-mon[159210]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 3978/145074 objects degraded (2.742%); 680/145074 objects misplaced (0.469%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:44:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3666064160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:33 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:33.700 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:44:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:44:33] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.33.4" 2023-09-23T22:44:33.700 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:33 smithi052 ceph-mon[187743]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 3978/145074 objects degraded (2.742%); 680/145074 objects misplaced (0.469%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:44:33.700 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3666064160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:33.700 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:33 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:33.701 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:33 smithi052 ceph-mon[180979]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 3978/145074 objects degraded (2.742%); 680/145074 objects misplaced (0.469%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:44:33.701 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3666064160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:33.701 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:33 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:44:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T22:44:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[187743]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:34 smithi052 ceph-mon[180979]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:34 smithi151 ceph-mon[159210]: failed to encode map e192 with expected crc 2023-09-23T22:44:34.814 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:34.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:34.935 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:35.016 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.162 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:35.192 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:35.342 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:35.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:35.451 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:35.543 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[187743]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 3978/145074 objects degraded (2.742%); 680/145074 objects misplaced (0.469%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:44:35.544 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:35.544 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:35.544 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3978/145074 objects degraded (2.742%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:35.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[180979]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 3978/145074 objects degraded (2.742%); 680/145074 objects misplaced (0.469%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:44:35.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:35.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:35.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3978/145074 objects degraded (2.742%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:35.554 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:35 smithi151 ceph-mon[159210]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 3978/145074 objects degraded (2.742%); 680/145074 objects misplaced (0.469%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:44:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3978/145074 objects degraded (2.742%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:35.735 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:44:35.786 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:44:35.843 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:44:35.844 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:35.900 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:44:35.949 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:44:35.953 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:35.985 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.988 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:44:36.034 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:44:36.066 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:36.082 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:44:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:36.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:36.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.106 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.106 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.106 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:44:36.183 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:36.223 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:36.294 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:36.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:36.401 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:36.485 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2207931525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:36.485 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:36 smithi052 ceph-mon[180979]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 3978/136761 objects degraded (2.909%); 440/136761 objects misplaced (0.322%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:44:36.486 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2207931525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:36.486 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:36 smithi052 ceph-mon[187743]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 3978/136761 objects degraded (2.909%); 440/136761 objects misplaced (0.322%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:44:36.492 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2207931525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:36 smithi151 ceph-mon[159210]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 3978/136761 objects degraded (2.909%); 440/136761 objects misplaced (0.322%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:44:36.910 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:44:37.754 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:37.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:37.863 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:37.948 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:38.249 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:38.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:38.355 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:38.384 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:38.441 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:38.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:38.544 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:38.642 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:39.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:39 smithi151 ceph-mon[159210]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3978/136140 objects degraded (2.922%); 440/136140 objects misplaced (0.323%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:44:39.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:39 smithi052 ceph-mon[180979]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3978/136140 objects degraded (2.922%); 440/136140 objects misplaced (0.323%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:44:39.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:39 smithi052 ceph-mon[187743]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3978/136140 objects degraded (2.922%); 440/136140 objects misplaced (0.323%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:44:39.496 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:39.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:39.541 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:39.613 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:39.716 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:39.914 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:39.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:40.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:40.174 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:40.353 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:44:40.470 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:40.470 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:40.489 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:40.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:40.606 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:40.762 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:40.762 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:40.762 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3978/136140 objects degraded (2.922%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:40.762 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[180979]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 3978/131121 objects degraded (3.034%); 133/131121 objects misplaced (0.101%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1865407752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3978/136140 objects degraded (2.922%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[187743]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 3978/131121 objects degraded (3.034%); 133/131121 objects misplaced (0.101%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1865407752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:44:40.763 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:40 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:44:40.766 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:40.876 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T22:44:41.185 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:41.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:41.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:41.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:41.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:40 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:41.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3978/136140 objects degraded (2.922%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:44:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:40 smithi151 ceph-mon[159210]: pgmap v476: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 3978/131121 objects degraded (3.034%); 133/131121 objects misplaced (0.101%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:44:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1865407752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:44:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:40 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:44:41.290 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:41.334 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:41.406 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:41.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:41.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:41.556 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:41.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:44:41.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:44:41.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:41.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:44:41.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T22:44:41.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: failed to encode map e193 with expected crc 2023-09-23T22:44:41.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[187743]: failed to encode map e193 with expected crc 2023-09-23T22:44:41.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:44:41.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:44:41.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:41.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:44:41.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T22:44:41.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: failed to encode map e193 with expected crc 2023-09-23T22:44:42.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:41 smithi052 ceph-mon[180979]: failed to encode map e193 with expected crc 2023-09-23T22:44:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:44:42.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:44:42.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:42.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:44:42.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T22:44:42.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: failed to encode map e193 with expected crc 2023-09-23T22:44:42.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:41 smithi151 ceph-mon[159210]: failed to encode map e193 with expected crc 2023-09-23T22:44:42.624 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:42.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:42.728 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:42.731 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:44:42.814 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:42.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:42 smithi052 ceph-mon[187743]: failed to encode map e193 with expected crc 2023-09-23T22:44:42.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:42 smithi052 ceph-mon[187743]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3978/127203 objects degraded (3.127%); 133/127203 objects misplaced (0.105%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:44:42.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:42 smithi052 ceph-mon[180979]: failed to encode map e193 with expected crc 2023-09-23T22:44:42.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:42 smithi052 ceph-mon[180979]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3978/127203 objects degraded (3.127%); 133/127203 objects misplaced (0.105%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:44:42.991 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:43.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:43.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:43.118 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:43.121 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:43.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:42 smithi151 ceph-mon[159210]: failed to encode map e193 with expected crc 2023-09-23T22:44:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:42 smithi151 ceph-mon[159210]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3978/127203 objects degraded (3.127%); 133/127203 objects misplaced (0.105%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:44:43.237 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:43.479 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:44:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:44:43] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.33.4" 2023-09-23T22:44:43.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:43.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:43.733 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:43.768 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:43.830 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:43.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:43.934 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:44.005 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:44.148 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:44.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:44.252 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:44.298 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:44.351 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:44.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:44.454 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:44.481 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:44.809 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:44.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:44.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:44.866 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:44.916 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:45.001 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:45.170 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[180979]: pgmap v479: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 3978/127203 objects degraded (3.127%); 133/127203 objects misplaced (0.105%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:44:45.170 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2225062461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:45.170 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:45.171 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3978/127203 objects degraded (3.127%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:45.186 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:45.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:45.215 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:44:45.296 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:45.388 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:45.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[187743]: pgmap v479: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 3978/127203 objects degraded (3.127%); 133/127203 objects misplaced (0.105%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:44:45.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2225062461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:45.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:45.432 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3978/127203 objects degraded (3.127%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:45.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:45 smithi151 ceph-mon[159210]: pgmap v479: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 3978/127203 objects degraded (3.127%); 133/127203 objects misplaced (0.105%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:44:45.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2225062461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:45.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:45 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:45.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3978/127203 objects degraded (3.127%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:45.779 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:44:45.818 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:44:45.884 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:44:45.914 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:44:45.977 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:44:46.070 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.071 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.072 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:44:46.267 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:46.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:46.283 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:46.283 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[180979]: failed to encode map e194 with expected crc 2023-09-23T22:44:46.283 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:46.283 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[180979]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T22:44:46.283 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[180979]: failed to encode map e194 with expected crc 2023-09-23T22:44:46.283 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/338415312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:46.284 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:46.284 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[187743]: failed to encode map e194 with expected crc 2023-09-23T22:44:46.284 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:46.284 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[187743]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T22:44:46.284 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[187743]: failed to encode map e194 with expected crc 2023-09-23T22:44:46.284 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/338415312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:46.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:46.424 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:46.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:46 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:46.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:46 smithi151 ceph-mon[159210]: failed to encode map e194 with expected crc 2023-09-23T22:44:46.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:46 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:46.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:46 smithi151 ceph-mon[159210]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T22:44:46.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:46 smithi151 ceph-mon[159210]: failed to encode map e194 with expected crc 2023-09-23T22:44:46.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/338415312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:46.499 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:46.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:46.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:46.608 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:46.642 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:46.715 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:46.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:46.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:46.823 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:46.853 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:47.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:47 smithi151 ceph-mon[159210]: pgmap v481: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3978/119151 objects degraded (3.339%); 3.4 MiB/s, 78 objects/s recovering 2023-09-23T22:44:47.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:47 smithi052 ceph-mon[180979]: pgmap v481: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3978/119151 objects degraded (3.339%); 3.4 MiB/s, 78 objects/s recovering 2023-09-23T22:44:47.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:47 smithi052 ceph-mon[187743]: pgmap v481: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3978/119151 objects degraded (3.339%); 3.4 MiB/s, 78 objects/s recovering 2023-09-23T22:44:47.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:47.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:47.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:47.782 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:44:47.782 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:44:47.782 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:44:47.782 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T22:44:47.817 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:47.985 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:48.265 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:48.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:48.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:48.370 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:48.467 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:48.613 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:44:48.613 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:44:48.613 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:44:48.613 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:44:48.613 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:44:48.620 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:48.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:48.730 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:48.781 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:48.857 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:48.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:48.970 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:49.009 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:49.094 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:49.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:49.207 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:49.243 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:49.312 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:49.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:49.416 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:49.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:49 smithi151 ceph-mon[159210]: pgmap v482: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 3978/118617 objects degraded (3.354%); 1.0 MiB/s, 39 objects/s recovering 2023-09-23T22:44:49.452 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:49.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:49 smithi052 ceph-mon[180979]: pgmap v482: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 3978/118617 objects degraded (3.354%); 1.0 MiB/s, 39 objects/s recovering 2023-09-23T22:44:49.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:49 smithi052 ceph-mon[187743]: pgmap v482: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 3978/118617 objects degraded (3.354%); 1.0 MiB/s, 39 objects/s recovering 2023-09-23T22:44:49.527 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:49.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:49.633 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:49.646 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:44:49.742 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:49.751 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:49.789 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:44:49.823 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:44:49.876 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:49.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:49.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:49.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:50.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:50.342 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:50.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:50.461 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:50.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:50.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:50.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3978/118617 objects degraded (3.354%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:50.531 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[180979]: pgmap v483: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 3978/113496 objects degraded (3.505%); 918 KiB/s, 73 objects/s recovering 2023-09-23T22:44:50.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:50.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:50.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3978/118617 objects degraded (3.354%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:50.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:50 smithi052 ceph-mon[187743]: pgmap v483: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 3978/113496 objects degraded (3.505%); 918 KiB/s, 73 objects/s recovering 2023-09-23T22:44:50.556 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:50.871 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:50.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:50.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:44:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3978/118617 objects degraded (3.354%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:50 smithi151 ceph-mon[159210]: pgmap v483: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 3978/113496 objects degraded (3.505%); 918 KiB/s, 73 objects/s recovering 2023-09-23T22:44:50.981 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:51.075 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.185 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:51.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:51.299 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:51.349 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.409 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:51.494 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:51.515 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:51.545 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.608 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:51.714 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:51.736 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:51.738 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:51.819 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:52.045 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:52.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:52.150 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:52.176 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:52.219 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:52.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:52.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:52.326 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:52.360 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:52.422 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:52.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:52.531 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:52.561 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:52.854 INFO:teuthology.orchestra.run.smithi052.stderr:marked out osd.7. 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: failed to encode map e195 with expected crc 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: failed to encode map e195 with expected crc 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 3978/109047 objects degraded (3.648%); 1.0 MiB/s, 84 objects/s recovering 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2161382549' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:44:52.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[187743]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 conmon[180954]: 2023-09-23T22:44:52.634+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e195 definitely_dead 0 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: failed to encode map e195 with expected crc 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: failed to encode map e195 with expected crc 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 3978/109047 objects degraded (3.648%); 1.0 MiB/s, 84 objects/s recovering 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2161382549' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:44:52.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:52 smithi052 ceph-mon[180979]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:44:52.965 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:52.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:52.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:53.068 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:53.097 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: failed to encode map e195 with expected crc 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: failed to encode map e195 with expected crc 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: pgmap v485: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 3978/109047 objects degraded (3.648%); 1.0 MiB/s, 84 objects/s recovering 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2161382549' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:44:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:52 smithi151 ceph-mon[159210]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:44:53.685 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:53.686 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:44:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:44:53] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.33.4" 2023-09-23T22:44:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:53.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:53.789 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:53.821 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:53.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:44:53.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T22:44:53.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[187743]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:53 smithi052 ceph-mon[180979]: failed to encode map e196 with expected crc 2023-09-23T22:44:53.986 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:53.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:54.090 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.114 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.115 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.115 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:53 smithi151 ceph-mon[159210]: failed to encode map e196 with expected crc 2023-09-23T22:44:54.116 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.157 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:54.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:54.261 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:54.287 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.329 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:54.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:54.434 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:54.461 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.505 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:54.613 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:54.646 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T22:44:54.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: failed to encode map e197 with expected crc 2023-09-23T22:44:54.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: pgmap v488: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 3978/109047 objects degraded (3.648%); 0 B/s, 59 objects/s recovering 2023-09-23T22:44:54.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3978/109047 objects degraded (3.648%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:54.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[187743]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T22:44:55.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T22:44:55.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: pgmap v488: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 3978/109047 objects degraded (3.648%); 0 B/s, 59 objects/s recovering 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3978/109047 objects degraded (3.648%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:55.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:54 smithi151 ceph-mon[159210]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: failed to encode map e197 with expected crc 2023-09-23T22:44:55.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: pgmap v488: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 3978/109047 objects degraded (3.648%); 0 B/s, 59 objects/s recovering 2023-09-23T22:44:55.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3978/109047 objects degraded (3.648%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:44:55.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:54 smithi052 ceph-mon[180979]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T22:44:55.846 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:55.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:55.958 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: 7.5 starting backfill to osd.1 from (0'0,178'6700] MIN to 195'9604 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: 3.e continuing backfill to osd.0 from (0'0,60'2] MIN to 60'2 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[180979]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.060 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: 7.5 starting backfill to osd.1 from (0'0,178'6700] MIN to 195'9604 2023-09-23T22:44:56.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: 3.e continuing backfill to osd.0 from (0'0,60'2] MIN to 60'2 2023-09-23T22:44:56.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:55 smithi052 ceph-mon[187743]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.063 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:56.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: 7.5 starting backfill to osd.1 from (0'0,178'6700] MIN to 195'9604 2023-09-23T22:44:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: 3.e continuing backfill to osd.0 from (0'0,60'2] MIN to 60'2 2023-09-23T22:44:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:55 smithi151 ceph-mon[159210]: failed to encode map e198 with expected crc 2023-09-23T22:44:56.847 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:56.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:56.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:56.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:57.001 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.050 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:57.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:57.153 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:57.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:56 smithi151 ceph-mon[159210]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:56 smithi151 ceph-mon[159210]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T22:44:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:56 smithi151 ceph-mon[159210]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:56 smithi151 ceph-mon[159210]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:56 smithi151 ceph-mon[159210]: pgmap v491: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 8298/104943 objects degraded (7.907%); 955/104943 objects misplaced (0.910%); 2.6 MiB/s, 2 keys/s, 91 objects/s recovering 2023-09-23T22:44:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[187743]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[187743]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T22:44:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[187743]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[187743]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[187743]: pgmap v491: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 8298/104943 objects degraded (7.907%); 955/104943 objects misplaced (0.910%); 2.6 MiB/s, 2 keys/s, 91 objects/s recovering 2023-09-23T22:44:57.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[180979]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[180979]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T22:44:57.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[180979]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[180979]: failed to encode map e199 with expected crc 2023-09-23T22:44:57.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:56 smithi052 ceph-mon[180979]: pgmap v491: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 8298/104943 objects degraded (7.907%); 955/104943 objects misplaced (0.910%); 2.6 MiB/s, 2 keys/s, 91 objects/s recovering 2023-09-23T22:44:57.261 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.554 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:57.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:57.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10170 6.1 6:8aa77601:::benchmark_data_smithi052_234456_object10169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.65977483, 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.264625532, 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10170 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199126+0000", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199153+0000", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7100000000000001e-05 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199170+0000", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6382e-05 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199177+0000", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7449999999999995e-06 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199217+0000", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9523000000000001e-05 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199406+0000", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018866 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199606+0000", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000200278 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201347+0000", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001740969 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463581+0000", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26223432499999999 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463651+0000", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9529999999999993e-05 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463753+0000", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000102389 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10180 6.1 6:8dfddb97:::benchmark_data_smithi052_234456_object10179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.659661051, 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26492964299999999, 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10180 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199272+0000", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1467999999999998e-05 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199288+0000", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5582000000000001e-05 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199294+0000", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200146+0000", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00085146099999999997 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200281+0000", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013534300000000001 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200378+0000", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6699000000000004e-05 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201357+0000", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097968399999999998 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464013+0000", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26265580399999999 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464077+0000", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4029999999999995e-05 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464171+0000", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3728999999999997e-05 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10183 6.1 6:8243606a:::benchmark_data_smithi052_234456_object10182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.659601729, 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265208887, 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10183 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199300+0000", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199322+0000", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1863999999999999e-05 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199329+0000", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4529999999999996e-06 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199335+0000", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6749999999999999e-06 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200743+0000", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001407569 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200839+0000", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5946999999999995e-05 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200957+0000", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011885200000000001 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201525+0000", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00056793000000000002 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464362+0000", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26283688599999999 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464416+0000", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4191999999999999e-05 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464509+0000", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3066000000000006e-05 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10194 6.1 6:818db385:::benchmark_data_smithi052_234456_object10193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.204091+0000", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.65481082700001, 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.261536294, 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10194 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204091+0000", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204091+0000", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204090+0000", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204114+0000", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4051999999999999e-05 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204123+0000", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6130000000000004e-06 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204130+0000", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0480000000000001e-06 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204174+0000", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4172000000000003e-05 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204288+0000", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113712 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.204452+0000", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163604 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.205205+0000", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00075335599999999995 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.465471+0000", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.260265737 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.465528+0000", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7648000000000001e-05 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.465628+0000", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9532999999999997e-05 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10221 6.1 6:8e2897f0:::benchmark_data_smithi052_234456_object10220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.217270+0000", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.641632687, 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24949115699999999, 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10221 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217270+0000", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217270+0000", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217268+0000", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217287+0000", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8828999999999999e-05 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217303+0000", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5421999999999999e-05 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217310+0000", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7859999999999997e-06 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217353+0000", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2451999999999999e-05 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217499+0000", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014595400000000001 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.217626+0000", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000126971 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.218176+0000", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00055033800000000004 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.466625+0000", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248449319 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.466673+0000", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7509999999999997e-05 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.466761+0000", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7847000000000001e-05 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36160 6.4 6:2dcd31ff:::benchmark_data_smithi052_234456_object14016:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 231.01063188399999, 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32418800800000003, 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36160 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848269+0000", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848277+0000", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2050000000000002e-06 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848301+0000", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3801000000000001e-05 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848311+0000", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0621e-05 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848367+0000", 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5881999999999998e-05 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848522+0000", 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000154544 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848651+0000", 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000129157 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850169+0000", 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015178349999999999 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850319+0000", 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015018400000000001 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172387+0000", 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32206747699999999 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172420+0000", 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3751999999999998e-05 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172458+0000", 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8139000000000002e-05 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:38747 6.4 6:20877948:::benchmark_data_smithi052_234456_object16603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:08.547567+0000", 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 229.31133579199999, 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.198348212, 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 38747 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.908 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547567+0000", 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547567+0000", 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547565+0000", 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547571+0000", 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5679999999999999e-06 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547586+0000", 2023-09-23T22:44:57.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4633e-05 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547593+0000", 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8189999999999992e-06 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547642+0000", 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8548999999999999e-05 2023-09-23T22:44:57.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547743+0000", 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101178 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.547824+0000", 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1098000000000002e-05 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.548983+0000", 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011587209999999999 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.549218+0000", 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023472200000000001 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.745831+0000", 2023-09-23T22:44:57.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19661288199999999 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.745877+0000", 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6539999999999998e-05 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:08.745915+0000", 2023-09-23T22:44:57.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.756e-05 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10377 7.d e154/146 7:b0a9409a:::benchmark_data_smithi052_277165_object10376:head v 154'659, mlcod=154'657)", 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.075588+0000", 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.78331442999999, 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.227911682, 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.915 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075588+0000", 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075588+0000", 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075587+0000", 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075629+0000", 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1495999999999997e-05 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075641+0000", 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2062000000000001e-05 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075648+0000", 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8820000000000001e-06 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075693+0000", 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4137000000000001e-05 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075725+0000", 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2147999999999998e-05 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078485+0000", 2023-09-23T22:44:57.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027598710000000001 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303500+0000", 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225014933 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.920 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10385 7.1 e154/146 7:87e353d9:::benchmark_data_smithi052_277165_object10384:head v 154'628, mlcod=154'626)", 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.78319985799999, 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22782445100000001, 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075702+0000", 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075735+0000", 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2697000000000002e-05 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075744+0000", 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6800000000000005e-06 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075750+0000", 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1940000000000003e-06 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075790+0000", 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9804e-05 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.075807+0000", 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6789000000000001e-05 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078501+0000", 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026934939999999998 2023-09-23T22:44:57.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303527+0000", 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22502640500000001 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10380 7.2 e154/146 7:4efbeca9:::benchmark_data_smithi052_277165_object10379:head v 154'647, mlcod=154'645)", 2023-09-23T22:44:57.926 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.076327+0000", 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.78257551199999, 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22720847199999999, 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076327+0000", 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076327+0000", 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076326+0000", 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076358+0000", 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1412999999999999e-05 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076368+0000", 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0597e-05 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076375+0000", 2023-09-23T22:44:57.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3940000000000001e-06 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076440+0000", 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5314999999999994e-05 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.076460+0000", 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0482000000000001e-05 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078511+0000", 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020503309999999999 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303535+0000", 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225024578 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10382 7.0 e154/146 7:08701358:::benchmark_data_smithi052_277165_object10381:head v 154'618, mlcod=154'615)", 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.077097+0000", 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.78180541899999, 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.227032811, 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077097+0000", 2023-09-23T22:44:57.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077097+0000", 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077096+0000", 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077144+0000", 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7518999999999997e-05 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077155+0000", 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1168999999999999e-05 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077162+0000", 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8979999999999999e-06 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077339+0000", 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017684600000000001 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077357+0000", 2023-09-23T22:44:57.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8813000000000001e-05 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304094+0000", 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.226736363 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304130+0000", 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5899000000000003e-05 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10384 7.0 e154/146 7:090486be:::benchmark_data_smithi052_277165_object10383:head v 154'619, mlcod=154'615)", 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.78172484999999, 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.226981144, 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077177+0000", 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077224+0000", 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6554000000000002e-05 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077234+0000", 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.006e-05 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077240+0000", 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8549999999999998e-06 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077811+0000", 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00057189800000000005 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077830+0000", 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8470000000000001e-05 2023-09-23T22:44:57.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304136+0000", 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22630588600000001 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304159+0000", 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2762999999999999e-05 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.941 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10390 7.0 e154/146 7:01932cc7:::benchmark_data_smithi052_277165_object10389:head v 154'620, mlcod=154'615)", 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.78165753100001, 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22694154499999999, 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077245+0000", 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077277+0000", 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2830000000000002e-05 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077284+0000", 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3439999999999997e-06 2023-09-23T22:44:57.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.077288+0000", 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3889999999999998e-06 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078158+0000", 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00087037699999999998 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.078175+0000", 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6781000000000001e-05 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304161+0000", 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225985827 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.304186+0000", 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5267000000000001e-05 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10393 7.9 e154/146 7:914dcfe4:::benchmark_data_smithi052_277165_object10392:head v 154'624, mlcod=154'622)", 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.079905+0000", 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.77899698799999, 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225263939, 2023-09-23T22:44:57.947 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079905+0000", 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079905+0000", 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079904+0000", 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.079930+0000", 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5576999999999999e-05 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303519+0000", 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22358936300000001 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303527+0000", 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8699999999999992e-06 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303571+0000", 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3480999999999999e-05 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303593+0000", 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2467999999999999e-05 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.305128+0000", 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001534934 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.305169+0000", 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1085999999999999e-05 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:10403 7.0 e154/146 7:08ef69f1:::benchmark_data_smithi052_277165_object10402:head v 154'621, mlcod=154'616)", 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:52.082968+0000", 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.77593384799999, 2023-09-23T22:44:57.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22345253500000001, 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.082968+0000", 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.082968+0000", 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.082967+0000", 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.083095+0000", 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012813500000000001 2023-09-23T22:44:57.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303474+0000", 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.220378193 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303485+0000", 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1378e-05 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303534+0000", 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9326000000000002e-05 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.303557+0000", 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3101000000000001e-05 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306380+0000", 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028225289999999998 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:52.306421+0000", 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0988999999999997e-05 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:91207 7.b 7:de891f47:::benchmark_data_smithi052_277165_object91206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:15.296105+0000", 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 102.562797096, 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15209299500000001, 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 91207 2023-09-23T22:44:57.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296105+0000", 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296105+0000", 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296104+0000", 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296134+0000", 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0147000000000001e-05 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296147+0000", 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3337e-05 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296156+0000", 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1859999999999998e-06 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296306+0000", 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015016399999999999 2023-09-23T22:44:57.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296443+0000", 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013616999999999999 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.296579+0000", 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000136582 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.302244+0000", 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056645910000000001 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.303562+0000", 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001318655 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.448107+0000", 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14454472099999999 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.448157+0000", 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9778999999999999e-05 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:15.448198+0000", 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1366999999999999e-05 2023-09-23T22:44:57.964 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:99607 7.b 7:d982319d:::benchmark_data_smithi052_277165_object7115:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:28.770575+0000", 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 89.088327629999995, 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14656804300000001, 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 99607 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770575+0000", 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770575+0000", 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770573+0000", 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770580+0000", 2023-09-23T22:44:57.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6869999999999997e-06 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770595+0000", 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4518999999999999e-05 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770604+0000", 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2040000000000003e-06 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770659+0000", 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4923999999999998e-05 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770814+0000", 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155279 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:44:57.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.770936+0000", 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012184300000000001 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.773932+0000", 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029955960000000001 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.774026+0000", 2023-09-23T22:44:57.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4233000000000004e-05 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.917029+0000", 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.143003246 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.917104+0000", 2023-09-23T22:44:57.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4884999999999999e-05 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:28.917143+0000", 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8862000000000002e-05 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:100762 7.b 7:d2e2e64e:::benchmark_data_smithi052_277165_object8270:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:44:57.972 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:30.558299+0000", 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 87.300603770999999, 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20031660300000001, 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 100762 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.558299+0000", 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.558299+0000", 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.558298+0000", 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.558301+0000", 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1e-06 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.558307+0000", 2023-09-23T22:44:57.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7289999999999997e-06 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.558312+0000", 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9089999999999999e-06 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.567131+0000", 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0088187609999999996 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.567266+0000", 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013569799999999999 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.567392+0000", 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012587699999999999 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.574527+0000", 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071349569999999999 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.588450+0000", 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013923082999999999 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.758468+0000", 2023-09-23T22:44:57.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.170017999 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.758565+0000", 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6448999999999998e-05 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.758615+0000", 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0473000000000003e-05 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.979 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:100773 7.b 7:d29120b0:::benchmark_data_smithi052_277165_object8281:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:30.576523+0000", 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 87.282378851000004, 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18233068199999999, 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.980 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 100773 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576523+0000", 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576523+0000", 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576522+0000", 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576530+0000", 2023-09-23T22:44:57.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8520000000000004e-06 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576544+0000", 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3842000000000001e-05 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576552+0000", 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5669999999999995e-06 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576602+0000", 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0287000000000002e-05 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576743+0000", 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000140414 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.576872+0000", 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012899799999999999 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.580116+0000", 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00324384 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.588488+0000", 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0083721439999999998 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.758764+0000", 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.170276492 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.758815+0000", 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0294000000000003e-05 2023-09-23T22:44:57.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.758854+0000", 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9564999999999998e-05 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:100785 7.b 7:d9a89d23:::benchmark_data_smithi052_277165_object8293:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:43:30.592224+0000", 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 87.266678725000006, 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17351058899999999, 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 100785 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.988 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.592224+0000", 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.592224+0000", 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.592222+0000", 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.592230+0000", 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7239999999999992e-06 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.592250+0000", 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0163999999999999e-05 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.592258+0000", 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5820000000000002e-06 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:44:57.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.602120+0000", 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098615330000000005 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.602204+0000", 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4153000000000006e-05 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.602274+0000", 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0182e-05 2023-09-23T22:44:57.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.605679+0000", 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034046739999999999 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.606161+0000", 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00048157499999999998 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.765646+0000", 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15948516200000001 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.765693+0000", 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7355000000000003e-05 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:43:30.765734+0000", 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1151e-05 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:44:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.996 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:57.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:57.997 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:44:58.038 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[180979]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[180979]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[180979]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[180979]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3036019052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:58.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:44:58.058 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[187743]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T22:44:58.058 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[187743]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.058 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[187743]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.058 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[187743]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.058 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3036019052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:58.088 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:44:58.097 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:58.125 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:58.128 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:44:58.163 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:44:58.183 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:58.186 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:57 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:44:58.186 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:57 smithi151 ceph-mon[159210]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T22:44:58.186 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:57 smithi151 ceph-mon[159210]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.187 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:57 smithi151 ceph-mon[159210]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.187 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:57 smithi151 ceph-mon[159210]: failed to encode map e200 with expected crc 2023-09-23T22:44:58.187 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3036019052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:58.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:58.202 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:44:58.236 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:44:58.272 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:44:58.289 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:58.321 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:44:58.327 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.328 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.377 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: failed to encode map e201 with expected crc 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3036019052' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: failed to encode map e201 with expected crc 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1860069066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:58 smithi151 ceph-mon[159210]: pgmap v494: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 8298/104760 objects degraded (7.921%); 955/104760 objects misplaced (0.912%); 2.6 MiB/s, 2 keys/s, 91 objects/s recovering 2023-09-23T22:44:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: failed to encode map e201 with expected crc 2023-09-23T22:44:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3036019052' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T22:44:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: failed to encode map e201 with expected crc 2023-09-23T22:44:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1860069066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:59.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[187743]: pgmap v494: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 8298/104760 objects degraded (7.921%); 955/104760 objects misplaced (0.912%); 2.6 MiB/s, 2 keys/s, 91 objects/s recovering 2023-09-23T22:44:59.252 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:59.252 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: failed to encode map e201 with expected crc 2023-09-23T22:44:59.253 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3036019052' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:59.253 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T22:44:59.253 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: failed to encode map e201 with expected crc 2023-09-23T22:44:59.253 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1860069066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:59.253 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:58 smithi052 ceph-mon[180979]: pgmap v494: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 8298/104760 objects degraded (7.921%); 955/104760 objects misplaced (0.912%); 2.6 MiB/s, 2 keys/s, 91 objects/s recovering 2023-09-23T22:45:00.183 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:44:59 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8298/104760 objects degraded (7.921%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:00.185 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:44:59 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8298/104760 objects degraded (7.921%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:00.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:44:59 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8298/104760 objects degraded (7.921%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:00.272 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:45:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:00.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:45:00.377 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:00.405 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:00.578 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:45:00.837 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:45:00.838 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:45:00.838 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T22:45:00.838 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T22:45:00.838 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:45:00.919 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:00.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:01.059 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:01.089 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.151 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:01.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:00 smithi052 ceph-mon[180979]: pgmap v495: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 12899/100512 objects degraded (12.833%); 2730/100512 objects misplaced (2.716%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2023-09-23T22:45:01.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:00 smithi052 ceph-mon[187743]: pgmap v495: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 12899/100512 objects degraded (12.833%); 2730/100512 objects misplaced (2.716%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2023-09-23T22:45:01.248 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:45:01 smithi052 systemd[1]: Stopping Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:45:01.255 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:01.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:00 smithi151 ceph-mon[159210]: pgmap v495: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 12899/100512 objects degraded (12.833%); 2730/100512 objects misplaced (2.716%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2023-09-23T22:45:01.285 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.347 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:01.456 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:01.518 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:01.566 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.700 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:01.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:01.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.919 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:01.966 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:45:01 smithi052 conmon[258962]: 2023-09-23T22:45:01.652+0000 7f4f21979700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:45:01.967 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:45:01 smithi052 conmon[258962]: 2023-09-23T22:45:01.652+0000 7f4f21979700 -1 osd.0 201 *** Got signal Terminated *** 2023-09-23T22:45:01.967 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:45:01 smithi052 conmon[258962]: 2023-09-23T22:45:01.652+0000 7f4f21979700 -1 osd.0 201 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:45:01.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:01 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:45:01.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:01 smithi052 ceph-mon[187743]: osd.0 marked itself down and dead 2023-09-23T22:45:02.028 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:02.055 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:02.096 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:02.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:01 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:45:02.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:01 smithi151 ceph-mon[159210]: osd.0 marked itself down and dead 2023-09-23T22:45:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:02.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:02.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:02.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:01 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:45:02.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:01 smithi052 ceph-mon[180979]: osd.0 marked itself down and dead 2023-09-23T22:45:02.241 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:02.343 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:02.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:02.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:02.474 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:02.518 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:02.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:02.623 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:02.648 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:02.689 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:02.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:02.797 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:02.829 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:02.860 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:45:02.881 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:02.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:02.953 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:45:02.987 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:03.024 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:03.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:45:03.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: osdmap e202: 8 total, 7 up, 7 in 2023-09-23T22:45:03.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[180979]: pgmap v497: 89 pgs: 1 stale+active+recovering+undersized+remapped, 14 stale+active+clean, 2 stale+active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 12899/97614 objects degraded (13.214%); 2730/97614 objects misplaced (2.797%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: osdmap e202: 8 total, 7 up, 7 in 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:02 smithi052 ceph-mon[187743]: pgmap v497: 89 pgs: 1 stale+active+recovering+undersized+remapped, 14 stale+active+clean, 2 stale+active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 12899/97614 objects degraded (13.214%); 2730/97614 objects misplaced (2.797%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:45:03.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: osdmap e202: 8 total, 7 up, 7 in 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: failed to encode map e202 with expected crc 2023-09-23T22:45:03.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:02 smithi151 ceph-mon[159210]: pgmap v497: 89 pgs: 1 stale+active+recovering+undersized+remapped, 14 stale+active+clean, 2 stale+active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 406 op/s; 12899/97614 objects degraded (13.214%); 2730/97614 objects misplaced (2.797%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:45:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:45:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:45:03] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.33.4" 2023-09-23T22:45:03.493 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:45:03 smithi052 bash[320389]: 0aacb3cd0674f8ffe1cf727b0cdc62f981aec69d4dec948502b26fb6b232f206 2023-09-23T22:45:04.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[187743]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[187743]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[187743]: osdmap e203: 8 total, 7 up, 7 in 2023-09-23T22:45:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[187743]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[187743]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[187743]: 7.5 continuing backfill to osd.1 from (154'1000,201'9842] MIN to 201'9842 2023-09-23T22:45:04.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[180979]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[180979]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[180979]: osdmap e203: 8 total, 7 up, 7 in 2023-09-23T22:45:04.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[180979]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[180979]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:03 smithi052 ceph-mon[180979]: 7.5 continuing backfill to osd.1 from (154'1000,201'9842] MIN to 201'9842 2023-09-23T22:45:04.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:03 smithi151 ceph-mon[159210]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:03 smithi151 ceph-mon[159210]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:03 smithi151 ceph-mon[159210]: osdmap e203: 8 total, 7 up, 7 in 2023-09-23T22:45:04.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:03 smithi151 ceph-mon[159210]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:03 smithi151 ceph-mon[159210]: failed to encode map e203 with expected crc 2023-09-23T22:45:04.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:03 smithi151 ceph-mon[159210]: 7.5 continuing backfill to osd.1 from (154'1000,201'9842] MIN to 201'9842 2023-09-23T22:45:05.040 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:05.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:05.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:05.146 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:05.178 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:05.231 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:05.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:05.338 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: failed to encode map e204 with expected crc 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: osdmap e204: 8 total, 7 up, 7 in 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: failed to encode map e204 with expected crc 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: pgmap v500: 89 pgs: 1 stale+active+recovering+undersized+remapped, 14 stale+active+clean, 2 stale+active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 12899/97614 objects degraded (13.214%); 2730/97614 objects misplaced (2.797%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:05.368 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 12899/97614 objects degraded (13.214%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:05.371 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:05.430 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:05.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:05.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: failed to encode map e204 with expected crc 2023-09-23T22:45:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: osdmap e204: 8 total, 7 up, 7 in 2023-09-23T22:45:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: failed to encode map e204 with expected crc 2023-09-23T22:45:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: pgmap v500: 89 pgs: 1 stale+active+recovering+undersized+remapped, 14 stale+active+clean, 2 stale+active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 12899/97614 objects degraded (13.214%); 2730/97614 objects misplaced (2.797%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:45:05.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:05.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:05.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 12899/97614 objects degraded (13.214%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: failed to encode map e204 with expected crc 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: osdmap e204: 8 total, 7 up, 7 in 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: failed to encode map e204 with expected crc 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: pgmap v500: 89 pgs: 1 stale+active+recovering+undersized+remapped, 14 stale+active+clean, 2 stale+active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 12899/97614 objects degraded (13.214%); 2730/97614 objects misplaced (2.797%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:05.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 12899/97614 objects degraded (13.214%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:05.541 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:05.582 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:05.642 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:05.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:05.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:05.747 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:05.867 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:05.937 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:05.937 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:06.181 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:06.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:06.293 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:06.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:06.405 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:06.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:06.512 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:06.560 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:06.630 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:06.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:06.745 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:06.803 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:07.189 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:07 smithi052 ceph-mon[180979]: pgmap v501: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 16 active+undersized, 1 active+clean+remapped, 4 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 52 active+clean; 1.9 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 418 op/s; 19639/92982 objects degraded (21.121%); 5122/92982 objects misplaced (5.509%); 3.8 MiB/s, 65 objects/s recovering 2023-09-23T22:45:07.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:07 smithi151 ceph-mon[159210]: pgmap v501: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 16 active+undersized, 1 active+clean+remapped, 4 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 52 active+clean; 1.9 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 418 op/s; 19639/92982 objects degraded (21.121%); 5122/92982 objects misplaced (5.509%); 3.8 MiB/s, 65 objects/s recovering 2023-09-23T22:45:07.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:07 smithi052 ceph-mon[187743]: pgmap v501: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 16 active+undersized, 1 active+clean+remapped, 4 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 52 active+clean; 1.9 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 418 op/s; 19639/92982 objects degraded (21.121%); 5122/92982 objects misplaced (5.509%); 3.8 MiB/s, 65 objects/s recovering 2023-09-23T22:45:08.296 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:08.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:08.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:08.401 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:08.503 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:45:08.503 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:09.148 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:09.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:09.210 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:09 smithi052 ceph-mon[187743]: pgmap v502: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 21444/91908 objects degraded (23.332%); 5122/91908 objects misplaced (5.573%); 3.7 MiB/s, 64 objects/s recovering 2023-09-23T22:45:09.211 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:09 smithi052 ceph-mon[180979]: pgmap v502: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 21444/91908 objects degraded (23.332%); 5122/91908 objects misplaced (5.573%); 3.7 MiB/s, 64 objects/s recovering 2023-09-23T22:45:09.254 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:09.352 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:09.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:09 smithi151 ceph-mon[159210]: pgmap v502: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 309 op/s; 21444/91908 objects degraded (23.332%); 5122/91908 objects misplaced (5.573%); 3.7 MiB/s, 64 objects/s recovering 2023-09-23T22:45:09.822 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:09.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:09.932 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:10.015 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:10.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 21444/91908 objects degraded (23.332%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:10.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 21444/91908 objects degraded (23.332%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:10.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 21444/91908 objects degraded (23.332%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:11.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:11 smithi052 ceph-mon[180979]: pgmap v503: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 20572/86838 objects degraded (23.690%); 4477/86838 objects misplaced (5.156%); 6.2 MiB/s, 129 objects/s recovering 2023-09-23T22:45:11.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:11 smithi052 ceph-mon[187743]: pgmap v503: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 20572/86838 objects degraded (23.690%); 4477/86838 objects misplaced (5.156%); 6.2 MiB/s, 129 objects/s recovering 2023-09-23T22:45:11.574 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:11.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:11.678 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:11.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:11 smithi151 ceph-mon[159210]: pgmap v503: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 20572/86838 objects degraded (23.690%); 4477/86838 objects misplaced (5.156%); 6.2 MiB/s, 129 objects/s recovering 2023-09-23T22:45:11.740 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:12.419 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T22:45:12.492 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:12.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:12.510 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:45:12.510 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T22:45:12.609 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:12.704 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:12.890 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:12.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:12.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:13.008 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:13.029 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:13.036 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:13.091 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:13.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:13.200 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:13.230 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:13 smithi052 ceph-mon[180979]: pgmap v504: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 19734/81696 objects degraded (24.155%); 4477/81696 objects misplaced (5.480%); 5.5 MiB/s, 113 objects/s recovering 2023-09-23T22:45:13.230 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3653673943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:13.230 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:13.236 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:45:13.292 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:13.294 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:45:13.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:13 smithi151 ceph-mon[159210]: pgmap v504: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 19734/81696 objects degraded (24.155%); 4477/81696 objects misplaced (5.480%); 5.5 MiB/s, 113 objects/s recovering 2023-09-23T22:45:13.301 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3653673943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:13.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:13.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:13.335 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:45:13.369 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:45:13.402 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:13.409 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:45:13.439 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:13.471 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:45:13.491 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:13.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:13.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:13.500 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:45:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:45:13] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.33.4" 2023-09-23T22:45:13.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:13 smithi052 ceph-mon[187743]: pgmap v504: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 19734/81696 objects degraded (24.155%); 4477/81696 objects misplaced (5.480%); 5.5 MiB/s, 113 objects/s recovering 2023-09-23T22:45:13.500 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3653673943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:13.526 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.527 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:45:13.596 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:13.682 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:14.283 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:45:14.306 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:14.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:14.420 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:14.454 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1419837700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:14.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:14 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:14.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1419837700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:14.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:14 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:14.520 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:14.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:14.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1419837700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:14.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:14 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:14.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:14.718 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:15.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: pgmap v505: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 19734/81696 objects degraded (24.155%); 4477/81696 objects misplaced (5.480%); 4.9 MiB/s, 102 objects/s recovering 2023-09-23T22:45:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: failed to encode map e205 with expected crc 2023-09-23T22:45:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: failed to encode map e205 with expected crc 2023-09-23T22:45:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:45:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: osdmap e205: 8 total, 7 up, 7 in 2023-09-23T22:45:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 19734/81696 objects degraded (24.155%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: pgmap v505: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 19734/81696 objects degraded (24.155%); 4477/81696 objects misplaced (5.480%); 4.9 MiB/s, 102 objects/s recovering 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: failed to encode map e205 with expected crc 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: failed to encode map e205 with expected crc 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: osdmap e205: 8 total, 7 up, 7 in 2023-09-23T22:45:15.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 19734/81696 objects degraded (24.155%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:15.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: pgmap v505: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 19734/81696 objects degraded (24.155%); 4477/81696 objects misplaced (5.480%); 4.9 MiB/s, 102 objects/s recovering 2023-09-23T22:45:15.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:15.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: failed to encode map e205 with expected crc 2023-09-23T22:45:15.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: failed to encode map e205 with expected crc 2023-09-23T22:45:15.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:45:15.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: osdmap e205: 8 total, 7 up, 7 in 2023-09-23T22:45:15.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 19734/81696 objects degraded (24.155%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:16.844 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:16.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:16.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:16.954 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:16.988 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:17.062 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:17.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:17.168 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:17.221 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:17.289 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:17.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:17.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:17.395 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:17.423 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:17.511 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [0] live_osds: [1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:45:17.511 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:45:17.511 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-09-23T22:45:17.511 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:45:17.585 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:17 smithi052 ceph-mon[180979]: pgmap v507: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 18364/73482 objects degraded (24.991%); 3653/73482 objects misplaced (4.971%); 6.5 MiB/s, 125 objects/s recovering 2023-09-23T22:45:17.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:17 smithi052 ceph-mon[187743]: pgmap v507: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 18364/73482 objects degraded (24.991%); 3653/73482 objects misplaced (4.971%); 6.5 MiB/s, 125 objects/s recovering 2023-09-23T22:45:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:45:17.619 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:45:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:17 smithi151 ceph-mon[159210]: pgmap v507: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 18364/73482 objects degraded (24.991%); 3653/73482 objects misplaced (4.971%); 6.5 MiB/s, 125 objects/s recovering 2023-09-23T22:45:17.807 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:17.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:17.928 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:18.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:18.292 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:18.320 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:18.379 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:18.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:18.481 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:45:18.482 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:45:18.482 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:45:18.482 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:45:18.482 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:45:18.487 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:18.518 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:18.528 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:18.596 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:18.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:18.702 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:18.746 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:18.821 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:18.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:18.928 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:18.964 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:19.035 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:19.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:19.140 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:19.191 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:19.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:19 smithi151 ceph-mon[159210]: pgmap v508: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 18178/72330 objects degraded (25.132%); 3653/72330 objects misplaced (5.050%); 6.5 MiB/s, 125 objects/s recovering 2023-09-23T22:45:19.341 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:19.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:19.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:19.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:19.485 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:19.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:19 smithi052 ceph-mon[180979]: pgmap v508: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 18178/72330 objects degraded (25.132%); 3653/72330 objects misplaced (5.050%); 6.5 MiB/s, 125 objects/s recovering 2023-09-23T22:45:19.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:19 smithi052 ceph-mon[187743]: pgmap v508: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 18178/72330 objects degraded (25.132%); 3653/72330 objects misplaced (5.050%); 6.5 MiB/s, 125 objects/s recovering 2023-09-23T22:45:19.580 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:19.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:19.619 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:45:19.686 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:19.719 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:19.766 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:19.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:19.860 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:45:19.874 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:19.923 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:19.998 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:20.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:20.104 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:20.214 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:20.358 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:20.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:20.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:45:20.458 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:20.458 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18178/72330 objects degraded (25.132%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:20.459 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:20.459 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18178/72330 objects degraded (25.132%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:20.477 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:20.512 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:20.586 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:20.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:20.591 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:20.591 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18178/72330 objects degraded (25.132%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:20.692 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:20.723 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:20.784 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:20.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:20.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:20.889 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:20.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:21.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:21 smithi151 ceph-mon[159210]: pgmap v509: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.4 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 17376/67509 objects degraded (25.739%); 2886/67509 objects misplaced (4.275%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:21.750 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:21 smithi052 ceph-mon[180979]: pgmap v509: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.4 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 17376/67509 objects degraded (25.739%); 2886/67509 objects misplaced (4.275%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:21.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:21 smithi052 ceph-mon[187743]: pgmap v509: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.4 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 17376/67509 objects degraded (25.739%); 2886/67509 objects misplaced (4.275%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:22.000 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:22.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:22.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:22.092 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:22.112 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:22.145 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:22.145 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:22.197 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:22.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:22.308 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:22.309 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:22.337 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:22.408 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:22.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:22.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:22.518 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:22.634 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:23.236 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:23.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:23.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:23.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:23 smithi052 ceph-mon[187743]: pgmap v510: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16665/63066 objects degraded (26.425%); 2886/63066 objects misplaced (4.576%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:23.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:23 smithi052 ceph-mon[180979]: pgmap v510: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16665/63066 objects degraded (26.425%); 2886/63066 objects misplaced (4.576%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:23.346 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:23.387 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:23.442 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:23.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:23.550 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:23 smithi151 ceph-mon[159210]: pgmap v510: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16665/63066 objects degraded (26.425%); 2886/63066 objects misplaced (4.576%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:23.589 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:23.652 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:23.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:23.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:23.716 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:45:23.740 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:45:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:45:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:45:23] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-09-23T22:45:23.761 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:23.854 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:25 smithi052 ceph-mon[180979]: from='client.55021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:45:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:25 smithi052 ceph-mon[180979]: pgmap v511: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16665/63066 objects degraded (26.425%); 2886/63066 objects misplaced (4.576%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:25.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 16665/63066 objects degraded (26.425%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:25.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:25 smithi052 ceph-mon[187743]: from='client.55021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:45:25.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:25 smithi052 ceph-mon[187743]: pgmap v511: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16665/63066 objects degraded (26.425%); 2886/63066 objects misplaced (4.576%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:25.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 16665/63066 objects degraded (26.425%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:25 smithi151 ceph-mon[159210]: from='client.55021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:45:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:25 smithi151 ceph-mon[159210]: pgmap v511: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16665/63066 objects degraded (26.425%); 2886/63066 objects misplaced (4.576%); 7.4 MiB/s, 131 objects/s recovering 2023-09-23T22:45:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 16665/63066 objects degraded (26.425%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:25.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:25.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:25.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:25.878 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:25.920 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:26.976 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:26.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:26.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:27.088 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:27.122 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:27.188 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:27.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:27.304 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:27.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:27.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:27 smithi151 ceph-mon[159210]: pgmap v512: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 15203/54672 objects degraded (27.808%); 2576/54672 objects misplaced (4.712%); 7.7 MiB/s, 163 objects/s recovering 2023-09-23T22:45:27.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:27 smithi052 ceph-mon[187743]: pgmap v512: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 15203/54672 objects degraded (27.808%); 2576/54672 objects misplaced (4.712%); 7.7 MiB/s, 163 objects/s recovering 2023-09-23T22:45:27.681 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:27 smithi052 ceph-mon[180979]: pgmap v512: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 15203/54672 objects degraded (27.808%); 2576/54672 objects misplaced (4.712%); 7.7 MiB/s, 163 objects/s recovering 2023-09-23T22:45:27.867 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:27.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:27.987 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:28.003 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":510,"stamp":"2023-09-23T22:45:22.131368+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1376655288,"num_objects":21022,"num_object_clones":0,"num_object_copies":63066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16665,"num_objects_misplaced":2886,"num_objects_unfound":0,"num_objects_dirty":21022,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":164242,"num_write_kb":5921742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13591,"num_bytes_recovered":715073740,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125874,"ondisk_log_size":125874,"up":228,"acting":229,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6017972,"kb_used_data":4487036,"kb_used_omap":126,"kb_used_meta":1530689,"kb_avail":650198092,"statfs":{"total":671965249536,"available":665802846208,"internally_reserved":0,"allocated":4594724864,"data_stored":4590090161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1567426171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,5,3,11,1,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-406978560,"num_objects":-6210,"num_object_clones":0,"num_object_copies":-18630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3069,"num_objects_misplaced":-1591,"num_objects_unfound":0,"num_objects_dirty":-6210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1315,"num_bytes_recovered":78118912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006209"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152066+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:45:04.152066+0000","last_peered":"2023-09-23T22:45:04.152066+0000","last_clean":"2023-09-23T22:45:04.152066+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:45:04.152066+0000","last_undegraded":"2023-09-23T22:45:04.152066+0000","last_fullsized":"2023-09-23T22:45:04.152066+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":175,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.991407+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:45:03.991407+0000","last_peered":"2023-09-23T22:45:03.991407+0000","last_clean":"2023-09-23T22:45:03.991407+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:45:03.991407+0000","last_undegraded":"2023-09-23T22:45:03.991407+0000","last_fullsized":"2023-09-23T22:45:03.991407+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.996522+0000","last_change":"2023-09-23T22:38:06.157097+0000","last_active":"2023-09-23T22:45:03.996522+0000","last_peered":"2023-09-23T22:45:03.996522+0000","last_clean":"2023-09-23T22:45:03.996522+0000","last_became_active":"2023-09-23T22:38:06.156735+0000","last_became_peered":"2023-09-23T22:38:06.156735+0000","last_unstale":"2023-09-23T22:45:03.996522+0000","last_undegraded":"2023-09-23T22:45:03.996522+0000","last_fullsized":"2023-09-23T22:45:03.996522+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:00.742765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.740172+0000","last_change":"2023-09-23T22:45:03.086327+0000","last_active":"2023-09-23T22:45:04.740172+0000","last_peered":"2023-09-23T22:45:04.740172+0000","last_clean":"2023-09-23T22:44:51.868713+0000","last_became_active":"2023-09-23T22:45:03.086327+0000","last_became_peered":"2023-09-23T22:45:03.086327+0000","last_unstale":"2023-09-23T22:45:04.740172+0000","last_undegraded":"2023-09-23T22:45:04.740172+0000","last_fullsized":"2023-09-23T22:45:03.040986+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:25.885396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":226,"reported_epoch":205,"state":"active+undersized","last_fresh":"2023-09-23T22:45:14.268416+0000","last_change":"2023-09-23T22:45:03.086425+0000","last_active":"2023-09-23T22:45:14.268416+0000","last_peered":"2023-09-23T22:45:14.268416+0000","last_clean":"2023-09-23T22:44:57.974580+0000","last_became_active":"2023-09-23T22:45:03.086425+0000","last_became_peered":"2023-09-23T22:45:03.086425+0000","last_unstale":"2023-09-23T22:45:14.268416+0000","last_undegraded":"2023-09-23T22:45:14.268416+0000","last_fullsized":"2023-09-23T22:45:02.986305+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:59.439724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":170,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.995216+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:45:03.995216+0000","last_peered":"2023-09-23T22:45:03.995216+0000","last_clean":"2023-09-23T22:45:03.995216+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:45:03.995216+0000","last_undegraded":"2023-09-23T22:45:03.995216+0000","last_fullsized":"2023-09-23T22:45:03.995216+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":89,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.993595+0000","last_change":"2023-09-23T22:45:03.055922+0000","last_active":"2023-09-23T22:45:03.993595+0000","last_peered":"2023-09-23T22:45:03.993595+0000","last_clean":"2023-09-23T22:37:35.261066+0000","last_became_active":"2023-09-23T22:45:03.055922+0000","last_became_peered":"2023-09-23T22:45:03.055922+0000","last_unstale":"2023-09-23T22:45:03.993595+0000","last_undegraded":"2023-09-23T22:45:03.993595+0000","last_fullsized":"2023-09-23T22:45:02.986034+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":203,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.733993+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:45:04.733993+0000","last_peered":"2023-09-23T22:45:04.733993+0000","last_clean":"2023-09-23T22:45:04.733993+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:45:04.733993+0000","last_undegraded":"2023-09-23T22:45:04.733993+0000","last_fullsized":"2023-09-23T22:45:04.733993+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":198,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993514+0000","last_change":"2023-09-23T22:44:53.852721+0000","last_active":"2023-09-23T22:45:03.993514+0000","last_peered":"2023-09-23T22:45:03.993514+0000","last_clean":"2023-09-23T22:45:03.993514+0000","last_became_active":"2023-09-23T22:44:53.852364+0000","last_became_peered":"2023-09-23T22:44:53.852364+0000","last_unstale":"2023-09-23T22:45:03.993514+0000","last_undegraded":"2023-09-23T22:45:03.993514+0000","last_fullsized":"2023-09-23T22:45:03.993514+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.734022+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:45:04.734022+0000","last_peered":"2023-09-23T22:45:04.734022+0000","last_clean":"2023-09-23T22:45:04.734022+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:45:04.734022+0000","last_undegraded":"2023-09-23T22:45:04.734022+0000","last_fullsized":"2023-09-23T22:45:04.734022+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":69,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:14.267703+0000","last_change":"2023-09-23T22:45:03.061626+0000","last_active":"2023-09-23T22:45:14.267703+0000","last_peered":"2023-09-23T22:45:14.267703+0000","last_clean":"2023-09-23T22:44:57.970387+0000","last_became_active":"2023-09-23T22:45:03.061626+0000","last_became_peered":"2023-09-23T22:45:03.061626+0000","last_unstale":"2023-09-23T22:45:14.267703+0000","last_undegraded":"2023-09-23T22:45:03.038230+0000","last_fullsized":"2023-09-23T22:45:03.038117+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:09.550759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.997000+0000","last_change":"2023-09-23T22:41:10.927399+0000","last_active":"2023-09-23T22:45:03.997000+0000","last_peered":"2023-09-23T22:45:03.997000+0000","last_clean":"2023-09-23T22:45:03.997000+0000","last_became_active":"2023-09-23T22:41:10.926366+0000","last_became_peered":"2023-09-23T22:41:10.926366+0000","last_unstale":"2023-09-23T22:45:03.997000+0000","last_undegraded":"2023-09-23T22:45:03.997000+0000","last_fullsized":"2023-09-23T22:45:03.997000+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:54.104169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":225,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152307+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:45:04.152307+0000","last_peered":"2023-09-23T22:45:04.152307+0000","last_clean":"2023-09-23T22:45:04.152307+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:45:04.152307+0000","last_undegraded":"2023-09-23T22:45:04.152307+0000","last_fullsized":"2023-09-23T22:45:04.152307+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152164+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:45:04.152164+0000","last_peered":"2023-09-23T22:45:04.152164+0000","last_clean":"2023-09-23T22:45:04.152164+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:45:04.152164+0000","last_undegraded":"2023-09-23T22:45:04.152164+0000","last_fullsized":"2023-09-23T22:45:04.152164+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":63,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152253+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:45:04.152253+0000","last_peered":"2023-09-23T22:45:04.152253+0000","last_clean":"2023-09-23T22:45:04.152253+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:45:04.152253+0000","last_undegraded":"2023-09-23T22:45:04.152253+0000","last_fullsized":"2023-09-23T22:45:04.152253+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":223,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.734133+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:45:04.734133+0000","last_peered":"2023-09-23T22:45:04.734133+0000","last_clean":"2023-09-23T22:45:04.734133+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:45:04.734133+0000","last_undegraded":"2023-09-23T22:45:04.734133+0000","last_fullsized":"2023-09-23T22:45:04.734133+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":163,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.734050+0000","last_change":"2023-09-23T22:45:03.085969+0000","last_active":"2023-09-23T22:45:04.734050+0000","last_peered":"2023-09-23T22:45:04.734050+0000","last_clean":"2023-09-23T22:44:51.847287+0000","last_became_active":"2023-09-23T22:45:03.085969+0000","last_became_peered":"2023-09-23T22:45:03.085969+0000","last_unstale":"2023-09-23T22:45:04.734050+0000","last_undegraded":"2023-09-23T22:45:04.734050+0000","last_fullsized":"2023-09-23T22:45:03.049509+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:16.530141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993342+0000","last_change":"2023-09-23T22:44:53.855184+0000","last_active":"2023-09-23T22:45:03.993342+0000","last_peered":"2023-09-23T22:45:03.993342+0000","last_clean":"2023-09-23T22:45:03.993342+0000","last_became_active":"2023-09-23T22:44:53.854767+0000","last_became_peered":"2023-09-23T22:44:53.854767+0000","last_unstale":"2023-09-23T22:45:03.993342+0000","last_undegraded":"2023-09-23T22:45:03.993342+0000","last_fullsized":"2023-09-23T22:45:03.993342+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.154337+0000","last_change":"2023-09-23T22:45:02.992343+0000","last_active":"2023-09-23T22:45:04.154337+0000","last_peered":"2023-09-23T22:45:04.154337+0000","last_clean":"2023-09-23T22:44:57.942235+0000","last_became_active":"2023-09-23T22:45:02.992343+0000","last_became_peered":"2023-09-23T22:45:02.992343+0000","last_unstale":"2023-09-23T22:45:04.154337+0000","last_undegraded":"2023-09-23T22:45:04.154337+0000","last_fullsized":"2023-09-23T22:45:02.987835+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.740233+0000","last_change":"2023-09-23T22:45:03.076867+0000","last_active":"2023-09-23T22:45:04.740233+0000","last_peered":"2023-09-23T22:45:04.740233+0000","last_clean":"2023-09-23T22:44:51.868792+0000","last_became_active":"2023-09-23T22:45:03.076867+0000","last_became_peered":"2023-09-23T22:45:03.076867+0000","last_unstale":"2023-09-23T22:45:04.740233+0000","last_undegraded":"2023-09-23T22:45:04.740233+0000","last_fullsized":"2023-09-23T22:45:03.041271+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:21.532111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":240,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:03.994170+0000","last_change":"2023-09-23T22:45:02.991143+0000","last_active":"2023-09-23T22:45:03.994170+0000","last_peered":"2023-09-23T22:45:03.994170+0000","last_clean":"2023-09-23T22:44:57.961656+0000","last_became_active":"2023-09-23T22:45:02.991143+0000","last_became_peered":"2023-09-23T22:45:02.991143+0000","last_unstale":"2023-09-23T22:45:03.994170+0000","last_undegraded":"2023-09-23T22:45:02.989867+0000","last_fullsized":"2023-09-23T22:45:02.989741+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":203,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:14.271640+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:45:14.271640+0000","last_peered":"2023-09-23T22:45:14.271640+0000","last_clean":"2023-09-23T22:45:14.271640+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:45:14.271640+0000","last_undegraded":"2023-09-23T22:45:14.271640+0000","last_fullsized":"2023-09-23T22:45:14.271640+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"205'10493","reported_seq":10562,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:21.946583+0000","last_change":"2023-09-23T22:41:36.969870+0000","last_active":"2023-09-23T22:45:21.946583+0000","last_peered":"2023-09-23T22:45:21.946583+0000","last_clean":"2023-09-23T22:45:21.946583+0000","last_became_active":"2023-09-23T22:41:36.969535+0000","last_became_peered":"2023-09-23T22:41:36.969535+0000","last_unstale":"2023-09-23T22:45:21.946583+0000","last_undegraded":"2023-09-23T22:45:21.946583+0000","last_fullsized":"2023-09-23T22:45:21.946583+0000","mapping_epoch":146,"log_start":"167'4186","ondisk_log_start":"167'4186","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:24.978626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86573056,"num_objects":1321,"num_object_clones":0,"num_object_copies":3963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10493,"num_write_kb":378048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":65,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:04.166123+0000","last_change":"2023-09-23T22:45:03.048339+0000","last_active":"2023-09-23T22:45:04.166123+0000","last_peered":"2023-09-23T22:45:04.166123+0000","last_clean":"2023-09-23T22:44:57.942625+0000","last_became_active":"2023-09-23T22:45:03.048339+0000","last_became_peered":"2023-09-23T22:45:03.048339+0000","last_unstale":"2023-09-23T22:45:04.166123+0000","last_undegraded":"2023-09-23T22:45:02.988358+0000","last_fullsized":"2023-09-23T22:45:02.988237+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.992938+0000","last_change":"2023-09-23T22:44:53.849837+0000","last_active":"2023-09-23T22:45:03.992938+0000","last_peered":"2023-09-23T22:45:03.992938+0000","last_clean":"2023-09-23T22:45:03.992938+0000","last_became_active":"2023-09-23T22:44:53.849375+0000","last_became_peered":"2023-09-23T22:44:53.849375+0000","last_unstale":"2023-09-23T22:45:03.992938+0000","last_undegraded":"2023-09-23T22:45:03.992938+0000","last_fullsized":"2023-09-23T22:45:03.992938+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"205'10180","reported_seq":10249,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:19.689602+0000","last_change":"2023-09-23T22:41:36.967245+0000","last_active":"2023-09-23T22:45:19.689602+0000","last_peered":"2023-09-23T22:45:19.689602+0000","last_clean":"2023-09-23T22:45:19.689602+0000","last_became_active":"2023-09-23T22:41:36.966938+0000","last_became_peered":"2023-09-23T22:41:36.966938+0000","last_unstale":"2023-09-23T22:45:19.689602+0000","last_undegraded":"2023-09-23T22:45:19.689602+0000","last_fullsized":"2023-09-23T22:45:19.689602+0000","mapping_epoch":146,"log_start":"164'3603","ondisk_log_start":"164'3603","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:31.285489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10180,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":75,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:04.152718+0000","last_change":"2023-09-23T22:45:03.048213+0000","last_active":"2023-09-23T22:45:04.152718+0000","last_peered":"2023-09-23T22:45:04.152718+0000","last_clean":"2023-09-23T22:44:57.942454+0000","last_became_active":"2023-09-23T22:45:03.048213+0000","last_became_peered":"2023-09-23T22:45:03.048213+0000","last_unstale":"2023-09-23T22:45:04.152718+0000","last_undegraded":"2023-09-23T22:45:02.988219+0000","last_fullsized":"2023-09-23T22:45:02.988098+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":185,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.154304+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:45:04.154304+0000","last_peered":"2023-09-23T22:45:04.154304+0000","last_clean":"2023-09-23T22:45:04.154304+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:45:04.154304+0000","last_undegraded":"2023-09-23T22:45:04.154304+0000","last_fullsized":"2023-09-23T22:45:04.154304+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"205'10226","reported_seq":10300,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:21.922359+0000","last_change":"2023-09-23T22:45:03.056368+0000","last_active":"2023-09-23T22:45:21.922359+0000","last_peered":"2023-09-23T22:45:21.922359+0000","last_clean":"2023-09-23T22:45:01.948145+0000","last_became_active":"2023-09-23T22:45:03.056368+0000","last_became_peered":"2023-09-23T22:45:03.056368+0000","last_unstale":"2023-09-23T22:45:21.922359+0000","last_undegraded":"2023-09-23T22:45:03.039446+0000","last_fullsized":"2023-09-23T22:45:02.987412+0000","mapping_epoch":202,"log_start":"166'3910","ondisk_log_start":"166'3910","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6316,"log_dups_size":0,"ondisk_log_size":6316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:23.875333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81264640,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1240,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10226,"num_write_kb":366912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1240}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":59,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993262+0000","last_change":"2023-09-23T22:44:53.905505+0000","last_active":"2023-09-23T22:45:03.993262+0000","last_peered":"2023-09-23T22:45:03.993262+0000","last_clean":"2023-09-23T22:45:03.993262+0000","last_became_active":"2023-09-23T22:44:53.854328+0000","last_became_peered":"2023-09-23T22:44:53.854328+0000","last_unstale":"2023-09-23T22:45:03.993262+0000","last_undegraded":"2023-09-23T22:45:03.993262+0000","last_fullsized":"2023-09-23T22:45:03.993262+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.994342+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:45:03.994342+0000","last_peered":"2023-09-23T22:45:03.994342+0000","last_clean":"2023-09-23T22:45:03.994342+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:45:03.994342+0000","last_undegraded":"2023-09-23T22:45:03.994342+0000","last_fullsized":"2023-09-23T22:45:03.994342+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"205'10253","reported_seq":12002,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:45:19.364486+0000","last_change":"2023-09-23T22:45:02.009177+0000","last_active":"2023-09-23T22:45:19.364486+0000","last_peered":"2023-09-23T22:45:19.364486+0000","last_clean":"2023-09-23T22:42:45.119339+0000","last_became_active":"2023-09-23T22:44:53.981194+0000","last_became_peered":"2023-09-23T22:44:53.981194+0000","last_unstale":"2023-09-23T22:45:19.364486+0000","last_undegraded":"2023-09-23T22:44:53.854717+0000","last_fullsized":"2023-09-23T22:44:53.837396+0000","mapping_epoch":196,"log_start":"153'200","ondisk_log_start":"153'200","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":10053,"log_dups_size":0,"ondisk_log_size":10053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:26.260630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88539136,"num_objects":1351,"num_object_clones":0,"num_object_copies":4053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":221,"num_objects_misplaced":79,"num_objects_unfound":0,"num_objects_dirty":1351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10253,"num_write_kb":371328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":828,"num_bytes_recovered":54132736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1051},{"shards":"2,4,7","objects":79}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993303+0000","last_change":"2023-09-23T22:38:06.155895+0000","last_active":"2023-09-23T22:45:03.993303+0000","last_peered":"2023-09-23T22:45:03.993303+0000","last_clean":"2023-09-23T22:45:03.993303+0000","last_became_active":"2023-09-23T22:38:06.155508+0000","last_became_peered":"2023-09-23T22:38:06.155508+0000","last_unstale":"2023-09-23T22:45:03.993303+0000","last_undegraded":"2023-09-23T22:45:03.993303+0000","last_fullsized":"2023-09-23T22:45:03.993303+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:06.966605+0000","last_change":"2023-09-23T22:45:03.053127+0000","last_active":"2023-09-23T22:45:06.966605+0000","last_peered":"2023-09-23T22:45:06.966605+0000","last_clean":"2023-09-23T22:35:51.474127+0000","last_became_active":"2023-09-23T22:45:03.053127+0000","last_became_peered":"2023-09-23T22:45:03.053127+0000","last_unstale":"2023-09-23T22:45:06.966605+0000","last_undegraded":"2023-09-23T22:45:06.966605+0000","last_fullsized":"2023-09-23T22:45:03.042244+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:16.631065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"205'10096","reported_seq":11933,"reported_epoch":205,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:45:19.812168+0000","last_change":"2023-09-23T22:45:04.206028+0000","last_active":"2023-09-23T22:45:19.812168+0000","last_peered":"2023-09-23T22:45:19.812168+0000","last_clean":"2023-09-23T22:42:45.086533+0000","last_became_active":"2023-09-23T22:45:04.169148+0000","last_became_peered":"2023-09-23T22:45:04.169148+0000","last_unstale":"2023-09-23T22:45:19.812168+0000","last_undegraded":"2023-09-23T22:45:04.082130+0000","last_fullsized":"2023-09-23T22:45:19.812168+0000","mapping_epoch":203,"log_start":"164'3523","ondisk_log_start":"164'3523","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:52.538820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721088,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3492,"num_objects_misplaced":1448,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10096,"num_write_kb":364928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":869,"num_bytes_recovered":56754176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,7],"avail_no_missing":["3"],"object_location_counts":[{"shards":"3,4","objects":1204},{"shards":"3,7","objects":104}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":195,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:03.991419+0000","last_change":"2023-09-23T22:45:03.073147+0000","last_active":"2023-09-23T22:45:03.991419+0000","last_peered":"2023-09-23T22:45:03.991419+0000","last_clean":"2023-09-23T22:44:57.969427+0000","last_became_active":"2023-09-23T22:45:03.073147+0000","last_became_peered":"2023-09-23T22:45:03.073147+0000","last_unstale":"2023-09-23T22:45:03.991419+0000","last_undegraded":"2023-09-23T22:45:02.986164+0000","last_fullsized":"2023-09-23T22:45:02.986046+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:39.312241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152219+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:45:04.152219+0000","last_peered":"2023-09-23T22:45:04.152219+0000","last_clean":"2023-09-23T22:45:04.152219+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:45:04.152219+0000","last_undegraded":"2023-09-23T22:45:04.152219+0000","last_fullsized":"2023-09-23T22:45:04.152219+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"205'10218","reported_seq":10292,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:19.371291+0000","last_change":"2023-09-23T22:45:03.043188+0000","last_active":"2023-09-23T22:45:19.371291+0000","last_peered":"2023-09-23T22:45:19.371291+0000","last_clean":"2023-09-23T22:45:01.902884+0000","last_became_active":"2023-09-23T22:45:03.043188+0000","last_became_peered":"2023-09-23T22:45:03.043188+0000","last_unstale":"2023-09-23T22:45:19.371291+0000","last_undegraded":"2023-09-23T22:45:02.986934+0000","last_fullsized":"2023-09-23T22:45:02.986817+0000","mapping_epoch":202,"log_start":"153'200","ondisk_log_start":"153'200","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:18.734797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89915392,"num_objects":1372,"num_object_clones":0,"num_object_copies":4116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1372,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10218,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1372}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":189,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.994459+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:45:03.994459+0000","last_peered":"2023-09-23T22:45:03.994459+0000","last_clean":"2023-09-23T22:45:03.994459+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:45:03.994459+0000","last_undegraded":"2023-09-23T22:45:03.994459+0000","last_fullsized":"2023-09-23T22:45:03.994459+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":200,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152286+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:45:04.152286+0000","last_peered":"2023-09-23T22:45:04.152286+0000","last_clean":"2023-09-23T22:45:04.152286+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:45:04.152286+0000","last_undegraded":"2023-09-23T22:45:04.152286+0000","last_fullsized":"2023-09-23T22:45:04.152286+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"205'10209","reported_seq":10287,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:20.722500+0000","last_change":"2023-09-23T22:45:03.120974+0000","last_active":"2023-09-23T22:45:20.722500+0000","last_peered":"2023-09-23T22:45:20.722500+0000","last_clean":"2023-09-23T22:45:01.964443+0000","last_became_active":"2023-09-23T22:45:03.056678+0000","last_became_peered":"2023-09-23T22:45:03.056678+0000","last_unstale":"2023-09-23T22:45:20.722500+0000","last_undegraded":"2023-09-23T22:45:02.994477+0000","last_fullsized":"2023-09-23T22:45:02.991343+0000","mapping_epoch":202,"log_start":"156'1557","ondisk_log_start":"156'1557","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8652,"log_dups_size":0,"ondisk_log_size":8652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886104,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1281,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10209,"num_write_kb":367617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":1281}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":213,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.154220+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:45:04.154220+0000","last_peered":"2023-09-23T22:45:04.154220+0000","last_clean":"2023-09-23T22:45:04.154220+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:45:04.154220+0000","last_undegraded":"2023-09-23T22:45:04.154220+0000","last_fullsized":"2023-09-23T22:45:04.154220+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152837+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:45:04.152837+0000","last_peered":"2023-09-23T22:45:04.152837+0000","last_clean":"2023-09-23T22:45:04.152837+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:45:04.152837+0000","last_undegraded":"2023-09-23T22:45:04.152837+0000","last_fullsized":"2023-09-23T22:45:04.152837+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"205'10125","reported_seq":18675,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:45:20.717763+0000","last_change":"2023-09-23T22:44:54.964879+0000","last_active":"2023-09-23T22:45:20.717763+0000","last_peered":"2023-09-23T22:45:20.717763+0000","last_clean":"2023-09-23T22:44:52.166660+0000","last_became_active":"2023-09-23T22:44:54.930154+0000","last_became_peered":"2023-09-23T22:44:54.930154+0000","last_unstale":"2023-09-23T22:45:20.717763+0000","last_undegraded":"2023-09-23T22:44:54.850277+0000","last_fullsized":"2023-09-23T22:44:54.839031+0000","mapping_epoch":197,"log_start":"155'1485","ondisk_log_start":"155'1485","created":146,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8640,"log_dups_size":0,"ondisk_log_size":8640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3734,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10125,"num_write_kb":364608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4223,"num_bytes_recovered":227672064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":345},{"shards":"3,6,7","objects":55}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":204,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.990107+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:45:03.990107+0000","last_peered":"2023-09-23T22:45:03.990107+0000","last_clean":"2023-09-23T22:45:03.990107+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:45:03.990107+0000","last_undegraded":"2023-09-23T22:45:03.990107+0000","last_fullsized":"2023-09-23T22:45:03.990107+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":129,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.994307+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:45:03.994307+0000","last_peered":"2023-09-23T22:45:03.994307+0000","last_clean":"2023-09-23T22:45:03.994307+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:45:03.994307+0000","last_undegraded":"2023-09-23T22:45:03.994307+0000","last_fullsized":"2023-09-23T22:45:03.994307+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"205'10459","reported_seq":15489,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:20.448156+0000","last_change":"2023-09-23T22:42:31.753433+0000","last_active":"2023-09-23T22:45:20.448156+0000","last_peered":"2023-09-23T22:45:20.448156+0000","last_clean":"2023-09-23T22:45:20.448156+0000","last_became_active":"2023-09-23T22:42:31.748584+0000","last_became_peered":"2023-09-23T22:42:31.748584+0000","last_unstale":"2023-09-23T22:45:20.448156+0000","last_undegraded":"2023-09-23T22:45:20.448156+0000","last_fullsized":"2023-09-23T22:45:20.448156+0000","mapping_epoch":165,"log_start":"163'3139","ondisk_log_start":"163'3139","created":146,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7320,"log_dups_size":0,"ondisk_log_size":7320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86441984,"num_objects":1319,"num_object_clones":0,"num_object_copies":3957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10459,"num_write_kb":376896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":162004992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":235,"reported_epoch":205,"state":"active+undersized","last_fresh":"2023-09-23T22:45:14.267358+0000","last_change":"2023-09-23T22:45:03.052408+0000","last_active":"2023-09-23T22:45:14.267358+0000","last_peered":"2023-09-23T22:45:14.267358+0000","last_clean":"2023-09-23T22:44:57.961086+0000","last_became_active":"2023-09-23T22:45:03.052408+0000","last_became_peered":"2023-09-23T22:45:03.052408+0000","last_unstale":"2023-09-23T22:45:14.267358+0000","last_undegraded":"2023-09-23T22:45:14.267358+0000","last_fullsized":"2023-09-23T22:45:02.987525+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:44.801955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":201,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993418+0000","last_change":"2023-09-23T22:41:10.929497+0000","last_active":"2023-09-23T22:45:03.993418+0000","last_peered":"2023-09-23T22:45:03.993418+0000","last_clean":"2023-09-23T22:45:03.993418+0000","last_became_active":"2023-09-23T22:41:10.929128+0000","last_became_peered":"2023-09-23T22:41:10.929128+0000","last_unstale":"2023-09-23T22:45:03.993418+0000","last_undegraded":"2023-09-23T22:45:03.993418+0000","last_fullsized":"2023-09-23T22:45:03.993418+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":147,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:14.272639+0000","last_change":"2023-09-23T22:40:14.755608+0000","last_active":"2023-09-23T22:45:14.272639+0000","last_peered":"2023-09-23T22:45:14.272639+0000","last_clean":"2023-09-23T22:45:14.272639+0000","last_became_active":"2023-09-23T22:40:14.755335+0000","last_became_peered":"2023-09-23T22:40:14.755335+0000","last_unstale":"2023-09-23T22:45:14.272639+0000","last_undegraded":"2023-09-23T22:45:14.272639+0000","last_fullsized":"2023-09-23T22:45:14.272639+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:31.014369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"205'10342","reported_seq":10414,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:20.709930+0000","last_change":"2023-09-23T22:45:03.077117+0000","last_active":"2023-09-23T22:45:20.709930+0000","last_peered":"2023-09-23T22:45:20.709930+0000","last_clean":"2023-09-23T22:45:01.971764+0000","last_became_active":"2023-09-23T22:45:03.077117+0000","last_became_peered":"2023-09-23T22:45:03.077117+0000","last_unstale":"2023-09-23T22:45:20.709930+0000","last_undegraded":"2023-09-23T22:45:02.990618+0000","last_fullsized":"2023-09-23T22:45:02.990469+0000","mapping_epoch":202,"log_start":"156'1675","ondisk_log_start":"156'1675","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983232,"num_objects":1312,"num_object_clones":0,"num_object_copies":3936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1312,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10342,"num_write_kb":372928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":1312}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":175,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993160+0000","last_change":"2023-09-23T22:37:03.805159+0000","last_active":"2023-09-23T22:45:03.993160+0000","last_peered":"2023-09-23T22:45:03.993160+0000","last_clean":"2023-09-23T22:45:03.993160+0000","last_became_active":"2023-09-23T22:37:03.804853+0000","last_became_peered":"2023-09-23T22:37:03.804853+0000","last_unstale":"2023-09-23T22:45:03.993160+0000","last_undegraded":"2023-09-23T22:45:03.993160+0000","last_fullsized":"2023-09-23T22:45:03.993160+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":192,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.152552+0000","last_change":"2023-09-23T22:45:02.990919+0000","last_active":"2023-09-23T22:45:04.152552+0000","last_peered":"2023-09-23T22:45:04.152552+0000","last_clean":"2023-09-23T22:44:57.942333+0000","last_became_active":"2023-09-23T22:45:02.990919+0000","last_became_peered":"2023-09-23T22:45:02.990919+0000","last_unstale":"2023-09-23T22:45:04.152552+0000","last_undegraded":"2023-09-23T22:45:04.152552+0000","last_fullsized":"2023-09-23T22:45:02.987844+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993187+0000","last_change":"2023-09-23T22:41:10.929964+0000","last_active":"2023-09-23T22:45:03.993187+0000","last_peered":"2023-09-23T22:45:03.993187+0000","last_clean":"2023-09-23T22:45:03.993187+0000","last_became_active":"2023-09-23T22:41:10.929689+0000","last_became_peered":"2023-09-23T22:41:10.929689+0000","last_unstale":"2023-09-23T22:45:03.993187+0000","last_undegraded":"2023-09-23T22:45:03.993187+0000","last_fullsized":"2023-09-23T22:45:03.993187+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"205'9995","reported_seq":10068,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:20.431997+0000","last_change":"2023-09-23T22:45:03.048251+0000","last_active":"2023-09-23T22:45:20.431997+0000","last_peered":"2023-09-23T22:45:20.431997+0000","last_clean":"2023-09-23T22:45:01.883498+0000","last_became_active":"2023-09-23T22:45:03.048251+0000","last_became_peered":"2023-09-23T22:45:03.048251+0000","last_unstale":"2023-09-23T22:45:20.431997+0000","last_undegraded":"2023-09-23T22:45:02.987862+0000","last_fullsized":"2023-09-23T22:45:02.987710+0000","mapping_epoch":202,"log_start":"161'2640","ondisk_log_start":"161'2640","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83034112,"num_objects":1267,"num_object_clones":0,"num_object_copies":3801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1267,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9995,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1267}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":185,"reported_epoch":205,"state":"active+undersized","last_fresh":"2023-09-23T22:45:14.256588+0000","last_change":"2023-09-23T22:45:03.074427+0000","last_active":"2023-09-23T22:45:14.256588+0000","last_peered":"2023-09-23T22:45:14.256588+0000","last_clean":"2023-09-23T22:44:57.947267+0000","last_became_active":"2023-09-23T22:45:03.074427+0000","last_became_peered":"2023-09-23T22:45:03.074427+0000","last_unstale":"2023-09-23T22:45:14.256588+0000","last_undegraded":"2023-09-23T22:45:14.256588+0000","last_fullsized":"2023-09-23T22:45:02.987780+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:23.238374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.996301+0000","last_change":"2023-09-23T22:41:10.927089+0000","last_active":"2023-09-23T22:45:03.996301+0000","last_peered":"2023-09-23T22:45:03.996301+0000","last_clean":"2023-09-23T22:45:03.996301+0000","last_became_active":"2023-09-23T22:41:10.926767+0000","last_became_peered":"2023-09-23T22:41:10.926767+0000","last_unstale":"2023-09-23T22:45:03.996301+0000","last_undegraded":"2023-09-23T22:45:03.996301+0000","last_fullsized":"2023-09-23T22:45:03.996301+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:03.097039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":203,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.996517+0000","last_change":"2023-09-23T22:45:03.049074+0000","last_active":"2023-09-23T22:45:03.996517+0000","last_peered":"2023-09-23T22:45:03.996517+0000","last_clean":"2023-09-23T22:44:57.960931+0000","last_became_active":"2023-09-23T22:45:03.049074+0000","last_became_peered":"2023-09-23T22:45:03.049074+0000","last_unstale":"2023-09-23T22:45:03.996517+0000","last_undegraded":"2023-09-23T22:45:03.996517+0000","last_fullsized":"2023-09-23T22:45:02.987251+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:17.890661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"205'10378","reported_seq":16986,"reported_epoch":205,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T22:45:19.817106+0000","last_change":"2023-09-23T22:45:04.150670+0000","last_active":"2023-09-23T22:45:19.817106+0000","last_peered":"2023-09-23T22:45:19.817106+0000","last_clean":"2023-09-23T22:42:45.097581+0000","last_became_active":"2023-09-23T22:45:04.092416+0000","last_became_peered":"2023-09-23T22:45:04.092416+0000","last_unstale":"2023-09-23T22:45:19.817106+0000","last_undegraded":"2023-09-23T22:45:04.090744+0000","last_fullsized":"2023-09-23T22:45:19.817106+0000","mapping_epoch":203,"log_start":"166'3811","ondisk_log_start":"166'3811","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:07.288407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90177536,"num_objects":1376,"num_object_clones":0,"num_object_copies":4128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10378,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3131,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1376}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":217,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993193+0000","last_change":"2023-09-23T22:44:53.858164+0000","last_active":"2023-09-23T22:45:03.993193+0000","last_peered":"2023-09-23T22:45:03.993193+0000","last_clean":"2023-09-23T22:45:03.993193+0000","last_became_active":"2023-09-23T22:44:53.857674+0000","last_became_peered":"2023-09-23T22:44:53.857674+0000","last_unstale":"2023-09-23T22:45:03.993193+0000","last_undegraded":"2023-09-23T22:45:03.993193+0000","last_fullsized":"2023-09-23T22:45:03.993193+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":283,"reported_epoch":204,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:45:04.123810+0000","last_change":"2023-09-23T22:45:04.123536+0000","last_active":"2023-09-23T22:45:04.123810+0000","last_peered":"2023-09-23T22:45:04.123810+0000","last_clean":"2023-09-23T22:45:04.123810+0000","last_became_active":"2023-09-23T22:45:03.997820+0000","last_became_peered":"2023-09-23T22:45:03.997820+0000","last_unstale":"2023-09-23T22:45:04.123810+0000","last_undegraded":"2023-09-23T22:45:04.123810+0000","last_fullsized":"2023-09-23T22:45:04.123810+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":2}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":134,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.990843+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:45:03.990843+0000","last_peered":"2023-09-23T22:45:03.990843+0000","last_clean":"2023-09-23T22:45:03.990843+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:45:03.990843+0000","last_undegraded":"2023-09-23T22:45:03.990843+0000","last_fullsized":"2023-09-23T22:45:03.990843+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.994900+0000","last_change":"2023-09-23T22:45:02.992922+0000","last_active":"2023-09-23T22:45:03.994900+0000","last_peered":"2023-09-23T22:45:03.994900+0000","last_clean":"2023-09-23T22:44:57.965301+0000","last_became_active":"2023-09-23T22:45:02.992922+0000","last_became_peered":"2023-09-23T22:45:02.992922+0000","last_unstale":"2023-09-23T22:45:03.994900+0000","last_undegraded":"2023-09-23T22:45:03.994900+0000","last_fullsized":"2023-09-23T22:45:02.989776+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"205'10461","reported_seq":14632,"reported_epoch":205,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T22:45:19.373174+0000","last_change":"2023-09-23T22:45:03.093339+0000","last_active":"2023-09-23T22:45:19.373174+0000","last_peered":"2023-09-23T22:45:19.373174+0000","last_clean":"2023-09-23T22:44:52.111120+0000","last_became_active":"2023-09-23T22:45:03.055831+0000","last_became_peered":"2023-09-23T22:45:03.055831+0000","last_unstale":"2023-09-23T22:45:19.373174+0000","last_undegraded":"2023-09-23T22:45:03.044250+0000","last_fullsized":"2023-09-23T22:45:02.987951+0000","mapping_epoch":202,"log_start":"154'1000","ondisk_log_start":"154'1000","created":146,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":9461,"log_dups_size":0,"ondisk_log_size":9461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:56.983800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88932352,"num_objects":1357,"num_object_clones":0,"num_object_copies":4071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1357,"num_objects_misplaced":1357,"num_objects_unfound":0,"num_objects_dirty":1357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10461,"num_write_kb":378176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2033,"num_bytes_recovered":81395712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1357}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.996629+0000","last_change":"2023-09-23T22:45:03.085275+0000","last_active":"2023-09-23T22:45:03.996629+0000","last_peered":"2023-09-23T22:45:03.996629+0000","last_clean":"2023-09-23T22:44:57.959854+0000","last_became_active":"2023-09-23T22:45:03.085275+0000","last_became_peered":"2023-09-23T22:45:03.085275+0000","last_unstale":"2023-09-23T22:45:03.996629+0000","last_undegraded":"2023-09-23T22:45:03.996629+0000","last_fullsized":"2023-09-23T22:45:02.986910+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:29.554613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":101,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.152733+0000","last_change":"2023-09-23T22:45:03.061067+0000","last_active":"2023-09-23T22:45:04.152733+0000","last_peered":"2023-09-23T22:45:04.152733+0000","last_clean":"2023-09-23T22:41:47.489210+0000","last_became_active":"2023-09-23T22:45:03.061067+0000","last_became_peered":"2023-09-23T22:45:03.061067+0000","last_unstale":"2023-09-23T22:45:04.152733+0000","last_undegraded":"2023-09-23T22:45:04.152733+0000","last_fullsized":"2023-09-23T22:45:02.987394+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":105,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:06.966685+0000","last_change":"2023-09-23T22:45:03.084557+0000","last_active":"2023-09-23T22:45:06.966685+0000","last_peered":"2023-09-23T22:45:06.966685+0000","last_clean":"2023-09-23T22:37:03.801144+0000","last_became_active":"2023-09-23T22:45:03.084557+0000","last_became_peered":"2023-09-23T22:45:03.084557+0000","last_unstale":"2023-09-23T22:45:06.966685+0000","last_undegraded":"2023-09-23T22:45:06.966685+0000","last_fullsized":"2023-09-23T22:45:03.038785+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:35.308580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"205'10214","reported_seq":10281,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:20.574402+0000","last_change":"2023-09-23T22:41:36.969378+0000","last_active":"2023-09-23T22:45:20.574402+0000","last_peered":"2023-09-23T22:45:20.574402+0000","last_clean":"2023-09-23T22:45:20.574402+0000","last_became_active":"2023-09-23T22:41:36.969153+0000","last_became_peered":"2023-09-23T22:41:36.969153+0000","last_unstale":"2023-09-23T22:45:20.574402+0000","last_undegraded":"2023-09-23T22:45:20.574402+0000","last_fullsized":"2023-09-23T22:45:20.574402+0000","mapping_epoch":146,"log_start":"153'200","ondisk_log_start":"153'200","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":10014,"log_dups_size":0,"ondisk_log_size":10014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983232,"num_objects":1312,"num_object_clones":0,"num_object_copies":3936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10214,"num_write_kb":368832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":136,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.152163+0000","last_change":"2023-09-23T22:45:03.049151+0000","last_active":"2023-09-23T22:45:04.152163+0000","last_peered":"2023-09-23T22:45:04.152163+0000","last_clean":"2023-09-23T22:43:31.824214+0000","last_became_active":"2023-09-23T22:45:03.049151+0000","last_became_peered":"2023-09-23T22:45:03.049151+0000","last_unstale":"2023-09-23T22:45:04.152163+0000","last_undegraded":"2023-09-23T22:45:04.152163+0000","last_fullsized":"2023-09-23T22:45:02.988063+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":177,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:14.267731+0000","last_change":"2023-09-23T22:41:10.930375+0000","last_active":"2023-09-23T22:45:14.267731+0000","last_peered":"2023-09-23T22:45:14.267731+0000","last_clean":"2023-09-23T22:45:14.267731+0000","last_became_active":"2023-09-23T22:41:10.930050+0000","last_became_peered":"2023-09-23T22:41:10.930050+0000","last_unstale":"2023-09-23T22:45:14.267731+0000","last_undegraded":"2023-09-23T22:45:14.267731+0000","last_fullsized":"2023-09-23T22:45:14.267731+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:29.593309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":136,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.993278+0000","last_change":"2023-09-23T22:45:03.049274+0000","last_active":"2023-09-23T22:45:03.993278+0000","last_peered":"2023-09-23T22:45:03.993278+0000","last_clean":"2023-09-23T22:44:51.843981+0000","last_became_active":"2023-09-23T22:45:03.049274+0000","last_became_peered":"2023-09-23T22:45:03.049274+0000","last_unstale":"2023-09-23T22:45:03.993278+0000","last_undegraded":"2023-09-23T22:45:03.993278+0000","last_fullsized":"2023-09-23T22:45:02.985707+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"205'10206","reported_seq":10275,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:19.669917+0000","last_change":"2023-09-23T22:41:36.966638+0000","last_active":"2023-09-23T22:45:19.669917+0000","last_peered":"2023-09-23T22:45:19.669917+0000","last_clean":"2023-09-23T22:45:19.669917+0000","last_became_active":"2023-09-23T22:41:36.966433+0000","last_became_peered":"2023-09-23T22:41:36.966433+0000","last_unstale":"2023-09-23T22:45:19.669917+0000","last_undegraded":"2023-09-23T22:45:19.669917+0000","last_fullsized":"2023-09-23T22:45:19.669917+0000","mapping_epoch":146,"log_start":"164'3630","ondisk_log_start":"164'3630","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6576,"log_dups_size":0,"ondisk_log_size":6576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:12.457010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10206,"num_write_kb":367552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":234,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:03.991693+0000","last_change":"2023-09-23T22:45:03.040904+0000","last_active":"2023-09-23T22:45:03.991693+0000","last_peered":"2023-09-23T22:45:03.991693+0000","last_clean":"2023-09-23T22:44:57.969880+0000","last_became_active":"2023-09-23T22:45:03.040904+0000","last_became_peered":"2023-09-23T22:45:03.040904+0000","last_unstale":"2023-09-23T22:45:03.991693+0000","last_undegraded":"2023-09-23T22:45:02.986129+0000","last_fullsized":"2023-09-23T22:45:02.986010+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:41.081712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":145,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.996759+0000","last_change":"2023-09-23T22:37:03.800801+0000","last_active":"2023-09-23T22:45:03.996759+0000","last_peered":"2023-09-23T22:45:03.996759+0000","last_clean":"2023-09-23T22:45:03.996759+0000","last_became_active":"2023-09-23T22:37:03.800491+0000","last_became_peered":"2023-09-23T22:37:03.800491+0000","last_unstale":"2023-09-23T22:45:03.996759+0000","last_undegraded":"2023-09-23T22:45:03.996759+0000","last_fullsized":"2023-09-23T22:45:03.996759+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:35.470709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":219,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.990178+0000","last_change":"2023-09-23T22:45:03.063010+0000","last_active":"2023-09-23T22:45:03.990178+0000","last_peered":"2023-09-23T22:45:03.990178+0000","last_clean":"2023-09-23T22:44:57.974507+0000","last_became_active":"2023-09-23T22:45:03.063010+0000","last_became_peered":"2023-09-23T22:45:03.063010+0000","last_unstale":"2023-09-23T22:45:03.990178+0000","last_undegraded":"2023-09-23T22:45:03.990178+0000","last_fullsized":"2023-09-23T22:45:02.985965+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:50.615249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"205'10127","reported_seq":10196,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T22:45:19.816457+0000","last_change":"2023-09-23T22:41:36.965968+0000","last_active":"2023-09-23T22:45:19.816457+0000","last_peered":"2023-09-23T22:45:19.816457+0000","last_clean":"2023-09-23T22:45:19.816457+0000","last_became_active":"2023-09-23T22:41:36.965582+0000","last_became_peered":"2023-09-23T22:41:36.965582+0000","last_unstale":"2023-09-23T22:45:19.816457+0000","last_undegraded":"2023-09-23T22:45:19.816457+0000","last_fullsized":"2023-09-23T22:45:19.816457+0000","mapping_epoch":146,"log_start":"164'3543","ondisk_log_start":"164'3543","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:35.944604+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.944604+0000","objects_scrubbed":0,"log_size":6584,"log_dups_size":0,"ondisk_log_size":6584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:51.048347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87228416,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10127,"num_write_kb":366656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":237,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.995159+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:45:03.995159+0000","last_peered":"2023-09-23T22:45:03.995159+0000","last_clean":"2023-09-23T22:45:03.995159+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:45:03.995159+0000","last_undegraded":"2023-09-23T22:45:03.995159+0000","last_fullsized":"2023-09-23T22:45:03.995159+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":67,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:06.966747+0000","last_change":"2023-09-23T22:45:03.051793+0000","last_active":"2023-09-23T22:45:06.966747+0000","last_peered":"2023-09-23T22:45:06.966747+0000","last_clean":"2023-09-23T22:28:44.694773+0000","last_became_active":"2023-09-23T22:45:03.051793+0000","last_became_peered":"2023-09-23T22:45:03.051793+0000","last_unstale":"2023-09-23T22:45:06.966747+0000","last_undegraded":"2023-09-23T22:45:03.042086+0000","last_fullsized":"2023-09-23T22:45:03.041965+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:21.997004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.154230+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:45:04.154230+0000","last_peered":"2023-09-23T22:45:04.154230+0000","last_clean":"2023-09-23T22:45:04.154230+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:45:04.154230+0000","last_undegraded":"2023-09-23T22:45:04.154230+0000","last_fullsized":"2023-09-23T22:45:04.154230+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":56,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.152440+0000","last_change":"2023-09-23T22:45:03.051978+0000","last_active":"2023-09-23T22:45:04.152440+0000","last_peered":"2023-09-23T22:45:04.152440+0000","last_clean":"2023-09-23T22:28:41.643121+0000","last_became_active":"2023-09-23T22:45:03.051978+0000","last_became_peered":"2023-09-23T22:45:03.051978+0000","last_unstale":"2023-09-23T22:45:04.152440+0000","last_undegraded":"2023-09-23T22:45:04.152440+0000","last_fullsized":"2023-09-23T22:45:02.988194+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:03.997737+0000","last_change":"2023-09-23T22:45:03.084554+0000","last_active":"2023-09-23T22:45:03.997737+0000","last_peered":"2023-09-23T22:45:03.997737+0000","last_clean":"2023-09-23T22:44:57.961260+0000","last_became_active":"2023-09-23T22:45:03.084554+0000","last_became_peered":"2023-09-23T22:45:03.084554+0000","last_unstale":"2023-09-23T22:45:03.997737+0000","last_undegraded":"2023-09-23T22:45:03.997737+0000","last_fullsized":"2023-09-23T22:45:03.041454+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:07.829590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993428+0000","last_change":"2023-09-23T22:37:03.805374+0000","last_active":"2023-09-23T22:45:03.993428+0000","last_peered":"2023-09-23T22:45:03.993428+0000","last_clean":"2023-09-23T22:45:03.993428+0000","last_became_active":"2023-09-23T22:37:03.804952+0000","last_became_peered":"2023-09-23T22:37:03.804952+0000","last_unstale":"2023-09-23T22:45:03.993428+0000","last_undegraded":"2023-09-23T22:45:03.993428+0000","last_fullsized":"2023-09-23T22:45:03.993428+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":204,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.152382+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:45:04.152382+0000","last_peered":"2023-09-23T22:45:04.152382+0000","last_clean":"2023-09-23T22:45:04.152382+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:45:04.152382+0000","last_undegraded":"2023-09-23T22:45:04.152382+0000","last_fullsized":"2023-09-23T22:45:04.152382+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.993518+0000","last_change":"2023-09-23T22:28:42.673866+0000","last_active":"2023-09-23T22:45:03.993518+0000","last_peered":"2023-09-23T22:45:03.993518+0000","last_clean":"2023-09-23T22:45:03.993518+0000","last_became_active":"2023-09-23T22:28:42.673559+0000","last_became_peered":"2023-09-23T22:28:42.673559+0000","last_unstale":"2023-09-23T22:45:03.993518+0000","last_undegraded":"2023-09-23T22:45:03.993518+0000","last_fullsized":"2023-09-23T22:45:03.993518+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":67,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-09-23T22:45:04.152176+0000","last_change":"2023-09-23T22:45:03.074156+0000","last_active":"2023-09-23T22:45:04.152176+0000","last_peered":"2023-09-23T22:45:04.152176+0000","last_clean":"2023-09-23T22:44:57.942156+0000","last_became_active":"2023-09-23T22:45:03.074156+0000","last_became_peered":"2023-09-23T22:45:03.074156+0000","last_unstale":"2023-09-23T22:45:04.152176+0000","last_undegraded":"2023-09-23T22:45:04.152176+0000","last_fullsized":"2023-09-23T22:45:02.987626+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:03.991596+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:45:03.991596+0000","last_peered":"2023-09-23T22:45:03.991596+0000","last_clean":"2023-09-23T22:45:03.991596+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:45:03.991596+0000","last_undegraded":"2023-09-23T22:45:03.991596+0000","last_fullsized":"2023-09-23T22:45:03.991596+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":226,"reported_epoch":205,"state":"active+undersized","last_fresh":"2023-09-23T22:45:14.267191+0000","last_change":"2023-09-23T22:45:03.060992+0000","last_active":"2023-09-23T22:45:14.267191+0000","last_peered":"2023-09-23T22:45:14.267191+0000","last_clean":"2023-09-23T22:44:57.970339+0000","last_became_active":"2023-09-23T22:45:03.060992+0000","last_became_peered":"2023-09-23T22:45:03.060992+0000","last_unstale":"2023-09-23T22:45:14.267191+0000","last_undegraded":"2023-09-23T22:45:14.267191+0000","last_fullsized":"2023-09-23T22:45:02.985966+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:04.429084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":72,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:45:03.993593+0000","last_change":"2023-09-23T22:45:02.991523+0000","last_active":"2023-09-23T22:45:03.993593+0000","last_peered":"2023-09-23T22:45:03.993593+0000","last_clean":"2023-09-23T22:28:44.694413+0000","last_became_active":"2023-09-23T22:45:02.991523+0000","last_became_peered":"2023-09-23T22:45:02.991523+0000","last_unstale":"2023-09-23T22:45:03.993593+0000","last_undegraded":"2023-09-23T22:45:02.986232+0000","last_fullsized":"2023-09-23T22:45:02.986117+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:45:04.151919+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:45:04.151919+0000","last_peered":"2023-09-23T22:45:04.151919+0000","last_clean":"2023-09-23T22:45:04.151919+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:45:04.151919+0000","last_undegraded":"2023-09-23T22:45:04.151919+0000","last_fullsized":"2023-09-23T22:45:04.151919+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1376059416,"num_objects":20998,"num_object_clones":0,"num_object_copies":62994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16652,"num_objects_misplaced":2884,"num_objects_unfound":0,"num_objects_dirty":20998,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163982,"num_write_kb":5919297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13557,"num_bytes_recovered":714473472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5029638144,"data_stored":5029625928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125680,"ondisk_log_size":125680,"up":40,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":85,"acting":85,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":9900,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":81,"acting":81,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430428,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626656,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":723964,"kb_used_data":422644,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93021188,"statfs":{"total":95995035648,"available":95253696512,"internally_reserved":0,"allocated":432787456,"data_stored":432296829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3999999999999999,"5min":2.331,"15min":1.069},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":18.634,"5min":52.377000000000002,"15min":52.377000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":4.9889999999999999,"5min":2.5369999999999999,"15min":1.232},"min":{"1min":0.26100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.481000000000002,"5min":51.970999999999997,"15min":51.970999999999997},"last":0.59499999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:44:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":4.3730000000000002,"15min":1.7989999999999999},"min":{"1min":0.28699999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.0300000000000002,"5min":153.78399999999999,"15min":153.78399999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.55,"5min":4.3570000000000002,"15min":1.833},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.1779999999999999,"5min":154.148,"15min":154.148},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:45:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2629999999999999,"5min":3.9870000000000001,"15min":3.1549999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.081,"5min":100.785,"15min":100.785},"last":5.4509999999999996},{"interface":"front","average":{"1min":3.2639999999999998,"5min":3.6989999999999998,"15min":3.0449999999999999},"min":{"1min":0.27300000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.68,"5min":100.77,"15min":100.77},"last":4.3810000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:44:22 2023","interfaces":[{"interface":"back","average":{"1min":6.8579999999999997,"5min":3.883,"15min":1.645},"min":{"1min":0.33900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":90.561999999999998,"5min":90.561999999999998,"15min":90.561999999999998},"last":3.2949999999999999},{"interface":"front","average":{"1min":7.5209999999999999,"5min":4.0990000000000002,"15min":1.7749999999999999},"min":{"1min":0.44500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":90.646000000000001,"5min":90.646000000000001,"15min":90.646000000000001},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:44:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.70899999999999996,"15min":0.57199999999999995},"min":{"1min":0.32500000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.927,"5min":18.085999999999999,"15min":18.085999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.67200000000000004,"15min":0.56999999999999995},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.5680000000000001,"5min":18.597000000000001,"15min":18.597000000000001},"last":4.4550000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:44:24 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.81699999999999995,"15min":0.63200000000000001},"min":{"1min":0.31900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.9249999999999998,"5min":18.972000000000001,"15min":18.972000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.75600000000000001,"15min":0.628},"min":{"1min":0.34599999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.5990000000000002,"5min":18.625,"15min":18.625},"last":4.4210000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.67400000000000004,"15min":0.61099999999999999},"min":{"1min":0.36199999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.6219999999999999,"5min":12.481999999999999,"15min":12.481999999999999},"last":4.5179999999999998},{"interface":"front","average":{"1min":1.1020000000000001,"5min":0.874,"15min":0.67400000000000004},"min":{"1min":0.35799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2690000000000001,"5min":16.661999999999999,"15min":16.661999999999999},"last":1.7549999999999999}]}]},{"osd":1,"up_from":84,"seq":360777252966,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649212,"kb_used_data":505268,"kb_used_omap":9,"kb_used_meta":143926,"kb_avail":93095940,"statfs":{"total":95995035648,"available":95330242560,"internally_reserved":0,"allocated":517394432,"data_stored":516615687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":147381225},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.2160000000000002,"5min":3.2799999999999998,"15min":1.6000000000000001},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.861000000000001,"5min":83.670000000000002,"15min":83.670000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":4.4059999999999997,"5min":3.5350000000000001,"15min":1.5149999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":29.591000000000001,"5min":84,"15min":84},"last":3.6659999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:45:13 2023","interfaces":[{"interface":"back","average":{"1min":3.8399999999999999,"5min":3.1560000000000001,"15min":1.742},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4770000000000003,"5min":37.889000000000003,"15min":37.889000000000003},"last":4.5289999999999999},{"interface":"front","average":{"1min":5.2560000000000002,"5min":3.2799999999999998,"15min":1.6899999999999999},"min":{"1min":0.58099999999999996,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.663,"5min":37.823,"15min":37.823},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 22:45:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":2.7269999999999999,"15min":1.1599999999999999},"min":{"1min":0.33900000000000002,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":7.335,"5min":50.033999999999999,"15min":50.033999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":4.3559999999999999,"5min":3.2440000000000002,"15min":1.335},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":29.565000000000001,"5min":52.491999999999997,"15min":52.491999999999997},"last":6.3479999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:45:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7839999999999998,"5min":2.6859999999999999,"15min":1.3180000000000001},"min":{"1min":0.311,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":29.466999999999999,"5min":69.344999999999999,"15min":69.344999999999999},"last":6.3179999999999996},{"interface":"front","average":{"1min":2.9940000000000002,"5min":2.7200000000000002,"15min":1.345},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.313},"max":{"1min":7.883,"5min":69.359999999999999,"15min":69.359999999999999},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:45:19 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.9219999999999999,"15min":1.069},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.4409999999999998,"5min":28.837,"15min":28.837},"last":3.3399999999999999},{"interface":"front","average":{"1min":3.855,"5min":2.1970000000000001,"15min":1.135},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":29.510999999999999,"5min":29.510999999999999,"15min":29.510999999999999},"last":6.2389999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:45:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":2.1619999999999999,"15min":1.1439999999999999},"min":{"1min":0.376,"5min":0.32500000000000001,"15min":0.317},"max":{"1min":7.9630000000000001,"5min":16.192,"15min":16.192},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.6919999999999999,"15min":0.99399999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.452,"5min":10.481999999999999,"15min":10.481999999999999},"last":0.92800000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:45:19 2023","interfaces":[{"interface":"back","average":{"1min":3.8780000000000001,"5min":3.915,"15min":1.7310000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":29.539000000000001,"5min":159.018,"15min":159.018},"last":6.2859999999999996},{"interface":"front","average":{"1min":3.456,"5min":4.1399999999999997,"15min":1.821},"min":{"1min":0.50800000000000001,"5min":0.379,"15min":0.37},"max":{"1min":8.6120000000000001,"5min":158.96000000000001,"15min":158.96000000000001},"last":0.42099999999999999}]}]},{"osd":0,"up_from":126,"seq":541165879347,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196692,"kb_used_data":1056332,"kb_used_omap":28,"kb_used_meta":140323,"kb_avail":92548460,"statfs":{"total":95995035648,"available":94769623040,"internally_reserved":0,"allocated":1081683968,"data_stored":1080852133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":143691600},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:44:53 2023","interfaces":[{"interface":"back","average":{"1min":11.599,"5min":4.1479999999999997,"15min":1.885},"min":{"1min":0.20899999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":135.81100000000001,"5min":135.81100000000001,"15min":135.81100000000001},"last":0.873},{"interface":"front","average":{"1min":10.467000000000001,"5min":3.335,"15min":1.613},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.219},"max":{"1min":132.83000000000001,"5min":132.83000000000001,"15min":132.83000000000001},"last":4.992}]},{"osd":2,"last update":"Sat Sep 23 22:44:14 2023","interfaces":[{"interface":"back","average":{"1min":5.96,"5min":6.9459999999999997,"15min":6.7709999999999999},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":22.106999999999999,"5min":105.818,"15min":105.818},"last":0.63200000000000001},{"interface":"front","average":{"1min":4.8680000000000003,"5min":6.2850000000000001,"15min":6.1559999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":13.76,"5min":105.77500000000001,"15min":105.77500000000001},"last":1.3260000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:44:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.2080000000000002,"15min":1.524},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.381,"5min":11.381,"15min":11.381},"last":7.2830000000000004},{"interface":"front","average":{"1min":8.2200000000000006,"5min":3.4350000000000001,"15min":1.9830000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":24.315000000000001,"5min":24.315000000000001,"15min":24.315000000000001},"last":0.52600000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:44:53 2023","interfaces":[{"interface":"back","average":{"1min":5.2279999999999998,"5min":2.2130000000000001,"15min":1.304},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.314},"max":{"1min":24.172000000000001,"5min":24.172000000000001,"15min":24.172000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":5.0750000000000002,"5min":2.29,"15min":1.3260000000000001},"min":{"1min":0.45400000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":14.007,"5min":14.007,"15min":14.007},"last":0.84199999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:44:53 2023","interfaces":[{"interface":"back","average":{"1min":5.4909999999999997,"5min":2.3420000000000001,"15min":1.3400000000000001},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":24.125,"5min":24.125,"15min":24.125},"last":0.58399999999999996},{"interface":"front","average":{"1min":5.0700000000000003,"5min":2.2480000000000002,"15min":1.3320000000000001},"min":{"1min":0.48899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":13.964,"5min":13.964,"15min":13.964},"last":0.77600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:44:53 2023","interfaces":[{"interface":"back","average":{"1min":5.3440000000000003,"5min":2.399,"15min":1.3899999999999999},"min":{"1min":0.52300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":14.045999999999999,"5min":14.045999999999999,"15min":14.045999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":3.077,"5min":1.6830000000000001,"15min":1.05},"min":{"1min":0.41399999999999998,"5min":0.377,"15min":0.377},"max":{"1min":12,"5min":12,"15min":12},"last":5.0469999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:44:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":1.6899999999999999,"15min":1.0800000000000001},"min":{"1min":0.45600000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":12.093,"5min":12.093,"15min":12.093},"last":5.0780000000000003},{"interface":"front","average":{"1min":5.915,"5min":2.5230000000000001,"15min":1.4590000000000001},"min":{"1min":0.433,"5min":0.372,"15min":0.372},"max":{"1min":24.202000000000002,"5min":24.202000000000002,"15min":24.202000000000002},"last":0.60499999999999998}]}]},{"osd":2,"up_from":131,"seq":562640715827,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842596,"kb_used_data":706268,"kb_used_omap":81,"kb_used_meta":136238,"kb_avail":92902556,"statfs":{"total":95995035648,"available":95132217344,"internally_reserved":0,"allocated":723218432,"data_stored":722405163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":139507930},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2069999999999999,"5min":4.8310000000000004,"15min":3.8759999999999999},"min":{"1min":0.24099999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":20.631,"5min":114.735,"15min":114.735},"last":6.165},{"interface":"front","average":{"1min":11.375,"5min":7.4950000000000001,"15min":5.6029999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":94.554000000000002,"5min":114.798,"15min":114.798},"last":4.9080000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":8.5120000000000005,"5min":4.165,"15min":3.4430000000000001},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":88.316000000000003,"5min":88.316000000000003,"15min":88.316000000000003},"last":3.4660000000000002},{"interface":"front","average":{"1min":9.2300000000000004,"5min":3.5209999999999999,"15min":2.5670000000000002},"min":{"1min":0.34899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":88.436999999999998,"5min":88.436999999999998,"15min":88.436999999999998},"last":1.2649999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":10.204000000000001,"5min":4.944,"15min":2.3639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":88.653000000000006,"5min":88.653000000000006,"15min":88.653000000000006},"last":12.679},{"interface":"front","average":{"1min":8.0549999999999997,"5min":4.0220000000000002,"15min":2.0449999999999999},"min":{"1min":0.372,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":88.474000000000004,"5min":88.474000000000004,"15min":88.474000000000004},"last":2.718}]},{"osd":4,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.1879999999999999,"15min":1},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.4340000000000002,"5min":7.3179999999999996,"15min":7.3179999999999996},"last":2.4140000000000001},{"interface":"front","average":{"1min":7.6299999999999999,"5min":2.867,"15min":1.833},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":88.459000000000003,"5min":88.459000000000003,"15min":88.459000000000003},"last":2.0920000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":8.4830000000000005,"5min":2.9809999999999999,"15min":1.8959999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":88.497,"5min":88.497,"15min":88.497},"last":2.1659999999999999},{"interface":"front","average":{"1min":7.0590000000000002,"5min":2.2570000000000001,"15min":1.2529999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":88.569000000000003,"5min":88.569000000000003,"15min":88.569000000000003},"last":1.3069999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":8.2260000000000009,"5min":2.5489999999999999,"15min":1.3740000000000001},"min":{"1min":0.378,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":88.606999999999999,"5min":88.606999999999999,"15min":88.606999999999999},"last":1.339},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.2989999999999999,"15min":1.0289999999999999},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.9000000000000004,"5min":9.9000000000000004,"15min":9.9000000000000004},"last":2.992}]},{"osd":7,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":1.3939999999999999,"15min":1.113},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.8580000000000005,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":2.4460000000000002},{"interface":"front","average":{"1min":8.5860000000000003,"5min":3.117,"15min":1.9419999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":88.584000000000003,"5min":88.584000000000003,"15min":88.584000000000003},"last":2.137}]}]},{"osd":3,"up_from":94,"seq":403726925913,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1063732,"kb_used_data":865580,"kb_used_omap":7,"kb_used_meta":198072,"kb_avail":92681420,"statfs":{"total":95995035648,"available":94905774080,"internally_reserved":0,"allocated":886353920,"data_stored":885581033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":202826344},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1,9],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:58 2023","interfaces":[{"interface":"back","average":{"1min":10.183999999999999,"5min":4.2210000000000001,"15min":1.7709999999999999},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":92.546999999999997,"5min":92.546999999999997,"15min":92.546999999999997},"last":0.218},{"interface":"front","average":{"1min":9.0670000000000002,"5min":3.8500000000000001,"15min":1.7909999999999999},"min":{"1min":0.27300000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":92.281000000000006,"5min":92.281000000000006,"15min":92.281000000000006},"last":0.23499999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:45:16 2023","interfaces":[{"interface":"back","average":{"1min":3.7040000000000002,"5min":2.4590000000000001,"15min":1.0489999999999999},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.20000000000000001},"max":{"1min":22.818000000000001,"5min":22.818000000000001,"15min":22.818000000000001},"last":17.231000000000002},{"interface":"front","average":{"1min":2.274,"5min":1.9810000000000001,"15min":0.91800000000000004},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":23.917999999999999,"5min":23.917999999999999,"15min":23.917999999999999},"last":2.6779999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:45:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3050000000000002,"5min":4.0759999999999996,"15min":2.1240000000000001},"min":{"1min":0.23899999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.51,"5min":125.446,"15min":125.446},"last":2.7090000000000001},{"interface":"front","average":{"1min":11.811,"5min":5.524,"15min":2.387},"min":{"1min":0.32100000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":182.483,"5min":182.483,"15min":182.483},"last":1.3879999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:45:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.8979999999999999,"15min":1.0369999999999999},"min":{"1min":0.39500000000000002,"5min":0.29199999999999998,"15min":0.25800000000000001},"max":{"1min":18.757000000000001,"5min":18.757000000000001,"15min":18.757000000000001},"last":12.896000000000001},{"interface":"front","average":{"1min":1.657,"5min":1.6279999999999999,"15min":0.98399999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.097,"5min":12.097,"15min":12.097},"last":2.7829999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:45:16 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.599,"15min":0.96199999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.127000000000001,"5min":12.127000000000001,"15min":12.127000000000001},"last":2.742},{"interface":"front","average":{"1min":2.0609999999999999,"5min":1.6080000000000001,"15min":0.93200000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.308},"max":{"1min":7.2969999999999997,"5min":8.8670000000000009,"15min":8.8670000000000009},"last":1.4179999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:45:16 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.5509999999999999,"15min":0.95799999999999996},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":12.16,"5min":12.16,"15min":12.16},"last":3.6749999999999998},{"interface":"front","average":{"1min":1.879,"5min":1.7230000000000001,"15min":1.0129999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.3380000000000001,"5min":10.755000000000001,"15min":10.755000000000001},"last":1.4399999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:45:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.6850000000000001,"15min":0.99299999999999999},"min":{"1min":0.40100000000000002,"5min":0.35999999999999999,"15min":0.34999999999999998},"max":{"1min":7.3710000000000004,"5min":8.7520000000000007,"15min":8.7520000000000007},"last":1.462},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.1619999999999999,"15min":1.131},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.33800000000000002},"max":{"1min":18.806999999999999,"5min":18.806999999999999,"15min":18.806999999999999},"last":12.85}]}]},{"osd":4,"up_from":30,"seq":128849019112,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796724,"kb_used_data":495404,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92948428,"statfs":{"total":95995035648,"available":95179190272,"internally_reserved":0,"allocated":507293696,"data_stored":506810181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:58 2023","interfaces":[{"interface":"back","average":{"1min":4.7610000000000001,"5min":1.627,"15min":0.95499999999999996},"min":{"1min":0.249,"5min":0.22,"15min":0.22},"max":{"1min":59.457000000000001,"5min":59.457000000000001,"15min":59.457000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":5.5750000000000002,"5min":1.9750000000000001,"15min":1.0820000000000001},"min":{"1min":0.36499999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":60.494,"5min":60.494,"15min":60.494},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:44:17 2023","interfaces":[{"interface":"back","average":{"1min":4.0110000000000001,"5min":1.8740000000000001,"15min":1.0229999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":34.972000000000001,"5min":45.320999999999998,"15min":45.320999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.3159999999999998,"5min":1.8380000000000001,"15min":0.97899999999999998},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":34.883000000000003,"5min":45.372999999999998,"15min":45.372999999999998},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:45:13 2023","interfaces":[{"interface":"back","average":{"1min":11.975,"5min":4.8520000000000003,"15min":3.4100000000000001},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":84.909000000000006,"5min":84.909000000000006,"15min":84.909000000000006},"last":0.436},{"interface":"front","average":{"1min":11.375,"5min":4.6539999999999999,"15min":3.2440000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":83.840000000000003,"5min":83.840000000000003,"15min":83.840000000000003},"last":0.45300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:44:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2629999999999999,"5min":2.1459999999999999,"15min":1.1579999999999999},"min":{"1min":0.315,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":40.906999999999996,"5min":44.994,"15min":44.994},"last":37.399000000000001},{"interface":"front","average":{"1min":4.891,"5min":2.3039999999999998,"15min":1.1599999999999999},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":40.854999999999997,"5min":44.765000000000001,"15min":44.765000000000001},"last":37.579000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.173,"15min":0.71399999999999997},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.984,"5min":43.189999999999998,"15min":43.189999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.196,"15min":0.73599999999999999},"min":{"1min":0.28199999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.5190000000000001,"5min":42.450000000000003,"15min":42.450000000000003},"last":0.28100000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:44:17 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.88100000000000001,"15min":0.65300000000000002},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.476,"5min":27.056000000000001,"15min":27.056000000000001},"last":0.311},{"interface":"front","average":{"1min":1.083,"5min":0.98899999999999999,"15min":0.70599999999999996},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2469999999999999,"5min":26.940999999999999,"15min":26.940999999999999},"last":0.32100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:44:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.77200000000000002,"15min":0.63800000000000001},"min":{"1min":0.36799999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.2889999999999997,"5min":5.2889999999999997,"15min":5.2889999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.70499999999999996,"15min":0.61099999999999999},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.2519999999999998,"5min":5.9409999999999998,"15min":5.9409999999999998},"last":0.33500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822884,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":745052,"kb_used_data":435540,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93000100,"statfs":{"total":95995035648,"available":95232102400,"internally_reserved":0,"allocated":445992960,"data_stored":445529135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:54 2023","interfaces":[{"interface":"back","average":{"1min":10.632999999999999,"5min":3.9769999999999999,"15min":1.7370000000000001},"min":{"1min":0.26600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":64.852000000000004,"5min":74.703999999999994,"15min":74.703999999999994},"last":2.6560000000000001},{"interface":"front","average":{"1min":9.9499999999999993,"5min":3.5030000000000001,"15min":1.5780000000000001},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":64.587000000000003,"5min":74.310000000000002,"15min":74.310000000000002},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:45:18 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":3.3660000000000001,"15min":1.4890000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":14.135,"5min":149.203,"15min":149.203},"last":3.6099999999999999},{"interface":"front","average":{"1min":2.7789999999999999,"5min":3.5720000000000001,"15min":1.5389999999999999},"min":{"1min":0.33200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.789,"5min":149.554,"15min":149.554},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Sep 23 22:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":6.0819999999999999,"15min":8.1430000000000007},"min":{"1min":0.38,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":10.853999999999999,"5min":116.123,"15min":116.123},"last":0.89800000000000002},{"interface":"front","average":{"1min":2.5409999999999999,"5min":6.1420000000000003,"15min":8.1639999999999997},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.766,"5min":116.14100000000001,"15min":116.14100000000001},"last":4.0410000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:45:18 2023","interfaces":[{"interface":"back","average":{"1min":5.5419999999999998,"5min":4.4240000000000004,"15min":1.827},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":55.646999999999998,"5min":121.94,"15min":121.94},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.5069999999999997,"5min":3.9540000000000002,"15min":1.718},"min":{"1min":0.34200000000000003,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":55.665999999999997,"5min":123.014,"15min":123.014},"last":0.55700000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:45:06 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.71399999999999997,"15min":0.55800000000000005},"min":{"1min":0.32000000000000001,"5min":0.27600000000000002,"15min":0.22800000000000001},"max":{"1min":3.7160000000000002,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":2.1739999999999999},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.629,"15min":0.53400000000000003},"min":{"1min":0.30399999999999999,"5min":0.25700000000000001,"15min":0.24099999999999999},"max":{"1min":4.6369999999999996,"5min":5.3780000000000001,"15min":5.3780000000000001},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:45:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":2.827,"15min":1.276},"min":{"1min":0.31,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":3.754,"5min":140.07499999999999,"15min":140.07499999999999},"last":1.242},{"interface":"front","average":{"1min":0.80600000000000005,"5min":2.8069999999999999,"15min":1.2929999999999999},"min":{"1min":0.314,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":3.778,"5min":139.739,"15min":139.739},"last":0.94099999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:44:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":0.83199999999999996,"15min":0.65400000000000003},"min":{"1min":0.36399999999999999,"5min":0.29999999999999999,"15min":0.29499999999999998},"max":{"1min":7.0010000000000003,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.71999999999999997,"15min":0.622},"min":{"1min":0.376,"5min":0.311,"15min":0.311},"max":{"1min":3.8119999999999998,"5min":4.056,"15min":4.056},"last":0.49299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1077743616,"data_stored":1077739544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":509939712,"data_stored":509935640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":705429504,"data_stored":705429504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":865009664,"data_stored":865009664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":501612544,"data_stored":501612544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":433651712,"data_stored":433651712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":428150784,"data_stored":428146712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":508100608,"data_stored":508100608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:45:28.009 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T22:45:28.009 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 26 2023-09-23T22:45:28.017 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:28.084 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:28.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:28.193 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:28.227 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:28.291 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:28.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:28.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:28.401 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:28.429 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:28.488 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:28.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:28.592 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:28.602 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:45:28.658 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:45:28.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:28 smithi052 ceph-mon[180979]: pgmap v513: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 14991/53349 objects degraded (28.100%); 2576/53349 objects misplaced (4.829%); 4.4 MiB/s, 111 objects/s recovering 2023-09-23T22:45:28.679 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:28 smithi052 ceph-mon[187743]: pgmap v513: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 14991/53349 objects degraded (28.100%); 2576/53349 objects misplaced (4.829%); 4.4 MiB/s, 111 objects/s recovering 2023-09-23T22:45:28.693 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:28.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:28 smithi151 ceph-mon[159210]: pgmap v513: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 14991/53349 objects degraded (28.100%); 2576/53349 objects misplaced (4.829%); 4.4 MiB/s, 111 objects/s recovering 2023-09-23T22:45:28.719 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:45:28.781 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:45:28.847 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:28.847 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:28.864 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:45:28.894 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:28.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:28.907 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:45:28.955 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:45:28.956 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:45:29.008 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:29.040 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:29.108 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:29.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:29.214 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:29.246 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:29.308 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:29.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:29.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:29.414 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:29.434 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:45:29.457 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:29.526 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:29.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:29.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:29.640 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:29.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3772528540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:29.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1269030550' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:29.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:29 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:29.687 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3772528540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1269030550' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:29 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:29.746 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:29.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3772528540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:29.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1269030550' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:29.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:29 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:29.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:29.862 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:29.955 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:30.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:45:30.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:30.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: osdmap e206: 8 total, 7 up, 7 in 2023-09-23T22:45:30.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: failed to encode map e206 with expected crc 2023-09-23T22:45:30.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: failed to encode map e206 with expected crc 2023-09-23T22:45:30.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 14991/53349 objects degraded (28.100%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:30.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:30 smithi151 ceph-mon[159210]: pgmap v515: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1012 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 13956/48624 objects degraded (28.702%); 2602/48624 objects misplaced (5.351%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:45:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: osdmap e206: 8 total, 7 up, 7 in 2023-09-23T22:45:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: failed to encode map e206 with expected crc 2023-09-23T22:45:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: failed to encode map e206 with expected crc 2023-09-23T22:45:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 14991/53349 objects degraded (28.100%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[180979]: pgmap v515: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1012 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 13956/48624 objects degraded (28.702%); 2602/48624 objects misplaced (5.351%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: osdmap e206: 8 total, 7 up, 7 in 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: failed to encode map e206 with expected crc 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: failed to encode map e206 with expected crc 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 14991/53349 objects degraded (28.100%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:30 smithi052 ceph-mon[187743]: pgmap v515: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 1012 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 13956/48624 objects degraded (28.702%); 2602/48624 objects misplaced (5.351%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:31.778 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:31.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:31.888 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:31.934 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:32.002 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:32.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:32.109 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:32.150 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:32.223 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:32.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:32.231 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:32.262 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:45:32.330 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:32.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:32.495 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:32.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:32.605 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:32.708 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:45:32.711 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:45:32.799 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:32.850 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:32.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:32.964 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:33.040 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:33.278 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:33.282 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:33 smithi052 ceph-mon[187743]: pgmap v516: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 914 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 13176/43929 objects degraded (29.994%); 2602/43929 objects misplaced (5.923%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:33.282 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:33 smithi052 ceph-mon[180979]: pgmap v516: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 914 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 13176/43929 objects degraded (29.994%); 2602/43929 objects misplaced (5.923%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:33.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:33.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:33.399 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:33 smithi151 ceph-mon[159210]: pgmap v516: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 914 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 13176/43929 objects degraded (29.994%); 2602/43929 objects misplaced (5.923%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:33.467 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:33.549 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:33.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:33.653 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:33.736 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:45:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:45:33] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-09-23T22:45:33.746 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:33.796 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:33.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:33.899 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:33.926 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:33.969 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:33.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:34.072 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:34.099 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:34.185 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:34.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:34.300 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:34.379 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:34.502 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:34.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:34.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:34.568 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:45:34.624 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:34.712 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:45:34.771 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:34.976 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:34.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:34.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:35.078 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:35.223 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:35.336 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:35.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:35.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:35.349 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:35.349 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:35.379 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:35 smithi151 ceph-mon[159210]: pgmap v517: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 914 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 13176/43929 objects degraded (29.994%); 2602/43929 objects misplaced (5.923%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:35.379 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:35.379 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:35.379 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 13176/43929 objects degraded (29.994%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:35.443 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:35.452 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:35.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[180979]: pgmap v517: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 914 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 13176/43929 objects degraded (29.994%); 2602/43929 objects misplaced (5.923%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:35.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:35.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:35.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 13176/43929 objects degraded (29.994%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:35.503 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[187743]: pgmap v517: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 914 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 13176/43929 objects degraded (29.994%); 2602/43929 objects misplaced (5.923%); 3.0 MiB/s, 136 objects/s recovering 2023-09-23T22:45:35.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:35.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:35.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 13176/43929 objects degraded (29.994%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:35.537 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:35.637 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:35.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:35.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:35.685 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:35.757 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:35.795 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:36.097 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:45:36.106 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:36.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:36.220 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:36.301 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 7 pg_num to 26 2023-09-23T22:45:36.309 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:36.387 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1164415693' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:45:36.387 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:45:36.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1164415693' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:45:36.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:45:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1164415693' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:45:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:45:36.631 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:37.403 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:37.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:37.509 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:37.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: pgmap v518: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 740 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 11558/35553 objects degraded (32.509%); 2603/35553 objects misplaced (7.321%); 2.6 MiB/s, 139 objects/s recovering 2023-09-23T22:45:37.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T22:45:37.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: osdmap e207: 8 total, 7 up, 7 in 2023-09-23T22:45:37.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.535 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.536 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.536 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.536 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.536 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:37 smithi151 ceph-mon[159210]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.537 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:37.580 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:37.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:37.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:37.683 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: pgmap v518: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 740 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 11558/35553 objects degraded (32.509%); 2603/35553 objects misplaced (7.321%); 2.6 MiB/s, 139 objects/s recovering 2023-09-23T22:45:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T22:45:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: osdmap e207: 8 total, 7 up, 7 in 2023-09-23T22:45:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[187743]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: pgmap v518: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 740 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 11558/35553 objects degraded (32.509%); 2603/35553 objects misplaced (7.321%); 2.6 MiB/s, 139 objects/s recovering 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: osdmap e207: 8 total, 7 up, 7 in 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:37 smithi052 ceph-mon[180979]: failed to encode map e207 with expected crc 2023-09-23T22:45:37.775 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:38.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:38 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T22:45:38.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:38 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T22:45:38.622 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:38.622 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:38.643 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:38 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T22:45:38.895 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:38.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:38.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:38.999 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:39.030 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:39.510 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:45:39.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: pgmap v520: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 714 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 11378/34326 objects degraded (33.147%); 2603/34326 objects misplaced (7.583%); 2.9 MiB/s, 144 objects/s recovering 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: osdmap e208: 8 total, 7 up, 7 in 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:39 smithi151 ceph-mon[159210]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: pgmap v520: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 714 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 11378/34326 objects degraded (33.147%); 2603/34326 objects misplaced (7.583%); 2.9 MiB/s, 144 objects/s recovering 2023-09-23T22:45:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T22:45:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: osdmap e208: 8 total, 7 up, 7 in 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[180979]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: pgmap v520: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 48 active+clean; 714 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 11378/34326 objects degraded (33.147%); 2603/34326 objects misplaced (7.583%); 2.9 MiB/s, 144 objects/s recovering 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: osdmap e208: 8 total, 7 up, 7 in 2023-09-23T22:45:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:39 smithi052 ceph-mon[187743]: failed to encode map e208 with expected crc 2023-09-23T22:45:39.951 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:39.985 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:39.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:40.090 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:40.119 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:40.170 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:40.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:40.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:40.276 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:40.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.315 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: osdmap e209: 8 total, 7 up, 7 in 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.318 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: 7.5 starting backfill to osd.1 from (154'1200,207'11216] MIN to 207'11217 2023-09-23T22:45:40.318 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11378/34326 objects degraded (33.147%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:40.318 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: osdmap e210: 8 total, 7 up, 7 in 2023-09-23T22:45:40.423 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:40.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:40.439 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.439 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.439 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: osdmap e209: 8 total, 7 up, 7 in 2023-09-23T22:45:40.439 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: 7.5 starting backfill to osd.1 from (154'1200,207'11216] MIN to 207'11217 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11378/34326 objects degraded (33.147%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: osdmap e210: 8 total, 7 up, 7 in 2023-09-23T22:45:40.440 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:40 smithi151 ceph-mon[159210]: failed to encode map e210 with expected crc 2023-09-23T22:45:40.524 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:45:40.530 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:40.548 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:45:40.563 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:40.571 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:45:40.600 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:45:40.621 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:40.622 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[187743]: failed to encode map e210 with expected crc 2023-09-23T22:45:40.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:40.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:40.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: osdmap e209: 8 total, 7 up, 7 in 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e209 with expected crc 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: 7.5 starting backfill to osd.1 from (154'1200,207'11216] MIN to 207'11217 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11378/34326 objects degraded (33.147%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: osdmap e210: 8 total, 7 up, 7 in 2023-09-23T22:45:40.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:40 smithi052 ceph-mon[180979]: failed to encode map e210 with expected crc 2023-09-23T22:45:40.625 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:45:40.643 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:45:40.662 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:45:40.724 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:40.753 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:40.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:40.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:40.897 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:45:40.903 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:40.936 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:41.000 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:41.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:41.106 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:41.145 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:41.249 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:41.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: pgmap v524: 99 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 7 unknown, 1 active+clean+remapped, 22 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 638 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 9940/30681 objects degraded (32.398%); 1470/30681 objects misplaced (4.791%); 3.7 MiB/s, 150 objects/s recovering 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/615765930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1610855312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:45:41.337 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: 7.0 scrub starts 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: failed to encode map e211 with expected crc 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: failed to encode map e211 with expected crc 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: osdmap e211: 8 total, 7 up, 7 in 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: 7.15 starting backfill to osd.1 from (154'1200,207'11216] MIN to 207'11217 2023-09-23T22:45:41.338 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:45:41.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:41.456 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:41.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: pgmap v524: 99 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 7 unknown, 1 active+clean+remapped, 22 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 638 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 9940/30681 objects degraded (32.398%); 1470/30681 objects misplaced (4.791%); 3.7 MiB/s, 150 objects/s recovering 2023-09-23T22:45:41.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/615765930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1610855312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: 7.0 scrub starts 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: failed to encode map e211 with expected crc 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: failed to encode map e211 with expected crc 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: osdmap e211: 8 total, 7 up, 7 in 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: 7.15 starting backfill to osd.1 from (154'1200,207'11216] MIN to 207'11217 2023-09-23T22:45:41.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: failed to encode map e210 with expected crc 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: pgmap v524: 99 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 7 unknown, 1 active+clean+remapped, 22 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 638 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 9940/30681 objects degraded (32.398%); 1470/30681 objects misplaced (4.791%); 3.7 MiB/s, 150 objects/s recovering 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/615765930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1610855312' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: 7.0 scrub starts 2023-09-23T22:45:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:41.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: failed to encode map e211 with expected crc 2023-09-23T22:45:41.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: failed to encode map e211 with expected crc 2023-09-23T22:45:41.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:45:41.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: osdmap e211: 8 total, 7 up, 7 in 2023-09-23T22:45:41.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: 7.15 starting backfill to osd.1 from (154'1200,207'11216] MIN to 207'11217 2023-09-23T22:45:41.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:45:42.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:42.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:42.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:42.249 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:42.347 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:42 smithi052 ceph-mon[180979]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.347 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:42 smithi052 ceph-mon[180979]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.347 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:42 smithi052 ceph-mon[180979]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:42 smithi151 ceph-mon[159210]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:42 smithi151 ceph-mon[159210]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:42 smithi151 ceph-mon[159210]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40802 6.e 6:7b05ccce:::benchmark_data_smithi052_234456_object18658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 272.59394733599999, 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533, 2023-09-23T22:45:42.460 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40802 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.461 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862704+0000", 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0949999999999996e-06 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862712+0000", 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2490000000000001e-05 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862725+0000", 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.041e-06 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862732+0000", 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036689000000000001 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866401+0000", 2023-09-23T22:45:42.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0216e-05 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866411+0000", 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99412088600000004 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860532+0000", 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3456e-05 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860545+0000", 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2107602909999999 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071306+0000", 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014411400000000001 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071450+0000", 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017796199999999999 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071628+0000", 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036812979999999999 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075309+0000", 2023-09-23T22:45:42.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001368813 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076678+0000", 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064926749999999998 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083170+0000", 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9931999999999994e-05 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083240+0000", 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034446200000000002 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083585+0000", 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40804 6.e 6:7187807f:::benchmark_data_smithi052_234456_object18660:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 272.59120176699997, 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001, 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:42.470 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40804 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:45:42.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865450+0000", 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4629999999999998e-06 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865457+0000", 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1907e-05 2023-09-23T22:45:42.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865469+0000", 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1515000000000001e-05 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865480+0000", 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096401899999999999 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866444+0000", 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8879999999999997e-06 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866452+0000", 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413119599999999 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860583+0000", 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7849999999999997e-06 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860592+0000", 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.212291711 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072884+0000", 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011587900000000001 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073000+0000", 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073105+0000", 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002208621 2023-09-23T22:45:42.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075313+0000", 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020589470000000002 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.077372+0000", 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062710200000000004 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083643+0000", 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9950000000000002e-05 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083683+0000", 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000162177 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083845+0000", 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40805 6.e 6:798273ba:::benchmark_data_smithi052_234456_object18661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 272.591164827, 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999, 2023-09-23T22:45:42.479 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40805 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:45:42.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865488+0000", 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6610000000000002e-06 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865492+0000", 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7939999999999997e-06 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865499+0000", 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7509999999999999e-06 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865504+0000", 2023-09-23T22:45:42.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097241299999999999 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866476+0000", 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6120000000000001e-06 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866484+0000", 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413304199999997 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860617+0000", 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3080000000000002e-06 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860624+0000", 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.213177682 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073802+0000", 2023-09-23T22:45:42.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8152000000000005e-05 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073900+0000", 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106062 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.074006+0000", 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022883959999999998 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076294+0000", 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002385603 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.078680+0000", 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052328410000000002 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083913+0000", 2023-09-23T22:45:42.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2397000000000005e-05 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083985+0000", 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026911300000000002 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.084254+0000", 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.487 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40809 6.2 6:400fe117:::benchmark_data_smithi052_234456_object18665:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 272.59028550599999, 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002, 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:42.488 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40809 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866367+0000", 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8289999999999996e-06 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866373+0000", 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1122e-05 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866384+0000", 2023-09-23T22:45:42.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8669999999999996e-06 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866391+0000", 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000202492 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866594+0000", 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7580000000000004e-06 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866603+0000", 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.994511967 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861115+0000", 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6169999999999996e-06 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861125+0000", 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.209770244 2023-09-23T22:45:42.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.070895+0000", 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013600900000000001 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071031+0000", 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121377 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071153+0000", 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016472430000000001 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072800+0000", 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000377547 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073177+0000", 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0096328860000000002 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082810+0000", 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6335999999999996e-05 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082877+0000", 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000259743 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083136+0000", 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39271 7.1 7:803b19ab:::benchmark_data_smithi052_277165_object39270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.70106383999999, 2023-09-23T22:45:42.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002, 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39271 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:45:42.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755588+0000", 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100054 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755689+0000", 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755701+0000", 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3199999999999999e-06 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755707+0000", 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9419000000000002e-05 2023-09-23T22:45:42.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755756+0000", 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5290000000000008e-06 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755765+0000", 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99249923500000004 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748264+0000", 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6199999999999994e-06 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748273+0000", 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.538374364 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286648+0000", 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000173475 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286821+0000", 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024065800000000001 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.287062+0000", 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003676038 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290738+0000", 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3826999999999998e-05 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290792+0000", 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2153e-05 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290804+0000", 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5701000000000002e-05 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290850+0000", 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093742900000000004 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291787+0000", 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002 2023-09-23T22:45:42.504 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39291 7.1 7:8a0225f8:::benchmark_data_smithi052_277165_object39290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.686355066, 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001, 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.505 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39291 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:45:42.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770297+0000", 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5182e-05 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770362+0000", 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498999999999998e-05 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770378+0000", 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0129999999999996e-06 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770385+0000", 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2045000000000007e-05 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770447+0000", 2023-09-23T22:45:42.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1433999999999999e-05 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770458+0000", 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97785107400000004 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748309+0000", 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8459999999999996e-06 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748315+0000", 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5410234169999999 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289338+0000", 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117113 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289455+0000", 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000150991 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289606+0000", 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023599210000000001 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291966+0000", 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4410000000000004e-06 2023-09-23T22:45:42.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291974+0000", 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8967999999999994e-05 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292043+0000", 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7804000000000003e-05 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292111+0000", 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00075040600000000001 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292861+0000", 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.513 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39293 7.1 7:8d7e8869:::benchmark_data_smithi052_277165_object39292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.68293692200001, 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908, 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39293 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.514 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773716+0000", 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2105000000000003e-05 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773778+0000", 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2449000000000001e-05 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773790+0000", 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6099999999999997e-06 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773796+0000", 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9072000000000001e-05 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773835+0000", 2023-09-23T22:45:42.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773843+0000", 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97449834000000002 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748341+0000", 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5279999999999997e-06 2023-09-23T22:45:42.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748346+0000", 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.544637525 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292983+0000", 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016144799999999999 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293145+0000", 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015888799999999999 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293304+0000", 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039379849999999997 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297242+0000", 2023-09-23T22:45:42.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6013000000000001e-05 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297288+0000", 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003665958 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300954+0000", 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7888999999999999e-05 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.301032+0000", 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012321579999999999 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302264+0000", 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.522 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39318 7.1 7:887a21fd:::benchmark_data_smithi052_277165_object39317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.66140329800001, 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997, 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.523 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39318 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:42.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795249+0000", 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1755000000000002e-05 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795301+0000", 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3233e-05 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795314+0000", 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1940000000000001e-06 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795321+0000", 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1218999999999999e-05 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795372+0000", 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0147e-05 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795383+0000", 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.95298535699999998 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748368+0000", 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2440000000000004e-06 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748372+0000", 2023-09-23T22:45:42.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5462721529999999 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294644+0000", 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112796 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294757+0000", 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018578900000000001 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294943+0000", 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023897250000000001 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297333+0000", 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297340+0000", 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050090029999999997 2023-09-23T22:45:42.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302349+0000", 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3208999999999999e-05 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302392+0000", 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00080982299999999999 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303202+0000", 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39334 7.1 7:83eed48f:::benchmark_data_smithi052_277165_object39333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.650243711, 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001, 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.531 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39334 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806408+0000", 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8466e-05 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806457+0000", 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3062e-05 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806470+0000", 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2000000000000002e-06 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806475+0000", 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9246999999999998e-05 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806524+0000", 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0499999999999997e-06 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806533+0000", 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94188278000000003 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748416+0000", 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.52e-06 2023-09-23T22:45:42.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748423+0000", 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5475971120000001 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296021+0000", 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015821300000000001 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296179+0000", 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013869899999999999 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296317+0000", 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045890180000000003 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300906+0000", 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023854169999999999 2023-09-23T22:45:42.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303292+0000", 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028062899999999999 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306098+0000", 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2596999999999999e-05 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306141+0000", 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068390599999999997 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306825+0000", 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.539 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39346 7.1 7:8a73189a:::benchmark_data_smithi052_277165_object39345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.64528800400001, 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001, 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39346 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.540 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811364+0000", 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012578900000000001 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811490+0000", 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8567999999999999e-05 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811508+0000", 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9329999999999996e-06 2023-09-23T22:45:42.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811518+0000", 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8912000000000001e-05 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811577+0000", 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1018e-05 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811588+0000", 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93686738199999997 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748455+0000", 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0320000000000003e-06 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748460+0000", 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5488962179999999 2023-09-23T22:45:42.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297357+0000", 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010246 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297459+0000", 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012592199999999999 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297585+0000", 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004696787 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302282+0000", 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001097813 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303380+0000", 2023-09-23T22:45:42.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034811529999999999 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306861+0000", 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9079000000000001e-05 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306920+0000", 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074140899999999995 2023-09-23T22:45:42.547 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.307661+0000", 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39360 7.1 7:8f91ddcd:::benchmark_data_smithi052_277165_object39359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.63389560900001, 2023-09-23T22:45:42.548 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001, 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39360 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.549 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822756+0000", 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0686999999999998e-05 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.550 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822807+0000", 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4600000000000001e-05 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822821+0000", 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2920000000000001e-06 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822828+0000", 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2878000000000003e-05 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.551 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822901+0000", 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9390000000000003e-06 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822909+0000", 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92559491500000002 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748504+0000", 2023-09-23T22:45:42.552 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4299999999999997e-06 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748510+0000", 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498821280000001 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298392+0000", 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1963e-05 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.553 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298484+0000", 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011828300000000001 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298602+0000", 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036917579999999998 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302294+0000", 2023-09-23T22:45:42.554 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011534850000000001 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303447+0000", 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049678559999999997 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308415+0000", 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0241000000000001e-05 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.555 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308476+0000", 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00084742900000000002 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309323+0000", 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.556 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39366 7.1 7:86dcbc64:::benchmark_data_smithi052_277165_object39365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.630266412, 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001, 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39366 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.557 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826386+0000", 2023-09-23T22:45:42.558 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4935999999999997e-05 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826441+0000", 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2864000000000001e-05 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826454+0000", 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7969999999999997e-06 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.559 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826460+0000", 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5853000000000003e-05 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826505+0000", 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0890000000000006e-06 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826514+0000", 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92203426200000005 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.560 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748549+0000", 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9800000000000003e-06 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748555+0000", 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.55081775 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299372+0000", 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155606 2023-09-23T22:45:42.561 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299528+0000", 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021651799999999999 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299745+0000", 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035645 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.562 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303309+0000", 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021267000000000001 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303522+0000", 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0058789619999999997 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309401+0000", 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0636999999999995e-05 2023-09-23T22:45:42.563 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309471+0000", 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00099162699999999992 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310463+0000", 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.564 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39373 7.1 7:85f7eb6c:::benchmark_data_smithi052_277165_object39372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.627188891, 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812, 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39373 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.565 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.566 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829464+0000", 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1137000000000002e-05 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829515+0000", 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3482000000000001e-05 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829528+0000", 2023-09-23T22:45:42.567 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.959e-06 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829534+0000", 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5408999999999999e-05 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829570+0000", 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2763e-05 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.568 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829582+0000", 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.91900465799999997 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748587+0000", 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7450000000000003e-06 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748592+0000", 2023-09-23T22:45:42.569 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.554972391 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303564+0000", 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012435699999999999 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303689+0000", 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000131853 2023-09-23T22:45:42.570 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303820+0000", 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004451455 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308272+0000", 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0795000000000003e-05 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.571 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308333+0000", 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022271669999999999 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310560+0000", 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7315000000000001e-05 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.573 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310637+0000", 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083538299999999998 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311473+0000", 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.574 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39414 7.1 7:8075013a:::benchmark_data_smithi052_277165_object39413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.595028216, 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001, 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.575 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39414 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:42.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861624+0000", 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4787000000000002e-05 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861678+0000", 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3568e-05 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861692+0000", 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1750000000000002e-06 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861698+0000", 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9217e-05 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861747+0000", 2023-09-23T22:45:42.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0080000000000002e-06 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861755+0000", 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88686667100000005 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748622+0000", 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0063e-05 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.579 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748632+0000", 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.562879379 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311512+0000", 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018041500000000001 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.580 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311692+0000", 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020370699999999999 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311896+0000", 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077409310000000004 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319637+0000", 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1062000000000006e-05 2023-09-23T22:45:42.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319698+0000", 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103898 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319802+0000", 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7586e-05 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319869+0000", 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017220569999999999 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321591+0000", 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39416 7.1 7:83076ecc:::benchmark_data_smithi052_277165_object39415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.583 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.59363895300001, 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074, 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39416 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.584 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863014+0000", 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106535 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863120+0000", 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1805e-05 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863132+0000", 2023-09-23T22:45:42.586 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4430000000000002e-06 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863138+0000", 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0098999999999997e-05 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863178+0000", 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0545e-05 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.587 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863188+0000", 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88547513300000003 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748663+0000", 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.507e-06 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748669+0000", 2023-09-23T22:45:42.588 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5646240410000001 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313293+0000", 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110319 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313403+0000", 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196856 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313600+0000", 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060436789999999997 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319644+0000", 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011535 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319759+0000", 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019084060000000001 2023-09-23T22:45:42.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321668+0000", 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2410999999999996e-05 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321730+0000", 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011838980000000001 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322914+0000", 2023-09-23T22:45:42.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39440 7.1 7:846ccd4b:::benchmark_data_smithi052_277165_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.573163647, 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002, 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.592 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39440 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:45:42.593 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883489+0000", 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0798999999999997e-05 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883569+0000", 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5083999999999999e-05 2023-09-23T22:45:42.594 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883594+0000", 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1099999999999999e-06 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883601+0000", 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5002e-05 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883646+0000", 2023-09-23T22:45:42.595 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5539999999999998e-06 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883654+0000", 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86504454099999994 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748699+0000", 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0530000000000003e-06 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748705+0000", 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.566234793 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.314939+0000", 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014568900000000001 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315085+0000", 2023-09-23T22:45:42.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019602700000000001 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315281+0000", 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004426915 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319708+0000", 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032665569999999998 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322975+0000", 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5554e-05 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323070+0000", 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1078999999999998e-05 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323111+0000", 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095274200000000002 2023-09-23T22:45:42.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324064+0000", 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39442 7.1 7:821661b1:::benchmark_data_smithi052_277165_object39441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.57012605, 2023-09-23T22:45:42.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999, 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39442 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999895 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886517+0000", 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0640000000000002e-05 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886578+0000", 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4535999999999999e-05 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886592+0000", 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.024e-06 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.603 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886599+0000", 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5583e-05 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886655+0000", 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9129999999999992e-06 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886664+0000", 2023-09-23T22:45:42.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86206608799999995 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748730+0000", 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6200000000000001e-06 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748737+0000", 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.567891876 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316628+0000", 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014458900000000001 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316773+0000", 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000198579 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316972+0000", 2023-09-23T22:45:42.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027449810000000001 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319717+0000", 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033197000000000001 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323036+0000", 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001091297 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324128+0000", 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0636999999999997e-05 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324168+0000", 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071410600000000005 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324882+0000", 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.608 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39446 7.1 7:8475cdd4:::benchmark_data_smithi052_277165_object39445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.56684682299999, 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001, 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.609 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39446 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889806+0000", 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7126000000000002e-05 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889853+0000", 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2593e-05 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889865+0000", 2023-09-23T22:45:42.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889871+0000", 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7049999999999998e-05 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889918+0000", 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5479999999999998e-06 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889924+0000", 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.858839715 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748764+0000", 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5000000000000001e-06 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748768+0000", 2023-09-23T22:45:42.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5693916369999998 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318160+0000", 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016406600000000001 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318324+0000", 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000199247 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318523+0000", 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044604579999999996 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322984+0000", 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019935700000000001 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324977+0000", 2023-09-23T22:45:42.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4416e-05 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325052+0000", 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9657000000000003e-05 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325092+0000", 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00089603700000000003 2023-09-23T22:45:42.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325988+0000", 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39458 7.1 7:846e660e:::benchmark_data_smithi052_277165_object39457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:45:42.617 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.55077015500001, 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002, 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39458 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905882+0000", 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7948000000000002e-05 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905940+0000", 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1592e-05 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905952+0000", 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4750000000000001e-06 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905957+0000", 2023-09-23T22:45:42.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2416000000000001e-05 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906010+0000", 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4849999999999992e-06 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906018+0000", 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.84277395099999997 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748792+0000", 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748796+0000", 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5772432570000001 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326040+0000", 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015276800000000001 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326193+0000", 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018607500000000001 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326379+0000", 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038150480000000001 2023-09-23T22:45:42.623 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.330194+0000", 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001090159 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331284+0000", 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0065274460000000001 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.624 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337811+0000", 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1335999999999999e-05 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337853+0000", 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014175889999999999 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339270+0000", 2023-09-23T22:45:42.625 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39462 7.1 7:8cf1a796:::benchmark_data_smithi052_277165_object39461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 211.54020944300001, 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001, 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:42.626 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39462 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:42.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916441+0000", 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5293000000000001e-05 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916497+0000", 2023-09-23T22:45:42.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3080000000000002e-05 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916530+0000", 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4319e-05 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916544+0000", 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018227799999999999 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916726+0000", 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3705999999999999e-05 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916760+0000", 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.83205244499999997 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748813+0000", 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5040000000000004e-06 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748817+0000", 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.578915206 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327732+0000", 2023-09-23T22:45:42.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115865 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327848+0000", 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000123075 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327971+0000", 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033208170000000002 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331292+0000", 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0080585329999999997 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339351+0000", 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1529e-05 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:42.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339412+0000", 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7031000000000002e-05 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339479+0000", 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010848730000000001 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.340564+0000", 2023-09-23T22:45:42.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001 2023-09-23T22:45:42.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.635 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:42.635 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:42.635 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:45:42.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:42 smithi052 ceph-mon[187743]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:42 smithi052 ceph-mon[187743]: failed to encode map e211 with expected crc 2023-09-23T22:45:42.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:42 smithi052 ceph-mon[187743]: failed to encode map e211 with expected crc 2023-09-23T22:45:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:43 smithi151 ceph-mon[159210]: pgmap v526: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 unknown, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 573 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 6601/27543 objects degraded (23.966%); 1470/27543 objects misplaced (5.337%); 3.7 MiB/s, 205 objects/s recovering 2023-09-23T22:45:43.745 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:45:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:45:43] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-09-23T22:45:43.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:43 smithi052 ceph-mon[187743]: pgmap v526: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 unknown, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 573 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 6601/27543 objects degraded (23.966%); 1470/27543 objects misplaced (5.337%); 3.7 MiB/s, 205 objects/s recovering 2023-09-23T22:45:43.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:43 smithi052 ceph-mon[180979]: pgmap v526: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 unknown, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 573 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 6601/27543 objects degraded (23.966%); 1470/27543 objects misplaced (5.337%); 3.7 MiB/s, 205 objects/s recovering 2023-09-23T22:45:43.890 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:45:43.936 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:43.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:45:43.980 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:45:44.042 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:44.108 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:44.217 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:44.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:44.327 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:44.406 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:44.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:44 smithi052 ceph-mon[187743]: pgmap v527: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 unknown, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 573 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 387 op/s; 6601/27543 objects degraded (23.966%); 1470/27543 objects misplaced (5.337%); 2.5 MiB/s, 141 objects/s recovering 2023-09-23T22:45:44.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:44 smithi052 ceph-mon[180979]: pgmap v527: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 unknown, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 573 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 387 op/s; 6601/27543 objects degraded (23.966%); 1470/27543 objects misplaced (5.337%); 2.5 MiB/s, 141 objects/s recovering 2023-09-23T22:45:44.527 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:44.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:44.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:44.570 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:44.640 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:44 smithi151 ceph-mon[159210]: pgmap v527: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 unknown, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 573 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 387 op/s; 6601/27543 objects degraded (23.966%); 1470/27543 objects misplaced (5.337%); 2.5 MiB/s, 141 objects/s recovering 2023-09-23T22:45:44.708 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:44.885 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:44.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:44.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:44.945 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [0] live_osds: [1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:45:44.945 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:45:44.945 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:45:44.945 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-09-23T22:45:44.946 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-09-23T22:45:45.000 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:45.087 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:45.236 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:45.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:45.344 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:45.420 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:45.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6601/27543 objects degraded (23.966%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:45.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6601/27543 objects degraded (23.966%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:45.545 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:45.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:45.652 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:45:45.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6601/27543 objects degraded (23.966%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:45.703 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:45.748 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:45.847 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:45:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:45.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:45:45.950 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:45.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:45:45.998 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:46.038 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:45:46.054 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:46.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:46.158 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:46.316 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:46.389 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:46 smithi052 ceph-mon[180979]: pgmap v528: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 357 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 7104/17205 objects degraded (41.290%); 2643/17205 objects misplaced (15.362%); 5.1 MiB/s, 200 objects/s recovering 2023-09-23T22:45:46.526 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:46.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:46.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:46.630 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:46.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:46 smithi151 ceph-mon[159210]: pgmap v528: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 357 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 7104/17205 objects degraded (41.290%); 2643/17205 objects misplaced (15.362%); 5.1 MiB/s, 200 objects/s recovering 2023-09-23T22:45:46.674 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:46.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:46 smithi052 ceph-mon[187743]: pgmap v528: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 357 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 7104/17205 objects degraded (41.290%); 2643/17205 objects misplaced (15.362%); 5.1 MiB/s, 200 objects/s recovering 2023-09-23T22:45:46.802 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:46.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:46.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:46.931 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:47.004 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.35001.0:181788 7.e e211/202 7:77946225:::benchmark_data_smithi052_277165_object89296:head v 211'11380, mlcod=211'11378)", 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:47.505803+0000", 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0040707199999999999, 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041004789999999998, 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:45:47.511 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505803+0000", 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505803+0000", 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505802+0000", 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1559999999999995e-06 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505810+0000", 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3823e-05 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505824+0000", 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.417e-06 2023-09-23T22:45:47.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505829+0000", 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6189000000000002e-05 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505865+0000", 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9864000000000002e-05 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:47.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:47.505885+0000", 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2610999999999999e-05 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 1 2023-09-23T22:45:47.515 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:48.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:48.680 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:48.715 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:48.803 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:48.869 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:48.931 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:45:48.938 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:48.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:49.045 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:49.091 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:49.187 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:49.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:49.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:49.299 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:49.405 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:49 smithi052 ceph-mon[180979]: pgmap v529: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 323 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 6834/15537 objects degraded (43.985%); 2643/15537 objects misplaced (17.011%); 4.2 MiB/s, 147 objects/s recovering 2023-09-23T22:45:49.405 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:49.405 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:49.406 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:49 smithi052 ceph-mon[187743]: pgmap v529: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 323 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 6834/15537 objects degraded (43.985%); 2643/15537 objects misplaced (17.011%); 4.2 MiB/s, 147 objects/s recovering 2023-09-23T22:45:49.406 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:49.406 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:49.411 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:49.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:49 smithi151 ceph-mon[159210]: pgmap v529: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 323 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 6834/15537 objects degraded (43.985%); 2643/15537 objects misplaced (17.011%); 4.2 MiB/s, 147 objects/s recovering 2023-09-23T22:45:49.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:49.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:49.544 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:49.745 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:49.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:49.854 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:49.918 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:50.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:50.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:50.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6834/15537 objects degraded (43.985%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:50.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1385158697' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T22:45:50.490 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[180979]: pgmap v530: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 185 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 5694/8931 objects degraded (63.755%); 2354/8931 objects misplaced (26.358%); 3.0 MiB/s, 100 objects/s recovering 2023-09-23T22:45:50.498 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.5 primary-affinity to 0 (802) 2023-09-23T22:45:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6834/15537 objects degraded (43.985%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1385158697' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T22:45:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:50 smithi052 ceph-mon[187743]: pgmap v530: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 185 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 5694/8931 objects degraded (63.755%); 2354/8931 objects misplaced (26.358%); 3.0 MiB/s, 100 objects/s recovering 2023-09-23T22:45:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6834/15537 objects degraded (43.985%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1385158697' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-09-23T22:45:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:50 smithi151 ceph-mon[159210]: pgmap v530: 99 pgs: 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 52 active+clean; 185 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 5694/8931 objects degraded (63.755%); 2354/8931 objects misplaced (26.358%); 3.0 MiB/s, 100 objects/s recovering 2023-09-23T22:45:51.689 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:45:51.689 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:45:51.689 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:45:51.689 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:45:51.689 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1385158697' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: osdmap e212: 8 total, 7 up, 7 in 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:51 smithi151 ceph-mon[159210]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1385158697' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: osdmap e212: 8 total, 7 up, 7 in 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[187743]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1385158697' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: osdmap e212: 8 total, 7 up, 7 in 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:51 smithi052 ceph-mon[180979]: failed to encode map e212 with expected crc 2023-09-23T22:45:51.923 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Removed 92489 objects 2023-09-23T22:45:51.923 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Clean up completed and total clean up time :154.375 2023-09-23T22:45:51.945 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:51.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:51.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:52.052 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:52.082 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:52.795 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.795 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: osdmap e213: 8 total, 7 up, 7 in 2023-09-23T22:45:52.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.796 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:52 smithi151 ceph-mon[159210]: pgmap v533: 99 pgs: 3 peering, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 50 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 4648/2454 objects degraded (189.405%); 2354/2454 objects misplaced (95.925%); 3.7 MiB/s, 129 objects/s recovering 2023-09-23T22:45:52.814 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:45:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: osdmap e213: 8 total, 7 up, 7 in 2023-09-23T22:45:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[187743]: pgmap v533: 99 pgs: 3 peering, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 50 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 4648/2454 objects degraded (189.405%); 2354/2454 objects misplaced (95.925%); 3.7 MiB/s, 129 objects/s recovering 2023-09-23T22:45:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: osdmap e213: 8 total, 7 up, 7 in 2023-09-23T22:45:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: failed to encode map e213 with expected crc 2023-09-23T22:45:52.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:52 smithi052 ceph-mon[180979]: pgmap v533: 99 pgs: 3 peering, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 50 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 4648/2454 objects degraded (189.405%); 2354/2454 objects misplaced (95.925%); 3.7 MiB/s, 129 objects/s recovering 2023-09-23T22:45:53.195 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:53 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:45:53.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:45:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:45:53] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.33.4" 2023-09-23T22:45:53.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:53 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:45:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:53 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:45:53.852 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:53.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:53.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:53.956 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:53.986 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:54.033 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:54.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:54.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:54.136 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:54.162 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:54.286 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:54.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:54.389 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:54.418 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:54.462 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:54.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:54.566 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:54.595 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:54.693 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:54.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:54 smithi151 ceph-mon[159210]: pgmap v534: 99 pgs: 3 peering, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 50 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 4648/2454 objects degraded (189.405%); 2354/2454 objects misplaced (95.925%); 1.2 MiB/s, 61 objects/s recovering 2023-09-23T22:45:54.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:54.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:54 smithi052 ceph-mon[187743]: pgmap v534: 99 pgs: 3 peering, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 50 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 4648/2454 objects degraded (189.405%); 2354/2454 objects misplaced (95.925%); 1.2 MiB/s, 61 objects/s recovering 2023-09-23T22:45:54.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:54 smithi052 ceph-mon[180979]: pgmap v534: 99 pgs: 3 peering, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 50 active+clean; 50 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 4648/2454 objects degraded (189.405%); 2354/2454 objects misplaced (95.925%); 1.2 MiB/s, 61 objects/s recovering 2023-09-23T22:45:54.756 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:45:54.756 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:45:54.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10192 6.a 6:5a145b56:::benchmark_data_smithi052_234456_object10191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.204050+0000", 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 300.549294809, 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26064942200000002, 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:54.757 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10192 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204050+0000", 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204050+0000", 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204049+0000", 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5916e-05 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204085+0000", 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1909999999999999e-05 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204097+0000", 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.007e-06 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204104+0000", 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4982000000000003e-05 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204159+0000", 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143879 2023-09-23T22:45:54.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204303+0000", 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016021399999999999 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204463+0000", 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017162919999999999 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.761 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206179+0000", 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0668e-05 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206190+0000", 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25841117400000002 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464601+0000", 2023-09-23T22:45:54.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8266000000000001e-05 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464659+0000", 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0209000000000001e-05 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464700+0000", 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26064942200000002 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.763 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:36138 6.a 6:5384730e:::benchmark_data_smithi052_234456_object13994:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.833909+0000", 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.919436426, 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33453486900000001, 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:54.764 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 36138 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833909+0000", 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833909+0000", 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:54.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833907+0000", 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7300000000000005e-06 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833915+0000", 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2337999999999999e-05 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833927+0000", 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1489999999999997e-06 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833934+0000", 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0534000000000001e-05 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833985+0000", 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012457899999999999 2023-09-23T22:45:54.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.834109+0000", 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012462500000000001 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.834234+0000", 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33402164899999998 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168256+0000", 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.13e-05 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168307+0000", 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2104999999999997e-05 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168359+0000", 2023-09-23T22:45:54.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5541999999999999e-05 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168405+0000", 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8788000000000002e-05 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168444+0000", 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33453486900000001 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.770 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:36141 6.d 6:b1c013a8:::benchmark_data_smithi052_234456_object13997:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.833951+0000", 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.91939434900002, 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33469530800000002, 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:54.771 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 36141 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833951+0000", 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833951+0000", 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:54.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833950+0000", 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4170000000000001e-06 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833954+0000", 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3013999999999999e-05 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833967+0000", 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3380000000000001e-06 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833973+0000", 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5982000000000006e-05 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.834039+0000", 2023-09-23T22:45:54.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8293000000000005e-05 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.834107+0000", 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8912000000000005e-05 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.834196+0000", 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014121050000000001 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835608+0000", 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33291701400000001 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168525+0000", 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2333999999999999e-05 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168558+0000", 2023-09-23T22:45:54.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0726999999999997e-05 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168608+0000", 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7669000000000001e-05 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168646+0000", 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33469530800000002 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.777 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:36142 6.a 6:51be9769:::benchmark_data_smithi052_234456_object13998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.833986+0000", 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.91935898499997, 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33727516400000002, 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:54.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 36142 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833986+0000", 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833986+0000", 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:54.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833986+0000", 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1779999999999999e-06 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833989+0000", 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3430000000000001e-06 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833995+0000", 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3479999999999997e-06 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.833999+0000", 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00134168 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835341+0000", 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100825 2023-09-23T22:45:54.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835442+0000", 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011312 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835555+0000", 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33273462799999998 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168290+0000", 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019643899999999999 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168486+0000", 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026907799999999998 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.171177+0000", 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7219000000000003e-05 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.171224+0000", 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7076000000000002e-05 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.171261+0000", 2023-09-23T22:45:54.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33727516400000002 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:36140 6.c e129/99 6:354f3782:::benchmark_data_smithi052_234456_object13996:head v 129'2150, mlcod=129'2148)", 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.835609+0000", 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.91773590600002, 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33473119499999998, 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.785 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835609+0000", 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835609+0000", 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835608+0000", 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3599999999999996e-06 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835617+0000", 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1124000000000001e-05 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835628+0000", 2023-09-23T22:45:54.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1970000000000001e-06 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835634+0000", 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0744999999999997e-05 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835675+0000", 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33121207899999999 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166887+0000", 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034221339999999999 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.170309+0000", 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1331000000000002e-05 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.170340+0000", 2023-09-23T22:45:54.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33473119499999998 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop_reply(client.34680.0:36138 6.a e129/126 ondisk, result = 0)", 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.835817+0000", 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.91752823799999, 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33244083299999999, 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:45:54.790 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835817+0000", 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835817+0000", 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835816+0000", 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5130000000000002e-06 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.835822+0000", 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.331043486 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166866+0000", 2023-09-23T22:45:54.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4714e-05 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166881+0000", 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00136155 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168242+0000", 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9059999999999996e-06 2023-09-23T22:45:54.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168249+0000", 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.507e-06 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168258+0000", 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33244083299999999 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.794 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:36144 6.7 6:e2faa942:::benchmark_data_smithi052_234456_object14000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.837506+0000", 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.91583896700001, 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33254340199999999, 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:45:54.795 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 36144 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.837506+0000", 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.837506+0000", 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:54.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.837505+0000", 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1260000000000003e-06 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.837511+0000", 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.32935457699999998 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166865+0000", 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7139999999999999e-05 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166882+0000", 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6594000000000001e-05 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166939+0000", 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000133509 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.167073+0000", 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012935999999999999 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.167202+0000", 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027268750000000001 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.169929+0000", 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8639000000000001e-05 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.169957+0000", 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4192000000000001e-05 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.169972+0000", 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2663000000000001e-05 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.170014+0000", 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5228000000000002e-05 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.170049+0000", 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.33254340199999999 2023-09-23T22:45:54.801 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:36151 6.c e129/99 6:368aa1e9:::benchmark_data_smithi052_234456_object14007:head v 129'2151, mlcod=129'2148)", 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:06.841829+0000", 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 287.91151578199998, 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.32955251699999999, 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.802 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.841829+0000", 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.841829+0000", 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.841828+0000", 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0860000000000002e-06 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:06.841836+0000", 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.32504673299999998 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166883+0000", 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8750000000000002e-06 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.166890+0000", 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013125540000000001 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168203+0000", 2023-09-23T22:45:54.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1908000000000001e-05 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.168224+0000", 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031242840000000002 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.171349+0000", 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3043999999999997e-05 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:07.171382+0000", 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.32955251699999999 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10382 7.0 7:08701358:::benchmark_data_smithi052_277165_object10381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.074682+0000", 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.67866262499999, 2023-09-23T22:45:54.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22984989, 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10382 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074682+0000", 2023-09-23T22:45:54.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074682+0000", 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074681+0000", 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115702 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074797+0000", 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7728e-05 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074814+0000", 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5209999999999997e-06 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074822+0000", 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7989000000000003e-05 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074880+0000", 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000145506 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075025+0000", 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017123299999999999 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075197+0000", 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00129065 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.076487+0000", 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014871940000000001 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.077974+0000", 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22648754300000001 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304462+0000", 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5624999999999998e-05 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304508+0000", 2023-09-23T22:45:54.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4793000000000001e-05 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304532+0000", 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22984989 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10384 7.0 7:090486be:::benchmark_data_smithi052_277165_object10383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.074837+0000", 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.67850763499999, 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22981749600000001, 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10384 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074837+0000", 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074837+0000", 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074837+0000", 2023-09-23T22:45:54.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3939999999999997e-05 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074871+0000", 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8980000000000002e-06 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074880+0000", 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3299999999999998e-06 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.074885+0000", 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00082726100000000003 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075712+0000", 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000135081 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075847+0000", 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196372 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.076044+0000", 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011419480000000001 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.077186+0000", 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00081960400000000004 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.078005+0000", 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22658318499999999 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304588+0000", 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9541e-05 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304628+0000", 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7018000000000001e-05 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304655+0000", 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22981749600000001 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.821 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10390 7.0 7:01932cc7:::benchmark_data_smithi052_277165_object10389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.075918+0000", 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.677426803, 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22881372699999999, 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10390 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.822 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075918+0000", 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075918+0000", 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075917+0000", 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2499000000000002e-05 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075950+0000", 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0689e-05 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075960+0000", 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6980000000000001e-06 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.075967+0000", 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054466200000000001 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.076512+0000", 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013060400000000001 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.076642+0000", 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000141447 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.076784+0000", 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00056268300000000002 2023-09-23T22:45:54.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.077347+0000", 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00089066199999999996 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.078237+0000", 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22644353 2023-09-23T22:45:54.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304681+0000", 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9807000000000001e-05 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304711+0000", 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1450999999999998e-05 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.304732+0000", 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22881372699999999 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10403 7.0 7:08ef69f1:::benchmark_data_smithi052_277165_object10402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.082463+0000", 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.67088203899999, 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22437167999999999, 2023-09-23T22:45:54.829 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10403 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082463+0000", 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082463+0000", 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082462+0000", 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000128362 2023-09-23T22:45:54.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082590+0000", 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3406e-05 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082603+0000", 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0220000000000004e-06 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082608+0000", 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6470000000000001e-05 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082655+0000", 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013821999999999999 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082793+0000", 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019972699999999999 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.082993+0000", 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000929618 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.083922+0000", 2023-09-23T22:45:54.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001753721 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.085676+0000", 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22109457399999999 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.306771+0000", 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9765000000000003e-05 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.306810+0000", 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4244e-05 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.306835+0000", 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22437167999999999 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.837 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10412 7.0 7:0555264b:::benchmark_data_smithi052_277165_object10411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.086250+0000", 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.66709498899999, 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22071242399999999, 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.838 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10412 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086250+0000", 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086250+0000", 2023-09-23T22:45:54.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086249+0000", 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7809999999999998e-05 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086297+0000", 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2662e-05 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086309+0000", 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1150000000000002e-06 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086314+0000", 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7521000000000001e-05 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086362+0000", 2023-09-23T22:45:54.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000136868 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086499+0000", 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017683799999999999 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.086676+0000", 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00055726799999999998 2023-09-23T22:45:54.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087233+0000", 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085246499999999995 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088085+0000", 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21879194900000001 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.306877+0000", 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3771999999999999e-05 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.306931+0000", 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1316999999999998e-05 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.306962+0000", 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22071242399999999 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10414 7.0 7:0ceb95bd:::benchmark_data_smithi052_277165_object10413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.087301+0000", 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.66604435400001, 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21981136900000001, 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.845 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10414 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087301+0000", 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087301+0000", 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087300+0000", 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9957000000000002e-05 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087360+0000", 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0987e-05 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087371+0000", 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8239999999999996e-06 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087377+0000", 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1723e-05 2023-09-23T22:45:54.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087418+0000", 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000127126 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087546+0000", 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017050799999999999 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.087716+0000", 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00044645299999999998 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088162+0000", 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001000578 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.089163+0000", 2023-09-23T22:45:54.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.217861315 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.307024+0000", 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0330000000000001e-05 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.307075+0000", 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7376000000000003e-05 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.307112+0000", 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21981136900000001 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:10417 7.0 7:07c516e4:::benchmark_data_smithi052_277165_object10416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:52.088422+0000", 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 242.66492333400001, 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.218783329, 2023-09-23T22:45:54.852 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10417 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088422+0000", 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088422+0000", 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088421+0000", 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0710000000000001e-05 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088472+0000", 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2971e-05 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088484+0000", 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1580000000000002e-06 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088491+0000", 2023-09-23T22:45:54.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0139999999999999e-05 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088531+0000", 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012856799999999999 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088659+0000", 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016094899999999999 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.088820+0000", 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00049546300000000002 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.089316+0000", 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00041073699999999999 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.089727+0000", 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.217426542 2023-09-23T22:45:54.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.307153+0000", 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8776e-05 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.307182+0000", 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3232999999999999e-05 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:52.307205+0000", 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.218783329 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101957 7.0 7:0053af50:::benchmark_data_smithi052_277165_object9465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.526400+0000", 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 142.226945242, 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226700761, 2023-09-23T22:45:54.859 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101957 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526400+0000", 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526400+0000", 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526398+0000", 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7970000000000001e-06 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526406+0000", 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4841000000000001e-05 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526421+0000", 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9680000000000003e-06 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526427+0000", 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2902000000000003e-05 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526480+0000", 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014171499999999999 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526622+0000", 2023-09-23T22:45:54.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012000100000000001 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526742+0000", 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006025875 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.532768+0000", 2023-09-23T22:45:54.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6030000000000004e-06 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.532773+0000", 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22024379699999999 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753017+0000", 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7750000000000003e-05 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753055+0000", 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5847999999999999e-05 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753101+0000", 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226700761 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101967 7.0 7:0d54f750:::benchmark_data_smithi052_277165_object9475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 142.22624761700001, 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226169548, 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101967 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5950000000000001e-06 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527104+0000", 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1766000000000001e-05 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527116+0000", 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7259999999999996e-06 2023-09-23T22:45:54.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527123+0000", 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028034370000000002 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.529926+0000", 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000130782 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530057+0000", 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012912700000000001 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530186+0000", 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002686627 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.532873+0000", 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00060950499999999999 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.533482+0000", 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.219671964 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753154+0000", 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6359000000000006e-05 2023-09-23T22:45:54.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753221+0000", 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6399000000000001e-05 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753267+0000", 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226169548 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.874 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164705 7.0 7:0e09ba28:::benchmark_data_smithi052_277165_object72213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:45:54.874 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:45:54.874 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 33.356948404999997, 2023-09-23T22:45:54.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31253646400000001, 2023-09-23T22:45:54.874 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164705 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396395+0000", 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4019999999999999e-06 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396402+0000", 2023-09-23T22:45:54.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1674999999999999e-05 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396414+0000", 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6180000000000003e-06 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396419+0000", 2023-09-23T22:45:54.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4564e-05 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396494+0000", 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013145799999999999 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396625+0000", 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000161607 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396787+0000", 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013774191999999999 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410561+0000", 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0606999999999998e-05 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410612+0000", 2023-09-23T22:45:54.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29822478200000002 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708836+0000", 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1789999999999998e-05 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708878+0000", 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4976999999999999e-05 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708933+0000", 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31253646400000001 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164707 7.0 7:017e1e7c:::benchmark_data_smithi052_277165_object72215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 33.356912387999998, 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31516750300000002, 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.881 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164707 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396432+0000", 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.994e-06 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396435+0000", 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0390000000000001e-06 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396441+0000", 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.155e-06 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396445+0000", 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052570480000000003 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401702+0000", 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000104561 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401807+0000", 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012558299999999999 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401933+0000", 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0087188879999999993 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410651+0000", 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2320000000000004e-06 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410659+0000", 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30079543800000003 2023-09-23T22:45:54.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711454+0000", 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100597 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711555+0000", 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5464999999999999e-05 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711600+0000", 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31516750300000002 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164711 7.0 7:0f1a7674:::benchmark_data_smithi052_277165_object72219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 33.356895172000002, 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31527841600000001, 2023-09-23T22:45:54.888 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164711 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396449+0000", 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6979999999999998e-06 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:45:54.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396452+0000", 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0100000000000001e-06 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396458+0000", 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3000000000000002e-06 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396461+0000", 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099721069999999992 2023-09-23T22:45:54.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406434+0000", 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012163399999999999 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406555+0000", 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000215744 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406771+0000", 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039129949999999998 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410684+0000", 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8912999999999998e-05 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410723+0000", 2023-09-23T22:45:54.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30092187100000001 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711645+0000", 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9745e-05 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711685+0000", 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3800000000000001e-05 2023-09-23T22:45:54.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711728+0000", 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31527841600000001 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:45:54.895 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:45:54.896 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:54.931 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:55.244 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:55.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:55.319 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:45:55.342 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:45:55.348 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:55.361 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:45:55.372 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:45:55.380 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:45:55.382 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:55.400 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:45:55.420 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:45:55.431 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:55.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:55.438 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:45:55.439 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:45:55.439 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.439 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.439 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.439 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.439 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.440 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.440 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:45:55.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4648/2454 objects degraded (189.405%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:55.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3832445811' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:55.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:55.504 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1708089482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:55.538 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:45:55.566 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:55.616 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:45:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:55.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:45:55.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4648/2454 objects degraded (189.405%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:55.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3832445811' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:55.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:55 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:55.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1708089482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:55.722 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:55.752 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4648/2454 objects degraded (189.405%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:45:55.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3832445811' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:55.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:55.753 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1708089482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:55.754 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:55.758 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:55.888 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:45:56.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:56 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:56.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:56 smithi151 ceph-mon[159210]: failed to encode map e214 with expected crc 2023-09-23T22:45:56.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:56 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:56.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:56 smithi151 ceph-mon[159210]: osdmap e214: 8 total, 7 up, 7 in 2023-09-23T22:45:56.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:56 smithi151 ceph-mon[159210]: pgmap v536: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 25 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 186 objects/s recovering 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[180979]: failed to encode map e214 with expected crc 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[180979]: osdmap e214: 8 total, 7 up, 7 in 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[180979]: pgmap v536: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 25 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 186 objects/s recovering 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:56.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[187743]: failed to encode map e214 with expected crc 2023-09-23T22:45:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[187743]: osdmap e214: 8 total, 7 up, 7 in 2023-09-23T22:45:56.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:56 smithi052 ceph-mon[187743]: pgmap v536: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 25 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 186 objects/s recovering 2023-09-23T22:45:57.211 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:57.212 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:57.576 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T22:45:57.576 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T22:45:57.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:57 smithi151 ceph-mon[159210]: failed to encode map e214 with expected crc 2023-09-23T22:45:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:57 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T22:45:57.818 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:45:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:57 smithi052 ceph-mon[187743]: failed to encode map e214 with expected crc 2023-09-23T22:45:57.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:57 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T22:45:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:57 smithi052 ceph-mon[180979]: failed to encode map e214 with expected crc 2023-09-23T22:45:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:57 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T22:45:58.194 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:58.287 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:58.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:58.310 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:45:58.311 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:45:58.393 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:45:58.422 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:58.472 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:45:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:58.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:45:58.576 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:58.605 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:58.654 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:58.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:58.758 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:58.785 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:58.832 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:58.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:58.936 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:58.964 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:59.012 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:59.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:59.118 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:45:59.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.144 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:59 smithi151 ceph-mon[159210]: pgmap v537: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 25 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 146 objects/s recovering 2023-09-23T22:45:59.144 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:45:59.144 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:45:59.144 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:45:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.146 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[180979]: pgmap v537: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 25 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 146 objects/s recovering 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[187743]: pgmap v537: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 25 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 146 objects/s recovering 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:45:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:45:59.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:45:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:45:59.266 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:45:59.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:59.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:45:59.372 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:59.385 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [0] live_osds: [1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:45:59.385 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:45:59.386 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-09-23T22:45:59.386 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-09-23T22:45:59.386 DEBUG:teuthology.orchestra.run.smithi052:osd.0> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T22:45:59.388 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl start ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:45:59.401 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:59.417 INFO:journalctl@ceph.osd.0.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:45:59.448 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:59.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:59.552 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:59.583 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:59.667 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:59.667 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:59.743 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:45:59 smithi052 systemd[1]: Starting Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:45:59.962 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:59.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:00.066 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:00.104 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:00.132 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:00.162 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:00.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:00.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6277/291 objects degraded (2157.045%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:46:00.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6277/291 objects degraded (2157.045%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:46:00.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:00.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6277/291 objects degraded (2157.045%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:46:00.296 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:00.311 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:46:00.328 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:46:00.342 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:00.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:00.447 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:00.474 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:00.519 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:00.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:00.627 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:00.655 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:00.703 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:00.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:00.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:00.725 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:46:00.809 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:00.842 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:00.993 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:00.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:01.097 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:01.124 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:01.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:01 smithi151 ceph-mon[159210]: pgmap v538: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 26 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 269 objects/s recovering 2023-09-23T22:46:01.275 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:01.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:01.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:01.334 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:01 smithi052 ceph-mon[187743]: pgmap v538: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 26 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 269 objects/s recovering 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:46:01.335 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:01 smithi052 bash[330831]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T22:46:01.336 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:01 smithi052 ceph-mon[180979]: pgmap v538: 99 pgs: 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 26 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 269 objects/s recovering 2023-09-23T22:46:01.378 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:01.405 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:01.449 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:01.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:01.552 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:01.579 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:01.712 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:01.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:01.818 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:01.846 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:01.890 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:01.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:01.994 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:02.022 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:02.027 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:46:02.119 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:02.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:02.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:02.201 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:46:02.201 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:46:02.223 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:02.255 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:02.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:46:03.100 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:03.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:03.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:03.159 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:03.208 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:03.235 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:46:03.246 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:03.247 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:03.343 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:03 smithi052 ceph-mon[187743]: pgmap v539: 99 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 27 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 233 objects/s recovering 2023-09-23T22:46:03.344 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:03 smithi052 ceph-mon[180979]: pgmap v539: 99 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 27 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 233 objects/s recovering 2023-09-23T22:46:03.377 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:03.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:03.464 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:03 smithi151 ceph-mon[159210]: pgmap v539: 99 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 27 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 233 objects/s recovering 2023-09-23T22:46:03.481 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:03.510 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:03.608 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:03.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:03.687 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:46:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:46:03] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.33.4" 2023-09-23T22:46:03.711 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:03.747 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:03.952 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:03.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:04.057 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:04.086 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:04.172 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:04.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:04.219 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_1' removed 2023-09-23T22:46:04.275 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:04.303 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:04.357 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:04.359 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2490893317' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:46:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:04.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:04.461 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:04.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2490893317' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:46:04.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2490893317' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:46:04.531 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:04.650 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:46:04.650 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[187743]: pgmap v540: 99 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 27 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 233 objects/s recovering 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[187743]: failed to encode map e215 with expected crc 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2490893317' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[187743]: osdmap e215: 8 total, 7 up, 7 in 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6277/291 objects degraded (2157.045%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:05.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[180979]: pgmap v540: 99 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 27 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 233 objects/s recovering 2023-09-23T22:46:05.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[180979]: failed to encode map e215 with expected crc 2023-09-23T22:46:05.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2490893317' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:46:05.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[180979]: osdmap e215: 8 total, 7 up, 7 in 2023-09-23T22:46:05.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:05.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6277/291 objects degraded (2157.045%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:05.276 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:46:05.622 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:46:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:05 smithi151 ceph-mon[159210]: pgmap v540: 99 pgs: 1 active+clean+remapped, 1 active+recovering+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+remapped, 27 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 52 active+clean; 5.1 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 6277/291 objects degraded (2157.045%); 2280/291 objects misplaced (783.505%); 0 B/s, 233 objects/s recovering 2023-09-23T22:46:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:05 smithi151 ceph-mon[159210]: failed to encode map e215 with expected crc 2023-09-23T22:46:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2490893317' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:46:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:05 smithi151 ceph-mon[159210]: osdmap e215: 8 total, 7 up, 7 in 2023-09-23T22:46:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6277/291 objects degraded (2157.045%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:05.771 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:46:05.772 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:46:05.772 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T22:46:05.772 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T22:46:06.063 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:06.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:06.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:06.169 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:06.199 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:06.242 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:06.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:06.345 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:06.381 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:06.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:06 smithi151 ceph-mon[159210]: failed to encode map e215 with expected crc 2023-09-23T22:46:06.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:06 smithi052 ceph-mon[187743]: failed to encode map e215 with expected crc 2023-09-23T22:46:06.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:06 smithi052 ceph-mon[180979]: failed to encode map e215 with expected crc 2023-09-23T22:46:06.495 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:46:06.514 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:46:06.533 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:46:06.542 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:06.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:06.554 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:46:06.573 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:46:06.592 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.611 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.612 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.612 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:46:06.647 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:06.676 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:06.719 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:06.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:06.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:06.728 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:46:06.795 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:06 smithi052 ceph-osd[332682]: -- 172.21.15.52:0/777582054 <== mon.0 v2:172.21.15.52:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bcc67844e0 con 0x55bcc6757000 2023-09-23T22:46:06.795 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:06 smithi052 bash[332291]: db01d2d31a60c927d426b4187f1616f5bf6d3b7c5306687c710e5f5a4f0bfe23 2023-09-23T22:46:06.795 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:06 smithi052 systemd[1]: Started Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:46:06.823 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:06.851 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:06.897 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:06.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:07.000 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:07.030 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:07.056 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:46:07.056 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:46:07.057 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:46:07.057 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:46:07.057 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:46:07.078 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:07.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:07.182 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:07.214 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:07 smithi151 ceph-mon[159210]: pgmap v542: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:07.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3772562072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:07.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:07 smithi052 ceph-mon[187743]: pgmap v542: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:07.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3772562072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:07.494 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:07 smithi052 conmon[332594]: 2023-09-23T22:46:07.241+0000 7f2fe09b6640 -1 Falling back to public interface 2023-09-23T22:46:07.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:07 smithi052 ceph-mon[180979]: pgmap v542: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:07.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3772562072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:07.519 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:07.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:07.596 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:46:07.622 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:46:07.623 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:07.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:46:07.650 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:07.693 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:07.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:07.714 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:07.798 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:07.835 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:07.886 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:07.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:07.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:07.994 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:46:08.019 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:08.232 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:46:08.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3579233676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:08.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3579233676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:08.534 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3579233676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:08.534 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:08.820 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:08.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:08.925 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:08.954 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:09.001 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:09.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:09.108 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:09.135 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:09.224 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[187743]: pgmap v543: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:09.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:09.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:46:09.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[187743]: osdmap e216: 8 total, 7 up, 7 in 2023-09-23T22:46:09.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[187743]: failed to encode map e216 with expected crc 2023-09-23T22:46:09.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[187743]: failed to encode map e216 with expected crc 2023-09-23T22:46:09.232 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:09.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:09.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:09.337 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:09.371 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:09.396 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:46:09.396 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1177 6.8 6:1fae9040:::benchmark_data_smithi052_234456_object1176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.08176186700001, 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16501295899999999, 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.397 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1177 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.398 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312081+0000", 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312080+0000", 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6277999999999999e-05 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312137+0000", 2023-09-23T22:46:09.399 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9989999999999997e-06 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312146+0000", 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6670000000000004e-06 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312150+0000", 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8374999999999998e-05 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.400 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312189+0000", 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8557e-05 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312267+0000", 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011574900000000001 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:46:09.401 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312383+0000", 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16315368599999999 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475537+0000", 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013937680000000001 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476930+0000", 2023-09-23T22:46:09.402 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1847000000000006e-05 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476992+0000", 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010188600000000001 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477094+0000", 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16501295899999999 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.403 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1180 6.3 6:c4de4eba:::benchmark_data_smithi052_234456_object1179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.08127993099998, 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.165229389, 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.404 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1180 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.405 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312563+0000", 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312562+0000", 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000111852 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312674+0000", 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3390999999999999e-05 2023-09-23T22:46:09.406 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312687+0000", 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312694+0000", 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9179999999999999e-05 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.407 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312753+0000", 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16226790299999999 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475021+0000", 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018092 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.408 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475202+0000", 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020914169999999999 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477293+0000", 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035115500000000002 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477644+0000", 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2717999999999999e-05 2023-09-23T22:46:09.409 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477697+0000", 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5310999999999996e-05 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477792+0000", 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.165229389 2023-09-23T22:46:09.410 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1182 6.8 6:12f46353:::benchmark_data_smithi052_234456_object1181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.08113209499999, 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16475775100000001, 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.411 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1182 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.412 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312711+0000", 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312710+0000", 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5873e-05 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312746+0000", 2023-09-23T22:46:09.413 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4910000000000005e-06 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312755+0000", 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8200000000000002e-06 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312760+0000", 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9837000000000002e-05 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.414 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312830+0000", 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.162295933 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475126+0000", 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015127000000000001 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:46:09.415 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475277+0000", 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001836475 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477114+0000", 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000174838 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477289+0000", 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9021000000000003e-05 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.416 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477348+0000", 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120896 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.477469+0000", 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16475775100000001 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.417 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:1184 6.5 6:ad9e7e6c:::benchmark_data_smithi052_234456_object1183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.081068523, 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16539384100000001, 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 1184 2023-09-23T22:46:09.418 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.419 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312774+0000", 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6648000000000001e-05 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312861+0000", 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2448e-05 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.420 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312873+0000", 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7630000000000003e-06 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.312880+0000", 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16255894300000001 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.421 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475439+0000", 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129219 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475568+0000", 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000164919 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475733+0000", 2023-09-23T22:46:09.422 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022748180000000001 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478008+0000", 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478015+0000", 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4327999999999997e-05 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.423 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478089+0000", 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2700000000000001e-05 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478132+0000", 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6504999999999997e-05 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.478168+0000", 2023-09-23T22:46:09.424 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16539384100000001 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:1191 6.6 e127/99 6:684d433b:::benchmark_data_smithi052_234456_object1190:head v 127'68, mlcod=127'65)", 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.08065023199998, 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16367246499999999, 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.425 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313193+0000", 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.426 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313192+0000", 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8267000000000001e-05 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.313270+0000", 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16176015999999999 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475030+0000", 2023-09-23T22:46:09.427 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7800000000000006e-06 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475039+0000", 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00056310300000000002 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475602+0000", 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0098e-05 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.428 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.475622+0000", 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012034210000000001 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476826+0000", 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9504e-05 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:51.476865+0000", 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16367246499999999 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.429 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10178 6.3 6:cbb8686c:::benchmark_data_smithi052_234456_object10177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.19422850799998, 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26379366399999998, 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.430 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10178 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.431 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199614+0000", 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8393999999999994e-05 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199702+0000", 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0538e-05 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199713+0000", 2023-09-23T22:46:09.432 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5840000000000002e-06 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199718+0000", 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7531000000000001e-05 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199786+0000", 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012609700000000001 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.433 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199912+0000", 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024177399999999999 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200154+0000", 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029848309999999999 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203139+0000", 2023-09-23T22:46:09.434 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.260079748 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463218+0000", 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3586999999999999e-05 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463292+0000", 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116289 2023-09-23T22:46:09.435 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463408+0000", 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26379366399999998 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10181 6.3 6:c0620c37:::benchmark_data_smithi052_234456_object10180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:46:09.436 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.19402566999997, 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.265707154, 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10181 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.437 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.438 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199817+0000", 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8221e-05 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199845+0000", 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3889999999999999e-06 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.439 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199852+0000", 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9130000000000002e-06 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.199857+0000", 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085247700000000003 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200710+0000", 2023-09-23T22:46:09.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001666491 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202376+0000", 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124387 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202501+0000", 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00086751799999999996 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203368+0000", 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26193364000000002 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465302+0000", 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106265 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465408+0000", 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116496 2023-09-23T22:46:09.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465524+0000", 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.265707154 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10191 6.5 6:a7cca2de:::benchmark_data_smithi052_234456_object10190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:46:09.443 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.18996894000003, 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26015834900000001, 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10191 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.444 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203874+0000", 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203873+0000", 2023-09-23T22:46:09.445 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010616300000000001 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203979+0000", 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2459999999999992e-06 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203989+0000", 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129999999999996e-06 2023-09-23T22:46:09.446 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.203993+0000", 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5849000000000002e-05 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204029+0000", 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105581 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.447 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204135+0000", 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000160994 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204296+0000", 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00127407 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.448 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205570+0000", 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052499199999999995 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206095+0000", 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25784317800000001 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.449 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463938+0000", 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5868000000000001e-05 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463994+0000", 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8639000000000001e-05 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464032+0000", 2023-09-23T22:46:09.450 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26015834900000001 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10193 6.5 6:a177b7dd:::benchmark_data_smithi052_234456_object10192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.18960368699999, 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26049174200000003, 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.451 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10193 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.452 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204239+0000", 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6399999999999997e-05 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.453 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204275+0000", 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1413000000000001e-05 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204286+0000", 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9870000000000002e-06 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204293+0000", 2023-09-23T22:46:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00054160700000000005 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204835+0000", 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013009600000000001 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.204965+0000", 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016865000000000001 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.455 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205134+0000", 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00065124099999999999 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205785+0000", 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037879500000000002 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.206164+0000", 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25846449399999999 2023-09-23T22:46:09.456 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464628+0000", 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9333000000000003e-05 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464688+0000", 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3455999999999999e-05 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.457 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464731+0000", 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26049174200000003 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10206 6.3 6:cca2051f:::benchmark_data_smithi052_234456_object10205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.18476932800002, 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25668903900000001, 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.458 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10206 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.459 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209074+0000", 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209072+0000", 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1259000000000002e-05 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.460 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209154+0000", 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3302000000000001e-05 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209167+0000", 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3980000000000002e-06 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209174+0000", 2023-09-23T22:46:09.461 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4868000000000003e-05 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209239+0000", 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018571700000000001 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209425+0000", 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018386100000000001 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.209609+0000", 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00066804500000000003 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.210277+0000", 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.255318247 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465595+0000", 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6978999999999998e-05 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465652+0000", 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110548 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465763+0000", 2023-09-23T22:46:09.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25668903900000001 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10215 6.3 6:c04bb586:::benchmark_data_smithi052_234456_object10214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.18053535899998, 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25266233500000002, 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.465 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10215 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213308+0000", 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213306+0000", 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3293000000000001e-05 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213340+0000", 2023-09-23T22:46:09.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0866000000000001e-05 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213350+0000", 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2249999999999997e-06 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213357+0000", 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1418999999999997e-05 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213408+0000", 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6898999999999995e-05 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213505+0000", 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132287 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.213637+0000", 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00064820999999999998 2023-09-23T22:46:09.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214285+0000", 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25152426300000003 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465810+0000", 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4280999999999997e-05 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465864+0000", 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105955 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.465970+0000", 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25266233500000002 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10217 6.3 6:c781daf6:::benchmark_data_smithi052_234456_object10216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.471 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.17898664099999, 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25134704800000002, 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10217 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214856+0000", 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214855+0000", 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8257999999999994e-05 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214944+0000", 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1997999999999999e-05 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214956+0000", 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1810000000000004e-06 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.214963+0000", 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9869e-05 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215013+0000", 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015027000000000001 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215163+0000", 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021892400000000001 2023-09-23T22:46:09.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.215382+0000", 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073708599999999997 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.216119+0000", 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.249917901 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466037+0000", 2023-09-23T22:46:09.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8545999999999998e-05 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466095+0000", 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010809199999999999 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466203+0000", 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25134704800000002 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.477 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:10224 6.3 6:c3676ada:::benchmark_data_smithi052_234456_object10223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 315.17343913500002, 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.246054935, 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.478 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 10224 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220404+0000", 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220402+0000", 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7369e-05 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220449+0000", 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6140000000000001e-05 2023-09-23T22:46:09.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220466+0000", 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8979999999999996e-06 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220473+0000", 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2861000000000002e-05 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220516+0000", 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011724500000000001 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220634+0000", 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013401900000000001 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 6", 2023-09-23T22:46:09.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.220768+0000", 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00059656599999999998 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.221364+0000", 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24489038599999999 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466255+0000", 2023-09-23T22:46:09.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6962e-05 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466321+0000", 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013730400000000001 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.466459+0000", 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.246054935 2023-09-23T22:46:09.484 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40799 6.8 6:14756be4:::benchmark_data_smithi052_234456_object18655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 299.53227353800003, 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2385245110000001, 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.485 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40799 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861569+0000", 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861568+0000", 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9182999999999999e-05 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861587+0000", 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3654e-05 2023-09-23T22:46:09.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861600+0000", 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.588e-06 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861608+0000", 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114259 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861722+0000", 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.251e-06 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861731+0000", 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028264639999999999 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864558+0000", 2023-09-23T22:46:09.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4350000000000003e-06 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864567+0000", 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99719855899999998 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861766+0000", 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9449999999999998e-05 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861785+0000", 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2282697419999999 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090055+0000", 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000194957 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090250+0000", 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000107145 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090357+0000", 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030987689999999999 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.093456+0000", 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023293989999999998 2023-09-23T22:46:09.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.095785+0000", 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038349410000000001 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.099620+0000", 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1438000000000004e-05 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.099682+0000", 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00041221600000000003 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.100094+0000", 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2385245110000001 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.494 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40800 6.3 6:c0d13c02:::benchmark_data_smithi052_234456_object18656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 299.53220951200001, 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.236326756, 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:46:09.495 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40800 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:46:09.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861633+0000", 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.693e-06 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861637+0000", 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6470000000000001e-06 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861643+0000", 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.408e-06 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861648+0000", 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4624999999999998e-05 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861722+0000", 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8899999999999996e-06 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op must wait for map", 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.861731+0000", 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003144053 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864875+0000", 2023-09-23T22:46:09.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6930000000000002e-06 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.864885+0000", 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99906347699999998 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.863948+0000", 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2204999999999999e-05 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.863961+0000", 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2265645000000001 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090525+0000", 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016863800000000001 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090694+0000", 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018472599999999999 2023-09-23T22:46:09.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.090878+0000", 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018197459999999999 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.092698+0000", 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.679e-06 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.092708+0000", 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048923589999999998 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097600+0000", 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7243000000000007e-05 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097667+0000", 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00029281000000000001 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.097960+0000", 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.236326756 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41338 7.5 7:a210bb19:::benchmark_data_smithi052_277165_object41337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:46:09.504 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.665781+0000", 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 234.72806154400001, 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21540489700000001, 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41338 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.505 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665781+0000", 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665781+0000", 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665780+0000", 2023-09-23T22:46:09.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012445499999999999 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665905+0000", 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3737000000000001e-05 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665918+0000", 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5450000000000003e-06 2023-09-23T22:46:09.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665924+0000", 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8445e-05 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.665982+0000", 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016478 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666147+0000", 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000182738 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.666330+0000", 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002808867 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.669139+0000", 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005151704 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.674291+0000", 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20678310799999999 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881074+0000", 2023-09-23T22:46:09.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6613999999999997e-05 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881130+0000", 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6079000000000003e-05 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.881186+0000", 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21540489700000001 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:41350 7.5 7:ab47f499:::benchmark_data_smithi052_277165_object41349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e161)", 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:14.672107+0000", 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 234.72173558700001, 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21391692500000001, 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 41350 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.512 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672107+0000", 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672107+0000", 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672106+0000", 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7646999999999995e-05 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672203+0000", 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2326e-05 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672216+0000", 2023-09-23T22:46:09.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5760000000000002e-06 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672222+0000", 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0977999999999999e-05 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672273+0000", 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016777 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672441+0000", 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000201454 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.672642+0000", 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002397255 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.675040+0000", 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014347139999999999 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.676474+0000", 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20943123999999999 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.885906+0000", 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7896000000000002e-05 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.885973+0000", 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0927000000000002e-05 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:14.886024+0000", 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21391692500000001 2023-09-23T22:46:09.518 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61224 7.5 7:ad9959c8:::benchmark_data_smithi052_277165_object61223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.072846+0000", 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 216.320996979, 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22912544700000001, 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.519 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61224 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072846+0000", 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072846+0000", 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072844+0000", 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4922000000000001e-05 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072889+0000", 2023-09-23T22:46:09.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5446000000000001e-05 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072904+0000", 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.409e-06 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072911+0000", 2023-09-23T22:46:09.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6582000000000002e-05 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.072957+0000", 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000147684 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073105+0000", 2023-09-23T22:46:09.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000195095 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073300+0000", 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020691300000000002 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075369+0000", 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00065103100000000005 2023-09-23T22:46:09.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076020+0000", 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22586408299999999 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301884+0000", 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9163999999999999e-05 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301934+0000", 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7830999999999997e-05 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301971+0000", 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22912544700000001 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61242 7.5 7:a582954c:::benchmark_data_smithi052_277165_object61241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.085367+0000", 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 216.30847572600001, 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.219389263, 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61242 2023-09-23T22:46:09.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085367+0000", 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085367+0000", 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085366+0000", 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3773999999999997e-05 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085420+0000", 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3003000000000001e-05 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085433+0000", 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8730000000000004e-06 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085439+0000", 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3528000000000001e-05 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085482+0000", 2023-09-23T22:46:09.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012782900000000001 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085610+0000", 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000135647 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.085746+0000", 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001705193 2023-09-23T22:46:09.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.087451+0000", 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00082960900000000003 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.088280+0000", 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.216402284 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304683+0000", 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6258000000000003e-05 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304729+0000", 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7606e-05 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304757+0000", 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.219389263 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:146618 7.5 7:a4c505da:::benchmark_data_smithi052_277165_object54126:head [delete] snapc 0=[] ondisk+write+known_if_redirected e194)", 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:44:46.761698+0000", 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 82.632144839999995, 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.170848156, 2023-09-23T22:46:09.534 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 146618 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761698+0000", 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:46:09.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761698+0000", 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761696+0000", 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5080000000000005e-06 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761705+0000", 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4559000000000001e-05 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761720+0000", 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6919999999999999e-06 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761725+0000", 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1068e-05 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761776+0000", 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013838000000000001 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.761915+0000", 2023-09-23T22:46:09.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016238799999999999 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.762077+0000", 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033071630000000001 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.765384+0000", 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00048579100000000001 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.765870+0000", 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16659237499999999 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.932462+0000", 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0562999999999998e-05 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.932503+0000", 2023-09-23T22:46:09.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3334999999999998e-05 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:44:46.932546+0000", 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.170848156 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:46:09.541 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:46:09.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[180979]: pgmap v543: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:09.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:09.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:46:09.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[180979]: osdmap e216: 8 total, 7 up, 7 in 2023-09-23T22:46:09.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[180979]: failed to encode map e216 with expected crc 2023-09-23T22:46:09.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:09 smithi052 ceph-mon[180979]: failed to encode map e216 with expected crc 2023-09-23T22:46:09.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:09 smithi151 ceph-mon[159210]: pgmap v543: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:09.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:09 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:09.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:46:09.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:09 smithi151 ceph-mon[159210]: osdmap e216: 8 total, 7 up, 7 in 2023-09-23T22:46:09.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:09 smithi151 ceph-mon[159210]: failed to encode map e216 with expected crc 2023-09-23T22:46:09.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:09 smithi151 ceph-mon[159210]: failed to encode map e216 with expected crc 2023-09-23T22:46:09.624 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:46:09.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:09.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:09.875 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:09.915 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:09.970 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:09.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:09.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:10.000 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:46:10.074 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:10.102 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:10.145 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:10.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:10.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:10.226 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:10 smithi052 ceph-mon[187743]: failed to encode map e216 with expected crc 2023-09-23T22:46:10.227 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:10 smithi052 ceph-mon[187743]: failed to encode map e216 with expected crc 2023-09-23T22:46:10.227 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:10.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:10 smithi052 ceph-mon[180979]: failed to encode map e216 with expected crc 2023-09-23T22:46:10.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:10 smithi052 ceph-mon[180979]: failed to encode map e216 with expected crc 2023-09-23T22:46:10.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:10.237 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:10.250 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:10.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:10 smithi151 ceph-mon[159210]: failed to encode map e216 with expected crc 2023-09-23T22:46:10.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:10 smithi151 ceph-mon[159210]: failed to encode map e216 with expected crc 2023-09-23T22:46:10.275 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:10.277 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:10.319 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:10.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:10.427 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:10.453 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:10.494 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:10.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:10.589 INFO:teuthology.orchestra.run.smithi052.stderr:no valid command found; 10 closest matches: 2023-09-23T22:46:10.589 INFO:teuthology.orchestra.run.smithi052.stderr:0 2023-09-23T22:46:10.589 INFO:teuthology.orchestra.run.smithi052.stderr:1 2023-09-23T22:46:10.589 INFO:teuthology.orchestra.run.smithi052.stderr:2 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:abort 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:assert 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs debug_inject_read_zeros 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs files list 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs stats 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:bluestore allocator dump block 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:bluestore allocator fragmentation block 2023-09-23T22:46:10.590 INFO:teuthology.orchestra.run.smithi052.stderr:admin_socket: invalid command 2023-09-23T22:46:10.601 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:10.627 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:10.668 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:10.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:10.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:10.775 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:10.801 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:10.843 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:10.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:10.948 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:10.982 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:11.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:11 smithi052 ceph-mon[180979]: pgmap v545: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:11.238 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3458016585' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:46:11.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:11 smithi052 ceph-mon[187743]: pgmap v545: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:11.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3458016585' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:46:11.241 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_2' created 2023-09-23T22:46:11.385 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:11.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:11.489 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:11.515 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:11 smithi151 ceph-mon[159210]: pgmap v545: 73 pgs: 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:11.515 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3458016585' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:46:11.518 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:11.562 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:11.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:11.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:11.587 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:46:11.671 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:11.697 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:11.740 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:11.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:11.847 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:11.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:11.914 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:11.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:12.020 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:12.050 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:12.425 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:12 smithi052 conmon[332594]: 2023-09-23T22:46:12.134+0000 7f2fe09b6640 -1 osd.0 201 log_to_monitors true 2023-09-23T22:46:12.425 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:46:12 smithi052 conmon[332594]: 2023-09-23T22:46:12.258+0000 7f2fcde1b700 -1 osd.0 201 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3458016585' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: osdmap e217: 8 total, 7 up, 7 in 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.426 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[180979]: osdmap e218: 8 total, 7 up, 7 in 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3458016585' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: osdmap e217: 8 total, 7 up, 7 in 2023-09-23T22:46:12.427 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:46:12.428 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:12 smithi052 ceph-mon[187743]: osdmap e218: 8 total, 7 up, 7 in 2023-09-23T22:46:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:46:12.442 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2023-09-23T22:46:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3458016585' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T22:46:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: osdmap e217: 8 total, 7 up, 7 in 2023-09-23T22:46:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: failed to encode map e217 with expected crc 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:46:12.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:12 smithi151 ceph-mon[159210]: osdmap e218: 8 total, 7 up, 7 in 2023-09-23T22:46:12.799 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T22:46:12.803 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:12.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:12.821 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:12.909 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:12.941 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:12.989 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:12.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:13.094 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:13.123 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:13.264 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: pgmap v547: 89 pgs: 4 creating+peering, 12 unknown, 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:13.264 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.264 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760] boot 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T22:46:13.265 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:46:13.325 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:13.325 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:46:13.329 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:13.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:13.382 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:46:13.408 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:46:13.433 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:46:13.443 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:13.452 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:46:13.470 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:46:13.474 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:13.488 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:46:13.508 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:46:13.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.510 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.510 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.510 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.510 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:46:13.523 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: pgmap v547: 89 pgs: 4 creating+peering, 12 unknown, 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:46:13.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:46:13.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:46:13.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760] boot 2023-09-23T22:46:13.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T22:46:13.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:46:13.531 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:46:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:46:13] "GET /metrics HTTP/1.1" 200 34191 "" "Prometheus/2.33.4" 2023-09-23T22:46:13.531 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: pgmap v547: 89 pgs: 4 creating+peering, 12 unknown, 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:13.531 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.531 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: failed to encode map e218 with expected crc 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: osd.0 [v2:172.21.15.52:6802/4058083760,v1:172.21.15.52:6803/4058083760] boot 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T22:46:13.532 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:46:13.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:13.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:13.634 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:13.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:13.704 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:13.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:13.809 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:13.839 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:14.291 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:14.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:14.394 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:14.423 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:14.468 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:14.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:14.575 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:14.580 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[180979]: failed to encode map e219 with expected crc 2023-09-23T22:46:14.580 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[180979]: failed to encode map e219 with expected crc 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3449629174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[180979]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[180979]: failed to encode map e220 with expected crc 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[187743]: failed to encode map e219 with expected crc 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[187743]: failed to encode map e219 with expected crc 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3449629174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:14.581 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[187743]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T22:46:14.582 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:14 smithi052 ceph-mon[187743]: failed to encode map e220 with expected crc 2023-09-23T22:46:14.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:14 smithi151 ceph-mon[159210]: failed to encode map e219 with expected crc 2023-09-23T22:46:14.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:14 smithi151 ceph-mon[159210]: failed to encode map e219 with expected crc 2023-09-23T22:46:14.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3449629174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:14.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:14 smithi151 ceph-mon[159210]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T22:46:14.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:14 smithi151 ceph-mon[159210]: failed to encode map e220 with expected crc 2023-09-23T22:46:14.602 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:14.643 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:14.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:14.749 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:14.778 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:15.018 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:15.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: pgmap v550: 89 pgs: 4 creating+peering, 12 unknown, 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:15.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: failed to encode map e221 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[180979]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: pgmap v550: 89 pgs: 4 creating+peering, 12 unknown, 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: failed to encode map e221 with expected crc 2023-09-23T22:46:15.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:15 smithi052 ceph-mon[187743]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T22:46:15.632 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:15.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: pgmap v550: 89 pgs: 4 creating+peering, 12 unknown, 1 active+clean+remapped, 22 active+undersized, 8 active+undersized+degraded, 42 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 13/72 objects degraded (18.056%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: failed to encode map e220 with expected crc 2023-09-23T22:46:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: failed to encode map e221 with expected crc 2023-09-23T22:46:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:15 smithi151 ceph-mon[159210]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T22:46:15.735 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:15.763 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:15.809 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:15.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:15.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:15.914 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:15.941 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:15.984 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:15.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:16.038 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:46:16.038 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:46:16.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:16.117 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:16.159 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:16.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:16.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:16.292 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:16.334 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:16.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:16.440 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:16 smithi151 ceph-mon[159210]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:16 smithi151 ceph-mon[159210]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:16 smithi151 ceph-mon[159210]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.468 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:16 smithi052 ceph-mon[187743]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.468 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:16 smithi052 ceph-mon[187743]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.468 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:16 smithi052 ceph-mon[187743]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:16 smithi052 ceph-mon[180979]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:16 smithi052 ceph-mon[180979]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:16 smithi052 ceph-mon[180979]: failed to encode map e221 with expected crc 2023-09-23T22:46:16.471 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:17.042 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:17.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:17.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:17.106 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:46:17.147 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:17.180 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:17.283 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:46:17.388 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:17.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:17.443 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:46:17.491 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:17.520 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:17 smithi151 ceph-mon[159210]: pgmap v553: 89 pgs: 4 creating+peering, 3 unknown, 1 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 65 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 3/72 objects degraded (4.167%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:17.521 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:17 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:17.521 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3076342762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:17.521 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:17.523 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:17.572 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:17.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:17.598 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[187743]: pgmap v553: 89 pgs: 4 creating+peering, 3 unknown, 1 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 65 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 3/72 objects degraded (4.167%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3076342762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[180979]: pgmap v553: 89 pgs: 4 creating+peering, 3 unknown, 1 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 65 active+clean; 582 KiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 3/72 objects degraded (4.167%); 2/72 objects misplaced (2.778%) 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3076342762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:17.623 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:17.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:17.706 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:17.750 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:17.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:17.857 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:17.884 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:17.926 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:17.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:18.031 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:18.059 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:18.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:46:18.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[187743]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:46:18.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[187743]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T22:46:18.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[187743]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.324 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[187743]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:46:18.327 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[180979]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:46:18.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[180979]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T22:46:18.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[180979]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.328 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:18 smithi052 ceph-mon[180979]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.541 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:18.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:18.645 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:18.677 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:18.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:18 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:46:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:18 smithi151 ceph-mon[159210]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:18 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:46:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:18 smithi151 ceph-mon[159210]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T22:46:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:18 smithi151 ceph-mon[159210]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:18 smithi151 ceph-mon[159210]: failed to encode map e222 with expected crc 2023-09-23T22:46:18.705 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:18.727 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:18.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:18.757 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:18.831 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:18.865 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:18.912 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:18.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:19.020 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:19.050 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:19.133 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:19.284 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[187743]: failed to encode map e222 with expected crc 2023-09-23T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[187743]: pgmap v555: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded, 13 pgs undersized) 2023-09-23T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2393471371' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:46:19.490 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[180979]: failed to encode map e222 with expected crc 2023-09-23T22:46:19.491 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[180979]: pgmap v555: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:46:19.491 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded, 13 pgs undersized) 2023-09-23T22:46:19.491 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:46:19.491 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2393471371' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:46:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:19 smithi151 ceph-mon[159210]: failed to encode map e222 with expected crc 2023-09-23T22:46:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:19 smithi151 ceph-mon[159210]: pgmap v555: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:46:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:19 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded, 13 pgs undersized) 2023-09-23T22:46:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:19 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:46:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2393471371' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:46:20.350 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:46:20.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2393471371' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:46:20.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[180979]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T22:46:20.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:20.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:20.538 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2393471371' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:46:20.538 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[187743]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T22:46:20.539 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:20.539 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:20.655 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:20.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:20.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2393471371' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:46:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:20 smithi151 ceph-mon[159210]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T22:46:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:20.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:20.792 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:20.839 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:20.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:20.945 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:20.974 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:21.019 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:21.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:21.127 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:21.154 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:21.196 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:21.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:21.301 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:21.327 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:21.368 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:21.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:21 smithi151 ceph-mon[159210]: pgmap v557: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:46:21.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:21 smithi151 ceph-mon[159210]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:21 smithi151 ceph-mon[159210]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:21 smithi151 ceph-mon[159210]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:21.511 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[187743]: pgmap v557: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[187743]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[187743]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[187743]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[180979]: pgmap v557: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[180979]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[180979]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.573 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:21 smithi052 ceph-mon[180979]: failed to encode map e223 with expected crc 2023-09-23T22:46:21.604 DEBUG:teuthology.orchestra.run.smithi052:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T22:46:21.606 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:46:21.606 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:46:21.652 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:hints = 1 2023-09-23T22:46:21.652 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:46:21.653 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object prefix: benchmark_data_smithi052_337357 2023-09-23T22:46:21.653 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:46:21.653 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:46:22.090 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:22.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:22.199 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:22.279 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:22.304 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:46:22.305 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:46:22.645 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:22.652 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:22 smithi052 ceph-mon[187743]: failed to encode map e223 with expected crc 2023-09-23T22:46:22.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:22 smithi052 ceph-mon[180979]: failed to encode map e223 with expected crc 2023-09-23T22:46:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:22.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:22.654 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 1 16 2964 2948 184.231 184.25 0.0110368 0.00541127 2023-09-23T22:46:22.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:22 smithi151 ceph-mon[159210]: failed to encode map e223 with expected crc 2023-09-23T22:46:22.711 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:46:22.754 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:22.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:46:22.790 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:22.795 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:46:22.839 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:46:22.852 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:22.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:22.878 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:46:22.919 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:46:22.958 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:46:22.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:22.998 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:23.001 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:46:23.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.026 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.026 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.026 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.026 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:46:23.063 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:23.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:23.169 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:23.208 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:23.279 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:23.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:23.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:23.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:23 smithi052 ceph-mon[180979]: pgmap v558: 89 pgs: 89 active+clean; 21 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-09-23T22:46:23.294 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2945456684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:23.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:23.415 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:23 smithi151 ceph-mon[159210]: pgmap v558: 89 pgs: 89 active+clean; 21 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-09-23T22:46:23.415 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2945456684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:23.419 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:23.481 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:23.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:23.588 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:23.629 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:23.654 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 2 16 6115 6099 190.516 196.938 0.00496241 0.00523792 2023-09-23T22:46:23.693 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:23.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:23.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:46:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:46:23] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-09-23T22:46:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:23 smithi052 ceph-mon[187743]: pgmap v558: 89 pgs: 89 active+clean; 21 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-09-23T22:46:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2945456684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:23.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:23.841 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:23.905 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:23.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:24.012 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:24.042 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:24.515 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:24.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:24.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:24.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:24.658 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 3 16 8809 8793 182.906 168.375 0.00549074 0.00545473 2023-09-23T22:46:24.687 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:25.658 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 4 15 11424 11409 178.056 163.5 0.00823699 0.00560953 2023-09-23T22:46:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:25 smithi151 ceph-mon[159210]: pgmap v559: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 5.3 MiB/s wr, 84 op/s 2023-09-23T22:46:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:25 smithi052 ceph-mon[180979]: pgmap v559: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 5.3 MiB/s wr, 84 op/s 2023-09-23T22:46:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:25 smithi052 ceph-mon[187743]: pgmap v559: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 5.3 MiB/s wr, 84 op/s 2023-09-23T22:46:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:26.084 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:26.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:26.131 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:46:26.189 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:26.220 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:26.282 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:26.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:26.388 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:26.402 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:26.499 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:26.549 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:46:26.549 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:46:26.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.549 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:13447 8.9 e223/219 8:9454d2de:::benchmark_data_smithi052_337357_object13446:head v 223'881, mlcod=223'879)", 2023-09-23T22:46:26.549 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:26.546174+0000", 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.001240054, 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001272144, 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546174+0000", 2023-09-23T22:46:26.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546174+0000", 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546174+0000", 2023-09-23T22:46:26.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546208+0000", 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4801000000000001e-05 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546231+0000", 2023-09-23T22:46:26.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2184e-05 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546239+0000", 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4630000000000001e-06 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546304+0000", 2023-09-23T22:46:26.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4869999999999994e-05 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.546332+0000", 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7813999999999998e-05 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:13444 8.6 8.c4f4536 (undecoded) ondisk+write+known_if_redirected e223)", 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:26.547423+0000", 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4294967295.9999914, 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012694599999999999, 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:46:26.555 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13444 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547423+0000", 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547423+0000", 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547421+0000", 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547451+0000", 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9413000000000001e-05 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547469+0000", 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8459000000000001e-05 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547478+0000", 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6079999999999996e-06 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:13445 8.6 8.473414b6 (undecoded) ondisk+write+known_if_redirected e223)", 2023-09-23T22:46:26.559 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:26.547496+0000", 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4294967295.9999185, 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012437200000000001, 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13445 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.560 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547496+0000", 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547496+0000", 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:26.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547495+0000", 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547519+0000", 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3569000000000001e-05 2023-09-23T22:46:26.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547531+0000", 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2169e-05 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:26.547537+0000", 2023-09-23T22:46:26.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.297e-06 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 3 2023-09-23T22:46:26.564 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:46:26.658 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 5 16 13662 13646 170.41 139.812 0.00798377 0.00585898 2023-09-23T22:46:26.903 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:26.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:27.007 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:27.050 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:27.119 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:27.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:27.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:27.227 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:27.264 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:27.306 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:27 smithi151 ceph-mon[159210]: pgmap v560: 89 pgs: 89 active+clean; 407 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T22:46:27.331 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:27.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:27.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:27.470 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:27.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:27 smithi052 ceph-mon[180979]: pgmap v560: 89 pgs: 89 active+clean; 407 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T22:46:27.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:27 smithi052 ceph-mon[187743]: pgmap v560: 89 pgs: 89 active+clean; 407 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T22:46:27.531 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:27.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:27.636 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:27.658 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 6 16 16167 16151 168.101 156.562 0.00323537 0.00594095 2023-09-23T22:46:27.673 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:27.743 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:27.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:27.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:27.849 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:27.887 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:27.940 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:27.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:27.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:28.046 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:28.085 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:28.151 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:28.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:28.160 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:46:28.257 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:28.297 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:28.357 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:28.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:28.463 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:28.529 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:28.658 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 7 16 18335 18319 163.445 135.5 0.00236551 0.00609061 2023-09-23T22:46:29.660 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 8 16 19659 19643 153.337 82.75 0.008391 0.00651317 2023-09-23T22:46:29.693 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:29 smithi052 ceph-mon[187743]: pgmap v561: 89 pgs: 89 active+clean; 766 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:46:29.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:29 smithi052 ceph-mon[180979]: pgmap v561: 89 pgs: 89 active+clean; 766 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:46:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:29 smithi151 ceph-mon[159210]: pgmap v561: 89 pgs: 89 active+clean; 766 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:46:29.796 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:29.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:29.855 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:29.886 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:29.910 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:29.944 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:30.001 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:30.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:30.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:30.107 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:30.171 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:30.329 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:30.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:30.441 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:30.507 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:30.627 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:30.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:30.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:30.660 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 9 16 21461 21445 148.816 112.625 0.00601905 0.00671297 2023-09-23T22:46:30.735 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:30.804 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:30.976 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:30.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:31.083 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:31.100 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:30 smithi052 ceph-mon[187743]: pgmap v562: 89 pgs: 89 active+clean; 970 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:46:31.100 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:30 smithi052 ceph-mon[180979]: pgmap v562: 89 pgs: 89 active+clean; 970 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:46:31.120 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:31.184 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:31.189 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:30 smithi151 ceph-mon[159210]: pgmap v562: 89 pgs: 89 active+clean; 970 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T22:46:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:31.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:31.291 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:31.335 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:31.396 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:31.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:31.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:31.501 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:31.526 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:31.593 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:31.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:31.660 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 10 16 23190 23174 144.729 108.062 0.00551666 0.00690358 2023-09-23T22:46:31.700 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:31.734 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:31.789 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:31.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:31.895 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:31.932 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:31.948 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:46:32.003 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:32.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:32.110 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:32.167 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2069939019' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:46:32.168 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2069939019' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:46:32.174 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:32.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2069939019' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:46:32.662 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 11 16 24964 24948 141.64 110.875 0.00528022 0.00705451 2023-09-23T22:46:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:32 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:32 smithi151 ceph-mon[159210]: failed to encode map e224 with expected crc 2023-09-23T22:46:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2069939019' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:46:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:32 smithi151 ceph-mon[159210]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T22:46:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:32 smithi151 ceph-mon[159210]: pgmap v564: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 125 MiB/s wr, 2k op/s 2023-09-23T22:46:33.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:33.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[187743]: failed to encode map e224 with expected crc 2023-09-23T22:46:33.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2069939019' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:46:33.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[187743]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T22:46:33.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[187743]: pgmap v564: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 125 MiB/s wr, 2k op/s 2023-09-23T22:46:33.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:33.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[180979]: failed to encode map e224 with expected crc 2023-09-23T22:46:33.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2069939019' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:46:33.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[180979]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T22:46:33.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:32 smithi052 ceph-mon[180979]: pgmap v564: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 125 MiB/s wr, 2k op/s 2023-09-23T22:46:33.647 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:46:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:46:33] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-09-23T22:46:33.662 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 12 16 26509 26493 137.885 96.5625 0.00571433 0.0072443 2023-09-23T22:46:33.672 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:33.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:33.778 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:33.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:34.322 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:34.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:34.428 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:34.465 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:34.518 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:34.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:34.626 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:34.661 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:34.662 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 13 16 28036 28020 134.619 95.4375 0.0060589 0.00742203 2023-09-23T22:46:34.697 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:46:34.697 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:46:34.697 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:46:34.697 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-09-23T22:46:34.697 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-09-23T22:46:34.724 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:34.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:34.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:34.974 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40792 6.f 6.8b4b57ef (undecoded) ondisk+retry+write+known_if_redirected e131)", 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:46:35.025 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 325.159403951, 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602794099999999, 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40792 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.026 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:46:35.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861820+0000", 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861828+0000", 2023-09-23T22:46:35.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861841+0000", 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2709e-05 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861851+0000", 2023-09-23T22:46:35.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861957+0000", 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106518 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:46:35.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861972+0000", 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5226e-05 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875022+0000", 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013050091999999999 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875034+0000", 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1232e-05 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922087+0000", 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047052836 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922101+0000", 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4476e-05 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40806 6.f 6.e80b2dbf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 325.159344847, 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602518750000001, 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:46:35.034 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40806 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.035 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861880+0000", 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.036 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861885+0000", 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.561e-06 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862005+0000", 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120701 2023-09-23T22:46:35.037 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862017+0000", 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862139+0000", 2023-09-23T22:46:35.038 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012248000000000001 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862145+0000", 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.039 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881337+0000", 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019192607 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881351+0000", 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3879999999999999e-05 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.040 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922124+0000", 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0407723719999999 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922133+0000", 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.041 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40815 6.f 6.8b4fc53f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 325.13696370700001, 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037899828, 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.042 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40815 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.043 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884260+0000", 2023-09-23T22:46:35.044 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884266+0000", 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6060000000000002e-06 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884276+0000", 2023-09-23T22:46:35.045 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7180000000000008e-06 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884284+0000", 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6170000000000007e-06 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884403+0000", 2023-09-23T22:46:35.046 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011853099999999999 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884415+0000", 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1683e-05 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922153+0000", 2023-09-23T22:46:35.047 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037738136 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922162+0000", 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9279999999999999e-06 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40816 6.f 6.4f46a35f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 325.136922841, 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378988650000001, 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:35.049 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40816 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884302+0000", 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884309+0000", 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884317+0000", 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5660000000000007e-06 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.052 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884324+0000", 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0859999999999996e-06 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884441+0000", 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011671 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:46:35.053 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884449+0000", 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.571e-06 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922191+0000", 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037741888 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.054 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922202+0000", 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.092e-05 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40818 6.f 6.431951cf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:46:35.055 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 325.13100551600002, 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.032011485, 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40818 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.056 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.057 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890219+0000", 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890224+0000", 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7840000000000003e-06 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.058 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890235+0000", 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0899e-05 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890243+0000", 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.225e-06 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.059 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890301+0000", 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890312+0000", 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1235e-05 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.060 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922221+0000", 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.031909277 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922232+0000", 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0465000000000001e-05 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.061 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40826 6.f 6.3e4c7bff (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 325.11882003699998, 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.019876078, 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:46:35.062 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40826 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902404+0000", 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902413+0000", 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5449999999999992e-06 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902432+0000", 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8488999999999999e-05 2023-09-23T22:46:35.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902441+0000", 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902502+0000", 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1064999999999999e-05 2023-09-23T22:46:35.066 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902511+0000", 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7350000000000002e-06 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922270+0000", 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0197598290000001 2023-09-23T22:46:35.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922282+0000", 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.068 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161600 7.8 7:101b9260:::benchmark_data_smithi052_277165_object69108:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 78.774791277999995, 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28372276299999999, 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161600 2023-09-23T22:46:35.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246437+0000", 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.666e-06 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246443+0000", 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2940000000000002e-06 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246448+0000", 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5199999999999997e-06 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251386+0000", 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00493751 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251523+0000", 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013705799999999999 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251637+0000", 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000114521 2023-09-23T22:46:35.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263851+0000", 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012213869 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263860+0000", 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.529403+0000", 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26554292099999999 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.529465+0000", 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1758000000000001e-05 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530157+0000", 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069202700000000001 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161606 7.8 7:189adb79:::benchmark_data_smithi052_277165_object69114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.077 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 78.774747503, 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28428864599999998, 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161606 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.078 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246480+0000", 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6749999999999998e-06 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246488+0000", 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0269999999999995e-06 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246494+0000", 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6579999999999999e-06 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255454+0000", 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089600449999999998 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255609+0000", 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015489900000000001 2023-09-23T22:46:35.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255761+0000", 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015231500000000001 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263866+0000", 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081051539999999998 2023-09-23T22:46:35.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.264798+0000", 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000931608 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530292+0000", 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26549436100000001 2023-09-23T22:46:35.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530346+0000", 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3328000000000002e-05 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530767+0000", 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00042096800000000003 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.085 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161607 7.8 7:17f6cc07:::benchmark_data_smithi052_277165_object69115:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 78.768864264000001, 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27881840099999999, 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.086 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161607 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252360+0000", 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252365+0000", 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4920000000000001e-06 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252374+0000", 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.49e-06 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252379+0000", 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2750000000000003e-06 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259765+0000", 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073862559999999999 2023-09-23T22:46:35.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259893+0000", 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012820900000000001 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259999+0000", 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105535 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263874+0000", 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003875295 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.264894+0000", 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010201089999999999 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530815+0000", 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26592050099999998 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530865+0000", 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9648999999999998e-05 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.531180+0000", 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031517999999999997 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162195 7.2 e205/146 7:4bcbd502:::benchmark_data_smithi052_277165_object69703:head v 205'10032, mlcod=205'10027)", 2023-09-23T22:46:35.094 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 77.772789915999994, 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.263093829, 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248435+0000", 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248443+0000", 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7219999999999999e-06 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248449+0000", 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8039999999999998e-06 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248453+0000", 2023-09-23T22:46:35.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4100000000000001e-06 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.252722+0000", 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004268679 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.252744+0000", 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1648e-05 2023-09-23T22:46:35.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.261335+0000", 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085910190000000001 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511529+0000", 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25019480999999999 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.099 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162201 7.c e205/197 7:3f19fea2:::benchmark_data_smithi052_277165_object69709:head v 205'9991, mlcod=164'3559)", 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 77.765656608, 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27094163999999998, 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255568+0000", 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255577+0000", 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1099999999999992e-06 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255589+0000", 2023-09-23T22:46:35.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255597+0000", 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7239999999999992e-06 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255657+0000", 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0096000000000002e-05 2023-09-23T22:46:35.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255685+0000", 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7429000000000001e-05 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526472+0000", 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.270787481 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526511+0000", 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8352000000000001e-05 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:162200 7.8 7:1ffee83e:::benchmark_data_smithi052_277165_object69708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:46:35.105 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 77.765041076000003, 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28005922700000002, 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 162200 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256183+0000", 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256190+0000", 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8719999999999996e-06 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256200+0000", 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7520000000000008e-06 2023-09-23T22:46:35.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256207+0000", 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4189999999999996e-06 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259499+0000", 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032920950000000001 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259640+0000", 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000141433 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259790+0000", 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014928000000000001 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516611+0000", 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25682131899999999 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516650+0000", 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8952000000000002e-05 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.535505+0000", 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018855212999999999 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.535560+0000", 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5226999999999998e-05 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.536244+0000", 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00068324599999999998 2023-09-23T22:46:35.112 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162202 7.2 e205/146 7:4736e59a:::benchmark_data_smithi052_277165_object69710:head v 205'10033, mlcod=205'10030)", 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 77.758040222999995, 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.255326575, 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.113 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263184+0000", 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263192+0000", 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3390000000000004e-06 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511467+0000", 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24827568999999999 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511483+0000", 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5209e-05 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511557+0000", 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4202999999999994e-05 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511576+0000", 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8870000000000001e-05 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.518481+0000", 2023-09-23T22:46:35.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0069055719999999996 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.518512+0000", 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0574999999999998e-05 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.35001.0:162200 7.8 e205/196 ondisk, result = 0)", 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.271586+0000", 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 77.749639396999996, 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24502732899999999, 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271586+0000", 2023-09-23T22:46:35.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271586+0000", 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271585+0000", 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271594+0000", 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6479999999999992e-06 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511551+0000", 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23995783400000001 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511559+0000", 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0330000000000008e-06 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516595+0000", 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00503566 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516604+0000", 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8410000000000006e-06 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516613+0000", 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5259999999999999e-06 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162214 7.c e205/197 7:33fa1cb0:::benchmark_data_smithi052_277165_object69722:head v 205'9992, mlcod=164'3559)", 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 77.749582489999995, 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25489373100000001, 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271642+0000", 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271650+0000", 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1620000000000002e-06 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511482+0000", 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239832029 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511494+0000", 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511527+0000", 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2679999999999999e-05 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511557+0000", 2023-09-23T22:46:35.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0304e-05 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526516+0000", 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014958767 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526537+0000", 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.12e-05 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.128 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164704 7.8 7:160987c2:::benchmark_data_smithi052_277165_object72212:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 73.624994655999998, 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31601756800000003, 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.129 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164704 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396230+0000", 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396236+0000", 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4389999999999997e-06 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396252+0000", 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6412000000000001e-05 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396260+0000", 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1160000000000001e-06 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396306+0000", 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6183999999999997e-05 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396424+0000", 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118237 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396585+0000", 2023-09-23T22:46:35.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016076799999999999 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.401552+0000", 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049672090000000002 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.405610+0000", 2023-09-23T22:46:35.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040580670000000003 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711571+0000", 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30596056300000002 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711627+0000", 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6586999999999999e-05 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.712248+0000", 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00062132300000000001 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168547 7.e 7:7817ab24:::benchmark_data_smithi052_277165_object76055:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 67.695301569999998, 2023-09-23T22:46:35.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248803264, 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168547 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:46:35.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325922+0000", 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:35.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325931+0000", 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4120000000000007e-06 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325949+0000", 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7567999999999998e-05 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325959+0000", 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8749999999999995e-06 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326592+0000", 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00063299999999999999 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326706+0000", 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113925 2023-09-23T22:46:35.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326821+0000", 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115184 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341186+0000", 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014364745 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574591+0000", 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.233404956 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574635+0000", 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4357000000000002e-05 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574727+0000", 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2262999999999994e-05 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168550 7.e 7:7960d031:::benchmark_data_smithi052_277165_object76058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 67.690015157000005, 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24916470700000001, 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.143 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168550 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331209+0000", 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331217+0000", 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0609999999999996e-06 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331230+0000", 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3159000000000001e-05 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331249+0000", 2023-09-23T22:46:35.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9205999999999999e-05 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332625+0000", 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001375498 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332749+0000", 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012410500000000001 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332864+0000", 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115203 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574774+0000", 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.241910027 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580246+0000", 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054718789999999998 2023-09-23T22:46:35.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580287+0000", 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0991000000000001e-05 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580375+0000", 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8368999999999994e-05 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.149 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168552 7.e 7:7af651e0:::benchmark_data_smithi052_277165_object76060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 67.689958841000006, 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24921663199999999, 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:46:35.150 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168552 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331266+0000", 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331269+0000", 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2389999999999998e-06 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331276+0000", 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8900000000000001e-06 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331281+0000", 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0660000000000003e-06 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341213+0000", 2023-09-23T22:46:35.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0099318059999999996 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341329+0000", 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115771 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341443+0000", 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113826 2023-09-23T22:46:35.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574809+0000", 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23336596900000001 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580383+0000", 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055746950000000002 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580417+0000", 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3581999999999998e-05 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580483+0000", 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6353999999999995e-05 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.156 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:168556 7.a e205/146 7:562c5ed1:::benchmark_data_smithi052_277165_object76064:head v 205'10476, mlcod=205'10474)", 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.349327+0000", 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 67.671898937999998, 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24479543000000001, 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.349327+0000", 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.349327+0000", 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.349326+0000", 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.349337+0000", 2023-09-23T22:46:35.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1633e-05 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574515+0000", 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22517794299999999 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574523+0000", 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4059999999999999e-06 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574574+0000", 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1958e-05 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574597+0000", 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2249e-05 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.594087+0000", 2023-09-23T22:46:35.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019489810999999999 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.594122+0000", 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5500000000000002e-05 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:46:35.161 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:46:35.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:35 smithi052 ceph-mon[180979]: pgmap v565: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T22:46:35.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:35.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:35.337 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:35.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:35.349 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:46:35.384 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:46:35.429 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:46:35.444 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:35.470 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:46:35.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:35 smithi151 ceph-mon[159210]: pgmap v565: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T22:46:35.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:35.478 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:35.480 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:35.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:35 smithi052 ceph-mon[187743]: pgmap v565: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-09-23T22:46:35.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:46:35.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:35.511 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:46:35.544 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:35.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:35.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:35.554 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:46:35.594 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:46:35.629 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:46:35.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:46:35.649 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:35.662 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 14 16 29585 29569 131.92 96.8125 0.00915963 0.00757409 2023-09-23T22:46:35.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:46:35.681 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:35.737 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:35.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:35.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:35.842 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:35.879 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:35.938 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:35.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:36.045 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:36.084 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:36.152 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:36.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:36.258 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:36.298 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:36.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/486931480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:36.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/486931480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:36.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/486931480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:36.662 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 15 16 30827 30811 128.301 77.625 0.00400933 0.00778947 2023-09-23T22:46:37.227 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:37 smithi052 ceph-mon[180979]: pgmap v566: 89 pgs: 89 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-09-23T22:46:37.280 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:46:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:37 smithi052 ceph-mon[187743]: pgmap v566: 89 pgs: 89 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-09-23T22:46:37.535 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:37.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:37.646 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:37.664 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 16 16 32191 32175 125.596 85.25 0.00841831 0.00795466 2023-09-23T22:46:37.673 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:37 smithi151 ceph-mon[159210]: pgmap v566: 89 pgs: 89 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-09-23T22:46:37.746 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:37.746 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.858 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:37.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:37.969 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:38.004 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:38.061 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:38.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:38.167 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:38.228 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:38.425 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:38.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:38.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:38.531 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:38.602 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:38.664 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 17 16 33727 33711 123.856 96 0.00795579 0.00806749 2023-09-23T22:46:38.806 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:38.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:38.911 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:38.974 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:39.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:39 smithi052 ceph-mon[180979]: pgmap v567: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T22:46:39.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:39 smithi052 ceph-mon[187743]: pgmap v567: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T22:46:39.548 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:46:39.666 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 18 16 35487 35471 123.075 110 0.00940008 0.00811971 2023-09-23T22:46:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:39 smithi151 ceph-mon[159210]: pgmap v567: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T22:46:40.224 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:40.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:40.333 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:40.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:40.435 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:40.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:40.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:40.615 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:40.662 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:40.666 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 19 16 36668 36652 120.483 73.8125 0.0101428 0.00829242 2023-09-23T22:46:40.765 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:40.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:40.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:40.878 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:40.910 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:40.926 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:40 smithi052 ceph-mon[180979]: pgmap v568: 89 pgs: 89 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T22:46:40.978 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:40.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:40.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:41.086 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:41.164 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:41.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:40 smithi151 ceph-mon[159210]: pgmap v568: 89 pgs: 89 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T22:46:41.202 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:40 smithi052 ceph-mon[187743]: pgmap v568: 89 pgs: 89 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T22:46:41.569 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:41.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:41.628 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:41.666 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:46:41.666627+0000 min lat: 0.0012697 max lat: 0.212738 avg lat: 0.00840891 2023-09-23T22:46:41.667 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:46:41.667 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 20 16 38069 38053 118.838 87.5625 0.00342259 0.00840891 2023-09-23T22:46:41.679 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:41.714 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:41.764 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:41.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:41.872 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:41.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:42.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2059751186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:42.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:42.018 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:46:42.062 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:42.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:42.164 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:42.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2059751186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:42.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:42.222 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2059751186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:42.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:46:42.227 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:42.337 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:42.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:42.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:42.444 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:42.470 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:42.511 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:42.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:42.616 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:42.642 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:42.666 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 21 16 39101 39085 116.251 64.5 0.249959 0.00855161 2023-09-23T22:46:42.687 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:42.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:42.792 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:42.829 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:42.894 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:42.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:42.999 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:43.040 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.1 primary-affinity to 0 (802) 2023-09-23T22:46:43.069 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: osdmap e225: 8 total, 8 up, 7 in 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: failed to encode map e225 with expected crc 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1844154312' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T22:46:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:43 smithi151 ceph-mon[159210]: pgmap v570: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:46:43.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:43.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:46:43.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: osdmap e225: 8 total, 8 up, 7 in 2023-09-23T22:46:43.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: failed to encode map e225 with expected crc 2023-09-23T22:46:43.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1844154312' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T22:46:43.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[187743]: pgmap v570: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: osdmap e225: 8 total, 8 up, 7 in 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: failed to encode map e225 with expected crc 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1844154312' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T22:46:43.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T22:46:43.346 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:43 smithi052 ceph-mon[180979]: pgmap v570: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:46:43.667 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 22 16 39627 39611 112.463 32.875 0.00281119 0.00863549 2023-09-23T22:46:43.669 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:43.671 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:46:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:46:43] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.33.4" 2023-09-23T22:46:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:43.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:43.774 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:43.805 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:44.008 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:44.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:44.112 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:44.158 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:44.223 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:44.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:44.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:44.334 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:44.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T22:46:44.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:44 smithi151 ceph-mon[159210]: osdmap e226: 8 total, 8 up, 7 in 2023-09-23T22:46:44.367 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:44 smithi151 ceph-mon[159210]: failed to encode map e226 with expected crc 2023-09-23T22:46:44.367 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T22:46:44.367 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:44 smithi052 ceph-mon[180979]: osdmap e226: 8 total, 8 up, 7 in 2023-09-23T22:46:44.367 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:44 smithi052 ceph-mon[180979]: failed to encode map e226 with expected crc 2023-09-23T22:46:44.368 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T22:46:44.368 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:44 smithi052 ceph-mon[187743]: osdmap e226: 8 total, 8 up, 7 in 2023-09-23T22:46:44.368 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:44 smithi052 ceph-mon[187743]: failed to encode map e226 with expected crc 2023-09-23T22:46:44.370 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:44.438 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:44.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:44.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:44.544 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:44.580 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:44.640 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:44.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:44.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:44.674 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 23 16 40672 40656 110.376 65.3125 0.0100176 0.0090548 2023-09-23T22:46:44.748 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:44.779 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:44.845 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:44.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:44.956 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:44.991 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:45.032 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:45.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:45.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:45.139 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:45.165 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:45.214 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:45.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:45.318 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[187743]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.318 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[187743]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[187743]: osdmap e227: 8 total, 8 up, 7 in 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[187743]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[187743]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[187743]: pgmap v573: 89 pgs: 89 active+clean; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[180979]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[180979]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[180979]: osdmap e227: 8 total, 8 up, 7 in 2023-09-23T22:46:45.319 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[180979]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[180979]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.320 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:45 smithi052 ceph-mon[180979]: pgmap v573: 89 pgs: 89 active+clean; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T22:46:45.322 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:45.395 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:45.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:45 smithi151 ceph-mon[159210]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:45 smithi151 ceph-mon[159210]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:45 smithi151 ceph-mon[159210]: osdmap e227: 8 total, 8 up, 7 in 2023-09-23T22:46:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:45 smithi151 ceph-mon[159210]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:45 smithi151 ceph-mon[159210]: failed to encode map e227 with expected crc 2023-09-23T22:46:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:45 smithi151 ceph-mon[159210]: pgmap v573: 89 pgs: 89 active+clean; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T22:46:45.671 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:45.675 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 24 16 41892 41876 108.955 76.25 0.00524397 0.00917064 2023-09-23T22:46:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:45.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:45.752 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:46:45.752 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:46:45.783 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:45.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:46:45.854 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:46.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:46 smithi052 ceph-mon[187743]: failed to encode map e227 with expected crc 2023-09-23T22:46:46.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:46 smithi052 ceph-mon[180979]: failed to encode map e227 with expected crc 2023-09-23T22:46:46.329 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:46.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:46.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:46.437 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:46.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:46 smithi151 ceph-mon[159210]: failed to encode map e227 with expected crc 2023-09-23T22:46:46.468 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:46.539 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:46.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:46.645 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:46.674 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 25 16 43203 43187 107.875 81.9375 0.0170751 0.00926552 2023-09-23T22:46:46.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:46.914 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:46:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:46.955 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:46.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:47.056 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:47.090 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:47.145 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:47.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:47.252 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:47.260 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:46:47.269 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:47.314 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:46:47.334 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:47 smithi052 ceph-mon[187743]: pgmap v574: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T22:46:47.334 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:47 smithi052 ceph-mon[180979]: pgmap v574: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T22:46:47.335 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:47.358 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:46:47.405 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:46:47.443 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:46:47.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:47 smithi151 ceph-mon[159210]: pgmap v574: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T22:46:47.463 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:47.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:47.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:47.483 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:46:47.521 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:46:47.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:46:47.570 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:46:47.655 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:47.674 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 26 16 44401 44385 106.606 74.875 0.0154194 0.00937356 2023-09-23T22:46:48.050 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:48.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:48.161 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:48.193 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:48.253 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:48.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:48.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:48.361 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:48.399 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2758045374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:48.402 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:48.466 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:48.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2758045374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:48.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2758045374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:48.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:48.605 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:48.662 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:48.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:48.675 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 27 16 45579 45563 105.385 73.625 0.00513211 0.00948101 2023-09-23T22:46:48.767 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:48.833 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:49.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:49 smithi052 ceph-mon[187743]: pgmap v575: 89 pgs: 89 active+clean; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 31 KiB/s, 0 objects/s recovering 2023-09-23T22:46:49.079 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:49 smithi052 ceph-mon[180979]: pgmap v575: 89 pgs: 89 active+clean; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 31 KiB/s, 0 objects/s recovering 2023-09-23T22:46:49.400 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:49.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:49.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:49 smithi151 ceph-mon[159210]: pgmap v575: 89 pgs: 89 active+clean; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 31 KiB/s, 0 objects/s recovering 2023-09-23T22:46:49.506 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:49.578 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:49.622 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:46:49.622 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:46:49.622 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-09-23T22:46:49.622 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:46:49.672 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:49.682 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 28 16 46724 46708 104.151 71.5625 0.00804522 0.00959573 2023-09-23T22:46:49.774 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:49.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:49.886 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:49.978 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:50.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-09-23T22:46:50.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:50.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-09-23T22:46:50.249 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:50.258 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:50.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:50.363 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:46:50.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-09-23T22:46:50.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:50.401 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:50.464 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:46:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:50.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:46:50.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:50.607 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:50.670 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:50.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:50.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:50.682 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 29 16 47943 47927 103.188 76.1875 0.0125886 0.00968608 2023-09-23T22:46:50.779 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:50.814 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:50.877 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:50.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:50.983 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:51.009 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:51.049 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:51.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:51.103 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:46:51.154 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:51.183 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:51.234 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:51.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:51.340 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:51.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:51 smithi151 ceph-mon[159210]: failed to encode map e228 with expected crc 2023-09-23T22:46:51.372 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:51 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-09-23T22:46:51.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:51 smithi151 ceph-mon[159210]: osdmap e228: 8 total, 8 up, 7 in 2023-09-23T22:46:51.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:51 smithi151 ceph-mon[159210]: pgmap v577: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T22:46:51.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4124074486' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:51.373 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:51 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:51.375 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:51.444 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:51.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[187743]: failed to encode map e228 with expected crc 2023-09-23T22:46:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-09-23T22:46:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[187743]: osdmap e228: 8 total, 8 up, 7 in 2023-09-23T22:46:51.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[187743]: pgmap v577: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T22:46:51.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4124074486' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:51.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:51.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[180979]: failed to encode map e228 with expected crc 2023-09-23T22:46:51.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-09-23T22:46:51.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[180979]: osdmap e228: 8 total, 8 up, 7 in 2023-09-23T22:46:51.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[180979]: pgmap v577: 89 pgs: 89 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 27 KiB/s, 0 objects/s recovering 2023-09-23T22:46:51.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4124074486' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:51.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:51 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:46:51.521 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:46:51.551 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:51.584 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:51.647 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:51.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:51.682 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 30 16 48907 48891 101.758 60.25 0.00277392 0.00981306 2023-09-23T22:46:51.755 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:51.788 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:51.847 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:51.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:51.954 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:51.992 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:52.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:52.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:46:52.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: osdmap e229: 8 total, 8 up, 7 in 2023-09-23T22:46:52.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[180979]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:52.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:46:52.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: osdmap e229: 8 total, 8 up, 7 in 2023-09-23T22:46:52.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:52 smithi052 ceph-mon[187743]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.277 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:52.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:52.389 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:52.416 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: osdmap e229: 8 total, 8 up, 7 in 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:52 smithi151 ceph-mon[159210]: failed to encode map e229 with expected crc 2023-09-23T22:46:52.453 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:52.630 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:52.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:52.682 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 31 16 50035 50019 100.75 70.5 0.00958859 0.00991927 2023-09-23T22:46:52.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:52.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:52.902 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:52.944 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:52.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:52.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:53.054 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:53.064 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:46:53.065 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:46:53.065 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:46:53.065 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:46:53.065 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:46:53.094 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:53.149 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:53.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:53.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:53 smithi052 ceph-mon[180979]: pgmap v579: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 24 KiB/s, 0 objects/s recovering 2023-09-23T22:46:53.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:53 smithi052 ceph-mon[187743]: pgmap v579: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 24 KiB/s, 0 objects/s recovering 2023-09-23T22:46:53.255 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:46:53.298 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:53.359 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:46:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:53.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:46:53.370 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:53 smithi151 ceph-mon[159210]: pgmap v579: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 24 KiB/s, 0 objects/s recovering 2023-09-23T22:46:53.470 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:53.517 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:53.595 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:53.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:53.682 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 32 16 51324 51308 100.119 80.5625 0.0101327 0.00998247 2023-09-23T22:46:53.701 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:53.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:46:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:46:53] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-09-23T22:46:53.784 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:54.398 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:46:54.682 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 33 16 52236 52220 98.8135 57 0.00718964 0.0101147 2023-09-23T22:46:54.696 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:54.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:54.807 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:46:54.879 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:54.925 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:46:55.248 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:46:55.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:55.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:46:55.353 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:55.413 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:55 smithi052 ceph-mon[187743]: pgmap v580: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 57 MiB/s wr, 914 op/s; 16 KiB/s, 0 objects/s recovering 2023-09-23T22:46:55.413 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:55 smithi052 ceph-mon[180979]: pgmap v580: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 57 MiB/s wr, 914 op/s; 16 KiB/s, 0 objects/s recovering 2023-09-23T22:46:55.426 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:55.640 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:55.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:55.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:55.683 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 34 16 53341 53325 97.9362 69.0625 0.016883 0.0102055 2023-09-23T22:46:55.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:55 smithi151 ceph-mon[159210]: pgmap v580: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 57 MiB/s wr, 914 op/s; 16 KiB/s, 0 objects/s recovering 2023-09-23T22:46:55.747 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:55.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:55.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:55.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:55.924 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:46:55.926 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:46:55.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:55.993 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:56.475 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:56.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:56.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:56.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:46:56.631 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:56.684 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 35 16 54366 54350 96.9668 64.0625 0.0430259 0.010301 2023-09-23T22:46:57.264 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:46:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:57.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:46:57.349 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:57 smithi052 ceph-mon[180979]: from='client.55153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:46:57.350 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:57 smithi052 ceph-mon[180979]: pgmap v581: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T22:46:57.350 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:57 smithi052 ceph-mon[187743]: from='client.55153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:46:57.350 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:57 smithi052 ceph-mon[187743]: pgmap v581: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T22:46:57.379 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":580,"stamp":"2023-09-23T22:46:54.172136+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3142522784,"num_objects":47966,"num_object_clones":0,"num_object_copies":143898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47966,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":48202,"num_write_kb":3070733,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":206368,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48136,"ondisk_log_size":48136,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11070164,"kb_used_data":9326556,"kb_used_omap":129,"kb_used_meta":1743422,"kb_avail":645145900,"statfs":{"total":671965249536,"available":660629401600,"internally_reserved":0,"allocated":9550393344,"data_stored":9545435553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":1785264861},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,0,1,3,2,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":479592448,"num_objects":7318,"num_object_clones":0,"num_object_copies":21954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7318,"num_write_kb":468352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002224"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":225,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018762+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:46:44.018762+0000","last_peered":"2023-09-23T22:46:44.018762+0000","last_clean":"2023-09-23T22:46:44.018762+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:46:44.018762+0000","last_undegraded":"2023-09-23T22:46:44.018762+0000","last_fullsized":"2023-09-23T22:46:44.018762+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":198,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024923+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:46:43.024923+0000","last_peered":"2023-09-23T22:46:43.024923+0000","last_clean":"2023-09-23T22:46:43.024923+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:46:43.024923+0000","last_undegraded":"2023-09-23T22:46:43.024923+0000","last_fullsized":"2023-09-23T22:46:43.024923+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.024593+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:46:44.024593+0000","last_peered":"2023-09-23T22:46:44.024593+0000","last_clean":"2023-09-23T22:46:44.024593+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:46:44.024593+0000","last_undegraded":"2023-09-23T22:46:44.024593+0000","last_fullsized":"2023-09-23T22:46:44.024593+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":195,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T22:46:51.235547+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:46:51.235547+0000","last_peered":"2023-09-23T22:46:51.235547+0000","last_clean":"2023-09-23T22:46:51.235547+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:46:51.235547+0000","last_undegraded":"2023-09-23T22:46:51.235547+0000","last_fullsized":"2023-09-23T22:46:51.235547+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":257,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024528+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:46:43.024528+0000","last_peered":"2023-09-23T22:46:43.024528+0000","last_clean":"2023-09-23T22:46:43.024528+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:46:43.024528+0000","last_undegraded":"2023-09-23T22:46:43.024528+0000","last_fullsized":"2023-09-23T22:46:43.024528+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":193,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020564+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:46:44.020564+0000","last_peered":"2023-09-23T22:46:44.020564+0000","last_clean":"2023-09-23T22:46:44.020564+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:46:44.020564+0000","last_undegraded":"2023-09-23T22:46:44.020564+0000","last_fullsized":"2023-09-23T22:46:44.020564+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":127,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.105295+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:46:50.105295+0000","last_peered":"2023-09-23T22:46:50.105295+0000","last_clean":"2023-09-23T22:46:50.105295+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:46:50.105295+0000","last_undegraded":"2023-09-23T22:46:50.105295+0000","last_fullsized":"2023-09-23T22:46:50.105295+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":225,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024616+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:46:43.024616+0000","last_peered":"2023-09-23T22:46:43.024616+0000","last_clean":"2023-09-23T22:46:43.024616+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:46:43.024616+0000","last_undegraded":"2023-09-23T22:46:43.024616+0000","last_fullsized":"2023-09-23T22:46:43.024616+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":195,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.019582+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:46:44.019582+0000","last_peered":"2023-09-23T22:46:44.019582+0000","last_clean":"2023-09-23T22:46:44.019582+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:46:44.019582+0000","last_undegraded":"2023-09-23T22:46:44.019582+0000","last_fullsized":"2023-09-23T22:46:44.019582+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":183,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.023557+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:46:43.023557+0000","last_peered":"2023-09-23T22:46:43.023557+0000","last_clean":"2023-09-23T22:46:43.023557+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:46:43.023557+0000","last_undegraded":"2023-09-23T22:46:43.023557+0000","last_fullsized":"2023-09-23T22:46:43.023557+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":100,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024857+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:46:43.024857+0000","last_peered":"2023-09-23T22:46:43.024857+0000","last_clean":"2023-09-23T22:46:43.024857+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:46:43.024857+0000","last_undegraded":"2023-09-23T22:46:43.024857+0000","last_fullsized":"2023-09-23T22:46:43.024857+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.026501+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:46:44.026501+0000","last_peered":"2023-09-23T22:46:44.026501+0000","last_clean":"2023-09-23T22:46:44.026501+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:46:44.026501+0000","last_undegraded":"2023-09-23T22:46:44.026501+0000","last_fullsized":"2023-09-23T22:46:44.026501+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":248,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018718+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:46:44.018718+0000","last_peered":"2023-09-23T22:46:44.018718+0000","last_clean":"2023-09-23T22:46:44.018718+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:46:44.018718+0000","last_undegraded":"2023-09-23T22:46:44.018718+0000","last_fullsized":"2023-09-23T22:46:44.018718+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":193,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018211+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:46:44.018211+0000","last_peered":"2023-09-23T22:46:44.018211+0000","last_clean":"2023-09-23T22:46:44.018211+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:46:44.018211+0000","last_undegraded":"2023-09-23T22:46:44.018211+0000","last_fullsized":"2023-09-23T22:46:44.018211+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":86,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018878+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:46:44.018878+0000","last_peered":"2023-09-23T22:46:44.018878+0000","last_clean":"2023-09-23T22:46:44.018878+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:46:44.018878+0000","last_undegraded":"2023-09-23T22:46:44.018878+0000","last_fullsized":"2023-09-23T22:46:44.018878+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.023766+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:46:43.023766+0000","last_peered":"2023-09-23T22:46:43.023766+0000","last_clean":"2023-09-23T22:46:43.023766+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:46:43.023766+0000","last_undegraded":"2023-09-23T22:46:43.023766+0000","last_fullsized":"2023-09-23T22:46:43.023766+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":183,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.100941+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:46:50.100941+0000","last_peered":"2023-09-23T22:46:50.100941+0000","last_clean":"2023-09-23T22:46:50.100941+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:46:50.100941+0000","last_undegraded":"2023-09-23T22:46:50.100941+0000","last_fullsized":"2023-09-23T22:46:50.100941+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":206,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024992+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:46:43.024992+0000","last_peered":"2023-09-23T22:46:43.024992+0000","last_clean":"2023-09-23T22:46:43.024992+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:46:43.024992+0000","last_undegraded":"2023-09-23T22:46:43.024992+0000","last_fullsized":"2023-09-23T22:46:43.024992+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018535+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:46:44.018535+0000","last_peered":"2023-09-23T22:46:44.018535+0000","last_clean":"2023-09-23T22:46:44.018535+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:46:44.018535+0000","last_undegraded":"2023-09-23T22:46:44.018535+0000","last_fullsized":"2023-09-23T22:46:44.018535+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":184,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.106142+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:46:50.106142+0000","last_peered":"2023-09-23T22:46:50.106142+0000","last_clean":"2023-09-23T22:46:50.106142+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:46:50.106142+0000","last_undegraded":"2023-09-23T22:46:50.106142+0000","last_fullsized":"2023-09-23T22:46:50.106142+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":272,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020213+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:46:44.020213+0000","last_peered":"2023-09-23T22:46:44.020213+0000","last_clean":"2023-09-23T22:46:44.020213+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:46:44.020213+0000","last_undegraded":"2023-09-23T22:46:44.020213+0000","last_fullsized":"2023-09-23T22:46:44.020213+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":225,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.026085+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:46:43.026085+0000","last_peered":"2023-09-23T22:46:43.026085+0000","last_clean":"2023-09-23T22:46:43.026085+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:46:43.026085+0000","last_undegraded":"2023-09-23T22:46:43.026085+0000","last_fullsized":"2023-09-23T22:46:43.026085+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"228'2926","reported_seq":2945,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.459740+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:46:50.459740+0000","last_peered":"2023-09-23T22:46:50.459740+0000","last_clean":"2023-09-23T22:46:50.459740+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:46:50.459740+0000","last_undegraded":"2023-09-23T22:46:50.459740+0000","last_fullsized":"2023-09-23T22:46:50.459740+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2926,"log_dups_size":0,"ondisk_log_size":2926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":187264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":97,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.019872+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:46:44.019872+0000","last_peered":"2023-09-23T22:46:44.019872+0000","last_clean":"2023-09-23T22:46:44.019872+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:46:44.019872+0000","last_undegraded":"2023-09-23T22:46:44.019872+0000","last_fullsized":"2023-09-23T22:46:44.019872+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":192,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020427+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:46:44.020427+0000","last_peered":"2023-09-23T22:46:44.020427+0000","last_clean":"2023-09-23T22:46:44.020427+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:46:44.020427+0000","last_undegraded":"2023-09-23T22:46:44.020427+0000","last_fullsized":"2023-09-23T22:46:44.020427+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"228'3008","reported_seq":3033,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.761697+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:46:50.761697+0000","last_peered":"2023-09-23T22:46:50.761697+0000","last_clean":"2023-09-23T22:46:50.761697+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:46:50.761697+0000","last_undegraded":"2023-09-23T22:46:50.761697+0000","last_fullsized":"2023-09-23T22:46:50.761697+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3008,"log_dups_size":0,"ondisk_log_size":3008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197132288,"num_objects":3008,"num_object_clones":0,"num_object_copies":9024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3008,"num_write_kb":192512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":107,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.019787+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:46:44.019787+0000","last_peered":"2023-09-23T22:46:44.019787+0000","last_clean":"2023-09-23T22:46:44.019787+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:46:44.019787+0000","last_undegraded":"2023-09-23T22:46:44.019787+0000","last_fullsized":"2023-09-23T22:46:44.019787+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":208,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018384+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:46:44.018384+0000","last_peered":"2023-09-23T22:46:44.018384+0000","last_clean":"2023-09-23T22:46:44.018384+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:46:44.018384+0000","last_undegraded":"2023-09-23T22:46:44.018384+0000","last_fullsized":"2023-09-23T22:46:44.018384+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"229'3007","reported_seq":3027,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T22:46:52.242575+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:46:52.242575+0000","last_peered":"2023-09-23T22:46:52.242575+0000","last_clean":"2023-09-23T22:46:52.242575+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:46:52.242575+0000","last_undegraded":"2023-09-23T22:46:52.242575+0000","last_fullsized":"2023-09-23T22:46:52.242575+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3007,"log_dups_size":0,"ondisk_log_size":3007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197066752,"num_objects":3007,"num_object_clones":0,"num_object_copies":9021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3007,"num_write_kb":192448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":38,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020685+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:46:44.020685+0000","last_peered":"2023-09-23T22:46:44.020685+0000","last_clean":"2023-09-23T22:46:44.020685+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:46:44.020685+0000","last_undegraded":"2023-09-23T22:46:44.020685+0000","last_fullsized":"2023-09-23T22:46:44.020685+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":220,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020440+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:46:44.020440+0000","last_peered":"2023-09-23T22:46:44.020440+0000","last_clean":"2023-09-23T22:46:44.020440+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:46:44.020440+0000","last_undegraded":"2023-09-23T22:46:44.020440+0000","last_fullsized":"2023-09-23T22:46:44.020440+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"227'3083","reported_seq":3110,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:49.939319+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:46:49.939319+0000","last_peered":"2023-09-23T22:46:49.939319+0000","last_clean":"2023-09-23T22:46:49.939319+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:46:49.939319+0000","last_undegraded":"2023-09-23T22:46:49.939319+0000","last_fullsized":"2023-09-23T22:46:49.939319+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3083,"log_dups_size":0,"ondisk_log_size":3083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202047488,"num_objects":3083,"num_object_clones":0,"num_object_copies":9249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3083,"num_write_kb":197312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":78,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024608+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:46:43.024608+0000","last_peered":"2023-09-23T22:46:43.024608+0000","last_clean":"2023-09-23T22:46:43.024608+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:46:43.024608+0000","last_undegraded":"2023-09-23T22:46:43.024608+0000","last_fullsized":"2023-09-23T22:46:43.024608+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.100970+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:46:50.100970+0000","last_peered":"2023-09-23T22:46:50.100970+0000","last_clean":"2023-09-23T22:46:50.100970+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:46:50.100970+0000","last_undegraded":"2023-09-23T22:46:50.100970+0000","last_fullsized":"2023-09-23T22:46:50.100970+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"227'2943","reported_seq":2957,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.029781+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:46:50.029781+0000","last_peered":"2023-09-23T22:46:50.029781+0000","last_clean":"2023-09-23T22:46:50.029781+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:46:50.029781+0000","last_undegraded":"2023-09-23T22:46:50.029781+0000","last_fullsized":"2023-09-23T22:46:50.029781+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2943,"log_dups_size":0,"ondisk_log_size":2943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192872448,"num_objects":2943,"num_object_clones":0,"num_object_copies":8829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2943,"num_write_kb":188352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":226,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.023548+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:46:43.023548+0000","last_peered":"2023-09-23T22:46:43.023548+0000","last_clean":"2023-09-23T22:46:43.023548+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:46:43.023548+0000","last_undegraded":"2023-09-23T22:46:43.023548+0000","last_fullsized":"2023-09-23T22:46:43.023548+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":193,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018632+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:46:44.018632+0000","last_peered":"2023-09-23T22:46:44.018632+0000","last_clean":"2023-09-23T22:46:44.018632+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:46:44.018632+0000","last_undegraded":"2023-09-23T22:46:44.018632+0000","last_fullsized":"2023-09-23T22:46:44.018632+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"229'3146","reported_seq":3188,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T22:46:52.200031+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:46:52.200031+0000","last_peered":"2023-09-23T22:46:52.200031+0000","last_clean":"2023-09-23T22:46:52.200031+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:46:52.200031+0000","last_undegraded":"2023-09-23T22:46:52.200031+0000","last_fullsized":"2023-09-23T22:46:52.200031+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3146,"log_dups_size":0,"ondisk_log_size":3146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206176256,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3146,"num_write_kb":201344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":212,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020548+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:46:44.020548+0000","last_peered":"2023-09-23T22:46:44.020548+0000","last_clean":"2023-09-23T22:46:44.020548+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:46:44.020548+0000","last_undegraded":"2023-09-23T22:46:44.020548+0000","last_fullsized":"2023-09-23T22:46:44.020548+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":223,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.019046+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:46:44.019046+0000","last_peered":"2023-09-23T22:46:44.019046+0000","last_clean":"2023-09-23T22:46:44.019046+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:46:44.019046+0000","last_undegraded":"2023-09-23T22:46:44.019046+0000","last_fullsized":"2023-09-23T22:46:44.019046+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"228'3030","reported_seq":3055,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.467190+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:46:50.467190+0000","last_peered":"2023-09-23T22:46:50.467190+0000","last_clean":"2023-09-23T22:46:50.467190+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:46:50.467190+0000","last_undegraded":"2023-09-23T22:46:50.467190+0000","last_fullsized":"2023-09-23T22:46:50.467190+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3030,"log_dups_size":0,"ondisk_log_size":3030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":193920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":236,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018256+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:46:44.018256+0000","last_peered":"2023-09-23T22:46:44.018256+0000","last_clean":"2023-09-23T22:46:44.018256+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:46:44.018256+0000","last_undegraded":"2023-09-23T22:46:44.018256+0000","last_fullsized":"2023-09-23T22:46:44.018256+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":211,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.019878+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:46:44.019878+0000","last_peered":"2023-09-23T22:46:44.019878+0000","last_clean":"2023-09-23T22:46:44.019878+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:46:44.019878+0000","last_undegraded":"2023-09-23T22:46:44.019878+0000","last_fullsized":"2023-09-23T22:46:44.019878+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"228'2912","reported_seq":2948,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.766090+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:46:50.766090+0000","last_peered":"2023-09-23T22:46:50.766090+0000","last_clean":"2023-09-23T22:46:50.766090+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:46:50.766090+0000","last_undegraded":"2023-09-23T22:46:50.766090+0000","last_fullsized":"2023-09-23T22:46:50.766090+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2912,"log_dups_size":0,"ondisk_log_size":2912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190840832,"num_objects":2912,"num_object_clones":0,"num_object_copies":8736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2912,"num_write_kb":186368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":227,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.013071+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:46:44.013071+0000","last_peered":"2023-09-23T22:46:44.013071+0000","last_clean":"2023-09-23T22:46:44.013071+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:46:44.013071+0000","last_undegraded":"2023-09-23T22:46:44.013071+0000","last_fullsized":"2023-09-23T22:46:44.013071+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":152,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.026041+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:46:43.026041+0000","last_peered":"2023-09-23T22:46:43.026041+0000","last_clean":"2023-09-23T22:46:43.026041+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:46:43.026041+0000","last_undegraded":"2023-09-23T22:46:43.026041+0000","last_fullsized":"2023-09-23T22:46:43.026041+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"229'3087","reported_seq":3107,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T22:46:52.215932+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:46:52.215932+0000","last_peered":"2023-09-23T22:46:52.215932+0000","last_clean":"2023-09-23T22:46:52.215932+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:46:52.215932+0000","last_undegraded":"2023-09-23T22:46:52.215932+0000","last_fullsized":"2023-09-23T22:46:52.215932+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3087,"log_dups_size":0,"ondisk_log_size":3087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202309632,"num_objects":3087,"num_object_clones":0,"num_object_copies":9261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3087,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":264,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.105409+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:46:50.105409+0000","last_peered":"2023-09-23T22:46:50.105409+0000","last_clean":"2023-09-23T22:46:50.105409+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:46:50.105409+0000","last_undegraded":"2023-09-23T22:46:50.105409+0000","last_fullsized":"2023-09-23T22:46:50.105409+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024791+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:46:43.024791+0000","last_peered":"2023-09-23T22:46:43.024791+0000","last_clean":"2023-09-23T22:46:43.024791+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:46:43.024791+0000","last_undegraded":"2023-09-23T22:46:43.024791+0000","last_fullsized":"2023-09-23T22:46:43.024791+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.024328+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:46:44.024328+0000","last_peered":"2023-09-23T22:46:44.024328+0000","last_clean":"2023-09-23T22:46:44.024328+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:46:44.024328+0000","last_undegraded":"2023-09-23T22:46:44.024328+0000","last_fullsized":"2023-09-23T22:46:44.024328+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"227'2963","reported_seq":2988,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.038996+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:46:50.038996+0000","last_peered":"2023-09-23T22:46:50.038996+0000","last_clean":"2023-09-23T22:46:50.038996+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:46:50.038996+0000","last_undegraded":"2023-09-23T22:46:50.038996+0000","last_fullsized":"2023-09-23T22:46:50.038996+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2963,"log_dups_size":0,"ondisk_log_size":2963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194183168,"num_objects":2963,"num_object_clones":0,"num_object_copies":8889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2963,"num_write_kb":189632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":62,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.025170+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:46:44.025170+0000","last_peered":"2023-09-23T22:46:44.025170+0000","last_clean":"2023-09-23T22:46:44.025170+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:46:44.025170+0000","last_undegraded":"2023-09-23T22:46:44.025170+0000","last_fullsized":"2023-09-23T22:46:44.025170+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":224,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018334+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:46:44.018334+0000","last_peered":"2023-09-23T22:46:44.018334+0000","last_clean":"2023-09-23T22:46:44.018334+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:46:44.018334+0000","last_undegraded":"2023-09-23T22:46:44.018334+0000","last_fullsized":"2023-09-23T22:46:44.018334+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":141,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.028409+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:46:43.028409+0000","last_peered":"2023-09-23T22:46:43.028409+0000","last_clean":"2023-09-23T22:46:43.028409+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:46:43.028409+0000","last_undegraded":"2023-09-23T22:46:43.028409+0000","last_fullsized":"2023-09-23T22:46:43.028409+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"227'2976","reported_seq":2995,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.033614+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:46:50.033614+0000","last_peered":"2023-09-23T22:46:50.033614+0000","last_clean":"2023-09-23T22:46:50.033614+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:46:50.033614+0000","last_undegraded":"2023-09-23T22:46:50.033614+0000","last_fullsized":"2023-09-23T22:46:50.033614+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2976,"log_dups_size":0,"ondisk_log_size":2976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195035136,"num_objects":2976,"num_object_clones":0,"num_object_copies":8928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2976,"num_write_kb":190464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":216,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.026841+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:46:43.026841+0000","last_peered":"2023-09-23T22:46:43.026841+0000","last_clean":"2023-09-23T22:46:43.026841+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:46:43.026841+0000","last_undegraded":"2023-09-23T22:46:43.026841+0000","last_fullsized":"2023-09-23T22:46:43.026841+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.026179+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:46:44.026179+0000","last_peered":"2023-09-23T22:46:44.026179+0000","last_clean":"2023-09-23T22:46:44.026179+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:46:44.026179+0000","last_undegraded":"2023-09-23T22:46:44.026179+0000","last_fullsized":"2023-09-23T22:46:44.026179+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":233,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.105202+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:46:50.105202+0000","last_peered":"2023-09-23T22:46:50.105202+0000","last_clean":"2023-09-23T22:46:50.105202+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:46:50.105202+0000","last_undegraded":"2023-09-23T22:46:50.105202+0000","last_fullsized":"2023-09-23T22:46:50.105202+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"228'2969","reported_seq":2994,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.759477+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:46:50.759477+0000","last_peered":"2023-09-23T22:46:50.759477+0000","last_clean":"2023-09-23T22:46:50.759477+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:46:50.759477+0000","last_undegraded":"2023-09-23T22:46:50.759477+0000","last_fullsized":"2023-09-23T22:46:50.759477+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2969,"log_dups_size":0,"ondisk_log_size":2969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194576384,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2969,"num_write_kb":190016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":226,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.025781+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:46:43.025781+0000","last_peered":"2023-09-23T22:46:43.025781+0000","last_clean":"2023-09-23T22:46:43.025781+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:46:43.025781+0000","last_undegraded":"2023-09-23T22:46:43.025781+0000","last_fullsized":"2023-09-23T22:46:43.025781+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":314,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.103278+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:46:50.103278+0000","last_peered":"2023-09-23T22:46:50.103278+0000","last_clean":"2023-09-23T22:46:50.103278+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:46:50.103278+0000","last_undegraded":"2023-09-23T22:46:50.103278+0000","last_fullsized":"2023-09-23T22:46:50.103278+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024506+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:46:43.024506+0000","last_peered":"2023-09-23T22:46:43.024506+0000","last_clean":"2023-09-23T22:46:43.024506+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:46:43.024506+0000","last_undegraded":"2023-09-23T22:46:43.024506+0000","last_fullsized":"2023-09-23T22:46:43.024506+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":220,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020767+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:46:44.020767+0000","last_peered":"2023-09-23T22:46:44.020767+0000","last_clean":"2023-09-23T22:46:44.020767+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:46:44.020767+0000","last_undegraded":"2023-09-23T22:46:44.020767+0000","last_fullsized":"2023-09-23T22:46:44.020767+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"227'2904","reported_seq":2931,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:49.443284+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:46:49.443284+0000","last_peered":"2023-09-23T22:46:49.443284+0000","last_clean":"2023-09-23T22:46:49.443284+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:46:49.443284+0000","last_undegraded":"2023-09-23T22:46:49.443284+0000","last_fullsized":"2023-09-23T22:46:49.443284+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2904,"log_dups_size":0,"ondisk_log_size":2904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190316544,"num_objects":2904,"num_object_clones":0,"num_object_copies":8712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2904,"num_write_kb":185856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":193,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.103411+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:46:50.103411+0000","last_peered":"2023-09-23T22:46:50.103411+0000","last_clean":"2023-09-23T22:46:50.103411+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:46:50.103411+0000","last_undegraded":"2023-09-23T22:46:50.103411+0000","last_fullsized":"2023-09-23T22:46:50.103411+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":164,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.100755+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:46:50.100755+0000","last_peered":"2023-09-23T22:46:50.100755+0000","last_clean":"2023-09-23T22:46:50.100755+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:46:50.100755+0000","last_undegraded":"2023-09-23T22:46:50.100755+0000","last_fullsized":"2023-09-23T22:46:50.100755+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":151,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.105915+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:46:50.105915+0000","last_peered":"2023-09-23T22:46:50.105915+0000","last_clean":"2023-09-23T22:46:50.105915+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:46:50.105915+0000","last_undegraded":"2023-09-23T22:46:50.105915+0000","last_fullsized":"2023-09-23T22:46:50.105915+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"227'2977","reported_seq":2996,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.022325+0000","last_change":"2023-09-23T22:46:12.238270+0000","last_active":"2023-09-23T22:46:50.022325+0000","last_peered":"2023-09-23T22:46:50.022325+0000","last_clean":"2023-09-23T22:46:50.022325+0000","last_became_active":"2023-09-23T22:46:12.237953+0000","last_became_peered":"2023-09-23T22:46:12.237953+0000","last_unstale":"2023-09-23T22:46:50.022325+0000","last_undegraded":"2023-09-23T22:46:50.022325+0000","last_fullsized":"2023-09-23T22:46:50.022325+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2977,"log_dups_size":0,"ondisk_log_size":2977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:12.147455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2977,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":156,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.107606+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:46:50.107606+0000","last_peered":"2023-09-23T22:46:50.107606+0000","last_clean":"2023-09-23T22:46:50.107606+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:46:50.107606+0000","last_undegraded":"2023-09-23T22:46:50.107606+0000","last_fullsized":"2023-09-23T22:46:50.107606+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":104,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.026155+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:46:44.026155+0000","last_peered":"2023-09-23T22:46:44.026155+0000","last_clean":"2023-09-23T22:46:44.026155+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:46:44.026155+0000","last_undegraded":"2023-09-23T22:46:44.026155+0000","last_fullsized":"2023-09-23T22:46:44.026155+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":156,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.101262+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:46:50.101262+0000","last_peered":"2023-09-23T22:46:50.101262+0000","last_clean":"2023-09-23T22:46:50.101262+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:46:50.101262+0000","last_undegraded":"2023-09-23T22:46:50.101262+0000","last_fullsized":"2023-09-23T22:46:50.101262+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"229'3125","reported_seq":3145,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T22:46:52.235815+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:46:52.235815+0000","last_peered":"2023-09-23T22:46:52.235815+0000","last_clean":"2023-09-23T22:46:52.235815+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:46:52.235815+0000","last_undegraded":"2023-09-23T22:46:52.235815+0000","last_fullsized":"2023-09-23T22:46:52.235815+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":3125,"log_dups_size":0,"ondisk_log_size":3125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204800000,"num_objects":3125,"num_object_clones":0,"num_object_copies":9375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3125,"num_write_kb":200000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":266,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.013929+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:46:44.013929+0000","last_peered":"2023-09-23T22:46:44.013929+0000","last_clean":"2023-09-23T22:46:44.013929+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:46:44.013929+0000","last_undegraded":"2023-09-23T22:46:44.013929+0000","last_fullsized":"2023-09-23T22:46:44.013929+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.025157+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:46:44.025157+0000","last_peered":"2023-09-23T22:46:44.025157+0000","last_clean":"2023-09-23T22:46:44.025157+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:46:44.025157+0000","last_undegraded":"2023-09-23T22:46:44.025157+0000","last_fullsized":"2023-09-23T22:46:44.025157+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":251,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.024711+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:46:43.024711+0000","last_peered":"2023-09-23T22:46:43.024711+0000","last_clean":"2023-09-23T22:46:43.024711+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:46:43.024711+0000","last_undegraded":"2023-09-23T22:46:43.024711+0000","last_fullsized":"2023-09-23T22:46:43.024711+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"227'2886","reported_seq":2905,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:49.448067+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:46:49.448067+0000","last_peered":"2023-09-23T22:46:49.448067+0000","last_clean":"2023-09-23T22:46:49.448067+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:46:49.448067+0000","last_undegraded":"2023-09-23T22:46:49.448067+0000","last_fullsized":"2023-09-23T22:46:49.448067+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":2886,"log_dups_size":0,"ondisk_log_size":2886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189136896,"num_objects":2886,"num_object_clones":0,"num_object_copies":8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2886,"num_write_kb":184704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":260,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.020217+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:46:44.020217+0000","last_peered":"2023-09-23T22:46:44.020217+0000","last_clean":"2023-09-23T22:46:44.020217+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:46:44.020217+0000","last_undegraded":"2023-09-23T22:46:44.020217+0000","last_fullsized":"2023-09-23T22:46:44.020217+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":123,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.105989+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:46:50.105989+0000","last_peered":"2023-09-23T22:46:50.105989+0000","last_clean":"2023-09-23T22:46:50.105989+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:46:50.105989+0000","last_undegraded":"2023-09-23T22:46:50.105989+0000","last_fullsized":"2023-09-23T22:46:50.105989+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":178,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018494+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:46:44.018494+0000","last_peered":"2023-09-23T22:46:44.018494+0000","last_clean":"2023-09-23T22:46:44.018494+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:46:44.018494+0000","last_undegraded":"2023-09-23T22:46:44.018494+0000","last_fullsized":"2023-09-23T22:46:44.018494+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":159,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.101127+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:46:50.101127+0000","last_peered":"2023-09-23T22:46:50.101127+0000","last_clean":"2023-09-23T22:46:50.101127+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:46:50.101127+0000","last_undegraded":"2023-09-23T22:46:50.101127+0000","last_fullsized":"2023-09-23T22:46:50.101127+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":181,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.024758+0000","last_change":"2023-09-23T22:46:44.024557+0000","last_active":"2023-09-23T22:46:44.024758+0000","last_peered":"2023-09-23T22:46:44.024758+0000","last_clean":"2023-09-23T22:46:44.024758+0000","last_became_active":"2023-09-23T22:46:44.024240+0000","last_became_peered":"2023-09-23T22:46:44.024240+0000","last_unstale":"2023-09-23T22:46:44.024758+0000","last_undegraded":"2023-09-23T22:46:44.024758+0000","last_fullsized":"2023-09-23T22:46:44.024758+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:51.251965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":38,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018451+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:46:44.018451+0000","last_peered":"2023-09-23T22:46:44.018451+0000","last_clean":"2023-09-23T22:46:44.018451+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:46:44.018451+0000","last_undegraded":"2023-09-23T22:46:44.018451+0000","last_fullsized":"2023-09-23T22:46:44.018451+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018422+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:46:44.018422+0000","last_peered":"2023-09-23T22:46:44.018422+0000","last_clean":"2023-09-23T22:46:44.018422+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:46:44.018422+0000","last_undegraded":"2023-09-23T22:46:44.018422+0000","last_fullsized":"2023-09-23T22:46:44.018422+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":38,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018873+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:46:44.018873+0000","last_peered":"2023-09-23T22:46:44.018873+0000","last_clean":"2023-09-23T22:46:44.018873+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:46:44.018873+0000","last_undegraded":"2023-09-23T22:46:44.018873+0000","last_fullsized":"2023-09-23T22:46:44.018873+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":99,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018857+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:46:44.018857+0000","last_peered":"2023-09-23T22:46:44.018857+0000","last_clean":"2023-09-23T22:46:44.018857+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:46:44.018857+0000","last_undegraded":"2023-09-23T22:46:44.018857+0000","last_fullsized":"2023-09-23T22:46:44.018857+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":219,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.026885+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:46:43.026885+0000","last_peered":"2023-09-23T22:46:43.026885+0000","last_clean":"2023-09-23T22:46:43.026885+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:46:43.026885+0000","last_undegraded":"2023-09-23T22:46:43.026885+0000","last_fullsized":"2023-09-23T22:46:43.026885+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":257,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-09-23T22:46:43.023478+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:46:43.023478+0000","last_peered":"2023-09-23T22:46:43.023478+0000","last_clean":"2023-09-23T22:46:43.023478+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:46:43.023478+0000","last_undegraded":"2023-09-23T22:46:43.023478+0000","last_fullsized":"2023-09-23T22:46:43.023478+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":157,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T22:46:50.105384+0000","last_change":"2023-09-23T22:46:14.250421+0000","last_active":"2023-09-23T22:46:50.105384+0000","last_peered":"2023-09-23T22:46:50.105384+0000","last_clean":"2023-09-23T22:46:50.105384+0000","last_became_active":"2023-09-23T22:46:14.249148+0000","last_became_peered":"2023-09-23T22:46:14.249148+0000","last_unstale":"2023-09-23T22:46:50.105384+0000","last_undegraded":"2023-09-23T22:46:50.105384+0000","last_fullsized":"2023-09-23T22:46:50.105384+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:05.157601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":193,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-09-23T22:46:44.018283+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:46:44.018283+0000","last_peered":"2023-09-23T22:46:44.018283+0000","last_clean":"2023-09-23T22:46:44.018283+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:46:44.018283+0000","last_undegraded":"2023-09-23T22:46:44.018283+0000","last_fullsized":"2023-09-23T22:46:44.018283+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":9760,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3141926912,"num_objects":47942,"num_object_clones":0,"num_object_copies":143826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":47942,"num_write_kb":3068288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9457696768,"data_stored":9457696768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47942,"ondisk_log_size":47942,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430446,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626674,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1441716,"kb_used_data":1132204,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92303436,"statfs":{"total":95995035648,"available":94518718464,"internally_reserved":0,"allocated":1159376896,"data_stored":1158881337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3999999999999999,"5min":2.331,"15min":1.069},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":18.634,"5min":52.377000000000002,"15min":52.377000000000002},"last":0.28000000000000003},{"interface":"front","average":{"1min":4.9889999999999999,"5min":2.5369999999999999,"15min":1.232},"min":{"1min":0.26100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.481000000000002,"5min":51.970999999999997,"15min":51.970999999999997},"last":1.6970000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":3.6379999999999999,"15min":2.0710000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.8260000000000005,"5min":153.78399999999999,"15min":153.78399999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":1.665,"5min":3.5289999999999999,"15min":2.0489999999999999},"min":{"1min":0.28899999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.8160000000000007,"5min":154.148,"15min":154.148},"last":1.6200000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":3.8839999999999999,"15min":3.1200000000000001},"min":{"1min":0.309,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.536,"5min":100.785,"15min":100.785},"last":1.0980000000000001},{"interface":"front","average":{"1min":2.3599999999999999,"5min":3.6269999999999998,"15min":3.0209999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.586,"5min":100.77,"15min":100.77},"last":2.3180000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":4.5010000000000003,"15min":1.9219999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.3190000000000008,"5min":90.561999999999998,"15min":90.561999999999998},"last":2.4220000000000002},{"interface":"front","average":{"1min":1.9430000000000001,"5min":4.649,"15min":2.0449999999999999},"min":{"1min":0.35599999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.5099999999999998,"5min":90.646000000000001,"15min":90.646000000000001},"last":2.1869999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.85899999999999999,"15min":0.61599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":3.782,"5min":18.085999999999999,"15min":18.085999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.91900000000000004,"15min":0.64100000000000001},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":9.9740000000000002,"5min":18.597000000000001,"15min":18.597000000000001},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.96899999999999997,"15min":0.67300000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":4.0700000000000003,"5min":18.972000000000001,"15min":18.972000000000001},"last":0.85999999999999999},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.0429999999999999,"15min":0.70899999999999996},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.26200000000000001},"max":{"1min":9.7899999999999991,"5min":18.625,"15min":18.625},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.94599999999999995,"15min":0.69899999999999995},"min":{"1min":0.36499999999999999,"5min":0.316,"15min":0.316},"max":{"1min":9.8800000000000008,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96999999999999997,"15min":0.69799999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":5.1790000000000003,"5min":16.661999999999999,"15min":16.661999999999999},"last":0.752}]}]},{"osd":1,"up_from":84,"seq":360777252984,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1585988,"kb_used_data":1391996,"kb_used_omap":9,"kb_used_meta":193974,"kb_avail":92159164,"statfs":{"total":95995035648,"available":94370983936,"internally_reserved":0,"allocated":1425403904,"data_stored":1424555114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":198630377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.2160000000000002,"5min":3.2799999999999998,"15min":1.6000000000000001},"min":{"1min":0.19600000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.861000000000001,"5min":83.670000000000002,"15min":83.670000000000002},"last":2.6299999999999999},{"interface":"front","average":{"1min":4.4059999999999997,"5min":3.5350000000000001,"15min":1.5149999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":29.591000000000001,"5min":84,"15min":84},"last":2.742}]},{"osd":2,"last update":"Sat Sep 23 22:46:13 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":3.379,"15min":1.8169999999999999},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.561,"5min":37.889000000000003,"15min":37.889000000000003},"last":0.85599999999999998},{"interface":"front","average":{"1min":2.254,"5min":3.552,"15min":1.7809999999999999},"min":{"1min":0.27700000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":22.489000000000001,"5min":37.823,"15min":37.823},"last":2.589}]},{"osd":3,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":2.9129999999999998,"15min":1.23},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":10.933,"5min":50.033999999999999,"15min":50.033999999999999},"last":5.4820000000000002},{"interface":"front","average":{"1min":1.8560000000000001,"5min":3.5289999999999999,"15min":1.4330000000000001},"min":{"1min":0.28299999999999997,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.1869999999999994,"5min":52.491999999999997,"15min":52.491999999999997},"last":3.012}]},{"osd":4,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.0920000000000001,"15min":1.379},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":6.6900000000000004,"5min":29.466999999999999,"15min":69.344999999999999},"last":2.9590000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.8959999999999999,"15min":1.343},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.313},"max":{"1min":2.2589999999999999,"5min":8.1630000000000003,"15min":69.359999999999999},"last":0.60299999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.7709999999999999,"15min":1.1220000000000001},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.641999999999999,"5min":10.641999999999999,"15min":28.837},"last":1.3140000000000001},{"interface":"front","average":{"1min":1.5309999999999999,"5min":2.0830000000000002,"15min":1.1970000000000001},"min":{"1min":0.35599999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.6369999999999996,"5min":29.510999999999999,"15min":29.510999999999999},"last":2.919}]},{"osd":6,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.1219999999999999,"15min":1.139},"min":{"1min":0.34799999999999998,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":1.377,"5min":16.192,"15min":16.192},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.837,"15min":1.0429999999999999},"min":{"1min":0.36899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":10.693,"5min":10.693,"15min":10.693},"last":1.2709999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":2.2120000000000002,"15min":1.8029999999999999},"min":{"1min":0.45000000000000001,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":6.7220000000000004,"5min":29.539000000000001,"15min":159.018},"last":2.9870000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":2.2360000000000002,"15min":1.8340000000000001},"min":{"1min":0.42099999999999999,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":5.5899999999999999,"5min":8.6120000000000001,"15min":158.96000000000001},"last":0.64500000000000002}]}]},{"osd":0,"up_from":219,"seq":940597837833,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1837396,"kb_used_data":1739404,"kb_used_omap":31,"kb_used_meta":97952,"kb_avail":91907756,"statfs":{"total":95995035648,"available":94113542144,"internally_reserved":0,"allocated":1781149696,"data_stored":1780227201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":100303794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.4779999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34300000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38700000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]}]},{"osd":2,"up_from":131,"seq":562640715845,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1298328,"kb_used_data":1108240,"kb_used_omap":81,"kb_used_meta":189998,"kb_avail":92446824,"statfs":{"total":95995035648,"available":94665547776,"internally_reserved":0,"allocated":1134837760,"data_stored":1133958401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":194558170},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2069999999999999,"5min":4.8310000000000004,"15min":3.8759999999999999},"min":{"1min":0.24099999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":20.631,"5min":114.735,"15min":114.735},"last":4.7800000000000002},{"interface":"front","average":{"1min":11.375,"5min":7.4950000000000001,"15min":5.6029999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":94.554000000000002,"5min":114.798,"15min":114.798},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":3.758,"15min":3.3069999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.0910000000000002,"5min":88.316000000000003,"15min":88.316000000000003},"last":0.436},{"interface":"front","average":{"1min":2.415,"5min":3.9350000000000001,"15min":2.7050000000000001},"min":{"1min":0.22700000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":20.210000000000001,"5min":88.436999999999998,"15min":88.436999999999998},"last":2.1629999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":5.9660000000000002,"15min":2.7040000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":7.4770000000000003,"5min":88.653000000000006,"15min":88.653000000000006},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.70199999999999996,"5min":4.3090000000000002,"15min":2.141},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.355,"5min":88.474000000000004,"15min":88.474000000000004},"last":0.84499999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":7.0229999999999997,"5min":2.5539999999999998,"15min":1.4550000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":108.878,"5min":108.878,"15min":108.878},"last":0.89100000000000001},{"interface":"front","average":{"1min":7.5549999999999997,"5min":4.2080000000000002,"15min":2.2799999999999998},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":108.93000000000001,"5min":108.93000000000001,"15min":108.93000000000001},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":6.8710000000000004,"5min":4.2679999999999998,"15min":2.3250000000000002},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":97.322999999999993,"5min":97.322999999999993,"15min":97.322999999999993},"last":0.68300000000000005},{"interface":"front","average":{"1min":6.7910000000000004,"5min":3.8500000000000001,"15min":1.784},"min":{"1min":0.35399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":97.301000000000002,"5min":97.301000000000002,"15min":97.301000000000002},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":3.016,"15min":1.53},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.7050000000000001,"5min":88.606999999999999,"15min":88.606999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.874,"5min":1.456,"15min":1.0820000000000001},"min":{"1min":0.36799999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.9900000000000002,"5min":9.9000000000000004,"15min":9.9000000000000004},"last":0.92300000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.4950000000000001,"15min":1.147},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.282,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.7250000000000001,"5min":3.4390000000000001,"15min":2.0489999999999999},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":7.0350000000000001,"5min":88.584000000000003,"15min":88.584000000000003},"last":0.54300000000000004}]}]},{"osd":3,"up_from":94,"seq":403726925931,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456816,"kb_used_data":1127912,"kb_used_omap":7,"kb_used_meta":328888,"kb_avail":92288336,"statfs":{"total":95995035648,"available":94503256064,"internally_reserved":0,"allocated":1154981888,"data_stored":1154135381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":336781928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:58 2023","interfaces":[{"interface":"back","average":{"1min":10.183999999999999,"5min":4.2210000000000001,"15min":1.7709999999999999},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":92.546999999999997,"5min":92.546999999999997,"15min":92.546999999999997},"last":2.0049999999999999},{"interface":"front","average":{"1min":9.0670000000000002,"5min":3.8500000000000001,"15min":1.7909999999999999},"min":{"1min":0.27300000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":92.281000000000006,"5min":92.281000000000006,"15min":92.281000000000006},"last":2.3170000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1379999999999999,"5min":2.8130000000000002,"15min":1.1679999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":17.231000000000002,"5min":22.818000000000001,"15min":22.818000000000001},"last":2.0430000000000001},{"interface":"front","average":{"1min":1.198,"5min":2.1469999999999998,"15min":0.97099999999999997},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.222},"max":{"1min":6.9189999999999996,"5min":23.917999999999999,"15min":23.917999999999999},"last":2.9380000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":4.0579999999999998,"15min":2.1179999999999999},"min":{"1min":0.24199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.1210000000000004,"5min":125.446,"15min":125.446},"last":2.9039999999999999},{"interface":"front","average":{"1min":1.9419999999999999,"5min":5.7489999999999997,"15min":2.4620000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":9.3049999999999997,"5min":182.483,"15min":182.483},"last":2.0569999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.0800000000000001,"15min":1.0960000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":12.896000000000001,"5min":18.757000000000001,"15min":18.757000000000001},"last":1.905},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.35599999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.7829999999999999,"5min":12.097,"15min":12.097},"last":2.3580000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":1.6379999999999999,"15min":0.96399999999999997},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.742,"5min":12.127000000000001,"15min":12.127000000000001},"last":2.423},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.7989999999999999,"15min":1.002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.0090000000000003,"5min":9.0090000000000003,"15min":9.0090000000000003},"last":1.9950000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":1.5940000000000001,"15min":0.96299999999999997},"min":{"1min":0.35399999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.6749999999999998,"5min":12.16,"15min":12.16},"last":2.3929999999999998},{"interface":"front","average":{"1min":1.677,"5min":1.9079999999999999,"15min":1.0800000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.1029999999999998,"5min":10.755000000000001,"15min":10.755000000000001},"last":1.9590000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:46:21 2023","interfaces":[{"interface":"back","average":{"1min":5.875,"5min":2.698,"15min":1.341},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":91.757999999999996,"5min":91.757999999999996,"15min":91.757999999999996},"last":2.02},{"interface":"front","average":{"1min":6.0670000000000002,"5min":3.2410000000000001,"15min":1.4930000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":91.709999999999994,"5min":91.709999999999994,"15min":91.709999999999994},"last":1.8680000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019130,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1442600,"kb_used_data":1133088,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92302552,"statfs":{"total":95995035648,"available":94517813248,"internally_reserved":0,"allocated":1160282112,"data_stored":1159789860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:58 2023","interfaces":[{"interface":"back","average":{"1min":4.7610000000000001,"5min":1.627,"15min":0.95499999999999996},"min":{"1min":0.249,"5min":0.22,"15min":0.22},"max":{"1min":59.457000000000001,"5min":59.457000000000001,"15min":59.457000000000001},"last":1.952},{"interface":"front","average":{"1min":5.5750000000000002,"5min":1.9750000000000001,"15min":1.0820000000000001},"min":{"1min":0.36499999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":60.494,"5min":60.494,"15min":60.494},"last":2.9969999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:46:25 2023","interfaces":[{"interface":"back","average":{"1min":3.609,"5min":2.649,"15min":1.276},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":41.643000000000001,"5min":45.320999999999998,"15min":45.320999999999998},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.855,"5min":2.3860000000000001,"15min":1.1639999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":41.712000000000003,"5min":45.372999999999998,"15min":45.372999999999998},"last":1.8600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:46:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2389999999999999,"5min":4.9619999999999997,"15min":3.4460000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":41.607999999999997,"5min":84.909000000000006,"15min":84.909000000000006},"last":0.33500000000000002},{"interface":"front","average":{"1min":3.2280000000000002,"5min":4.7919999999999998,"15min":3.29},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":41.670000000000002,"5min":83.840000000000003,"15min":83.840000000000003},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:46:25 2023","interfaces":[{"interface":"back","average":{"1min":4.7439999999999998,"5min":3.9620000000000002,"15min":1.879},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":42.020000000000003,"5min":59.552999999999997,"15min":59.552999999999997},"last":1.6399999999999999},{"interface":"front","average":{"1min":4.7699999999999996,"5min":4.1749999999999998,"15min":1.9299999999999999},"min":{"1min":0.40500000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":42.106000000000002,"5min":59.031999999999996,"15min":59.031999999999996},"last":1.806}]},{"osd":5,"last update":"Sat Sep 23 22:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.76000000000000001,"15min":0.71899999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.25900000000000001},"max":{"1min":4.3540000000000001,"5min":4.3540000000000001,"15min":43.189999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.80400000000000005,"15min":0.752},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.29,"5min":9.5190000000000001,"15min":42.450000000000003},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 22:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.79900000000000004,"15min":0.70599999999999996},"min":{"1min":0.314,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":4.5999999999999996,"5min":6.8109999999999999,"15min":27.056000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.75,"15min":0.71699999999999997},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.27600000000000002},"max":{"1min":0.63600000000000001,"5min":4.2469999999999999,"15min":26.940999999999999},"last":1.0880000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:46:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.72699999999999998,"15min":0.64300000000000002},"min":{"1min":0.33600000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.2609999999999999,"5min":5.2889999999999997,"15min":5.2889999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.72199999999999998,"15min":0.61199999999999999},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":1.5900000000000001,"5min":5.9409999999999998,"15min":5.9409999999999998},"last":0.40799999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822902,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2007320,"kb_used_data":1693712,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91737832,"statfs":{"total":95995035648,"available":93939539968,"internally_reserved":0,"allocated":1734361088,"data_stored":1733888259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:54 2023","interfaces":[{"interface":"back","average":{"1min":10.632999999999999,"5min":3.9769999999999999,"15min":1.7370000000000001},"min":{"1min":0.26600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":64.852000000000004,"5min":74.703999999999994,"15min":74.703999999999994},"last":2.5179999999999998},{"interface":"front","average":{"1min":9.9499999999999993,"5min":3.5030000000000001,"15min":1.5780000000000001},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":64.587000000000003,"5min":74.310000000000002,"15min":74.310000000000002},"last":1.9390000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":3.5030000000000001,"15min":1.544},"min":{"1min":0.24299999999999999,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":7.0620000000000003,"5min":149.203,"15min":149.203},"last":2.6030000000000002},{"interface":"front","average":{"1min":1.339,"5min":3.718,"15min":1.593},"min":{"1min":0.29499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":5.7949999999999999,"5min":149.554,"15min":149.554},"last":2.0270000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:46:18 2023","interfaces":[{"interface":"back","average":{"1min":9.8940000000000001,"5min":6.226,"15min":8.1910000000000007},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":158.92599999999999,"5min":158.92599999999999,"15min":158.92599999999999},"last":2.0059999999999998},{"interface":"front","average":{"1min":10.004,"5min":6.3079999999999998,"15min":8.2189999999999994},"min":{"1min":0.33200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":158.91399999999999,"5min":158.91399999999999,"15min":158.91399999999999},"last":1.9099999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:46:18 2023","interfaces":[{"interface":"back","average":{"1min":15.456,"5min":7.4000000000000004,"15min":2.8220000000000001},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":257.05000000000001,"5min":257.05000000000001,"15min":257.05000000000001},"last":1.9610000000000001},{"interface":"front","average":{"1min":15.186,"5min":6.8369999999999997,"15min":2.6890000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":257.35000000000002,"5min":257.35000000000002,"15min":257.35000000000002},"last":2.48}]},{"osd":4,"last update":"Sat Sep 23 22:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.82299999999999995,"15min":0.58799999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":4.1790000000000003,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":2.4660000000000002},{"interface":"front","average":{"1min":1.468,"5min":0.84499999999999997,"15min":0.60099999999999998},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.24099999999999999},"max":{"1min":10.535,"5min":10.535,"15min":10.535},"last":2.319}]},{"osd":6,"last update":"Sat Sep 23 22:46:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.2070000000000001,"15min":1.292},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":4.2329999999999997,"5min":48.360999999999997,"15min":140.07499999999999},"last":1.881},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.3360000000000001,"15min":1.355},"min":{"1min":0.33100000000000002,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":10.662000000000001,"5min":48.292999999999999,"15min":139.739},"last":2.5760000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.93999999999999995,"15min":0.68100000000000005},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":4.2510000000000003,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":2.548},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.77900000000000003,"15min":0.63300000000000001},"min":{"1min":0.34100000000000003,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":4.2720000000000002,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":1.9830000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1621229568,"data_stored":1621229568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1422589952,"data_stored":1422589952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1132986368,"data_stored":1132986368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1173422080,"data_stored":1173422080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1175453696,"data_stored":1175453696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1756889088,"data_stored":1756889088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1175126016,"data_stored":1175126016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:46:57.385 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 16 2023-09-23T22:46:57.386 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:57.417 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:57.425 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:57.473 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:57.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:57.578 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:46:57.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:57 smithi151 ceph-mon[159210]: from='client.55153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:46:57.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:57 smithi151 ceph-mon[159210]: pgmap v581: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T22:46:57.614 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:57.668 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:46:57.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:57.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:46:57.685 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 36 16 55575 55559 96.3725 75.5625 0.0104831 0.0103718 2023-09-23T22:46:57.773 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:46:57.808 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:57.862 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:58.001 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:46:58.017 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:46:58.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:58.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:46:58.052 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:46:58.097 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:46:58.125 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:46:58.142 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:46:58.160 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:58.188 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:46:58.222 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:46:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:58.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:46:58.229 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:46:58.266 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:46:58.315 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:46:58.328 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:58.346 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.346 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.346 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.347 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.347 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.347 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.347 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.347 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:46:58.435 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1817983328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:58.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1817983328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:58.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1817983328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:46:58.573 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:46:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:46:58.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:46:58.684 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:46:58.684 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 37 16 56651 56635 95.5859 67.25 0.0112441 0.0104569 2023-09-23T22:46:58.740 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:46:59.685 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 38 16 57817 57801 94.9884 72.875 0.0105835 0.0105221 2023-09-23T22:46:59.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:59 smithi151 ceph-mon[159210]: pgmap v582: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T22:46:59.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:46:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:46:59.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:59 smithi052 ceph-mon[180979]: pgmap v582: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T22:46:59.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:46:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:46:59.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:59 smithi052 ceph-mon[187743]: pgmap v582: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T22:46:59.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:46:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:47:00.506 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:00.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:00.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:00.611 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:00.658 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:00.694 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 39 16 58812 58796 94.1266 62.1875 0.00802728 0.0106182 2023-09-23T22:47:00.738 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:00.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:00.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:00.846 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:00.892 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:00.928 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:00 smithi151 ceph-mon[159210]: pgmap v583: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T22:47:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:00.959 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:00.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:01.066 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:01.126 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:01.182 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:00 smithi052 ceph-mon[180979]: pgmap v583: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T22:47:01.182 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:00 smithi052 ceph-mon[187743]: pgmap v583: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T22:47:01.204 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:01.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:01.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:01.398 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:01.564 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:01.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:01.675 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:01.694 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:47:01.694218+0000 min lat: 0.0012697 max lat: 1.05393 avg lat: 0.0106843 2023-09-23T22:47:01.694 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:47:01.694 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 40 16 59934 59918 93.5268 70.125 0.0286845 0.0106843 2023-09-23T22:47:01.715 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:01.895 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:01.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:01.985 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:47:01.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3840610938' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:01.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:01.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3840610938' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:01.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:01.999 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3840610938' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:02.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:01 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:02.032 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:02.144 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:02.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:02.252 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:02.360 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:02.567 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:47:02.568 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:47:02.694 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 41 16 61011 60995 92.8878 67.3125 0.0113533 0.0107605 2023-09-23T22:47:02.969 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:47:03.136 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:03.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T22:47:03.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: pgmap v585: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T22:47:03.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:02 smithi151 ceph-mon[159210]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:47:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:47:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: pgmap v585: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[187743]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:47:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: pgmap v585: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T22:47:03.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:02 smithi052 ceph-mon[180979]: failed to encode map e230 with expected crc 2023-09-23T22:47:03.247 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:03.280 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:03.331 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:03.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:03.436 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:03.463 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:03.542 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:03.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:03.650 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:03.699 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 42 16 61979 61963 92.1066 60.5 0.0100416 0.0108514 2023-09-23T22:47:03.722 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:47:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:47:03] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-09-23T22:47:03.741 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:03.992 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:04.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:04.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:04.110 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:04.162 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:04.207 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:04.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:04.314 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:04.365 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:04.365 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:04.369 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:04.369 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:04.374 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:04.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:04.697 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:04.699 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 43 16 62953 62937 91.3809 60.875 0.014805 0.010937 2023-09-23T22:47:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:04.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:04.757 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:47:04.806 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:04.932 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:05.220 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:05.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:05.308 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:05.323 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:05.355 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:05.420 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:05.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:05.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:05 smithi052 ceph-mon[187743]: pgmap v586: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:47:05.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:05.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:05.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:05 smithi052 ceph-mon[180979]: pgmap v586: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:47:05.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:05.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:05.527 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:05.556 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:05 smithi151 ceph-mon[159210]: pgmap v586: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:47:05.556 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:05.556 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:05.559 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:05.614 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:05.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:05.699 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 44 16 63855 63839 90.5859 56.375 0.0114342 0.0110326 2023-09-23T22:47:05.719 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:05.806 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:05.893 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:47:06.035 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:06.147 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:06.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:06.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:06.261 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:06.332 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 8 pgp_num to 16 2023-09-23T22:47:06.340 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1727591560' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:47:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:06 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:47:06.701 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 45 16 64932 64916 90.0641 67.3125 0.0147036 0.0110979 2023-09-23T22:47:06.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1727591560' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:47:06.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:06 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:47:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1727591560' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:47:06.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:06 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:47:07.283 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:07.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:07.396 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:07.432 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:07.500 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:07.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:07.605 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[187743]: pgmap v587: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[187743]: failed to encode map e231 with expected crc 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[187743]: failed to encode map e231 with expected crc 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[187743]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[180979]: pgmap v587: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[180979]: failed to encode map e231 with expected crc 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[180979]: failed to encode map e231 with expected crc 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:47:07.673 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:07 smithi052 ceph-mon[180979]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T22:47:07.676 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:07 smithi151 ceph-mon[159210]: pgmap v587: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T22:47:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:07 smithi151 ceph-mon[159210]: failed to encode map e231 with expected crc 2023-09-23T22:47:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:07 smithi151 ceph-mon[159210]: failed to encode map e231 with expected crc 2023-09-23T22:47:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:47:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:07 smithi151 ceph-mon[159210]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T22:47:07.701 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 46 16 65890 65874 89.4083 59.875 0.00926296 0.0111794 2023-09-23T22:47:08.205 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:08.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:08.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:08.349 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:08.410 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:08.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:08.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:08.518 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:08.551 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:08.627 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:08.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:08.702 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 47 16 67002 66986 88.9849 69.5 0.0112683 0.0112332 2023-09-23T22:47:08.732 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:08.771 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:08.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:08.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:08.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:08.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:09.334 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:09.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:09.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:09.438 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:09.522 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:09.672 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:09.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:09 smithi151 ceph-mon[159210]: pgmap v589: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T22:47:09.705 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 48 16 68086 68070 88.5361 67.75 0.0296706 0.0112885 2023-09-23T22:47:09.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:09 smithi052 ceph-mon[180979]: pgmap v589: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T22:47:09.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:09 smithi052 ceph-mon[187743]: pgmap v589: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T22:47:09.807 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:47:09.816 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:09.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:09.863 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:47:09.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:47:09.924 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:09.960 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:09.962 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:47:10.014 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:47:10.026 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:10.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:10.044 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:47:10.089 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:47:10.131 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:47:10.135 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:47:10.169 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:10.221 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:10.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:68529 8.7 e231/226 8:e098bde7:::benchmark_data_smithi052_337357_object68528:head v 231'4387, mlcod=231'4384)", 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:47:10.214871+0000", 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0036854589999999999, 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037259020000000001, 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:47:10.230 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214871+0000", 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214871+0000", 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:47:10.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214870+0000", 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214894+0000", 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3900000000000002e-05 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214906+0000", 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.166e-05 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214913+0000", 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8550000000000002e-06 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214957+0000", 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3698000000000001e-05 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:10.214977+0000", 2023-09-23T22:47:10.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0058999999999999e-05 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 1 2023-09-23T22:47:10.234 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:47:10.326 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:10.423 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:10.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3802005101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:10.501 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3802005101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:10.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3802005101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:10.705 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 49 16 69046 69030 87.9542 60 0.00716367 0.0113642 2023-09-23T22:47:10.989 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:10.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:10.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:11.099 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:11.130 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:11.307 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:11 smithi052 ceph-mon[180979]: pgmap v590: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T22:47:11.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:11 smithi151 ceph-mon[159210]: pgmap v590: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T22:47:11.705 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:11.705 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 50 16 70070 70054 87.4755 64 0.0289629 0.0114266 2023-09-23T22:47:11.712 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:11 smithi052 ceph-mon[187743]: pgmap v590: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T22:47:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:11.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:11.813 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:11.843 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:11.889 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:11.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:11.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:11.993 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:12.019 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:12.059 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:12.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:12.165 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:12.241 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:12.706 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 51 16 70878 70862 86.7511 50.5 0.01674 0.0115225 2023-09-23T22:47:12.724 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:12.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:12.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:12.834 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:12.884 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:12.944 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:12.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:12.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:13.050 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:13.087 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:13.163 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:13.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:13.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:13.175 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:47:13.175 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:47:13.175 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T22:47:13.176 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:47:13.176 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:47:13.176 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T22:47:13.268 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:13.300 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:13.356 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:13.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:13.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:13.415 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:47:13.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:13 smithi151 ceph-mon[159210]: pgmap v591: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:47:13.464 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:13.494 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:13.526 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:13 smithi052 ceph-mon[180979]: pgmap v591: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:47:13.527 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:47:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:47:13] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.33.4" 2023-09-23T22:47:13.527 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:13 smithi052 ceph-mon[187743]: pgmap v591: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:47:13.550 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:13.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:13.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:13.655 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:13.690 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:13.707 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 52 16 71896 71880 86.3037 63.625 0.00970665 0.0115815 2023-09-23T22:47:13.747 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:13.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:13.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:13.854 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:13.886 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:13.939 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:13.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:14.045 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:14.154 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:14.259 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:14.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:14.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:14.275 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:47:14.294 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:14.365 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:14.365 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:14.382 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:47:14.394 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:14.415 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/948941307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:14.415 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:14 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:14.418 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/948941307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:14.418 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:14 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:14.451 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:14.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:14.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:14.558 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:14.640 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:14.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/948941307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:14 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:14.708 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 53 16 72710 72694 85.6358 50.875 0.0169868 0.0116727 2023-09-23T22:47:15.458 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:15.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:15.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:15.565 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:15 smithi151 ceph-mon[159210]: pgmap v592: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:47:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:15 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:15 smithi151 ceph-mon[159210]: failed to encode map e232 with expected crc 2023-09-23T22:47:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:47:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:15 smithi151 ceph-mon[159210]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T22:47:15.595 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:15 smithi151 ceph-mon[159210]: failed to encode map e232 with expected crc 2023-09-23T22:47:15.598 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:15.665 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:15.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:15.710 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 54 16 73632 73616 85.1133 57.625 0.0153377 0.0117443 2023-09-23T22:47:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[187743]: pgmap v592: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:47:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[187743]: failed to encode map e232 with expected crc 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[187743]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[187743]: failed to encode map e232 with expected crc 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[180979]: pgmap v592: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[180979]: failed to encode map e232 with expected crc 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:47:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[180979]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T22:47:15.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:15 smithi052 ceph-mon[180979]: failed to encode map e232 with expected crc 2023-09-23T22:47:15.775 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:15.883 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:16.167 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:16.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:16.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:16.274 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:16.311 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:16.367 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:16.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:16.473 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:16.545 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:16.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 55 16 74542 74526 84.5985 56.875 0.0140023 0.0118156 2023-09-23T22:47:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:17 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:17 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:17 smithi151 ceph-mon[159210]: pgmap v594: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 993 op/s 2023-09-23T22:47:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:17 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:17 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:17.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:17 smithi052 ceph-mon[180979]: pgmap v594: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 993 op/s 2023-09-23T22:47:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:17 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:17 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:17.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:17 smithi052 ceph-mon[187743]: pgmap v594: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 993 op/s 2023-09-23T22:47:17.586 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:47:17.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 56 16 75417 75401 84.0648 54.6875 0.0212461 0.01189 2023-09-23T22:47:17.826 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:17.893 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:17.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:18.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:18.140 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:18.244 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:18.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:18.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:18.357 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:18.423 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:18 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:18.423 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:18 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:18.423 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:18 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:18.426 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:18 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:18.426 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:18 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:18.426 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:18 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:18.432 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:18.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:18 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:18.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:18 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:18.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:18 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:18.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 57 16 76267 76251 83.5224 53.125 0.0172431 0.0119685 2023-09-23T22:47:19.286 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:47:19.286 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:47:19.286 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:47:19.286 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:47:19.286 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:47:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:19 smithi151 ceph-mon[159210]: pgmap v595: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T22:47:19.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:19 smithi052 ceph-mon[180979]: pgmap v595: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T22:47:19.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:19 smithi052 ceph-mon[187743]: pgmap v595: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T22:47:19.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 58 16 77090 77074 82.9697 51.4375 0.0161087 0.0120472 2023-09-23T22:47:19.788 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:19.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:19.897 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:19.974 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:20.220 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:47:20.221 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "" 2023-09-23T22:47:20.221 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:47:20.579 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:20.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:20.610 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:47:20.688 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:20.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 59 16 77999 77983 82.5262 56.8125 0.00827876 0.012112 2023-09-23T22:47:20.724 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:20.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:20 smithi052 ceph-mon[180979]: pgmap v596: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-09-23T22:47:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:20 smithi052 ceph-mon[187743]: pgmap v596: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-09-23T22:47:20.779 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:20.787 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:20.787 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:20.787 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:20 smithi151 ceph-mon[159210]: pgmap v596: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-09-23T22:47:20.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:20.888 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:20.914 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:20.968 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:20.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:21.074 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:21.113 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:21.180 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:21.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:21.290 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:21.352 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:21.444 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:21.622 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:21.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:21.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:47:21.712898+0000 min lat: 0.0012697 max lat: 1.05393 avg lat: 0.0121751 2023-09-23T22:47:21.713 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:47:21.713 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 60 16 78916 78900 82.1063 57.3125 0.00600207 0.0121751 2023-09-23T22:47:21.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:21.771 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:21.832 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:21.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:21.933 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:21.939 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:21.969 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:47:22.018 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:47:22.059 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:22.072 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:47:22.118 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:47:22.164 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:47:22.181 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/664487078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:22.181 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/664487078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:22.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/664487078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:22.201 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:47:22.248 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:47:22.293 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.315 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:47:22.574 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:22.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:22.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:22.712 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 61 16 79842 79826 81.7093 57.875 0.0200193 0.012233 2023-09-23T22:47:22.722 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:22.797 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:22.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:22.904 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:22.943 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:23.028 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:23.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:23.134 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:23.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:22 smithi052 ceph-mon[187743]: pgmap v597: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-09-23T22:47:23.216 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:22 smithi052 ceph-mon[180979]: pgmap v597: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-09-23T22:47:23.229 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:22 smithi151 ceph-mon[159210]: pgmap v597: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-09-23T22:47:23.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:47:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:47:23] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.33.4" 2023-09-23T22:47:23.713 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 62 16 80665 80649 81.2214 51.4375 0.0115 0.0123067 2023-09-23T22:47:24.065 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:24.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:24.175 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:24.250 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:24.381 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:24.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:24.499 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:24.539 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:24.621 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:24.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:24.713 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 63 16 81646 81630 80.9055 61.3125 0.00539472 0.0123429 2023-09-23T22:47:24.728 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:24.754 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:24.826 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:24.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:24.938 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:25.006 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:25.367 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:25.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:25.479 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:25.519 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:25 smithi151 ceph-mon[159210]: pgmap v598: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-09-23T22:47:25.523 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:25.594 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:25.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:25.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:25.701 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:25.713 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 64 16 82530 82514 80.5049 55.25 0.00857032 0.0124153 2023-09-23T22:47:25.736 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:25 smithi052 ceph-mon[180979]: pgmap v598: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-09-23T22:47:25.745 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:47:25.745 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:47:25.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:25 smithi052 ceph-mon[187743]: pgmap v598: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 898 op/s 2023-09-23T22:47:25.810 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:25.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:25.916 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:25.955 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:26.025 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:26.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:26.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:47:26.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:26.175 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:26.249 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:26.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:26.294 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:47:26.355 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:26.388 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3350559974' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:26.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:26.450 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:26.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:26.485 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:47:26.485 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:47:26.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3350559974' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:26.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:26.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3350559974' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:26.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:26.557 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:26.586 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:26.645 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:26.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:26.713 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 65 16 83436 83420 80.1378 56.625 0.0116051 0.0124737 2023-09-23T22:47:26.751 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:26.844 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:26.984 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:47:27.024 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:27.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:27.128 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:27.162 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:27.205 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:27.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:27.310 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:27.364 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:27.437 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:27.445 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:27 smithi151 ceph-mon[159210]: pgmap v599: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T22:47:27.445 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:27 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:27.446 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:27 smithi151 ceph-mon[159210]: failed to encode map e233 with expected crc 2023-09-23T22:47:27.446 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:27.446 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:27 smithi151 ceph-mon[159210]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T22:47:27.446 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:27 smithi151 ceph-mon[159210]: failed to encode map e233 with expected crc 2023-09-23T22:47:27.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:27.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:27.547 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:27.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[180979]: pgmap v599: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T22:47:27.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:27.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[180979]: failed to encode map e233 with expected crc 2023-09-23T22:47:27.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:27.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[180979]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T22:47:27.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[180979]: failed to encode map e233 with expected crc 2023-09-23T22:47:27.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[187743]: pgmap v599: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T22:47:27.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:27.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[187743]: failed to encode map e233 with expected crc 2023-09-23T22:47:27.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:27.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[187743]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T22:47:27.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:27 smithi052 ceph-mon[187743]: failed to encode map e233 with expected crc 2023-09-23T22:47:27.626 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:27.714 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 66 16 84211 84195 79.6569 48.4375 0.0185127 0.0125471 2023-09-23T22:47:27.751 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:27.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:27.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:27.863 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:27.879 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:27.909 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:27.930 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:27.996 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:28.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:28.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:28.150 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:28.236 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:28.667 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:28.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:28.717 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 67 16 85073 85057 79.2679 53.875 0.0137123 0.0126101 2023-09-23T22:47:28.791 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:28.871 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:29.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:29 smithi052 ceph-mon[180979]: pgmap v601: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-09-23T22:47:29.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:29 smithi052 ceph-mon[187743]: pgmap v601: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-09-23T22:47:29.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:29 smithi151 ceph-mon[159210]: pgmap v601: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-09-23T22:47:29.717 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 68 16 86016 86000 78.9691 58.9375 0.0125635 0.0126584 2023-09-23T22:47:29.750 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:47:30.720 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 69 16 86716 86700 78.457 43.75 0.0176852 0.0127408 2023-09-23T22:47:31.108 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:31.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:31.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:30 smithi151 ceph-mon[159210]: pgmap v602: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2023-09-23T22:47:31.241 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:31.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:30 smithi052 ceph-mon[180979]: pgmap v602: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2023-09-23T22:47:31.250 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:30 smithi052 ceph-mon[187743]: pgmap v602: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2023-09-23T22:47:31.325 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:31.722 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 70 16 87573 87557 78.0988 53.5625 0.00843129 0.0127996 2023-09-23T22:47:31.772 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:31.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:31.885 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:31.977 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:32.042 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:47:32.042 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:47:32.121 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:32.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:32.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:32.229 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:32.253 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:32.315 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:32.460 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:47:32.481 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:32.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:32.507 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:47:32.536 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:47:32.574 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:47:32.589 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:32.619 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:47:32.659 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:47:32.661 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:32.722 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 71 16 88333 88317 77.6681 47.5 0.0246948 0.0128692 2023-09-23T22:47:32.743 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:32.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:32.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:32.840 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:47:32.847 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:32.879 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:32.933 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:32.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:33.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:33.083 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:33.153 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:33.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:33.266 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:33.307 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:47:33.349 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:33.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:33 smithi151 ceph-mon[159210]: pgmap v603: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-09-23T22:47:33.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1913920897' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:47:33.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1228020482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:33.724 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 72 16 89154 89138 77.3005 51.3125 0.00870808 0.0129313 2023-09-23T22:47:33.748 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:47:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:47:33] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.33.4" 2023-09-23T22:47:33.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:33 smithi052 ceph-mon[187743]: pgmap v603: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-09-23T22:47:33.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1913920897' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:47:33.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1228020482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:33.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:33 smithi052 ceph-mon[180979]: pgmap v603: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-09-23T22:47:33.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1913920897' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:47:33.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1228020482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:34.130 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:34.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:34.175 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:47:34.239 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:34.330 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:34.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[187743]: failed to encode map e234 with expected crc 2023-09-23T22:47:34.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1913920897' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:47:34.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[187743]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T22:47:34.360 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[187743]: failed to encode map e234 with expected crc 2023-09-23T22:47:34.360 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[180979]: failed to encode map e234 with expected crc 2023-09-23T22:47:34.360 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1913920897' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:47:34.361 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[180979]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T22:47:34.361 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:34 smithi052 ceph-mon[180979]: failed to encode map e234 with expected crc 2023-09-23T22:47:34.454 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:34.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:34.561 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:34.631 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:34 smithi151 ceph-mon[159210]: failed to encode map e234 with expected crc 2023-09-23T22:47:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1913920897' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:47:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:34 smithi151 ceph-mon[159210]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T22:47:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:34 smithi151 ceph-mon[159210]: failed to encode map e234 with expected crc 2023-09-23T22:47:34.724 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 73 16 90010 89994 76.9747 53.5 0.0109691 0.0129852 2023-09-23T22:47:35.348 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:35.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:35.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:35.457 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:35.483 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:35.525 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:35.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:35.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:35.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:35 smithi151 ceph-mon[159210]: pgmap v605: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-23T22:47:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:35.703 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:35 smithi052 ceph-mon[180979]: pgmap v605: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-23T22:47:35.703 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:35.707 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:35 smithi052 ceph-mon[187743]: pgmap v605: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-23T22:47:35.707 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:35.712 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:35.725 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 74 16 90764 90748 76.5704 47.125 0.027322 0.013055 2023-09-23T22:47:35.728 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:47:35.936 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:35.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:36.048 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:36.078 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:36.136 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:47:36.148 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:36.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:36.256 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:36.368 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:36.522 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:36.571 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:36.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:36.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:36.726 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 75 16 91483 91467 76.1489 44.9375 0.0109284 0.0131261 2023-09-23T22:47:36.732 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:36.789 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:37.133 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:37.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:37.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:37.249 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:37.288 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:37.352 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:47:37.355 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:37.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:37.393 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:37.465 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:37 smithi052 ceph-mon[180979]: pgmap v606: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2023-09-23T22:47:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2648816582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:47:37.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:47:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:37 smithi052 ceph-mon[187743]: pgmap v606: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2023-09-23T22:47:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2648816582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:47:37.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:47:37.498 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:37.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:37.570 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:37 smithi151 ceph-mon[159210]: pgmap v606: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2023-09-23T22:47:37.570 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2648816582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:47:37.570 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:47:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:37.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:37.672 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:37.710 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:37.725 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 76 15 92299 92284 75.8191 51.0625 0.0119834 0.0131837 2023-09-23T22:47:37.786 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:37.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:37.900 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:37.991 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:38.571 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:38.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:38.681 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:38 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:47:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:38 smithi151 ceph-mon[159210]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T22:47:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:38 smithi151 ceph-mon[159210]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:38 smithi151 ceph-mon[159210]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:38 smithi151 ceph-mon[159210]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.707 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:38.729 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 77 16 93018 93002 75.4144 44.875 0.153374 0.0132458 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[187743]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[187743]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[187743]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[187743]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:47:38.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:47:38.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[180979]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T22:47:38.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[180979]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[180979]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:38 smithi052 ceph-mon[180979]: failed to encode map e235 with expected crc 2023-09-23T22:47:38.763 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:38.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:38.869 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:38.911 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:38.998 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:39.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:39.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:39.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:39.130 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:39.170 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:39.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:39.277 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:39.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:39 smithi052 ceph-mon[187743]: pgmap v608: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T22:47:39.346 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:39 smithi052 ceph-mon[180979]: pgmap v608: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T22:47:39.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:39 smithi151 ceph-mon[159210]: pgmap v608: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T22:47:39.719 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:39.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:39.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:39.732 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 78 16 93873 93857 75.1293 53.4375 0.0134687 0.0133053 2023-09-23T22:47:39.830 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:39.862 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:39.928 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:39.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:40.035 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:40.141 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:40.399 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:40.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:40.510 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:40.590 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:40.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 79 16 94675 94659 74.8122 50.125 0.0134499 0.0133617 2023-09-23T22:47:40.932 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:47:40.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:40 smithi052 ceph-mon[187743]: pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s 2023-09-23T22:47:40.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3378876152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:47:40.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:40 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:47:40.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:40 smithi052 ceph-mon[180979]: pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s 2023-09-23T22:47:40.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3378876152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:47:40.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:40 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:47:41.079 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:47:41.079 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:47:41.079 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:47:41.079 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:47:41.079 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:47:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:40 smithi151 ceph-mon[159210]: pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s 2023-09-23T22:47:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3378876152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:47:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:40 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:47:41.323 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:41.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:41.427 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:41.469 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:41.534 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:41.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:41.640 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:41.653 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:47:41.676 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:41.718 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:41.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:41.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:41.733 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:47:41.734155+0000 min lat: 0.0012697 max lat: 1.05393 avg lat: 0.0134122 2023-09-23T22:47:41.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:47:41.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 80 16 95425 95409 74.4633 46.875 0.00550862 0.0134122 2023-09-23T22:47:41.823 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:41.850 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:41.897 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:41.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:41.937 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[180979]: failed to encode map e236 with expected crc 2023-09-23T22:47:41.937 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[180979]: failed to encode map e236 with expected crc 2023-09-23T22:47:41.937 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:47:41.937 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[180979]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T22:47:42.006 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:42.034 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:41 smithi151 ceph-mon[159210]: failed to encode map e236 with expected crc 2023-09-23T22:47:42.035 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:41 smithi151 ceph-mon[159210]: failed to encode map e236 with expected crc 2023-09-23T22:47:42.035 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:47:42.035 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:41 smithi151 ceph-mon[159210]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T22:47:42.037 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:42.078 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:42.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:42.184 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:42.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[187743]: failed to encode map e236 with expected crc 2023-09-23T22:47:42.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[187743]: failed to encode map e236 with expected crc 2023-09-23T22:47:42.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:47:42.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:41 smithi052 ceph-mon[187743]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T22:47:42.225 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:42.562 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:42.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:42.677 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:42.701 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:47:42.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 81 16 96091 96075 74.0582 41.625 0.034968 0.0134973 2023-09-23T22:47:42.764 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:47:42.788 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:43.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:43.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:43.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:42 smithi052 ceph-mon[180979]: pgmap v611: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-09-23T22:47:43.095 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:42 smithi052 ceph-mon[187743]: pgmap v611: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-09-23T22:47:43.145 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:43.259 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:43.323 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:43.356 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:47:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:47:43] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.33.4" 2023-09-23T22:47:43.397 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:43.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:43.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:43.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:42 smithi151 ceph-mon[159210]: pgmap v611: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-09-23T22:47:43.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:43.529 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:47:43.573 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:47:43.618 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:43.644 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:47:43.693 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:47:43.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 82 16 96737 96721 73.6477 40.375 0.0243204 0.0135708 2023-09-23T22:47:43.735 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:47:43.746 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:43.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:43.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:43.774 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:47:43.822 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:47:43.863 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:43.864 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:47:43.882 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.882 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.882 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.882 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.882 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.883 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.883 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.883 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:47:43.900 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:43.956 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:43.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:43.993 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:43.993 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:44.062 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:44.095 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:44.169 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:44.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1534937946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:44.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:44.238 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1534937946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:44.242 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1534937946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:44.276 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:44.358 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:44.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 83 16 97580 97564 73.3953 52.6875 0.0238569 0.0136205 2023-09-23T22:47:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:45 smithi151 ceph-mon[159210]: pgmap v612: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s 2023-09-23T22:47:45.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:45 smithi052 ceph-mon[180979]: pgmap v612: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s 2023-09-23T22:47:45.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:45 smithi052 ceph-mon[187743]: pgmap v612: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s 2023-09-23T22:47:45.587 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:45.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:45.698 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:45.733 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:45.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 84 16 98466 98450 73.1809 55.375 0.0138768 0.0136589 2023-09-23T22:47:45.787 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:45.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:45.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:45.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:45.980 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:46.320 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:46.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:46.427 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:46.505 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:46.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 85 16 99328 99312 72.9539 53.875 0.0148194 0.0137015 2023-09-23T22:47:47.403 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:47.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:47.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:47.593 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:47 smithi052 ceph-mon[187743]: pgmap v613: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T22:47:47.595 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:47 smithi052 ceph-mon[180979]: pgmap v613: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T22:47:47.598 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:47.656 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:47:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:47 smithi151 ceph-mon[159210]: pgmap v613: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2023-09-23T22:47:47.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 86 16 100101 100085 72.6676 48.3125 0.0227507 0.0137561 2023-09-23T22:47:47.971 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:47.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:48.078 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:48.125 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:48.188 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:48.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:48.293 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:48.322 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:48.377 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:48.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:48.489 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:48.608 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:48.635 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:47:48.636 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:47:48.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 87 16 100799 100783 72.334 43.625 0.0120315 0.0138192 2023-09-23T22:47:48.937 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:48.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:49.048 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:49.094 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:49.175 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:49.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:49.283 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:49.318 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:49.327 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:47:49.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:49.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:49.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:49 smithi151 ceph-mon[159210]: pgmap v614: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s 2023-09-23T22:47:49.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2934690410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:49.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:49 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:49.444 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:49.500 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:49.530 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:49.587 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:49.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:49.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:49.694 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:49.723 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:49.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 88 16 101628 101612 72.0998 51.8125 0.0210328 0.0138631 2023-09-23T22:47:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:49 smithi052 ceph-mon[180979]: pgmap v614: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s 2023-09-23T22:47:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2934690410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:49 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:49.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:49 smithi052 ceph-mon[187743]: pgmap v614: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s 2023-09-23T22:47:49.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2934690410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:49.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:49 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:47:49.785 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:49.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:49.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:49.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:50.402 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:47:50.528 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:47:50.534 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:50.548 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:47:50.548 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:47:50.548 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:50.548 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T22:47:50.548 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.548 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/707186164' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:47:50.549 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:50.550 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/707186164' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:47:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:50.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:50.652 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:50.696 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: failed to encode map e237 with expected crc 2023-09-23T22:47:50.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:47:50.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:47:50.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/707186164' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:47:50.699 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:50.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 89 16 102393 102377 71.8271 47.8125 0.00815276 0.0139044 2023-09-23T22:47:50.758 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:50.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:50.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:50.856 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:47:50.866 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:50.895 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:51.419 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:51 smithi151 ceph-mon[159210]: pgmap v616: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 36 MiB/s wr, 569 op/s 2023-09-23T22:47:51.419 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/707186164' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:47:51.419 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:51 smithi151 ceph-mon[159210]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T22:47:51.419 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:51 smithi151 ceph-mon[159210]: failed to encode map e238 with expected crc 2023-09-23T22:47:51.586 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[180979]: pgmap v616: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 36 MiB/s wr, 569 op/s 2023-09-23T22:47:51.586 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/707186164' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:47:51.586 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[180979]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T22:47:51.586 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[180979]: failed to encode map e238 with expected crc 2023-09-23T22:47:51.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[187743]: pgmap v616: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 36 MiB/s wr, 569 op/s 2023-09-23T22:47:51.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/707186164' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:47:51.587 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[187743]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T22:47:51.587 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:51 smithi052 ceph-mon[187743]: failed to encode map e238 with expected crc 2023-09-23T22:47:51.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 90 16 102448 102432 71.0679 3.4375 0.00468692 0.0139 2023-09-23T22:47:51.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T22:47:51.990 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:51.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:51.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:52.094 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:52.126 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:52.197 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:52.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:52.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:52.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:52.333 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:52.482 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:52.484 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[187743]: failed to encode map e238 with expected crc 2023-09-23T22:47:52.484 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[187743]: failed to encode map e238 with expected crc 2023-09-23T22:47:52.484 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[187743]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T22:47:52.484 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[187743]: failed to encode map e239 with expected crc 2023-09-23T22:47:52.484 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[187743]: failed to encode map e239 with expected crc 2023-09-23T22:47:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:52.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:52.486 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[180979]: failed to encode map e238 with expected crc 2023-09-23T22:47:52.486 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[180979]: failed to encode map e238 with expected crc 2023-09-23T22:47:52.486 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[180979]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T22:47:52.486 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[180979]: failed to encode map e239 with expected crc 2023-09-23T22:47:52.486 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:52 smithi052 ceph-mon[180979]: failed to encode map e239 with expected crc 2023-09-23T22:47:52.501 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:47:52.510 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:52.551 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:47:52.574 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:47:52.587 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:52.609 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:47:52.626 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:52.645 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:47:52.680 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:47:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:52 smithi151 ceph-mon[159210]: failed to encode map e238 with expected crc 2023-09-23T22:47:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:52 smithi151 ceph-mon[159210]: failed to encode map e238 with expected crc 2023-09-23T22:47:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:52 smithi151 ceph-mon[159210]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T22:47:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:52 smithi151 ceph-mon[159210]: failed to encode map e239 with expected crc 2023-09-23T22:47:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:52 smithi151 ceph-mon[159210]: failed to encode map e239 with expected crc 2023-09-23T22:47:52.699 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:47:52.729 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:47:52.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 91 16 102448 102432 70.2875 0 - 0.0139 2023-09-23T22:47:52.764 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:47:52.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:47:52.787 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:52.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:52.901 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:52.932 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:52.979 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:52.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:53.084 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:53.113 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:53.465 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:53.467 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:47:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:47:53] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.33.4" 2023-09-23T22:47:53.467 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[187743]: failed to encode map e239 with expected crc 2023-09-23T22:47:53.467 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[187743]: pgmap v619: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-23T22:47:53.467 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[187743]: failed to encode map e240 with expected crc 2023-09-23T22:47:53.467 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[187743]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2422026954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[187743]: failed to encode map e240 with expected crc 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[180979]: failed to encode map e239 with expected crc 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[180979]: pgmap v619: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[180979]: failed to encode map e240 with expected crc 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[180979]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2422026954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:53.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:53 smithi052 ceph-mon[180979]: failed to encode map e240 with expected crc 2023-09-23T22:47:53.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:53.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:53.570 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:47:53.599 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:53 smithi151 ceph-mon[159210]: failed to encode map e239 with expected crc 2023-09-23T22:47:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:53 smithi151 ceph-mon[159210]: pgmap v619: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 666 op/s 2023-09-23T22:47:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:53 smithi151 ceph-mon[159210]: failed to encode map e240 with expected crc 2023-09-23T22:47:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:53 smithi151 ceph-mon[159210]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T22:47:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2422026954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:47:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:53 smithi151 ceph-mon[159210]: failed to encode map e240 with expected crc 2023-09-23T22:47:53.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total time run: 91.2579 2023-09-23T22:47:53.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total writes made: 102448 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Write size: 65536 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object size: 65536 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Bandwidth (MB/sec): 70.1638 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Bandwidth: 33.4081 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max bandwidth (MB/sec): 196.938 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average IOPS: 1122 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev IOPS: 534.53 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max IOPS: 3151 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min IOPS: 0 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average Latency(s): 0.0142428 2023-09-23T22:47:53.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Latency(s): 0.0339124 2023-09-23T22:47:53.738 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max latency(s): 2.31988 2023-09-23T22:47:53.738 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min latency(s): 0.0012697 2023-09-23T22:47:53.741 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:47:53.994 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:47:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:54.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:47:54.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:54.159 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:54.233 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:54.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:54.337 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:54.367 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:54.436 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:102907 8.4 8:2359a180:::benchmark_data_smithi052_337357_object456:head [delete] snapc 0=[] ondisk+write+known_if_redirected e240)", 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:47:54.430880+0000", 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0051821810000000001, 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052063279999999997, 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:47:54.437 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 102907 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.430880+0000", 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.438 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.430880+0000", 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.430878+0000", 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7839999999999998e-06 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.439 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.430887+0000", 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3643e-05 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.430901+0000", 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2819999999999996e-06 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.430909+0000", 2023-09-23T22:47:54.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3219000000000003e-05 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.431002+0000", 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000138708 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.431141+0000", 2023-09-23T22:47:54.441 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012898900000000001 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.431270+0000", 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035800469999999998 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:47:54.434850+0000", 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039704700000000002 2023-09-23T22:47:54.442 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 1 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:47:54.443 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:54.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:54 smithi151 ceph-mon[159210]: failed to encode map e240 with expected crc 2023-09-23T22:47:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:54.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:54.546 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:54.588 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:54.654 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:54.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:54 smithi052 ceph-mon[187743]: failed to encode map e240 with expected crc 2023-09-23T22:47:54.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:54 smithi052 ceph-mon[180979]: failed to encode map e240 with expected crc 2023-09-23T22:47:54.762 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:54.789 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:54.831 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:54.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:54.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:54.858 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:54.935 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:54.968 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:55.205 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:47:55.206 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:55.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:55.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:55.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:55.347 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:55.347 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:55 smithi052 ceph-mon[180979]: failed to encode map e240 with expected crc 2023-09-23T22:47:55.347 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:55 smithi052 ceph-mon[180979]: pgmap v621: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 112 B/s, 0 objects/s recovering 2023-09-23T22:47:55.348 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:55 smithi052 ceph-mon[187743]: failed to encode map e240 with expected crc 2023-09-23T22:47:55.348 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:55 smithi052 ceph-mon[187743]: pgmap v621: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 112 B/s, 0 objects/s recovering 2023-09-23T22:47:55.409 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:55.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:55 smithi151 ceph-mon[159210]: failed to encode map e240 with expected crc 2023-09-23T22:47:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:55 smithi151 ceph-mon[159210]: pgmap v621: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 740 op/s; 112 B/s, 0 objects/s recovering 2023-09-23T22:47:55.517 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:47:55.599 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:55.642 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:47:55.808 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:47:55.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:55.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:47:55.902 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:47:55.915 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:55.952 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:56.059 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:56.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:56.166 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:47:56.200 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:56.291 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T22:47:56.311 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:47:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:56.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:47:56.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:56.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1156911472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:47:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:56 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:47:56.450 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:56.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1156911472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:47:56.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:56 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:47:56.670 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1156911472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:47:56.671 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:56 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:47:56.689 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:56.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:56.801 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:56.828 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:56.930 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:56.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:57.034 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:47:57.104 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:57.226 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:47:57.226 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:47:57.226 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:47:57.226 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:47:57.226 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:47:57.235 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:47:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:57.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:47:57.328 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:47:57.347 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:57.382 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:57.397 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[180979]: pgmap v622: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 170 B/s rd, 35 MiB/s wr, 697 op/s; 6539/304947 objects misplaced (2.144%); 842 KiB/s, 13 objects/s recovering 2023-09-23T22:47:57.397 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[180979]: failed to encode map e241 with expected crc 2023-09-23T22:47:57.398 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:47:57.398 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[180979]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T22:47:57.398 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[187743]: pgmap v622: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 170 B/s rd, 35 MiB/s wr, 697 op/s; 6539/304947 objects misplaced (2.144%); 842 KiB/s, 13 objects/s recovering 2023-09-23T22:47:57.398 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[187743]: failed to encode map e241 with expected crc 2023-09-23T22:47:57.398 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:47:57.398 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:57 smithi052 ceph-mon[187743]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T22:47:57.444 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:57.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:57.547 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:57.548 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:57.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:57 smithi151 ceph-mon[159210]: pgmap v622: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 170 B/s rd, 35 MiB/s wr, 697 op/s; 6539/304947 objects misplaced (2.144%); 842 KiB/s, 13 objects/s recovering 2023-09-23T22:47:57.575 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:57 smithi151 ceph-mon[159210]: failed to encode map e241 with expected crc 2023-09-23T22:47:57.575 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:57 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:47:57.575 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:57 smithi151 ceph-mon[159210]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T22:47:57.577 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:57.629 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:57.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:57.735 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:57.824 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:57.946 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:57.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:57.994 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:47:58.060 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:47:58.152 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:58.360 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:47:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:58.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:47:58.421 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:47:58.478 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:58.518 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:58.562 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:58.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:58.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:58.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:58.700 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:58.743 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:58.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:58.845 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:58.897 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:59.058 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:47:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:59.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:47:59.161 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:59.195 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:59.249 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:59.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:59.253 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:59.337 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:47:59.353 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:47:59.383 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:59 smithi052 ceph-mon[180979]: pgmap v624: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 152 B/s rd, 8.4 MiB/s wr, 342 op/s; 6539/303249 objects misplaced (2.156%); 754 KiB/s, 12 objects/s recovering 2023-09-23T22:47:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/60447156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:47:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:59.504 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:59 smithi052 ceph-mon[187743]: pgmap v624: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 152 B/s rd, 8.4 MiB/s wr, 342 op/s; 6539/303249 objects misplaced (2.156%); 754 KiB/s, 12 objects/s recovering 2023-09-23T22:47:59.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/60447156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:59.505 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:47:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:59.570 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:47:59.577 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:59 smithi151 ceph-mon[159210]: pgmap v624: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 152 B/s rd, 8.4 MiB/s wr, 342 op/s; 6539/303249 objects misplaced (2.156%); 754 KiB/s, 12 objects/s recovering 2023-09-23T22:47:59.577 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/60447156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:59.577 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:47:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:47:59.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:59.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:47:59.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:47:59.714 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:47:59.820 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:47:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:47:59.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:47:59.926 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:47:59.966 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:00.056 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:00.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:00.083 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:48:00.083 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:48:00.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:48:00.083 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:48:00.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40802 6.e 6:7b05ccce:::benchmark_data_smithi052_234456_object18658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 410.21678836400002, 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533, 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40802 2023-09-23T22:48:00.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862704+0000", 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0949999999999996e-06 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862712+0000", 2023-09-23T22:48:00.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2490000000000001e-05 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862725+0000", 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.041e-06 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862732+0000", 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036689000000000001 2023-09-23T22:48:00.087 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866401+0000", 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0216e-05 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866411+0000", 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99412088600000004 2023-09-23T22:48:00.088 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860532+0000", 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3456e-05 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860545+0000", 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2107602909999999 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.089 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071306+0000", 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014411400000000001 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071450+0000", 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017796199999999999 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071628+0000", 2023-09-23T22:48:00.090 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036812979999999999 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075309+0000", 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001368813 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076678+0000", 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064926749999999998 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.091 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083170+0000", 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9931999999999994e-05 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083240+0000", 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034446200000000002 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.092 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083585+0000", 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40804 6.e 6:7187807f:::benchmark_data_smithi052_234456_object18660:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 410.21404279500001, 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001, 2023-09-23T22:48:00.093 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40804 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.094 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865450+0000", 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4629999999999998e-06 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.095 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865457+0000", 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1907e-05 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865469+0000", 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1515000000000001e-05 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865480+0000", 2023-09-23T22:48:00.096 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096401899999999999 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866444+0000", 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8879999999999997e-06 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866452+0000", 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413119599999999 2023-09-23T22:48:00.097 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860583+0000", 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7849999999999997e-06 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860592+0000", 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.212291711 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.098 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072884+0000", 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011587900000000001 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073000+0000", 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073105+0000", 2023-09-23T22:48:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002208621 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075313+0000", 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020589470000000002 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.077372+0000", 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062710200000000004 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083643+0000", 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9950000000000002e-05 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083683+0000", 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000162177 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.101 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083845+0000", 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40805 6.e 6:798273ba:::benchmark_data_smithi052_234456_object18661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 410.21400585499998, 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999, 2023-09-23T22:48:00.102 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40805 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.103 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.104 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865488+0000", 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6610000000000002e-06 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865492+0000", 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7939999999999997e-06 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.105 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865499+0000", 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7509999999999999e-06 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865504+0000", 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097241299999999999 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866476+0000", 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6120000000000001e-06 2023-09-23T22:48:00.106 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866484+0000", 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413304199999997 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860617+0000", 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3080000000000002e-06 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.107 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860624+0000", 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.213177682 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073802+0000", 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8152000000000005e-05 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073900+0000", 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106062 2023-09-23T22:48:00.108 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.074006+0000", 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022883959999999998 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076294+0000", 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002385603 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.109 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.078680+0000", 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052328410000000002 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083913+0000", 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2397000000000005e-05 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083985+0000", 2023-09-23T22:48:00.110 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026911300000000002 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.084254+0000", 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.111 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40809 6.2 6:400fe117:::benchmark_data_smithi052_234456_object18665:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 410.21312653400003, 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002, 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40809 2023-09-23T22:48:00.112 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:48:00.113 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866367+0000", 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8289999999999996e-06 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866373+0000", 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1122e-05 2023-09-23T22:48:00.114 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866384+0000", 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8669999999999996e-06 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866391+0000", 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000202492 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.115 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866594+0000", 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7580000000000004e-06 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866603+0000", 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.994511967 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861115+0000", 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6169999999999996e-06 2023-09-23T22:48:00.116 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861125+0000", 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.209770244 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.070895+0000", 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013600900000000001 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.117 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071031+0000", 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121377 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071153+0000", 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016472430000000001 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072800+0000", 2023-09-23T22:48:00.118 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000377547 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073177+0000", 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0096328860000000002 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082810+0000", 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6335999999999996e-05 2023-09-23T22:48:00.119 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082877+0000", 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000259743 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083136+0000", 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.120 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39271 7.1 7:803b19ab:::benchmark_data_smithi052_277165_object39270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.323904868, 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002, 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39271 2023-09-23T22:48:00.121 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.122 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755588+0000", 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100054 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755689+0000", 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.123 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755701+0000", 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3199999999999999e-06 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755707+0000", 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9419000000000002e-05 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755756+0000", 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5290000000000008e-06 2023-09-23T22:48:00.124 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755765+0000", 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99249923500000004 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748264+0000", 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6199999999999994e-06 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.125 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748273+0000", 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.538374364 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286648+0000", 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000173475 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286821+0000", 2023-09-23T22:48:00.126 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024065800000000001 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.287062+0000", 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003676038 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290738+0000", 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3826999999999998e-05 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.127 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290792+0000", 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2153e-05 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290804+0000", 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5701000000000002e-05 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.128 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290850+0000", 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093742900000000004 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291787+0000", 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.129 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39291 7.1 7:8a0225f8:::benchmark_data_smithi052_277165_object39290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.30919609400001, 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001, 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39291 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.130 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.131 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770297+0000", 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5182e-05 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770362+0000", 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498999999999998e-05 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770378+0000", 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0129999999999996e-06 2023-09-23T22:48:00.132 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770385+0000", 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2045000000000007e-05 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770447+0000", 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1433999999999999e-05 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.133 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770458+0000", 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97785107400000004 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748309+0000", 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8459999999999996e-06 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.134 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748315+0000", 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5410234169999999 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289338+0000", 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117113 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289455+0000", 2023-09-23T22:48:00.135 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000150991 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289606+0000", 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023599210000000001 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291966+0000", 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4410000000000004e-06 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.136 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291974+0000", 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8967999999999994e-05 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292043+0000", 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7804000000000003e-05 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.137 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292111+0000", 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00075040600000000001 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292861+0000", 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.138 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39293 7.1 7:8d7e8869:::benchmark_data_smithi052_277165_object39292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.30577794999999, 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908, 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.139 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39293 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:48:00.140 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773716+0000", 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2105000000000003e-05 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773778+0000", 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2449000000000001e-05 2023-09-23T22:48:00.141 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773790+0000", 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6099999999999997e-06 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773796+0000", 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9072000000000001e-05 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.142 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773835+0000", 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773843+0000", 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97449834000000002 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.143 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748341+0000", 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5279999999999997e-06 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748346+0000", 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.544637525 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.144 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292983+0000", 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016144799999999999 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293145+0000", 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015888799999999999 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293304+0000", 2023-09-23T22:48:00.145 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039379849999999997 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297242+0000", 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6013000000000001e-05 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297288+0000", 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003665958 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.146 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300954+0000", 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7888999999999999e-05 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.301032+0000", 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012321579999999999 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.147 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302264+0000", 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39318 7.1 7:887a21fd:::benchmark_data_smithi052_277165_object39317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.28424432600002, 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997, 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.148 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39318 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.149 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795249+0000", 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1755000000000002e-05 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.150 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795301+0000", 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3233e-05 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795314+0000", 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1940000000000001e-06 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.151 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795321+0000", 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1218999999999999e-05 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795372+0000", 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0147e-05 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795383+0000", 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.95298535699999998 2023-09-23T22:48:00.152 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748368+0000", 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2440000000000004e-06 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748372+0000", 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5462721529999999 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.153 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294644+0000", 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112796 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294757+0000", 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018578900000000001 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.154 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294943+0000", 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023897250000000001 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297333+0000", 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297340+0000", 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050090029999999997 2023-09-23T22:48:00.155 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302349+0000", 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3208999999999999e-05 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302392+0000", 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00080982299999999999 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.156 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303202+0000", 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39334 7.1 7:83eed48f:::benchmark_data_smithi052_277165_object39333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.27308473900001, 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001, 2023-09-23T22:48:00.157 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39334 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.158 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806408+0000", 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8466e-05 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806457+0000", 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3062e-05 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806470+0000", 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2000000000000002e-06 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.160 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806475+0000", 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9246999999999998e-05 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806524+0000", 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0499999999999997e-06 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806533+0000", 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94188278000000003 2023-09-23T22:48:00.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748416+0000", 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.52e-06 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748423+0000", 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5475971120000001 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296021+0000", 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015821300000000001 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296179+0000", 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013869899999999999 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296317+0000", 2023-09-23T22:48:00.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045890180000000003 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300906+0000", 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023854169999999999 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303292+0000", 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028062899999999999 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306098+0000", 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2596999999999999e-05 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306141+0000", 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068390599999999997 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.165 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306825+0000", 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39346 7.1 7:8a73189a:::benchmark_data_smithi052_277165_object39345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.26812903199999, 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001, 2023-09-23T22:48:00.166 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39346 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.167 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811364+0000", 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012578900000000001 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.168 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811490+0000", 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8567999999999999e-05 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811508+0000", 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9329999999999996e-06 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.169 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811518+0000", 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8912000000000001e-05 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811577+0000", 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1018e-05 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811588+0000", 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93686738199999997 2023-09-23T22:48:00.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748455+0000", 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0320000000000003e-06 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748460+0000", 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5488962179999999 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297357+0000", 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010246 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297459+0000", 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012592199999999999 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297585+0000", 2023-09-23T22:48:00.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004696787 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302282+0000", 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001097813 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303380+0000", 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034811529999999999 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306861+0000", 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9079000000000001e-05 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306920+0000", 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074140899999999995 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.174 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.307661+0000", 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39360 7.1 7:8f91ddcd:::benchmark_data_smithi052_277165_object39359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.25673663700002, 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001, 2023-09-23T22:48:00.175 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39360 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.176 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822756+0000", 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0686999999999998e-05 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.177 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822807+0000", 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4600000000000001e-05 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822821+0000", 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2920000000000001e-06 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822828+0000", 2023-09-23T22:48:00.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2878000000000003e-05 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822901+0000", 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9390000000000003e-06 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822909+0000", 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92559491500000002 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748504+0000", 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4299999999999997e-06 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748510+0000", 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498821280000001 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298392+0000", 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1963e-05 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298484+0000", 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011828300000000001 2023-09-23T22:48:00.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298602+0000", 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036917579999999998 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302294+0000", 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011534850000000001 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303447+0000", 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049678559999999997 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308415+0000", 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0241000000000001e-05 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308476+0000", 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00084742900000000002 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309323+0000", 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.184 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39366 7.1 7:86dcbc64:::benchmark_data_smithi052_277165_object39365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.25310744000001, 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001, 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.185 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39366 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:48:00.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826386+0000", 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4935999999999997e-05 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826441+0000", 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2864000000000001e-05 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826454+0000", 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7969999999999997e-06 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826460+0000", 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5853000000000003e-05 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826505+0000", 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0890000000000006e-06 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826514+0000", 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92203426200000005 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748549+0000", 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9800000000000003e-06 2023-09-23T22:48:00.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748555+0000", 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.55081775 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299372+0000", 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155606 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299528+0000", 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021651799999999999 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299745+0000", 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035645 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303309+0000", 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021267000000000001 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303522+0000", 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0058789619999999997 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309401+0000", 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0636999999999995e-05 2023-09-23T22:48:00.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309471+0000", 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00099162699999999992 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310463+0000", 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.193 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39373 7.1 7:85f7eb6c:::benchmark_data_smithi052_277165_object39372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.25002991899999, 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812, 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.194 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39373 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:00.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829464+0000", 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1137000000000002e-05 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829515+0000", 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3482000000000001e-05 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829528+0000", 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.959e-06 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829534+0000", 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5408999999999999e-05 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829570+0000", 2023-09-23T22:48:00.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2763e-05 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829582+0000", 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.91900465799999997 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748587+0000", 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7450000000000003e-06 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748592+0000", 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.554972391 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303564+0000", 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012435699999999999 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303689+0000", 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000131853 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303820+0000", 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004451455 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308272+0000", 2023-09-23T22:48:00.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0795000000000003e-05 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308333+0000", 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022271669999999999 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310560+0000", 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7315000000000001e-05 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310637+0000", 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083538299999999998 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311473+0000", 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.202 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39414 7.1 7:8075013a:::benchmark_data_smithi052_277165_object39413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.21786924399998, 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001, 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.203 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39414 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:00.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861624+0000", 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4787000000000002e-05 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861678+0000", 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3568e-05 2023-09-23T22:48:00.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861692+0000", 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1750000000000002e-06 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861698+0000", 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9217e-05 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861747+0000", 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0080000000000002e-06 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861755+0000", 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88686667100000005 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748622+0000", 2023-09-23T22:48:00.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0063e-05 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748632+0000", 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.562879379 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311512+0000", 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018041500000000001 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311692+0000", 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020370699999999999 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311896+0000", 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077409310000000004 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319637+0000", 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1062000000000006e-05 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319698+0000", 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103898 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319802+0000", 2023-09-23T22:48:00.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7586e-05 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319869+0000", 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017220569999999999 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321591+0000", 2023-09-23T22:48:00.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39416 7.1 7:83076ecc:::benchmark_data_smithi052_277165_object39415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.21647998100002, 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074, 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.212 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39416 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863014+0000", 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106535 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863120+0000", 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1805e-05 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863132+0000", 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4430000000000002e-06 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.215 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863138+0000", 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0098999999999997e-05 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863178+0000", 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0545e-05 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863188+0000", 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88547513300000003 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748663+0000", 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.507e-06 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748669+0000", 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5646240410000001 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313293+0000", 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110319 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313403+0000", 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196856 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313600+0000", 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060436789999999997 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319644+0000", 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011535 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319759+0000", 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019084060000000001 2023-09-23T22:48:00.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321668+0000", 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2410999999999996e-05 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321730+0000", 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011838980000000001 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322914+0000", 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39440 7.1 7:846ccd4b:::benchmark_data_smithi052_277165_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.19600467499998, 2023-09-23T22:48:00.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002, 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39440 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.222 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883489+0000", 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0798999999999997e-05 2023-09-23T22:48:00.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883569+0000", 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5083999999999999e-05 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883594+0000", 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1099999999999999e-06 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883601+0000", 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5002e-05 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883646+0000", 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5539999999999998e-06 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883654+0000", 2023-09-23T22:48:00.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86504454099999994 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748699+0000", 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0530000000000003e-06 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748705+0000", 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.566234793 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.314939+0000", 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014568900000000001 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315085+0000", 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019602700000000001 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315281+0000", 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004426915 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319708+0000", 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032665569999999998 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322975+0000", 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5554e-05 2023-09-23T22:48:00.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323070+0000", 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1078999999999998e-05 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323111+0000", 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095274200000000002 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324064+0000", 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39442 7.1 7:821661b1:::benchmark_data_smithi052_277165_object39441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.19296707799998, 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999, 2023-09-23T22:48:00.230 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39442 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999895 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886517+0000", 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0640000000000002e-05 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886578+0000", 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4535999999999999e-05 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886592+0000", 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.024e-06 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886599+0000", 2023-09-23T22:48:00.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5583e-05 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886655+0000", 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9129999999999992e-06 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886664+0000", 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86206608799999995 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748730+0000", 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6200000000000001e-06 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748737+0000", 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.567891876 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316628+0000", 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014458900000000001 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316773+0000", 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000198579 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316972+0000", 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027449810000000001 2023-09-23T22:48:00.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319717+0000", 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033197000000000001 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323036+0000", 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001091297 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324128+0000", 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0636999999999997e-05 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324168+0000", 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071410600000000005 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324882+0000", 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39446 7.1 7:8475cdd4:::benchmark_data_smithi052_277165_object39445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.18968785099997, 2023-09-23T22:48:00.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001, 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39446 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:48:00.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889806+0000", 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7126000000000002e-05 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889853+0000", 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2593e-05 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889865+0000", 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889871+0000", 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7049999999999998e-05 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889918+0000", 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5479999999999998e-06 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889924+0000", 2023-09-23T22:48:00.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.858839715 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748764+0000", 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5000000000000001e-06 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748768+0000", 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5693916369999998 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318160+0000", 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016406600000000001 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318324+0000", 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000199247 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318523+0000", 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044604579999999996 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322984+0000", 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019935700000000001 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324977+0000", 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4416e-05 2023-09-23T22:48:00.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325052+0000", 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9657000000000003e-05 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325092+0000", 2023-09-23T22:48:00.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00089603700000000003 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325988+0000", 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39458 7.1 7:846e660e:::benchmark_data_smithi052_277165_object39457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.17361118299999, 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002, 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39458 2023-09-23T22:48:00.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:00.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905882+0000", 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7948000000000002e-05 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905940+0000", 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1592e-05 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905952+0000", 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4750000000000001e-06 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905957+0000", 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2416000000000001e-05 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906010+0000", 2023-09-23T22:48:00.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4849999999999992e-06 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906018+0000", 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.84277395099999997 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748792+0000", 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748796+0000", 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5772432570000001 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326040+0000", 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015276800000000001 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326193+0000", 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018607500000000001 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326379+0000", 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038150480000000001 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.330194+0000", 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001090159 2023-09-23T22:48:00.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331284+0000", 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0065274460000000001 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337811+0000", 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1335999999999999e-05 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337853+0000", 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014175889999999999 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339270+0000", 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39462 7.1 7:8cf1a796:::benchmark_data_smithi052_277165_object39461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 349.16305047100002, 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001, 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39462 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.259 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916441+0000", 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5293000000000001e-05 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916497+0000", 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3080000000000002e-05 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916530+0000", 2023-09-23T22:48:00.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4319e-05 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916544+0000", 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018227799999999999 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916726+0000", 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3705999999999999e-05 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916760+0000", 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.83205244499999997 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748813+0000", 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5040000000000004e-06 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748817+0000", 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.578915206 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327732+0000", 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115865 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327848+0000", 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000123075 2023-09-23T22:48:00.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327971+0000", 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033208170000000002 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331292+0000", 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0080585329999999997 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339351+0000", 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1529e-05 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339412+0000", 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7031000000000002e-05 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339479+0000", 2023-09-23T22:48:00.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010848730000000001 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.340564+0000", 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:00.267 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:48:00.268 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:00.300 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:00 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:48:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:00 smithi151 ceph-mon[159210]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T22:48:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:00 smithi151 ceph-mon[159210]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:00 smithi151 ceph-mon[159210]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:00 smithi151 ceph-mon[159210]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.699 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:48:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:48:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[180979]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T22:48:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[180979]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[180979]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[180979]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:48:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[187743]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T22:48:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[187743]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[187743]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:00 smithi052 ceph-mon[187743]: failed to encode map e242 with expected crc 2023-09-23T22:48:00.745 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:48:00.786 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:48:00.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:48:00.821 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:48:00.856 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:48:00.894 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:48:00.942 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:48:00.943 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:00.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:00.983 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:48:01.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.011 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.012 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.012 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.012 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.012 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:48:01.058 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:01.157 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:01.280 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:01.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[187743]: failed to encode map e242 with expected crc 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[187743]: failed to encode map e242 with expected crc 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[187743]: pgmap v626: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 132 B/s rd, 2.0 MiB/s wr, 266 op/s; 6539/302004 objects misplaced (2.165%); 656 KiB/s, 10 objects/s recovering 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/547436054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[180979]: failed to encode map e242 with expected crc 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[180979]: failed to encode map e242 with expected crc 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[180979]: pgmap v626: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 132 B/s rd, 2.0 MiB/s wr, 266 op/s; 6539/302004 objects misplaced (2.165%); 656 KiB/s, 10 objects/s recovering 2023-09-23T22:48:01.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/547436054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:01 smithi151 ceph-mon[159210]: failed to encode map e242 with expected crc 2023-09-23T22:48:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:01 smithi151 ceph-mon[159210]: failed to encode map e242 with expected crc 2023-09-23T22:48:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:01 smithi151 ceph-mon[159210]: pgmap v626: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 132 B/s rd, 2.0 MiB/s wr, 266 op/s; 6539/302004 objects misplaced (2.165%); 656 KiB/s, 10 objects/s recovering 2023-09-23T22:48:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/547436054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:02.209 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:02.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:02.320 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:02.356 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:02.424 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:02.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:02.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:02 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 99/296241 objects degraded (0.033%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:02.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:02 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 99/296241 objects degraded (0.033%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:02.531 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:02.568 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:02.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:02 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 99/296241 objects degraded (0.033%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:02.690 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:02.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:02.793 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:02.887 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:02.902 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:02.902 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:02.904 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:48:03.016 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:03.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:03.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:03.164 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:03.222 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:03.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:03.325 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:03.419 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:03.524 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:03.536 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:48:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:48:03] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.33.4" 2023-09-23T22:48:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:03.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:03.537 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:03 smithi052 ceph-mon[187743]: pgmap v627: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 1.9 MiB/s wr, 496 op/s; 99/296241 objects degraded (0.033%); 6360/296241 objects misplaced (2.147%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:03.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:03 smithi052 ceph-mon[180979]: pgmap v627: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 1.9 MiB/s wr, 496 op/s; 99/296241 objects degraded (0.033%); 6360/296241 objects misplaced (2.147%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:03.638 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:03.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:03 smithi151 ceph-mon[159210]: pgmap v627: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 1.9 MiB/s wr, 496 op/s; 99/296241 objects degraded (0.033%); 6360/296241 objects misplaced (2.147%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:03.677 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:48:03.745 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:03.798 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:03.897 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:03.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:03.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:04.005 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:04.042 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:04.124 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:04.124 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:04.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:04.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:04.234 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:04.329 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:04.460 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:48:04.488 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1337265268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:48:04.488 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:04 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:48:04.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1337265268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:48:04.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:04 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:48:04.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:48:04.675 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1337265268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:48:04.675 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:04 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:48:05.503 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: pgmap v628: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 99/293904 objects degraded (0.034%); 6360/293904 objects misplaced (2.164%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:05.504 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: failed to encode map e243 with expected crc 2023-09-23T22:48:05.504 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:48:05.504 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T22:48:05.504 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: failed to encode map e243 with expected crc 2023-09-23T22:48:05.504 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:05.504 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:05.730 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:05.730 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: pgmap v628: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 99/293904 objects degraded (0.034%); 6360/293904 objects misplaced (2.164%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: failed to encode map e243 with expected crc 2023-09-23T22:48:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: failed to encode map e243 with expected crc 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: pgmap v628: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 99/293904 objects degraded (0.034%); 6360/293904 objects misplaced (2.164%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: failed to encode map e243 with expected crc 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: failed to encode map e243 with expected crc 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:06.252 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:06.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:06.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:06.279 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:48:06.376 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:06.412 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:48:06.480 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:06.698 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:06.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:06.811 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:06.834 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:06.847 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:06.884 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:06.900 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:06.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:07.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:07.075 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:07.304 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:07.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:07.413 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:07.444 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:07 smithi052 ceph-mon[180979]: pgmap v630: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 230/286854 objects degraded (0.080%); 6149/286854 objects misplaced (2.144%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:07.444 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2886807009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:07.444 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:07.445 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:07 smithi052 ceph-mon[187743]: pgmap v630: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 230/286854 objects degraded (0.080%); 6149/286854 objects misplaced (2.144%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:07.445 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2886807009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:07.445 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:07.467 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:48:07.471 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:07.479 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:07 smithi151 ceph-mon[159210]: pgmap v630: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 230/286854 objects degraded (0.080%); 6149/286854 objects misplaced (2.144%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2886807009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:08.404 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:08.405 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:08.509 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:08.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:08.622 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:08 smithi151 ceph-mon[159210]: pgmap v632: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 230/284664 objects degraded (0.081%); 6149/284664 objects misplaced (2.160%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[187743]: pgmap v632: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 230/284664 objects degraded (0.081%); 6149/284664 objects misplaced (2.160%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: failed to encode map e244 with expected crc 2023-09-23T22:48:08.714 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:08 smithi052 ceph-mon[180979]: pgmap v632: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 230/284664 objects degraded (0.081%); 6149/284664 objects misplaced (2.160%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:08.719 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:09.118 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:48:09.339 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:09.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:09.444 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:09.532 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:09.568 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:09.576 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:09 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 230/284664 objects degraded (0.081%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:09.580 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:09 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 230/284664 objects degraded (0.081%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:09.793 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:09 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 230/284664 objects degraded (0.081%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:09.872 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:09.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:09.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:09.985 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:10.096 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:10.206 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:10.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:10.242 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:48:10.304 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:48:10.316 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:10.335 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:48:10.344 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:10.360 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:48:10.405 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:48:10.420 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:10.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:10.527 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:10.556 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:10.557 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:48:10.699 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:10.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:10.804 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:10.909 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:11.149 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:11.159 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3602337236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:11.159 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:10 smithi052 ceph-mon[180979]: pgmap v633: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 230/283632 objects degraded (0.081%); 6149/283632 objects misplaced (2.168%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:11.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3602337236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:11.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:10 smithi052 ceph-mon[187743]: pgmap v633: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 230/283632 objects degraded (0.081%); 6149/283632 objects misplaced (2.168%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:11.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:11.177 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:48:11.177 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:48:11.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.177 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:113152 8.5 8:ac9f7a39:::benchmark_data_smithi052_337357_object10701:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-09-23T22:48:11.177 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:48:11.166274+0000", 2023-09-23T22:48:11.177 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0053826409999999996, 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054054899999999998, 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 113152 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166274+0000", 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166274+0000", 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:11.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166273+0000", 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0555e-05 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166283+0000", 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3636e-05 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166297+0000", 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5740000000000001e-06 2023-09-23T22:48:11.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166303+0000", 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5198000000000003e-05 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166349+0000", 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0001281 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166477+0000", 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016962800000000001 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166646+0000", 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000372462 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:113156 8.9 8:9556e0b9:::benchmark_data_smithi052_337357_object10705:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:48:11.166314+0000", 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0053421279999999998, 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054231349999999999, 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 113156 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.183 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166314+0000", 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166314+0000", 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166314+0000", 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.371e-06 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166317+0000", 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2250000000000003e-06 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166324+0000", 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.702e-06 2023-09-23T22:48:11.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166329+0000", 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4966000000000002e-05 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166384+0000", 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013950099999999999 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166524+0000", 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015928700000000001 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166683+0000", 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00036851400000000002 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:113157 8.9 8:9476660f:::benchmark_data_smithi052_337357_object10706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:48:11.166349+0000", 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0053076290000000003, 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054718479999999996, 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 113157 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.188 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166349+0000", 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166349+0000", 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166348+0000", 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.298e-06 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166352+0000", 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6239999999999996e-06 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:11.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166358+0000", 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1520000000000002e-06 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.166362+0000", 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039176369999999999 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170280+0000", 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015004100000000001 2023-09-23T22:48:11.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170430+0000", 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000134091 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170564+0000", 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0042152520000000001 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:11.192 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45206.0:113148 8.6 e244/219 8:6a6d04d5:::benchmark_data_smithi052_337357_object10697:head v 244'6941, mlcod=244'6939)", 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:48:11.170367+0000", 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0012897309999999999, 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001508131, 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:11.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170367+0000", 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170367+0000", 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170365+0000", 2023-09-23T22:48:11.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0669999999999998e-06 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170374+0000", 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2309e-05 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170387+0000", 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0971999999999999e-05 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170398+0000", 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9089000000000003e-05 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170457+0000", 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.592e-05 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:11.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:48:11.170473+0000", 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105945 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 4 2023-09-23T22:48:11.197 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:48:11.197 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3602337236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:11.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:10 smithi151 ceph-mon[159210]: pgmap v633: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 230/283632 objects degraded (0.081%); 6149/283632 objects misplaced (2.168%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:11.260 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:11.299 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:11.361 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:11.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:11.456 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:48:11.464 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:11.532 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:11.932 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:48:11.985 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:11.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:12.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:12.133 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:12.205 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:12.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:12.311 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:12.385 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:12.397 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:12.417 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:12.551 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:12.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:12.663 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:12.697 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:12.959 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:12.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:13.063 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:13.097 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:13.149 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:13.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:13.255 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:13.339 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:48:13.344 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:13 smithi052 ceph-mon[187743]: pgmap v634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 339/278061 objects degraded (0.122%); 5960/278061 objects misplaced (2.143%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:13.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1817075150' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:48:13.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:48:13.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:13 smithi052 ceph-mon[180979]: pgmap v634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 339/278061 objects degraded (0.122%); 5960/278061 objects misplaced (2.143%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:13.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1817075150' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:48:13.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:48:13.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:13.479 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:13.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:13.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:13.587 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:13.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:13 smithi151 ceph-mon[159210]: pgmap v634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 339/278061 objects degraded (0.122%); 5960/278061 objects misplaced (2.143%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:13.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1817075150' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:48:13.616 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:48:13.620 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:13.675 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:13.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:48:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:48:13] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T22:48:13.780 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:13.821 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:13.895 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:48:13.904 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:13.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:13.983 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:48:13.983 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:48:13.983 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:48:13.984 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:48:13.984 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:48:14.011 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:14.041 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:14 smithi151 ceph-mon[159210]: failed to encode map e245 with expected crc 2023-09-23T22:48:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:14 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:48:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:14 smithi151 ceph-mon[159210]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T22:48:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:14 smithi151 ceph-mon[159210]: failed to encode map e245 with expected crc 2023-09-23T22:48:14.738 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[180979]: failed to encode map e245 with expected crc 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[180979]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[180979]: failed to encode map e245 with expected crc 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[187743]: failed to encode map e245 with expected crc 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[187743]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T22:48:14.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:14 smithi052 ceph-mon[187743]: failed to encode map e245 with expected crc 2023-09-23T22:48:14.939 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:14.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:15.035 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:48:15.056 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:15.164 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:15.183 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:15.305 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:15.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:15.418 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:15.443 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:15.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:15 smithi052 ceph-mon[180979]: pgmap v636: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 339/275790 objects degraded (0.123%); 5960/275790 objects misplaced (2.161%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:15.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 339/275790 objects degraded (0.123%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:15.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:15 smithi052 ceph-mon[187743]: pgmap v636: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 339/275790 objects degraded (0.123%); 5960/275790 objects misplaced (2.161%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:15.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 339/275790 objects degraded (0.123%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:15.536 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:15.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:15 smithi151 ceph-mon[159210]: pgmap v636: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 339/275790 objects degraded (0.123%); 5960/275790 objects misplaced (2.161%); 640 KiB/s, 9 objects/s recovering 2023-09-23T22:48:15.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 339/275790 objects degraded (0.123%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:15.668 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:15.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:15.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:15.772 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:15.885 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:15.968 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:15.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:16.071 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:16.173 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:16.300 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:16.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:16.367 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:48:16.404 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:16.449 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:16.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3061405875' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:48:16.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3061405875' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:48:16.515 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:16.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:16.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:16.588 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3061405875' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:48:16.622 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:16.666 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:16.739 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:16.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:16.838 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:48:16.838 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:48:16.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:48:16.838 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:48:16.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10176 6.c e128/99 6:30687e13:::benchmark_data_smithi052_234456_object10175:head v 128'611, mlcod=128'609)", 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.63567070800002, 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26259265100000001, 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:48:16.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200005+0000", 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4515999999999999e-05 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200049+0000", 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0868e-05 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200060+0000", 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7479999999999999e-06 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200069+0000", 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8922999999999999e-05 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200118+0000", 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1420000000000002e-05 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200139+0000", 2023-09-23T22:48:16.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26240534999999998 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462545+0000", 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3596e-05 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462598+0000", 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26259265100000001 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10178 6.3 e128/104 6:cbb8686c:::benchmark_data_smithi052_234456_object10177:head v 128'684, mlcod=128'682)", 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.635408799, 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26233074499999998, 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200267+0000", 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2943999999999999e-05 2023-09-23T22:48:16.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200290+0000", 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0489000000000001e-05 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200300+0000", 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4090000000000002e-06 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200305+0000", 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000490297 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200796+0000", 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2308999999999999e-05 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200818+0000", 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085734699999999995 2023-09-23T22:48:16.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201675+0000", 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26092291299999998 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462598+0000", 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26233074499999998 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10170 6.1 e128/104 6:8aa77601:::benchmark_data_smithi052_234456_object10169:head v 128'666, mlcod=128'664)", 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.63489186100003, 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26275960799999998, 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1285999999999997e-05 2023-09-23T22:48:16.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200825+0000", 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6779999999999995e-06 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200835+0000", 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.976e-06 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200841+0000", 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0885999999999999e-05 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200902+0000", 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1832999999999999e-05 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200923+0000", 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26257463399999997 2023-09-23T22:48:16.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463498+0000", 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6022000000000001e-05 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463544+0000", 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26275960799999998 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10180 6.1 e128/104 6:8dfddb97:::benchmark_data_smithi052_234456_object10179:head v 128'667, mlcod=128'664)", 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.63480422700002, 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26309316700000002, 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:48:16.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9139e-05 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200901+0000", 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2439999999999996e-06 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200908+0000", 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200913+0000", 2023-09-23T22:48:16.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045587999999999998 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201369+0000", 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8995e-05 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201388+0000", 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26253594800000002 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463924+0000", 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1671000000000001e-05 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463965+0000", 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26309316700000002 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10183 6.1 e128/104 6:8243606a:::benchmark_data_smithi052_234456_object10182:head v 128'668, mlcod=128'664)", 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.63356768099999, 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261887543, 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202108+0000", 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0642000000000002e-05 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202148+0000", 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26041415499999998 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462563+0000", 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6589999999999996e-06 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462571+0000", 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052397000000000001 2023-09-23T22:48:16.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463095+0000", 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2568000000000001e-05 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463118+0000", 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085566799999999997 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463973+0000", 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2731999999999999e-05 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463996+0000", 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261887543 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10181 6.3 e128/104 6:c0620c37:::benchmark_data_smithi052_234456_object10180:head v 128'685, mlcod=128'682)", 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.63308519600002, 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26238228600000002, 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202590+0000", 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5381999999999999e-05 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202616+0000", 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25993996400000002 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462556+0000", 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0231000000000001e-05 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462566+0000", 2023-09-23T22:48:16.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2098000000000001e-05 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462628+0000", 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0216999999999999e-05 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462658+0000", 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002274767 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464933+0000", 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0459e-05 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464973+0000", 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26238228600000002 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10192 6.a e128/126 6:5a145b56:::benchmark_data_smithi052_234456_object10191:head v 128'636, mlcod=128'634)", 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 442.63015582600002, 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.258119451, 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205519+0000", 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8529000000000001e-05 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205548+0000", 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25701668599999999 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462565+0000", 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9289999999999998e-06 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462573+0000", 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4702000000000004e-05 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462647+0000", 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1945000000000001e-05 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462669+0000", 2023-09-23T22:48:16.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093275200000000002 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463602+0000", 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8049000000000001e-05 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463640+0000", 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.258119451 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61219 7.d 7:b3f702b2:::benchmark_data_smithi052_277165_object61218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 343.76656636299998, 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23283454100000001, 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.874 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61219 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:16.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069109+0000", 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8376000000000001e-05 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069137+0000", 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0046e-05 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069147+0000", 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3259999999999997e-06 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069151+0000", 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9551999999999997e-05 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069191+0000", 2023-09-23T22:48:16.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112429 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069303+0000", 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114738 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069418+0000", 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003836182 2023-09-23T22:48:16.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073254+0000", 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022611850000000002 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075516+0000", 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226299794 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301815+0000", 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7392999999999995e-05 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301903+0000", 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1786000000000003e-05 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301945+0000", 2023-09-23T22:48:16.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23283454100000001 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61222 7.d 7:b9a46085:::benchmark_data_smithi052_277165_object61221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 343.76651737100002, 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23289720799999999, 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.881 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61222 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069158+0000", 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8464999999999998e-05 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069197+0000", 2023-09-23T22:48:16.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3069999999999999e-06 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069203+0000", 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0140000000000001e-06 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069210+0000", 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002195729 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071406+0000", 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010020499999999999 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071506+0000", 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013088800000000001 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071637+0000", 2023-09-23T22:48:16.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018844230000000001 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073521+0000", 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020741359999999999 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075596+0000", 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226399296 2023-09-23T22:48:16.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301995+0000", 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6539999999999999e-05 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.302031+0000", 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4808e-05 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.302056+0000", 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23289720799999999 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61226 7.7 7:ebd42417:::benchmark_data_smithi052_277165_object61225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 343.75956550199999, 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.227997757, 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.888 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61226 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076110+0000", 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9333999999999999e-05 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076159+0000", 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1613e-05 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076171+0000", 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129999999999996e-06 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076175+0000", 2023-09-23T22:48:16.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6888000000000002e-05 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076222+0000", 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129424 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076352+0000", 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8393000000000005e-05 2023-09-23T22:48:16.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076450+0000", 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023315779999999999 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.078782+0000", 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001019966 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.079801+0000", 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.224238032 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304040+0000", 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3377000000000002e-05 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304083+0000", 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5698e-05 2023-09-23T22:48:16.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304109+0000", 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.227997757 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100776 7.c 7:350d58b3:::benchmark_data_smithi052_277165_object8284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:48:16.895 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 286.25897139199998, 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999, 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100776 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576703+0000", 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0799999999999995e-06 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576712+0000", 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2679e-05 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576725+0000", 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6329999999999999e-06 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576731+0000", 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18361677800000001 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760348+0000", 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000140829 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760489+0000", 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016031800000000001 2023-09-23T22:48:16.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760649+0000", 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010853742 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.771503+0000", 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023432639000000002 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.794936+0000", 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035400660000000001 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798476+0000", 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5828999999999999e-05 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798511+0000", 2023-09-23T22:48:16.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012960090000000001 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.799807+0000", 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100781 7.c 7:39965936:::benchmark_data_smithi052_277165_object8289:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:48:16.902 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 286.25169274799998, 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999, 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100781 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583981+0000", 2023-09-23T22:48:16.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.59e-05 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583996+0000", 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.174193922 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758190+0000", 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2031000000000001e-05 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758202+0000", 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049640919999999998 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763167+0000", 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143024 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763310+0000", 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016401100000000001 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763474+0000", 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012004406 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.775478+0000", 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019532911 2023-09-23T22:48:16.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.795011+0000", 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072752770000000001 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802286+0000", 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0192000000000002e-05 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802346+0000", 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018569999999999999 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.804203+0000", 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101958 7.7 7:e819df4a:::benchmark_data_smithi052_277165_object9466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 284.30894328900001, 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001, 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101958 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.910 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526731+0000", 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.396e-06 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526739+0000", 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3361e-05 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526752+0000", 2023-09-23T22:48:16.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2920000000000003e-06 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526757+0000", 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1291000000000002e-05 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526799+0000", 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121802 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526920+0000", 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014018900000000001 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527061+0000", 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033778649999999999 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530439+0000", 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000232978 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530671+0000", 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21847482400000001 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749146+0000", 2023-09-23T22:48:16.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749200+0000", 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5618000000000003e-05 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749245+0000", 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162201 7.c 7:3f19fea2:::benchmark_data_smithi052_277165_object69709:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 179.58042365599999, 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.27262330099999998, 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.917 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162201 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:16.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255252+0000", 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.712e-06 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255255+0000", 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8510000000000001e-06 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255259+0000", 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5349999999999999e-06 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255261+0000", 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1329999999999999e-05 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255303+0000", 2023-09-23T22:48:16.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132652 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255435+0000", 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012794600000000001 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255563+0000", 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040626919999999997 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.259626+0000", 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060452930000000002 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.265671+0000", 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261175247 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.526847+0000", 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4647000000000005e-05 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.526921+0000", 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095477599999999995 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.527876+0000", 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.27262330099999998 2023-09-23T22:48:16.923 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162214 7.c 7:33fa1cb0:::benchmark_data_smithi052_277165_object69722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 179.56440241199999, 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25716776299999999, 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.924 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162214 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271272+0000", 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9519999999999994e-06 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271280+0000", 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2356e-05 2023-09-23T22:48:16.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271292+0000", 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9319999999999999e-06 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271299+0000", 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8116999999999998e-05 2023-09-23T22:48:16.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271377+0000", 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014364799999999999 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271521+0000", 2023-09-23T22:48:16.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014726999999999999 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271668+0000", 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089242190000000006 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280593+0000", 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6284000000000004e-05 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280659+0000", 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24732958699999999 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.527989+0000", 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4438999999999998e-05 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528043+0000", 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000398756 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528442+0000", 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25716776299999999 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.932 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162220 7.c 7:33dba7f1:::benchmark_data_smithi052_277165_object69728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 179.55964516700001, 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25290671399999998, 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162220 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276030+0000", 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2030000000000003e-06 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276037+0000", 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4914e-05 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276052+0000", 2023-09-23T22:48:16.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3910000000000003e-06 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276058+0000", 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010739300000000001 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276166+0000", 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018878300000000001 2023-09-23T22:48:16.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276355+0000", 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000217147 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276572+0000", 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041362409999999997 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280708+0000", 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2092e-05 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280720+0000", 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24777231399999999 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528492+0000", 2023-09-23T22:48:16.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9494999999999999e-05 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528552+0000", 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00038610499999999998 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528938+0000", 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25290671399999998 2023-09-23T22:48:16.939 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164712 7.7 7:e35a7399:::benchmark_data_smithi052_277165_object72220:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 175.43925884699999, 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29594756100000003, 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.940 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164712 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396416+0000", 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6749999999999997e-06 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396425+0000", 2023-09-23T22:48:16.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1613000000000001e-05 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396447+0000", 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6890000000000004e-06 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396455+0000", 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021833180000000001 2023-09-23T22:48:16.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398638+0000", 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139655 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398778+0000", 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015481100000000001 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398932+0000", 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045151879999999998 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.403448+0000", 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.288777124 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692225+0000", 2023-09-23T22:48:16.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9673999999999999e-05 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692274+0000", 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0642000000000004e-05 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692365+0000", 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29594756100000003 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.946 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:168558 7.c 7:3f3c3116:::benchmark_data_smithi052_277165_object76066:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 169.49576874900001, 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.244409458, 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.947 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 168558 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:48:16.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339906+0000", 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0329999999999997e-06 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339915+0000", 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3118999999999999e-05 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339928+0000", 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8990000000000001e-06 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339936+0000", 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8579999999999998e-05 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339994+0000", 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000148231 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.340142+0000", 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017634200000000001 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.340319+0000", 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037420520000000001 2023-09-23T22:48:16.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344061+0000", 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056251469999999996 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.349686+0000", 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23382455599999999 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.583510+0000", 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4010999999999993e-05 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.583584+0000", 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073256099999999995 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584317+0000", 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.244409458 2023-09-23T22:48:16.953 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:168562 7.c 7:3c1c89f1:::benchmark_data_smithi052_277165_object76070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 169.49113692700001, 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.240143207, 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.954 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 168562 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4640000000000002e-06 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344542+0000", 2023-09-23T22:48:16.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0480000000000004e-06 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344548+0000", 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1690000000000002e-06 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344553+0000", 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3362999999999996e-05 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344616+0000", 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014696300000000001 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344763+0000", 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018186 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:48:16.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344945+0000", 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039275380000000004 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.348872+0000", 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096992300000000003 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.349842+0000", 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.234521651 2023-09-23T22:48:16.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584364+0000", 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3619999999999998e-05 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584418+0000", 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026506700000000002 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584683+0000", 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.240143207 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:48297 8.9 8:9a4ce8fa:::benchmark_data_smithi052_337357_object48296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 85.887750971000003, 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25518236, 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:48:16.961 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 48297 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:48:16.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947924+0000", 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9459000000000002e-05 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947974+0000", 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9210000000000004e-06 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947984+0000", 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2860000000000001e-06 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:48:16.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947988+0000", 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4461000000000001e-05 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948022+0000", 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6387999999999999e-05 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948119+0000", 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9592999999999993e-05 2023-09-23T22:48:16.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948218+0000", 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001431501 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.949650+0000", 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00027939799999999998 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.949929+0000", 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25307412600000001 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203003+0000", 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0455000000000003e-05 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:48:16.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203064+0000", 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3832999999999999e-05 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203108+0000", 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25518236 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:48:16.968 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:48:16.969 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:17.014 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:17.084 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:17.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:17.188 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:17.224 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:17.429 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:17.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:17.535 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:17.549 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:17 smithi151 ceph-mon[159210]: pgmap v637: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 437/269163 objects degraded (0.162%); 5777/269163 objects misplaced (2.146%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:48:17.549 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:17 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:17.549 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:17 smithi151 ceph-mon[159210]: failed to encode map e246 with expected crc 2023-09-23T22:48:17.549 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3061405875' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:48:17.549 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:17 smithi151 ceph-mon[159210]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T22:48:17.562 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:48:17.563 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[180979]: pgmap v637: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 437/269163 objects degraded (0.162%); 5777/269163 objects misplaced (2.146%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:48:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[180979]: failed to encode map e246 with expected crc 2023-09-23T22:48:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3061405875' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:48:17.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[180979]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T22:48:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[187743]: pgmap v637: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 437/269163 objects degraded (0.162%); 5777/269163 objects misplaced (2.146%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:48:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[187743]: failed to encode map e246 with expected crc 2023-09-23T22:48:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3061405875' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:48:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:17 smithi052 ceph-mon[187743]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T22:48:17.908 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:17.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:18.014 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:18.020 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:18.031 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:18.045 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:18.115 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:18.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:18.223 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:18.265 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:18.343 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:18.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:18.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:18.488 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:18.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:18.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:18 smithi151 ceph-mon[159210]: failed to encode map e246 with expected crc 2023-09-23T22:48:18.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:18 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:18.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:18.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:18 smithi052 ceph-mon[187743]: failed to encode map e246 with expected crc 2023-09-23T22:48:18.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:18 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:18.663 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:18 smithi052 ceph-mon[180979]: failed to encode map e246 with expected crc 2023-09-23T22:48:18.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:18 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:48:18.672 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:18.685 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:48:18.702 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:18.725 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:48:18.765 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:18.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:18.782 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:48:18.829 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:48:18.863 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:48:18.874 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:18.901 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:18.903 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:48:18.940 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:48:18.953 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:18.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:18.967 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.000 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:48:19.056 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:19.158 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:19.350 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:19 smithi052 ceph-mon[180979]: pgmap v639: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 437/267015 objects degraded (0.164%); 5777/267015 objects misplaced (2.164%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:48:19.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:19 smithi151 ceph-mon[159210]: pgmap v639: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 437/267015 objects degraded (0.164%); 5777/267015 objects misplaced (2.164%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:48:19.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1552318440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:19.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:19 smithi052 ceph-mon[187743]: pgmap v639: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 437/267015 objects degraded (0.164%); 5777/267015 objects misplaced (2.164%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:48:19.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1552318440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:19.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1552318440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:19.786 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:19.786 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:19.841 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:19.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:19.950 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:19.981 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:20.060 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:48:20.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:20.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 437/267015 objects degraded (0.164%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:20.380 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:20.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 437/267015 objects degraded (0.164%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:20.498 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:20.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:20.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:20.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:48:20.607 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:20.627 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:20.627 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 437/267015 objects degraded (0.164%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:20.693 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:21.023 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:21.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:21.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:21.128 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:21.164 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:21.270 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:21.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:21.302 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:21.379 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:21.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:21 smithi052 ceph-mon[180979]: pgmap v640: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 437/265929 objects degraded (0.164%); 5777/265929 objects misplaced (2.172%); 680 KiB/s, 10 objects/s recovering 2023-09-23T22:48:21.410 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:21 smithi151 ceph-mon[159210]: pgmap v640: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 437/265929 objects degraded (0.164%); 5777/265929 objects misplaced (2.172%); 680 KiB/s, 10 objects/s recovering 2023-09-23T22:48:21.516 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:21.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:21.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:21.561 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:21.620 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:21.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:21 smithi052 ceph-mon[187743]: pgmap v640: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 437/265929 objects degraded (0.164%); 5777/265929 objects misplaced (2.172%); 680 KiB/s, 10 objects/s recovering 2023-09-23T22:48:21.718 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:21.840 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:21.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:21.956 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:22.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:22.176 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:22.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:22.282 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:22.317 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:22.413 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:48:22.505 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:48:22.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1570352032' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:48:22.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:22 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:48:22.568 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1570352032' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:48:22.568 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:22 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:48:22.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1570352032' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:48:22.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:22 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:48:23.288 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:23.289 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:23.568 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:23.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: pgmap v641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 542/260367 objects degraded (0.208%); 5587/260367 objects misplaced (2.146%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.581 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:23 smithi151 ceph-mon[159210]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: pgmap v641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 542/260367 objects degraded (0.208%); 5587/260367 objects misplaced (2.146%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[180979]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.662 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:48:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:48:23] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: pgmap v641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 542/260367 objects degraded (0.208%); 5587/260367 objects misplaced (2.146%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:23 smithi052 ceph-mon[187743]: failed to encode map e247 with expected crc 2023-09-23T22:48:23.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:23.713 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:24.098 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:24.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:24.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:24.187 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:48:24.201 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:24.257 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:24.414 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:48:24.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:48:24.440 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:24.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:24 smithi151 ceph-mon[159210]: failed to encode map e247 with expected crc 2023-09-23T22:48:24.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:24.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:24.529 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:24 smithi052 ceph-mon[180979]: failed to encode map e247 with expected crc 2023-09-23T22:48:24.529 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:24 smithi052 ceph-mon[187743]: failed to encode map e247 with expected crc 2023-09-23T22:48:24.561 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:24.619 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:24.696 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:24.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:24.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:24.735 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:48:24.764 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:24.804 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:24.900 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:25.050 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:25.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:25.161 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:25.290 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:25.430 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:25.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:25.470 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:25.549 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:25.594 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:25.660 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:25.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:25 smithi151 ceph-mon[159210]: pgmap v643: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 542/258099 objects degraded (0.210%); 5587/258099 objects misplaced (2.165%); 679 KiB/s, 10 objects/s recovering 2023-09-23T22:48:25.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:25.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:25.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 542/258099 objects degraded (0.210%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:25.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:25.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[187743]: pgmap v643: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 542/258099 objects degraded (0.210%); 5587/258099 objects misplaced (2.165%); 679 KiB/s, 10 objects/s recovering 2023-09-23T22:48:25.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:25.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:25.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 542/258099 objects degraded (0.210%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:25.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[180979]: pgmap v643: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 542/258099 objects degraded (0.210%); 5587/258099 objects misplaced (2.165%); 679 KiB/s, 10 objects/s recovering 2023-09-23T22:48:25.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:25.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:25.691 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 542/258099 objects degraded (0.210%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:25.765 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:25.850 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:26.389 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:26.389 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:26.504 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:48:26.554 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:26.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:26.666 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2038223227' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:26 smithi052 ceph-mon[187743]: pgmap v644: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 614/251850 objects degraded (0.244%); 5427/251850 objects misplaced (2.155%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2038223227' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:26.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:26.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:26 smithi052 ceph-mon[180979]: pgmap v644: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 614/251850 objects degraded (0.244%); 5427/251850 objects misplaced (2.155%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:26.765 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2038223227' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:26.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:26 smithi151 ceph-mon[159210]: pgmap v644: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 614/251850 objects degraded (0.244%); 5427/251850 objects misplaced (2.155%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:27.063 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:48:27.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:48:27.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:27.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[187743]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T22:48:27.491 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[187743]: failed to encode map e248 with expected crc 2023-09-23T22:48:27.603 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:27.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:27.722 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:27.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:48:27.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:27.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[180979]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T22:48:27.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:27 smithi052 ceph-mon[180979]: failed to encode map e248 with expected crc 2023-09-23T22:48:27.820 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:27 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:48:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:27 smithi151 ceph-mon[159210]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T22:48:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:27 smithi151 ceph-mon[159210]: failed to encode map e248 with expected crc 2023-09-23T22:48:28.249 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:28.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:28.329 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:48:28.368 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:28.380 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:48:28.426 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:48:28.460 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:48:28.467 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:28.507 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:48:28.539 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:48:28.575 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:48:28.614 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:48:28.636 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.636 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.636 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.636 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.637 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.637 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.637 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.637 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:48:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:28 smithi052 ceph-mon[187743]: pgmap v646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 614/249663 objects degraded (0.246%); 5427/249663 objects misplaced (2.174%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:48:28.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1259135091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:28 smithi052 ceph-mon[180979]: pgmap v646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 614/249663 objects degraded (0.246%); 5427/249663 objects misplaced (2.174%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:48:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1259135091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:28 smithi151 ceph-mon[159210]: pgmap v646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 614/249663 objects degraded (0.246%); 5427/249663 objects misplaced (2.174%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:48:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1259135091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:29.111 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:29.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:29.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:29.217 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:29.285 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:29.399 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:29.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:29.517 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:29.548 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:29.593 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:29.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:29.697 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:29.738 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:29.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:29.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:29.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:29.909 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:29.942 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:30.003 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:30.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:30.110 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:30.187 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:29 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 614/249663 objects degraded (0.246%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:30.229 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:48:30.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:29 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 614/249663 objects degraded (0.246%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:30.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:29 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 614/249663 objects degraded (0.246%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:30.275 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:30.532 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:30.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:30.643 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:30.672 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:30.718 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:30.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:30.825 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:30.948 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:31.103 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:48:31.128 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:48:31.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:31 smithi052 ceph-mon[187743]: pgmap v647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 614/248541 objects degraded (0.247%); 5427/248541 objects misplaced (2.184%); 704 KiB/s, 10 objects/s recovering 2023-09-23T22:48:31.132 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:31 smithi052 ceph-mon[180979]: pgmap v647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 614/248541 objects degraded (0.247%); 5427/248541 objects misplaced (2.184%); 704 KiB/s, 10 objects/s recovering 2023-09-23T22:48:31.234 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:31.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:31.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:31.347 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:31.416 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:31 smithi151 ceph-mon[159210]: pgmap v647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 614/248541 objects degraded (0.247%); 5427/248541 objects misplaced (2.184%); 704 KiB/s, 10 objects/s recovering 2023-09-23T22:48:32.251 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:48:32.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:32 smithi151 ceph-mon[159210]: from='client.45362 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:32.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:32.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:32.477 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:32 smithi052 ceph-mon[187743]: from='client.45362 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:32.477 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:32.477 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:32.478 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:32 smithi052 ceph-mon[180979]: from='client.45362 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:32.478 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:32.478 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:32.508 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:32.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:32.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:32.730 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:33.023 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:33.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:33.062 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":647,"stamp":"2023-09-23T22:48:30.225639+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5428418488,"num_objects":82847,"num_object_clones":0,"num_object_copies":248541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":614,"num_objects_misplaced":5427,"num_objects_unfound":0,"num_objects_dirty":82847,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":122335,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":612,"num_bytes_recovered":38021194,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119469,"ondisk_log_size":119469,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17960392,"kb_used_data":16028752,"kb_used_omap":129,"kb_used_meta":1931390,"kb_avail":638255672,"statfs":{"total":671965249536,"available":653573808128,"internally_reserved":0,"allocated":16413442048,"data_stored":16408288465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":1977744093},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-258342912,"num_objects":-3942,"num_object_clones":0,"num_object_copies":-11826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":72,"num_objects_misplaced":-160,"num_objects_unfound":0,"num_objects_dirty":-3942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3942,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":5767168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004880"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":244,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351343+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:48:16.351343+0000","last_peered":"2023-09-23T22:48:16.351343+0000","last_clean":"2023-09-23T22:48:16.351343+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:48:16.351343+0000","last_undegraded":"2023-09-23T22:48:16.351343+0000","last_fullsized":"2023-09-23T22:48:16.351343+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":219,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.356616+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:48:16.356616+0000","last_peered":"2023-09-23T22:48:16.356616+0000","last_clean":"2023-09-23T22:48:16.356616+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:48:16.356616+0000","last_undegraded":"2023-09-23T22:48:16.356616+0000","last_fullsized":"2023-09-23T22:48:16.356616+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355170+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:48:16.355170+0000","last_peered":"2023-09-23T22:48:16.355170+0000","last_clean":"2023-09-23T22:48:16.355170+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:48:16.355170+0000","last_undegraded":"2023-09-23T22:48:16.355170+0000","last_fullsized":"2023-09-23T22:48:16.355170+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":215,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440706+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:48:22.440706+0000","last_peered":"2023-09-23T22:48:22.440706+0000","last_clean":"2023-09-23T22:48:22.440706+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:48:22.440706+0000","last_undegraded":"2023-09-23T22:48:22.440706+0000","last_fullsized":"2023-09-23T22:48:22.440706+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":278,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355070+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:48:16.355070+0000","last_peered":"2023-09-23T22:48:16.355070+0000","last_clean":"2023-09-23T22:48:16.355070+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:48:16.355070+0000","last_undegraded":"2023-09-23T22:48:16.355070+0000","last_fullsized":"2023-09-23T22:48:16.355070+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355798+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:48:16.355798+0000","last_peered":"2023-09-23T22:48:16.355798+0000","last_clean":"2023-09-23T22:48:16.355798+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:48:16.355798+0000","last_undegraded":"2023-09-23T22:48:16.355798+0000","last_fullsized":"2023-09-23T22:48:16.355798+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440378+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:48:22.440378+0000","last_peered":"2023-09-23T22:48:22.440378+0000","last_clean":"2023-09-23T22:48:22.440378+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:48:22.440378+0000","last_undegraded":"2023-09-23T22:48:22.440378+0000","last_fullsized":"2023-09-23T22:48:22.440378+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":246,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355099+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:48:16.355099+0000","last_peered":"2023-09-23T22:48:16.355099+0000","last_clean":"2023-09-23T22:48:16.355099+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:48:16.355099+0000","last_undegraded":"2023-09-23T22:48:16.355099+0000","last_fullsized":"2023-09-23T22:48:16.355099+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":215,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.356798+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:48:16.356798+0000","last_peered":"2023-09-23T22:48:16.356798+0000","last_clean":"2023-09-23T22:48:16.356798+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:48:16.356798+0000","last_undegraded":"2023-09-23T22:48:16.356798+0000","last_fullsized":"2023-09-23T22:48:16.356798+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":204,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355073+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:48:16.355073+0000","last_peered":"2023-09-23T22:48:16.355073+0000","last_clean":"2023-09-23T22:48:16.355073+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:48:16.355073+0000","last_undegraded":"2023-09-23T22:48:16.355073+0000","last_fullsized":"2023-09-23T22:48:16.355073+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":121,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.356539+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:48:16.356539+0000","last_peered":"2023-09-23T22:48:16.356539+0000","last_clean":"2023-09-23T22:48:16.356539+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:48:16.356539+0000","last_undegraded":"2023-09-23T22:48:16.356539+0000","last_fullsized":"2023-09-23T22:48:16.356539+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355283+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:48:16.355283+0000","last_peered":"2023-09-23T22:48:16.355283+0000","last_clean":"2023-09-23T22:48:16.355283+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:48:16.355283+0000","last_undegraded":"2023-09-23T22:48:16.355283+0000","last_fullsized":"2023-09-23T22:48:16.355283+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":267,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351211+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:48:16.351211+0000","last_peered":"2023-09-23T22:48:16.351211+0000","last_clean":"2023-09-23T22:48:16.351211+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:48:16.351211+0000","last_undegraded":"2023-09-23T22:48:16.351211+0000","last_fullsized":"2023-09-23T22:48:16.351211+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351585+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:48:16.351585+0000","last_peered":"2023-09-23T22:48:16.351585+0000","last_clean":"2023-09-23T22:48:16.351585+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:48:16.351585+0000","last_undegraded":"2023-09-23T22:48:16.351585+0000","last_fullsized":"2023-09-23T22:48:16.351585+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":105,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351624+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:48:16.351624+0000","last_peered":"2023-09-23T22:48:16.351624+0000","last_clean":"2023-09-23T22:48:16.351624+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:48:16.351624+0000","last_undegraded":"2023-09-23T22:48:16.351624+0000","last_fullsized":"2023-09-23T22:48:16.351624+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":266,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.357118+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:48:16.357118+0000","last_peered":"2023-09-23T22:48:16.357118+0000","last_clean":"2023-09-23T22:48:16.357118+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:48:16.357118+0000","last_undegraded":"2023-09-23T22:48:16.357118+0000","last_fullsized":"2023-09-23T22:48:16.357118+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419810+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:48:22.419810+0000","last_peered":"2023-09-23T22:48:22.419810+0000","last_clean":"2023-09-23T22:48:22.419810+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:48:22.419810+0000","last_undegraded":"2023-09-23T22:48:22.419810+0000","last_fullsized":"2023-09-23T22:48:22.419810+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":227,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:20.655817+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:48:20.655817+0000","last_peered":"2023-09-23T22:48:20.655817+0000","last_clean":"2023-09-23T22:48:20.655817+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:48:20.655817+0000","last_undegraded":"2023-09-23T22:48:20.655817+0000","last_fullsized":"2023-09-23T22:48:20.655817+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351315+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:48:16.351315+0000","last_peered":"2023-09-23T22:48:16.351315+0000","last_clean":"2023-09-23T22:48:16.351315+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:48:16.351315+0000","last_undegraded":"2023-09-23T22:48:16.351315+0000","last_fullsized":"2023-09-23T22:48:16.351315+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":203,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440783+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:48:22.440783+0000","last_peered":"2023-09-23T22:48:22.440783+0000","last_clean":"2023-09-23T22:48:22.440783+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:48:22.440783+0000","last_undegraded":"2023-09-23T22:48:22.440783+0000","last_fullsized":"2023-09-23T22:48:22.440783+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":291,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.354915+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:48:16.354915+0000","last_peered":"2023-09-23T22:48:16.354915+0000","last_clean":"2023-09-23T22:48:16.354915+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:48:16.354915+0000","last_undegraded":"2023-09-23T22:48:16.354915+0000","last_fullsized":"2023-09-23T22:48:16.354915+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":246,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.360398+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:48:16.360398+0000","last_peered":"2023-09-23T22:48:16.360398+0000","last_clean":"2023-09-23T22:48:16.360398+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:48:16.360398+0000","last_undegraded":"2023-09-23T22:48:16.360398+0000","last_fullsized":"2023-09-23T22:48:16.360398+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"247'7532","reported_seq":7570,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.381829+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:48:25.381829+0000","last_peered":"2023-09-23T22:48:25.381829+0000","last_clean":"2023-09-23T22:48:25.381829+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:48:25.381829+0000","last_undegraded":"2023-09-23T22:48:25.381829+0000","last_fullsized":"2023-09-23T22:48:25.381829+0000","mapping_epoch":217,"log_start":"223'200","ondisk_log_start":"223'200","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7332,"log_dups_size":0,"ondisk_log_size":7332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342097920,"num_objects":5220,"num_object_clones":0,"num_object_copies":15660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7532,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":116,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.350806+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:48:16.350806+0000","last_peered":"2023-09-23T22:48:16.350806+0000","last_clean":"2023-09-23T22:48:16.350806+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:48:16.350806+0000","last_undegraded":"2023-09-23T22:48:16.350806+0000","last_fullsized":"2023-09-23T22:48:16.350806+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.354996+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:48:16.354996+0000","last_peered":"2023-09-23T22:48:16.354996+0000","last_clean":"2023-09-23T22:48:16.354996+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:48:16.354996+0000","last_undegraded":"2023-09-23T22:48:16.354996+0000","last_fullsized":"2023-09-23T22:48:16.354996+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"247'7473","reported_seq":7517,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.733737+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:48:25.733737+0000","last_peered":"2023-09-23T22:48:25.733737+0000","last_clean":"2023-09-23T22:48:25.733737+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:48:25.733737+0000","last_undegraded":"2023-09-23T22:48:25.733737+0000","last_fullsized":"2023-09-23T22:48:25.733737+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7473,"log_dups_size":0,"ondisk_log_size":7473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332988416,"num_objects":5081,"num_object_clones":0,"num_object_copies":15243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7473,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":126,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351103+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:48:16.351103+0000","last_peered":"2023-09-23T22:48:16.351103+0000","last_clean":"2023-09-23T22:48:16.351103+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:48:16.351103+0000","last_undegraded":"2023-09-23T22:48:16.351103+0000","last_fullsized":"2023-09-23T22:48:16.351103+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":227,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351266+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:48:16.351266+0000","last_peered":"2023-09-23T22:48:16.351266+0000","last_clean":"2023-09-23T22:48:16.351266+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:48:16.351266+0000","last_undegraded":"2023-09-23T22:48:16.351266+0000","last_fullsized":"2023-09-23T22:48:16.351266+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"248'7553","reported_seq":7593,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:27.534389+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:48:27.534389+0000","last_peered":"2023-09-23T22:48:27.534389+0000","last_clean":"2023-09-23T22:48:27.534389+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:48:27.534389+0000","last_undegraded":"2023-09-23T22:48:27.534389+0000","last_fullsized":"2023-09-23T22:48:27.534389+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330498048,"num_objects":5043,"num_object_clones":0,"num_object_copies":15129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7553,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":57,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355235+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:48:16.355235+0000","last_peered":"2023-09-23T22:48:16.355235+0000","last_clean":"2023-09-23T22:48:16.355235+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:48:16.355235+0000","last_undegraded":"2023-09-23T22:48:16.355235+0000","last_fullsized":"2023-09-23T22:48:16.355235+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355405+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:48:16.355405+0000","last_peered":"2023-09-23T22:48:16.355405+0000","last_clean":"2023-09-23T22:48:16.355405+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:48:16.355405+0000","last_undegraded":"2023-09-23T22:48:16.355405+0000","last_fullsized":"2023-09-23T22:48:16.355405+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"247'7774","reported_seq":7821,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.596692+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:48:25.596692+0000","last_peered":"2023-09-23T22:48:25.596692+0000","last_clean":"2023-09-23T22:48:25.596692+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:48:25.596692+0000","last_undegraded":"2023-09-23T22:48:25.596692+0000","last_fullsized":"2023-09-23T22:48:25.596692+0000","mapping_epoch":226,"log_start":"223'600","ondisk_log_start":"223'600","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7774,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.357277+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:48:16.357277+0000","last_peered":"2023-09-23T22:48:16.357277+0000","last_clean":"2023-09-23T22:48:16.357277+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:48:16.357277+0000","last_undegraded":"2023-09-23T22:48:16.357277+0000","last_fullsized":"2023-09-23T22:48:16.357277+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498588+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:48:26.498588+0000","last_peered":"2023-09-23T22:48:26.498588+0000","last_clean":"2023-09-23T22:48:26.498588+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:48:26.498588+0000","last_undegraded":"2023-09-23T22:48:26.498588+0000","last_fullsized":"2023-09-23T22:48:26.498588+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"247'7542","reported_seq":7582,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.613285+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:48:25.613285+0000","last_peered":"2023-09-23T22:48:25.613285+0000","last_clean":"2023-09-23T22:48:25.613285+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:48:25.613285+0000","last_undegraded":"2023-09-23T22:48:25.613285+0000","last_fullsized":"2023-09-23T22:48:25.613285+0000","mapping_epoch":217,"log_start":"223'300","ondisk_log_start":"223'300","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7542,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":247,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.358411+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:48:16.358411+0000","last_peered":"2023-09-23T22:48:16.358411+0000","last_clean":"2023-09-23T22:48:16.358411+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:48:16.358411+0000","last_undegraded":"2023-09-23T22:48:16.358411+0000","last_fullsized":"2023-09-23T22:48:16.358411+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351109+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:48:16.351109+0000","last_peered":"2023-09-23T22:48:16.351109+0000","last_clean":"2023-09-23T22:48:16.351109+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:48:16.351109+0000","last_undegraded":"2023-09-23T22:48:16.351109+0000","last_fullsized":"2023-09-23T22:48:16.351109+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"248'7638","reported_seq":7699,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:27.553029+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:48:27.553029+0000","last_peered":"2023-09-23T22:48:27.553029+0000","last_clean":"2023-09-23T22:48:27.553029+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:48:27.553029+0000","last_undegraded":"2023-09-23T22:48:27.553029+0000","last_fullsized":"2023-09-23T22:48:27.553029+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7638,"log_dups_size":0,"ondisk_log_size":7638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336330752,"num_objects":5132,"num_object_clones":0,"num_object_copies":15396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7638,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":231,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355495+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:48:16.355495+0000","last_peered":"2023-09-23T22:48:16.355495+0000","last_clean":"2023-09-23T22:48:16.355495+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:48:16.355495+0000","last_undegraded":"2023-09-23T22:48:16.355495+0000","last_fullsized":"2023-09-23T22:48:16.355495+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":242,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351043+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:48:16.351043+0000","last_peered":"2023-09-23T22:48:16.351043+0000","last_clean":"2023-09-23T22:48:16.351043+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:48:16.351043+0000","last_undegraded":"2023-09-23T22:48:16.351043+0000","last_fullsized":"2023-09-23T22:48:16.351043+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"247'7697","reported_seq":7741,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.388730+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:48:25.388730+0000","last_peered":"2023-09-23T22:48:25.388730+0000","last_clean":"2023-09-23T22:48:25.388730+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:48:25.388730+0000","last_undegraded":"2023-09-23T22:48:25.388730+0000","last_fullsized":"2023-09-23T22:48:25.388730+0000","mapping_epoch":219,"log_start":"223'300","ondisk_log_start":"223'300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347406336,"num_objects":5301,"num_object_clones":0,"num_object_copies":15903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7697,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":255,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.350997+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:48:16.350997+0000","last_peered":"2023-09-23T22:48:16.350997+0000","last_clean":"2023-09-23T22:48:16.350997+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:48:16.350997+0000","last_undegraded":"2023-09-23T22:48:16.350997+0000","last_fullsized":"2023-09-23T22:48:16.350997+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":230,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351372+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:48:16.351372+0000","last_peered":"2023-09-23T22:48:16.351372+0000","last_clean":"2023-09-23T22:48:16.351372+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:48:16.351372+0000","last_undegraded":"2023-09-23T22:48:16.351372+0000","last_fullsized":"2023-09-23T22:48:16.351372+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"247'7374","reported_seq":7429,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.749799+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:48:25.749799+0000","last_peered":"2023-09-23T22:48:25.749799+0000","last_clean":"2023-09-23T22:48:25.749799+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:48:25.749799+0000","last_undegraded":"2023-09-23T22:48:25.749799+0000","last_fullsized":"2023-09-23T22:48:25.749799+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332398592,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7374,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":247,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.354978+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:48:16.354978+0000","last_peered":"2023-09-23T22:48:16.354978+0000","last_clean":"2023-09-23T22:48:16.354978+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:48:16.354978+0000","last_undegraded":"2023-09-23T22:48:16.354978+0000","last_fullsized":"2023-09-23T22:48:16.354978+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.408097+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:48:22.408097+0000","last_peered":"2023-09-23T22:48:22.408097+0000","last_clean":"2023-09-23T22:48:22.408097+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:48:22.408097+0000","last_undegraded":"2023-09-23T22:48:22.408097+0000","last_fullsized":"2023-09-23T22:48:22.408097+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"248'7698","reported_seq":7737,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:27.565971+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:48:27.565971+0000","last_peered":"2023-09-23T22:48:27.565971+0000","last_clean":"2023-09-23T22:48:27.565971+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:48:27.565971+0000","last_undegraded":"2023-09-23T22:48:27.565971+0000","last_fullsized":"2023-09-23T22:48:27.565971+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7698,"log_dups_size":0,"ondisk_log_size":7698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336330752,"num_objects":5132,"num_object_clones":0,"num_object_copies":15396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7698,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":283,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.435234+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:48:22.435234+0000","last_peered":"2023-09-23T22:48:22.435234+0000","last_clean":"2023-09-23T22:48:22.435234+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:48:22.435234+0000","last_undegraded":"2023-09-23T22:48:22.435234+0000","last_fullsized":"2023-09-23T22:48:22.435234+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":166,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.356455+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:48:16.356455+0000","last_peered":"2023-09-23T22:48:16.356455+0000","last_clean":"2023-09-23T22:48:16.356455+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:48:16.356455+0000","last_undegraded":"2023-09-23T22:48:16.356455+0000","last_fullsized":"2023-09-23T22:48:16.356455+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351314+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:48:16.351314+0000","last_peered":"2023-09-23T22:48:16.351314+0000","last_clean":"2023-09-23T22:48:16.351314+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:48:16.351314+0000","last_undegraded":"2023-09-23T22:48:16.351314+0000","last_fullsized":"2023-09-23T22:48:16.351314+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"247'7503","reported_seq":7547,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.650039+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:48:25.650039+0000","last_peered":"2023-09-23T22:48:25.650039+0000","last_clean":"2023-09-23T22:48:25.650039+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:48:25.650039+0000","last_undegraded":"2023-09-23T22:48:25.650039+0000","last_fullsized":"2023-09-23T22:48:25.650039+0000","mapping_epoch":219,"log_start":"223'300","ondisk_log_start":"223'300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7203,"log_dups_size":0,"ondisk_log_size":7203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342556672,"num_objects":5227,"num_object_clones":0,"num_object_copies":15681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7503,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":81,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355451+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:48:16.355451+0000","last_peered":"2023-09-23T22:48:16.355451+0000","last_clean":"2023-09-23T22:48:16.355451+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:48:16.355451+0000","last_undegraded":"2023-09-23T22:48:16.355451+0000","last_fullsized":"2023-09-23T22:48:16.355451+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":243,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351673+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:48:16.351673+0000","last_peered":"2023-09-23T22:48:16.351673+0000","last_clean":"2023-09-23T22:48:16.351673+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:48:16.351673+0000","last_undegraded":"2023-09-23T22:48:16.351673+0000","last_fullsized":"2023-09-23T22:48:16.351673+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":163,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.408275+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:48:22.408275+0000","last_peered":"2023-09-23T22:48:22.408275+0000","last_clean":"2023-09-23T22:48:22.408275+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:48:22.408275+0000","last_undegraded":"2023-09-23T22:48:22.408275+0000","last_fullsized":"2023-09-23T22:48:22.408275+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"247'7663","reported_seq":7701,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.650797+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:48:25.650797+0000","last_peered":"2023-09-23T22:48:25.650797+0000","last_clean":"2023-09-23T22:48:25.650797+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:48:25.650797+0000","last_undegraded":"2023-09-23T22:48:25.650797+0000","last_fullsized":"2023-09-23T22:48:25.650797+0000","mapping_epoch":217,"log_start":"223'500","ondisk_log_start":"223'500","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7163,"log_dups_size":0,"ondisk_log_size":7163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047040,"num_objects":5265,"num_object_clones":0,"num_object_copies":15795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7663,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":238,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.407663+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:48:22.407663+0000","last_peered":"2023-09-23T22:48:22.407663+0000","last_clean":"2023-09-23T22:48:22.407663+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:48:22.407663+0000","last_undegraded":"2023-09-23T22:48:22.407663+0000","last_fullsized":"2023-09-23T22:48:22.407663+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":126,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.407470+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:48:22.407470+0000","last_peered":"2023-09-23T22:48:22.407470+0000","last_clean":"2023-09-23T22:48:22.407470+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:48:22.407470+0000","last_undegraded":"2023-09-23T22:48:22.407470+0000","last_fullsized":"2023-09-23T22:48:22.407470+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":252,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440290+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:48:22.440290+0000","last_peered":"2023-09-23T22:48:22.440290+0000","last_clean":"2023-09-23T22:48:22.440290+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:48:22.440290+0000","last_undegraded":"2023-09-23T22:48:22.440290+0000","last_fullsized":"2023-09-23T22:48:22.440290+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"247'7556","reported_seq":7599,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:25.710634+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:48:25.710634+0000","last_peered":"2023-09-23T22:48:25.710634+0000","last_clean":"2023-09-23T22:48:25.710634+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:48:25.710634+0000","last_undegraded":"2023-09-23T22:48:25.710634+0000","last_fullsized":"2023-09-23T22:48:25.710634+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331481088,"num_objects":5058,"num_object_clones":0,"num_object_copies":15174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7556,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":248,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.407752+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:48:22.407752+0000","last_peered":"2023-09-23T22:48:22.407752+0000","last_clean":"2023-09-23T22:48:22.407752+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:48:22.407752+0000","last_undegraded":"2023-09-23T22:48:22.407752+0000","last_fullsized":"2023-09-23T22:48:22.407752+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":334,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419700+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:48:22.419700+0000","last_peered":"2023-09-23T22:48:22.419700+0000","last_clean":"2023-09-23T22:48:22.419700+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:48:22.419700+0000","last_undegraded":"2023-09-23T22:48:22.419700+0000","last_fullsized":"2023-09-23T22:48:22.419700+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":177,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355104+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:48:16.355104+0000","last_peered":"2023-09-23T22:48:16.355104+0000","last_clean":"2023-09-23T22:48:16.355104+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:48:16.355104+0000","last_undegraded":"2023-09-23T22:48:16.355104+0000","last_fullsized":"2023-09-23T22:48:16.355104+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355366+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:48:16.355366+0000","last_peered":"2023-09-23T22:48:16.355366+0000","last_clean":"2023-09-23T22:48:16.355366+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:48:16.355366+0000","last_undegraded":"2023-09-23T22:48:16.355366+0000","last_fullsized":"2023-09-23T22:48:16.355366+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"248'7798","reported_seq":7846,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:29.479163+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:48:29.479163+0000","last_peered":"2023-09-23T22:48:29.479163+0000","last_clean":"2023-09-23T22:48:29.479163+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:48:29.479163+0000","last_undegraded":"2023-09-23T22:48:29.479163+0000","last_fullsized":"2023-09-23T22:48:29.479163+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7798,"log_dups_size":0,"ondisk_log_size":7798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337510400,"num_objects":5150,"num_object_clones":0,"num_object_copies":15450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7798,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419785+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:48:22.419785+0000","last_peered":"2023-09-23T22:48:22.419785+0000","last_clean":"2023-09-23T22:48:22.419785+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:48:22.419785+0000","last_undegraded":"2023-09-23T22:48:22.419785+0000","last_fullsized":"2023-09-23T22:48:22.419785+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419923+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:48:22.419923+0000","last_peered":"2023-09-23T22:48:22.419923+0000","last_clean":"2023-09-23T22:48:22.419923+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:48:22.419923+0000","last_undegraded":"2023-09-23T22:48:22.419923+0000","last_fullsized":"2023-09-23T22:48:22.419923+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":171,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.626308+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:48:22.626308+0000","last_peered":"2023-09-23T22:48:22.626308+0000","last_clean":"2023-09-23T22:48:22.626308+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:48:22.626308+0000","last_undegraded":"2023-09-23T22:48:22.626308+0000","last_fullsized":"2023-09-23T22:48:22.626308+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"247'7824","reported_seq":9026,"reported_epoch":247,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:48:25.651281+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:48:25.651281+0000","last_peered":"2023-09-23T22:48:25.651281+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:48:25.651281+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"223'600","ondisk_log_start":"223'600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7224,"log_dups_size":0,"ondisk_log_size":7224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356777984,"num_objects":5444,"num_object_clones":0,"num_object_copies":16332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":614,"num_objects_misplaced":5427,"num_objects_unfound":0,"num_objects_dirty":5444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7824,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":577,"num_bytes_recovered":37814272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-597},{"shards":"1,3,4","objects":5427}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440211+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:48:22.440211+0000","last_peered":"2023-09-23T22:48:22.440211+0000","last_clean":"2023-09-23T22:48:22.440211+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:48:22.440211+0000","last_undegraded":"2023-09-23T22:48:22.440211+0000","last_fullsized":"2023-09-23T22:48:22.440211+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.408156+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:48:22.408156+0000","last_peered":"2023-09-23T22:48:22.408156+0000","last_clean":"2023-09-23T22:48:22.408156+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:48:22.408156+0000","last_undegraded":"2023-09-23T22:48:22.408156+0000","last_fullsized":"2023-09-23T22:48:22.408156+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.420413+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:48:22.420413+0000","last_peered":"2023-09-23T22:48:22.420413+0000","last_clean":"2023-09-23T22:48:22.420413+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:48:22.420413+0000","last_undegraded":"2023-09-23T22:48:22.420413+0000","last_fullsized":"2023-09-23T22:48:22.420413+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"248'7696","reported_seq":7735,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:27.525311+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:48:27.525311+0000","last_peered":"2023-09-23T22:48:27.525311+0000","last_clean":"2023-09-23T22:48:27.525311+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:48:27.525311+0000","last_undegraded":"2023-09-23T22:48:27.525311+0000","last_fullsized":"2023-09-23T22:48:27.525311+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7696,"log_dups_size":0,"ondisk_log_size":7696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341573632,"num_objects":5212,"num_object_clones":0,"num_object_copies":15636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7696,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":286,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.360316+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:48:16.360316+0000","last_peered":"2023-09-23T22:48:16.360316+0000","last_clean":"2023-09-23T22:48:16.360316+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:48:16.360316+0000","last_undegraded":"2023-09-23T22:48:16.360316+0000","last_fullsized":"2023-09-23T22:48:16.360316+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":45,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355972+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:48:16.355972+0000","last_peered":"2023-09-23T22:48:16.355972+0000","last_clean":"2023-09-23T22:48:16.355972+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:48:16.355972+0000","last_undegraded":"2023-09-23T22:48:16.355972+0000","last_fullsized":"2023-09-23T22:48:16.355972+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":272,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355011+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:48:16.355011+0000","last_peered":"2023-09-23T22:48:16.355011+0000","last_clean":"2023-09-23T22:48:16.355011+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:48:16.355011+0000","last_undegraded":"2023-09-23T22:48:16.355011+0000","last_fullsized":"2023-09-23T22:48:16.355011+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"248'7754","reported_seq":7796,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:29.526192+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:48:29.526192+0000","last_peered":"2023-09-23T22:48:29.526192+0000","last_clean":"2023-09-23T22:48:29.526192+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:48:29.526192+0000","last_undegraded":"2023-09-23T22:48:29.526192+0000","last_fullsized":"2023-09-23T22:48:29.526192+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336265240,"num_objects":5132,"num_object_clones":0,"num_object_copies":15396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5132,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7754,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":279,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.355725+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:48:16.355725+0000","last_peered":"2023-09-23T22:48:16.355725+0000","last_clean":"2023-09-23T22:48:16.355725+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:48:16.355725+0000","last_undegraded":"2023-09-23T22:48:16.355725+0000","last_fullsized":"2023-09-23T22:48:16.355725+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":143,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.626327+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:48:22.626327+0000","last_peered":"2023-09-23T22:48:22.626327+0000","last_clean":"2023-09-23T22:48:22.626327+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:48:22.626327+0000","last_undegraded":"2023-09-23T22:48:22.626327+0000","last_fullsized":"2023-09-23T22:48:22.626327+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":197,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351082+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:48:16.351082+0000","last_peered":"2023-09-23T22:48:16.351082+0000","last_clean":"2023-09-23T22:48:16.351082+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:48:16.351082+0000","last_undegraded":"2023-09-23T22:48:16.351082+0000","last_fullsized":"2023-09-23T22:48:16.351082+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":178,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.420288+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:48:22.420288+0000","last_peered":"2023-09-23T22:48:22.420288+0000","last_clean":"2023-09-23T22:48:22.420288+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:48:22.420288+0000","last_undegraded":"2023-09-23T22:48:22.420288+0000","last_fullsized":"2023-09-23T22:48:22.420288+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":20,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.408103+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:48:22.408103+0000","last_peered":"2023-09-23T22:48:22.408103+0000","last_clean":"2023-09-23T22:48:22.408103+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:48:22.408103+0000","last_undegraded":"2023-09-23T22:48:22.408103+0000","last_fullsized":"2023-09-23T22:48:22.408103+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":57,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351023+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:48:16.351023+0000","last_peered":"2023-09-23T22:48:16.351023+0000","last_clean":"2023-09-23T22:48:16.351023+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:48:16.351023+0000","last_undegraded":"2023-09-23T22:48:16.351023+0000","last_fullsized":"2023-09-23T22:48:16.351023+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":246,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351704+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:48:16.351704+0000","last_peered":"2023-09-23T22:48:16.351704+0000","last_clean":"2023-09-23T22:48:16.351704+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:48:16.351704+0000","last_undegraded":"2023-09-23T22:48:16.351704+0000","last_fullsized":"2023-09-23T22:48:16.351704+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":57,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351191+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:48:16.351191+0000","last_peered":"2023-09-23T22:48:16.351191+0000","last_clean":"2023-09-23T22:48:16.351191+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:48:16.351191+0000","last_undegraded":"2023-09-23T22:48:16.351191+0000","last_fullsized":"2023-09-23T22:48:16.351191+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":118,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.351392+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:48:16.351392+0000","last_peered":"2023-09-23T22:48:16.351392+0000","last_clean":"2023-09-23T22:48:16.351392+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:48:16.351392+0000","last_undegraded":"2023-09-23T22:48:16.351392+0000","last_fullsized":"2023-09-23T22:48:16.351392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":240,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.356874+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:48:16.356874+0000","last_peered":"2023-09-23T22:48:16.356874+0000","last_clean":"2023-09-23T22:48:16.356874+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:48:16.356874+0000","last_undegraded":"2023-09-23T22:48:16.356874+0000","last_fullsized":"2023-09-23T22:48:16.356874+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":278,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.354973+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:48:16.354973+0000","last_peered":"2023-09-23T22:48:16.354973+0000","last_clean":"2023-09-23T22:48:16.354973+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:48:16.354973+0000","last_undegraded":"2023-09-23T22:48:16.354973+0000","last_fullsized":"2023-09-23T22:48:16.354973+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440466+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:48:22.440466+0000","last_peered":"2023-09-23T22:48:22.440466+0000","last_clean":"2023-09-23T22:48:22.440466+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:48:22.440466+0000","last_undegraded":"2023-09-23T22:48:22.440466+0000","last_fullsized":"2023-09-23T22:48:22.440466+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:48:16.350819+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:48:16.350819+0000","last_peered":"2023-09-23T22:48:16.350819+0000","last_clean":"2023-09-23T22:48:16.350819+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:48:16.350819+0000","last_undegraded":"2023-09-23T22:48:16.350819+0000","last_fullsized":"2023-09-23T22:48:16.350819+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5427822616,"num_objects":82823,"num_object_clones":0,"num_object_copies":248469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":614,"num_objects_misplaced":5427,"num_objects_unfound":0,"num_objects_dirty":82823,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122075,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":580,"num_bytes_recovered":38010880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16358846464,"data_stored":16358834248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119275,"ondisk_log_size":119275,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430465,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626693,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2283340,"kb_used_data":1986116,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91461812,"statfs":{"total":95995035648,"available":93656895488,"internally_reserved":0,"allocated":2033782784,"data_stored":2033281883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":4.6740000000000004,"5min":1.966,"15min":1.514},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":36.651000000000003,"5min":36.651000000000003,"15min":36.651000000000003},"last":0.30299999999999999},{"interface":"front","average":{"1min":5.5049999999999999,"5min":1.8819999999999999,"15min":1.278},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":36.558999999999997,"5min":36.558999999999997,"15min":36.558999999999997},"last":2.1070000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:47:33 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":2.1619999999999999,"15min":2.1200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.5970000000000004,"5min":34.695999999999998,"15min":153.78399999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.339,"5min":2.274,"15min":2.1669999999999998},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":14.028,"5min":34.307000000000002,"15min":154.148},"last":2.0819999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":2.3959999999999999,"15min":3.044},"min":{"1min":0.33800000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.802,"5min":12.802,"15min":100.785},"last":8.4570000000000007},{"interface":"front","average":{"1min":11.632999999999999,"5min":4.0060000000000002,"15min":3.5270000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":185.21799999999999,"5min":185.21799999999999,"15min":185.21799999999999},"last":8.3610000000000007}]},{"osd":3,"last update":"Sat Sep 23 22:47:33 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":3.2480000000000002,"15min":2.0099999999999998},"min":{"1min":0.316,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.9160000000000004,"5min":90.561999999999998,"15min":90.561999999999998},"last":0.378},{"interface":"front","average":{"1min":3.161,"5min":3.6280000000000001,"15min":2.2149999999999999},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":13.67,"5min":90.646000000000001,"15min":90.646000000000001},"last":10.241}]},{"osd":4,"last update":"Sat Sep 23 22:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.89300000000000002,"15min":0.626},"min":{"1min":0.29399999999999998,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":2.7189999999999999,"5min":18.085999999999999,"15min":18.085999999999999},"last":2.0489999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.008,"15min":0.67100000000000004},"min":{"1min":0.26500000000000001,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":4.1699999999999999,"5min":18.597000000000001,"15min":18.597000000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:47:33 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.98799999999999999,"15min":0.67700000000000005},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":1.925,"5min":18.972000000000001,"15min":18.972000000000001},"last":2.1299999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.0740000000000001,"15min":0.71899999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":3.323,"5min":18.625,"15min":18.625},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.0600000000000001,"15min":0.73699999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":4.2640000000000002,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.034,"15min":0.71599999999999997},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":4.165,"5min":16.661999999999999,"15min":16.661999999999999},"last":0.44400000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253003,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2607496,"kb_used_data":2327232,"kb_used_omap":9,"kb_used_meta":280182,"kb_avail":91137656,"statfs":{"total":95995035648,"available":93324959744,"internally_reserved":0,"allocated":2383085568,"data_stored":2382191047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":286907369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":9.6259999999999994,"5min":5.2590000000000003,"15min":4.5309999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":101.605,"5min":101.605,"15min":101.605},"last":15.827999999999999},{"interface":"front","average":{"1min":10.422000000000001,"5min":5.6959999999999997,"15min":4.9080000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":101.892,"5min":101.892,"15min":101.892},"last":9.3109999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":2.9689999999999999,"15min":2.0190000000000001},"min":{"1min":0.23000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":36.456000000000003,"5min":36.456000000000003,"15min":37.889000000000003},"last":15.936},{"interface":"front","average":{"1min":4.141,"5min":3.4209999999999998,"15min":2.04},"min":{"1min":0.315,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":36.427,"5min":36.427,"15min":37.823},"last":15.789999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6280000000000001,"5min":2.7839999999999998,"15min":1.5529999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.5259999999999998,"5min":16.631,"15min":50.033999999999999},"last":3.2160000000000002},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.4900000000000002,"15min":1.847},"min":{"1min":0.34399999999999997,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.8819999999999997,"5min":29.565000000000001,"15min":52.491999999999997},"last":14.016999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.5830000000000002,"5min":3.2080000000000002,"15min":1.8049999999999999},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":78.948999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":5.0250000000000004},{"interface":"front","average":{"1min":4.7999999999999998,"5min":2.4159999999999999,"15min":1.6220000000000001},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.313},"max":{"1min":78.963999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":5.1609999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.3620000000000001,"5min":2.556,"15min":1.472},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":84.254999999999995,"5min":84.254999999999995,"15min":84.254999999999995},"last":2.0659999999999998},{"interface":"front","average":{"1min":6.2939999999999996,"5min":3.2879999999999998,"15min":1.6719999999999999},"min":{"1min":0.32800000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":87.515000000000001,"5min":87.515000000000001,"15min":87.515000000000001},"last":4.9640000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0250000000000004,"5min":2.6819999999999999,"15min":1.45},"min":{"1min":0.436,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":79.311000000000007,"5min":79.311000000000007,"15min":79.311000000000007},"last":5.194},{"interface":"front","average":{"1min":5.4189999999999996,"5min":2.7029999999999998,"15min":1.405},"min":{"1min":0.34699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":79.328999999999994,"5min":79.328999999999994,"15min":79.328999999999994},"last":2.1120000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.5910000000000002,"15min":2.0059999999999998},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":8.5449999999999999,"5min":29.539000000000001,"15min":159.018},"last":5.056},{"interface":"front","average":{"1min":1.6519999999999999,"5min":2.0259999999999998,"15min":1.9179999999999999},"min":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":6.0810000000000004,"5min":8.6120000000000001,"15min":158.96000000000001},"last":5.2190000000000003}]}]},{"osd":0,"up_from":219,"seq":940597837852,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2778956,"kb_used_data":2630916,"kb_used_omap":31,"kb_used_meta":148000,"kb_avail":90966196,"statfs":{"total":95995035648,"available":93149384704,"internally_reserved":0,"allocated":2694057984,"data_stored":2693100887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":151552946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":3.556,"15min":3.6600000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":15.611000000000001,"5min":32.344999999999999,"15min":32.344999999999999},"last":7.3479999999999999},{"interface":"front","average":{"1min":3.6389999999999998,"5min":3.5750000000000002,"15min":3.5640000000000001},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.092000000000001,"5min":32.106000000000002,"15min":32.106000000000002},"last":5.7789999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":2.468,"15min":2.2210000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.170999999999999,"5min":12.170999999999999,"15min":12.170999999999999},"last":20.382000000000001},{"interface":"front","average":{"1min":3.27,"5min":2.0590000000000002,"15min":1.857},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9180000000000001,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":7.4379999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":2.1549999999999998,"15min":2.0289999999999999},"min":{"1min":0.34699999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":5.8330000000000002},{"interface":"front","average":{"1min":2.6360000000000001,"5min":1.4259999999999999,"15min":1.224},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.9730000000000008,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":2.085}]},{"osd":4,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7280000000000002,"5min":2.2280000000000002,"15min":1.978},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":15.432,"5min":15.432,"15min":15.432},"last":2.4620000000000002},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.7509999999999999,"15min":1.7490000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9790000000000001,"5min":7.5590000000000002,"15min":7.5590000000000002},"last":5.8049999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.601,"15min":1.4239999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":2.1539999999999999},{"interface":"front","average":{"1min":3.1019999999999999,"5min":1.8620000000000001,"15min":1.6559999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":15.536,"5min":15.536,"15min":15.536},"last":2.5139999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.6699999999999999,"15min":1.4710000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":2.1259999999999999},{"interface":"front","average":{"1min":3.952,"5min":2.1389999999999998,"15min":1.837},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.489000000000001,"5min":15.489000000000001,"15min":15.489000000000001},"last":2.4900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.9690000000000001,"15min":1.72},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":15.574,"5min":15.574,"15min":15.574},"last":2.4220000000000002},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.4139999999999999,"15min":1.256},"min":{"1min":0.46200000000000002,"5min":0.432,"15min":0.432},"max":{"1min":8.9000000000000004,"5min":8.9000000000000004,"15min":8.9000000000000004},"last":2.1859999999999999}]}]},{"osd":2,"up_from":131,"seq":562640715865,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2184692,"kb_used_data":1954412,"kb_used_omap":81,"kb_used_meta":230190,"kb_avail":91560460,"statfs":{"total":95995035648,"available":93757911040,"internally_reserved":0,"allocated":2001317888,"data_stored":2000384658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":235714778},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":5.1059999999999999,"5min":5.2359999999999998,"15min":5.258},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":41.723999999999997,"5min":57.811999999999998,"15min":57.811999999999998},"last":2.0409999999999999},{"interface":"front","average":{"1min":7.7679999999999998,"5min":5.8650000000000002,"15min":5.548},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":41.454000000000001,"5min":57.387999999999998,"15min":57.387999999999998},"last":1.9850000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:47:29 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":3.4569999999999999,"15min":3.2069999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.3310000000000004,"5min":88.316000000000003,"15min":88.316000000000003},"last":3.9129999999999998},{"interface":"front","average":{"1min":2.2410000000000001,"5min":3.9649999999999999,"15min":2.7149999999999999},"min":{"1min":0.254,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.0090000000000003,"5min":88.436999999999998,"15min":88.436999999999998},"last":0.95299999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:47:29 2023","interfaces":[{"interface":"back","average":{"1min":2.077,"5min":6.1669999999999998,"15min":2.7709999999999999},"min":{"1min":0.26800000000000002,"5min":0.223,"15min":0.223},"max":{"1min":6.9279999999999999,"5min":88.653000000000006,"15min":88.653000000000006},"last":9.8879999999999999},{"interface":"front","average":{"1min":2.4020000000000001,"5min":4.5780000000000003,"15min":2.23},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":6.8140000000000001,"5min":88.474000000000004,"15min":88.474000000000004},"last":1.341}]},{"osd":4,"last update":"Sat Sep 23 22:47:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8759999999999999,"5min":2.7480000000000002,"15min":1.52},"min":{"1min":0.30499999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.6020000000000003,"5min":108.878,"15min":108.878},"last":1.425},{"interface":"front","average":{"1min":1.3300000000000001,"5min":4.2110000000000003,"15min":2.2810000000000001},"min":{"1min":0.40200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.7679999999999998,"5min":108.93000000000001,"15min":108.93000000000001},"last":3.4910000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:47:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":4.274,"15min":2.327},"min":{"1min":0.36899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":4.7290000000000001,"5min":97.322999999999993,"15min":97.322999999999993},"last":1.5820000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":4.0609999999999999,"15min":1.855},"min":{"1min":0.36499999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.1500000000000004,"5min":97.301000000000002,"15min":97.301000000000002},"last":2.3180000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:47:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":3.198,"15min":1.5900000000000001},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.1920000000000002,"5min":88.606999999999999,"15min":88.606999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.7230000000000001,"15min":1.171},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.7149999999999999,"5min":9.9000000000000004,"15min":9.9000000000000004},"last":1.3939999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:47:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.704,"15min":1.216},"min":{"1min":0.40799999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.6399999999999997,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":1.448},{"interface":"front","average":{"1min":1.6140000000000001,"5min":3.4910000000000001,"15min":2.0670000000000002},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":5.202,"5min":88.584000000000003,"15min":88.584000000000003},"last":4.383}]}]},{"osd":3,"up_from":94,"seq":403726925950,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2405688,"kb_used_data":2032496,"kb_used_omap":7,"kb_used_meta":373176,"kb_avail":91339464,"statfs":{"total":95995035648,"available":93531611136,"internally_reserved":0,"allocated":2081275904,"data_stored":2080387114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":382132840},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:13 2023","interfaces":[{"interface":"back","average":{"1min":6.5970000000000004,"5min":2.8069999999999999,"15min":2.1749999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":80.099000000000004,"5min":80.099000000000004,"15min":80.099000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":6.5149999999999997,"5min":2.581,"15min":1.925},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":80.150000000000006,"5min":80.150000000000006,"15min":80.150000000000006},"last":2.4940000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.8639999999999999,"15min":1.3979999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":6.7110000000000003,"5min":22.818000000000001,"15min":22.818000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.472,"5min":2,"15min":1.1160000000000001},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4160000000000004,"5min":23.917999999999999,"15min":23.917999999999999},"last":2.5419999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.367,"5min":3.879,"15min":2.2509999999999999},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.431000000000001,"5min":125.446,"15min":125.446},"last":2.573},{"interface":"front","average":{"1min":3.476,"5min":5.7249999999999996,"15min":2.7330000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":14.670999999999999,"5min":182.483,"15min":182.483},"last":4.8129999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1150000000000002,"15min":1.2310000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":3.7959999999999998,"5min":18.757000000000001,"15min":18.757000000000001},"last":1.5920000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.5740000000000001,"15min":1.0649999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.516,"5min":12.097,"15min":12.097},"last":2.6520000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.4570000000000001,"15min":1.0229999999999999},"min":{"1min":0.30099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.4710000000000001,"5min":12.127000000000001,"15min":12.127000000000001},"last":2.5979999999999999},{"interface":"front","average":{"1min":2.52,"5min":2.105,"15min":1.2150000000000001},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.9950000000000001,"5min":9.0090000000000003,"15min":9.0090000000000003},"last":4.6829999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.589,"15min":1.0669999999999999},"min":{"1min":0.34000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.173,"5min":12.16,"15min":12.16},"last":2.6269999999999998},{"interface":"front","average":{"1min":2.5830000000000002,"5min":2.089,"15min":1.2869999999999999},"min":{"1min":0.47099999999999997,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.0259999999999998,"5min":9.1029999999999998,"15min":10.755000000000001},"last":4.7380000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8399999999999999,"5min":2.9940000000000002,"15min":1.5649999999999999},"min":{"1min":0.40100000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.0510000000000002,"5min":91.757999999999996,"15min":91.757999999999996},"last":4.7770000000000001},{"interface":"front","average":{"1min":1.7170000000000001,"5min":3.234,"15min":1.639},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.7709999999999999,"5min":91.709999999999994,"15min":91.709999999999994},"last":1.8560000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019149,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2395464,"kb_used_data":2098240,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91349688,"statfs":{"total":95995035648,"available":93542080512,"internally_reserved":0,"allocated":2148597760,"data_stored":2148100429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":6.6619999999999999,"15min":7.5640000000000001},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.0190000000000001,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.77700000000000002},{"interface":"front","average":{"1min":2.2759999999999998,"5min":7.2779999999999996,"15min":8.1120000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.7200000000000006,"5min":110.33799999999999,"15min":110.33799999999999},"last":1.0269999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:47:26 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":3.0049999999999999,"15min":1.409},"min":{"1min":0.255,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":31.888000000000002,"5min":45.320999999999998,"15min":45.320999999999998},"last":1.5389999999999999},{"interface":"front","average":{"1min":2.3650000000000002,"5min":2.6960000000000002,"15min":1.284},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":32.5,"5min":45.372999999999998,"15min":45.372999999999998},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":4.2560000000000002,"15min":3.2400000000000002},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.7030000000000003,"5min":84.909000000000006,"15min":84.909000000000006},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.1659999999999999,"5min":4.2590000000000003,"15min":3.1669999999999998},"min":{"1min":0.371,"5min":0.309,"15min":0.309},"max":{"1min":7.0570000000000004,"5min":83.840000000000003,"15min":83.840000000000003},"last":1.0449999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:47:26 2023","interfaces":[{"interface":"back","average":{"1min":14.002000000000001,"5min":6.6210000000000004,"15min":2.7679999999999998},"min":{"1min":0.38800000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":128.68299999999999,"5min":128.68299999999999,"15min":128.68299999999999},"last":5.827},{"interface":"front","average":{"1min":14.19,"5min":6.8730000000000002,"15min":2.8370000000000002},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":129.197,"5min":129.197,"15min":129.197},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.746,"15min":0.73999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.8979999999999997,"5min":5.8979999999999997,"15min":43.189999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.78000000000000003,"15min":0.76100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26500000000000001},"max":{"1min":2.7040000000000002,"5min":9.5190000000000001,"15min":42.450000000000003},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:47:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.81100000000000005,"15min":0.70799999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.5619999999999998,"5min":6.8109999999999999,"15min":27.056000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.80800000000000005,"15min":0.73699999999999999},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":5.8689999999999998,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.754,"15min":0.66400000000000003},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.8460000000000001,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.436},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.69199999999999995,"15min":0.628},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":2.3130000000000002,"5min":3.2519999999999998,"15min":5.9409999999999998},"last":0.42199999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822921,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3304756,"kb_used_data":2999340,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90440396,"statfs":{"total":95995035648,"available":92610965504,"internally_reserved":0,"allocated":3071324160,"data_stored":3070842447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":18.346,"5min":5.1150000000000002,"15min":2.9100000000000001},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":121.319,"5min":121.319,"15min":121.319},"last":0.39700000000000002},{"interface":"front","average":{"1min":18.654,"5min":5.0469999999999997,"15min":2.7789999999999999},"min":{"1min":0.32700000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":121.075,"5min":121.075,"15min":121.075},"last":0.58299999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8,"15min":1.716},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":28.658999999999999,"5min":28.658999999999999,"15min":149.203},"last":6.7779999999999996},{"interface":"front","average":{"1min":3.8969999999999998,"5min":2.3999999999999999,"15min":1.871},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":28.664999999999999,"5min":28.664999999999999,"15min":149.554},"last":0.53100000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":4.5599999999999996,"15min":7.141},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.2879999999999998,"5min":158.92599999999999,"15min":158.92599999999999},"last":2.8700000000000001},{"interface":"front","average":{"1min":1.861,"5min":4.7949999999999999,"15min":7.1959999999999997},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3899999999999997,"5min":158.91399999999999,"15min":158.91399999999999},"last":7.0430000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":14.204000000000001,"5min":8.2560000000000002,"15min":3.9020000000000001},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":108.58799999999999,"5min":257.05000000000001,"15min":257.05000000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":14.087999999999999,"5min":7.6820000000000004,"15min":3.7570000000000001},"min":{"1min":0.311,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":108.15600000000001,"5min":257.35000000000002,"15min":257.35000000000002},"last":6.8159999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.79300000000000004,"15min":0.60299999999999998},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":1.4790000000000001,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.79800000000000004,"15min":0.61299999999999999},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.619,"5min":10.535,"15min":10.535},"last":0.26600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.411,"15min":1.3879999999999999},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":12.615,"5min":48.360999999999997,"15min":140.07499999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.417,"5min":1.532,"15min":1.4399999999999999},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.254},"max":{"1min":12.74,"5min":48.292999999999999,"15min":139.739},"last":0.30499999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.81699999999999995,"15min":0.69599999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.3450000000000002,"5min":7.0010000000000003,"15min":7.9809999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.78500000000000003,"15min":0.66000000000000003},"min":{"1min":0.36699999999999999,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":1.9830000000000001,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.32100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2680815616,"data_stored":2680815616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2375745536,"data_stored":2375745536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1997606912,"data_stored":1997602840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2069889024,"data_stored":2069889024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2144010240,"data_stored":2144010240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3068858368,"data_stored":3068854296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2021920768,"data_stored":2021916696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:48:33.068 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:48:33.089 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:33.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:33.161 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:33.216 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:33.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:33.322 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:33.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:33 smithi052 ceph-mon[180979]: pgmap v648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 719/243084 objects degraded (0.296%); 5244/243084 objects misplaced (2.157%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T22:48:33.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:33 smithi052 ceph-mon[187743]: pgmap v648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 719/243084 objects degraded (0.296%); 5244/243084 objects misplaced (2.157%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T22:48:33.350 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:48:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:48:33] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-09-23T22:48:33.420 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:33.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:33 smithi151 ceph-mon[159210]: pgmap v648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 719/243084 objects degraded (0.296%); 5244/243084 objects misplaced (2.157%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T22:48:33.699 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:33.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:33.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:33.810 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:33.820 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:33.906 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:34.092 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:48:34.092 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:126274 8.a 8:5c8be9c5:::benchmark_data_smithi052_337357_object23823:head [delete] snapc 0=[] ondisk+write+known_if_redirected e248)", 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.063493+0000", 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.017735457999999999, 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017759013000000001, 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:48:34.093 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 126274 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063493+0000", 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063493+0000", 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063492+0000", 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063498+0000", 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063514+0000", 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5999999999999999e-05 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063524+0000", 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1209999999999996e-06 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063575+0000", 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1292999999999998e-05 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063713+0000", 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138038 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.063828+0000", 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000114749 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:126274 8.a e248/226)", 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.072399+0000", 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.008829468, 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.008902297, 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072399+0000", 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072399+0000", 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072398+0000", 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072405+0000", 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6699999999999994e-06 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072426+0000", 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0800000000000001e-05 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072435+0000", 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6039999999999993e-06 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.100 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:126258 8.f e248/239 8:ffa53544:::benchmark_data_smithi052_337357_object23807:head v 248'8112, mlcod=224'1475)", 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.068493+0000", 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.01273518, 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012839440000000001, 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.068493+0000", 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.068493+0000", 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.068492+0000", 2023-09-23T22:48:34.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.068500+0000", 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5989999999999996e-06 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072897+0000", 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004396652 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072907+0000", 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0027000000000001e-05 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.073010+0000", 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103379 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.073042+0000", 2023-09-23T22:48:34.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1724000000000003e-05 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:126260 8.f e248/239)", 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.072924+0000", 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0083042889999999994, 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0084473369999999992, 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:48:34.105 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072924+0000", 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072924+0000", 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072923+0000", 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072930+0000", 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3169999999999998e-06 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072941+0000", 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1694e-05 2023-09-23T22:48:34.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.072946+0000", 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9099999999999996e-06 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:126265 8.f e248/239)", 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.075122+0000", 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0061064209999999999, 2023-09-23T22:48:34.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062829339999999996, 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075122+0000", 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075122+0000", 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075121+0000", 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075131+0000", 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0011e-05 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075153+0000", 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2402999999999999e-05 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075162+0000", 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0359999999999995e-06 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:126270 8.f e248/239)", 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.075620+0000", 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0056077949999999996, 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.005812027, 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075620+0000", 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075620+0000", 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075620+0000", 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075627+0000", 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0450000000000003e-06 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075642+0000", 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4859e-05 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.075649+0000", 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0099999999999998e-06 2023-09-23T22:48:34.114 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45206.0:126279 8.f e248/239)", 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:34.080115+0000", 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.001113763, 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001359199, 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:34.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.080115+0000", 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.080115+0000", 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.080114+0000", 2023-09-23T22:48:34.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.080122+0000", 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0830000000000004e-06 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.080141+0000", 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9136000000000001e-05 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:34.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:34.080149+0000", 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1030000000000002e-06 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 7 2023-09-23T22:48:34.118 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:48:34.151 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:34.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:34.162 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:34.162 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:34.162 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:34.162 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:34.162 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:34.162 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:34.263 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:34.365 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:48:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:48:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:35 smithi151 ceph-mon[159210]: pgmap v649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 719/240984 objects degraded (0.298%); 5244/240984 objects misplaced (2.176%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:35.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 719/243084 objects degraded (0.296%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[180979]: pgmap v649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 719/240984 objects degraded (0.298%); 5244/240984 objects misplaced (2.176%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 719/243084 objects degraded (0.296%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[187743]: pgmap v649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 719/240984 objects degraded (0.298%); 5244/240984 objects misplaced (2.176%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:35.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 719/243084 objects degraded (0.296%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:35.737 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:35.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:35.853 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:35.876 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:48:35.928 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:36.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:36.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:36.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:36.269 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:36.288 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:36.388 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:37.601 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:37.611 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:37 smithi052 ceph-mon[187743]: pgmap v650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 793/234534 objects degraded (0.338%); 5084/234534 objects misplaced (2.168%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:37.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:37.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:37 smithi052 ceph-mon[180979]: pgmap v650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 793/234534 objects degraded (0.338%); 5084/234534 objects misplaced (2.168%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:37 smithi151 ceph-mon[159210]: pgmap v650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 793/234534 objects degraded (0.338%); 5084/234534 objects misplaced (2.168%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:37.715 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:37.750 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:37.812 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:37.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:37.919 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:38.004 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:38.007 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:38.284 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:48:38.304 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:38.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:38.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:38.344 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:48:38.376 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:48:38.407 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:48:38.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:38.480 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:38.488 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:48:38.495 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:48:38.523 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:48:38.561 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:48:38.592 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:48:38.632 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.633 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:48:38.662 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3793756470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:48:38.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:48:38.663 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1191940223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:38.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3793756470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:48:38.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:48:38.664 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1191940223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3793756470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:48:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:48:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1191940223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:39.315 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:39.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:39.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:39.420 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:39.495 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: pgmap v651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 793/232275 objects degraded (0.341%); 5084/232275 objects misplaced (2.189%); 893 KiB/s, 13 objects/s recovering 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:39 smithi151 ceph-mon[159210]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: pgmap v651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 793/232275 objects degraded (0.341%); 5084/232275 objects misplaced (2.189%); 893 KiB/s, 13 objects/s recovering 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[180979]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: pgmap v651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 793/232275 objects degraded (0.341%); 5084/232275 objects misplaced (2.189%); 893 KiB/s, 13 objects/s recovering 2023-09-23T22:48:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:48:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T22:48:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: failed to encode map e249 with expected crc 2023-09-23T22:48:39.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:39 smithi052 ceph-mon[187743]: failed to encode map e249 with expected crc 2023-09-23T22:48:40.469 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:40.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:40.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:40.580 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:40.609 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:40 smithi052 ceph-mon[187743]: failed to encode map e249 with expected crc 2023-09-23T22:48:40.609 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 793/232275 objects degraded (0.341%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:40.610 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:40 smithi151 ceph-mon[159210]: failed to encode map e249 with expected crc 2023-09-23T22:48:40.610 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 793/232275 objects degraded (0.341%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:40.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:40 smithi052 ceph-mon[180979]: failed to encode map e249 with expected crc 2023-09-23T22:48:40.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 793/232275 objects degraded (0.341%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:40.617 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:40.681 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:40.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:40.787 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:40.815 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:40.887 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:40.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:40.993 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:41.029 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:41.106 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:41.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:41.221 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:41.269 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:41.325 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:41.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:41.442 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:41.468 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:41.510 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:41.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:41.616 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:41.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:41 smithi151 ceph-mon[159210]: pgmap v653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 793/231075 objects degraded (0.343%); 5084/231075 objects misplaced (2.200%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:41.713 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:41 smithi052 ceph-mon[180979]: pgmap v653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 793/231075 objects degraded (0.343%); 5084/231075 objects misplaced (2.200%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:41.715 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:41 smithi052 ceph-mon[187743]: pgmap v653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 793/231075 objects degraded (0.343%); 5084/231075 objects misplaced (2.200%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:41.724 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:41.968 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:41.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:42.080 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:42.113 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:42.166 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:42.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:42.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:42.271 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:42.393 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:42.424 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:48:42.468 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:48:42.606 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:42.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:42.635 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:48:42.658 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:48:42.721 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:42.734 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:48:42.735 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:48:42.748 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:42.800 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:42.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:42.909 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:42.935 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:42.989 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:42.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:43.096 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:43.133 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:43.201 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:43.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:43.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:43.308 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:43.341 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:43.410 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:43.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:43.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:43.416 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:43 smithi151 ceph-mon[159210]: pgmap v654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 881/225495 objects degraded (0.391%); 4907/225495 objects misplaced (2.176%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:43.517 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:43.548 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:43.603 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:43.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:43.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:43.712 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:43.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:43 smithi052 ceph-mon[180979]: pgmap v654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 881/225495 objects degraded (0.391%); 4907/225495 objects misplaced (2.176%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:43.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:48:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:48:43] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-09-23T22:48:43.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:43 smithi052 ceph-mon[187743]: pgmap v654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 881/225495 objects degraded (0.391%); 4907/225495 objects misplaced (2.176%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:43.746 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:43.802 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:43.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:43.915 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:43.955 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:44.426 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:44.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:44.468 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":654,"stamp":"2023-09-23T22:48:42.232998+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4924970936,"num_objects":75165,"num_object_clones":0,"num_object_copies":225495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":881,"num_objects_misplaced":4907,"num_objects_unfound":0,"num_objects_dirty":75165,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":130017,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":54601802,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122851,"ondisk_log_size":122851,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16513448,"kb_used_data":14547056,"kb_used_omap":129,"kb_used_meta":1965822,"kb_avail":639702616,"statfs":{"total":671965249536,"available":655055478784,"internally_reserved":0,"allocated":14896185344,"data_stored":14891014437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2013002461},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,2,7,11,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384237568,"num_objects":-5863,"num_object_clones":0,"num_object_copies":-17589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":162,"num_objects_misplaced":-337,"num_objects_unfound":0,"num_objects_dirty":-5863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006795"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":246,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.501961+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:48:26.501961+0000","last_peered":"2023-09-23T22:48:26.501961+0000","last_clean":"2023-09-23T22:48:26.501961+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:48:26.501961+0000","last_undegraded":"2023-09-23T22:48:26.501961+0000","last_fullsized":"2023-09-23T22:48:26.501961+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":221,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482173+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:48:26.482173+0000","last_peered":"2023-09-23T22:48:26.482173+0000","last_clean":"2023-09-23T22:48:26.482173+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:48:26.482173+0000","last_undegraded":"2023-09-23T22:48:26.482173+0000","last_fullsized":"2023-09-23T22:48:26.482173+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.481594+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:48:26.481594+0000","last_peered":"2023-09-23T22:48:26.481594+0000","last_clean":"2023-09-23T22:48:26.481594+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:48:26.481594+0000","last_undegraded":"2023-09-23T22:48:26.481594+0000","last_fullsized":"2023-09-23T22:48:26.481594+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":215,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440706+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:48:22.440706+0000","last_peered":"2023-09-23T22:48:22.440706+0000","last_clean":"2023-09-23T22:48:22.440706+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:48:22.440706+0000","last_undegraded":"2023-09-23T22:48:22.440706+0000","last_fullsized":"2023-09-23T22:48:22.440706+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":280,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.483091+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:48:26.483091+0000","last_peered":"2023-09-23T22:48:26.483091+0000","last_clean":"2023-09-23T22:48:26.483091+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:48:26.483091+0000","last_undegraded":"2023-09-23T22:48:26.483091+0000","last_fullsized":"2023-09-23T22:48:26.483091+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":214,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493509+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:48:26.493509+0000","last_peered":"2023-09-23T22:48:26.493509+0000","last_clean":"2023-09-23T22:48:26.493509+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:48:26.493509+0000","last_undegraded":"2023-09-23T22:48:26.493509+0000","last_fullsized":"2023-09-23T22:48:26.493509+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":147,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.497893+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:48:26.497893+0000","last_peered":"2023-09-23T22:48:26.497893+0000","last_clean":"2023-09-23T22:48:26.497893+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:48:26.497893+0000","last_undegraded":"2023-09-23T22:48:26.497893+0000","last_fullsized":"2023-09-23T22:48:26.497893+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.500829+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:48:26.500829+0000","last_peered":"2023-09-23T22:48:26.500829+0000","last_clean":"2023-09-23T22:48:26.500829+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:48:26.500829+0000","last_undegraded":"2023-09-23T22:48:26.500829+0000","last_fullsized":"2023-09-23T22:48:26.500829+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482333+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:48:26.482333+0000","last_peered":"2023-09-23T22:48:26.482333+0000","last_clean":"2023-09-23T22:48:26.482333+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:48:26.482333+0000","last_undegraded":"2023-09-23T22:48:26.482333+0000","last_fullsized":"2023-09-23T22:48:26.482333+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":206,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.481514+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:48:26.481514+0000","last_peered":"2023-09-23T22:48:26.481514+0000","last_clean":"2023-09-23T22:48:26.481514+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:48:26.481514+0000","last_undegraded":"2023-09-23T22:48:26.481514+0000","last_fullsized":"2023-09-23T22:48:26.481514+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":123,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482104+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:48:26.482104+0000","last_peered":"2023-09-23T22:48:26.482104+0000","last_clean":"2023-09-23T22:48:26.482104+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:48:26.482104+0000","last_undegraded":"2023-09-23T22:48:26.482104+0000","last_fullsized":"2023-09-23T22:48:26.482104+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493255+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:48:26.493255+0000","last_peered":"2023-09-23T22:48:26.493255+0000","last_clean":"2023-09-23T22:48:26.493255+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:48:26.493255+0000","last_undegraded":"2023-09-23T22:48:26.493255+0000","last_fullsized":"2023-09-23T22:48:26.493255+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":269,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.501747+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:48:26.501747+0000","last_peered":"2023-09-23T22:48:26.501747+0000","last_clean":"2023-09-23T22:48:26.501747+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:48:26.501747+0000","last_undegraded":"2023-09-23T22:48:26.501747+0000","last_fullsized":"2023-09-23T22:48:26.501747+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":214,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499205+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:48:26.499205+0000","last_peered":"2023-09-23T22:48:26.499205+0000","last_clean":"2023-09-23T22:48:26.499205+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:48:26.499205+0000","last_undegraded":"2023-09-23T22:48:26.499205+0000","last_fullsized":"2023-09-23T22:48:26.499205+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":107,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499259+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:48:26.499259+0000","last_peered":"2023-09-23T22:48:26.499259+0000","last_clean":"2023-09-23T22:48:26.499259+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:48:26.499259+0000","last_undegraded":"2023-09-23T22:48:26.499259+0000","last_fullsized":"2023-09-23T22:48:26.499259+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":268,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482656+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:48:26.482656+0000","last_peered":"2023-09-23T22:48:26.482656+0000","last_clean":"2023-09-23T22:48:26.482656+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:48:26.482656+0000","last_undegraded":"2023-09-23T22:48:26.482656+0000","last_fullsized":"2023-09-23T22:48:26.482656+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":203,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499662+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:48:26.499662+0000","last_peered":"2023-09-23T22:48:26.499662+0000","last_clean":"2023-09-23T22:48:26.499662+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:48:26.499662+0000","last_undegraded":"2023-09-23T22:48:26.499662+0000","last_fullsized":"2023-09-23T22:48:26.499662+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482244+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:48:26.482244+0000","last_peered":"2023-09-23T22:48:26.482244+0000","last_clean":"2023-09-23T22:48:26.482244+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:48:26.482244+0000","last_undegraded":"2023-09-23T22:48:26.482244+0000","last_fullsized":"2023-09-23T22:48:26.482244+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498900+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:48:26.498900+0000","last_peered":"2023-09-23T22:48:26.498900+0000","last_clean":"2023-09-23T22:48:26.498900+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:48:26.498900+0000","last_undegraded":"2023-09-23T22:48:26.498900+0000","last_fullsized":"2023-09-23T22:48:26.498900+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":204,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498478+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:48:26.498478+0000","last_peered":"2023-09-23T22:48:26.498478+0000","last_clean":"2023-09-23T22:48:26.498478+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:48:26.498478+0000","last_undegraded":"2023-09-23T22:48:26.498478+0000","last_fullsized":"2023-09-23T22:48:26.498478+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":293,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493088+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:48:26.493088+0000","last_peered":"2023-09-23T22:48:26.493088+0000","last_clean":"2023-09-23T22:48:26.493088+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:48:26.493088+0000","last_undegraded":"2023-09-23T22:48:26.493088+0000","last_fullsized":"2023-09-23T22:48:26.493088+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.500742+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:48:26.500742+0000","last_peered":"2023-09-23T22:48:26.500742+0000","last_clean":"2023-09-23T22:48:26.500742+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:48:26.500742+0000","last_undegraded":"2023-09-23T22:48:26.500742+0000","last_fullsized":"2023-09-23T22:48:26.500742+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"249'8034","reported_seq":8074,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.480155+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:48:40.480155+0000","last_peered":"2023-09-23T22:48:40.480155+0000","last_clean":"2023-09-23T22:48:40.480155+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:48:40.480155+0000","last_undegraded":"2023-09-23T22:48:40.480155+0000","last_fullsized":"2023-09-23T22:48:40.480155+0000","mapping_epoch":217,"log_start":"223'700","ondisk_log_start":"223'700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309198848,"num_objects":4718,"num_object_clones":0,"num_object_copies":14154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8034,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":118,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498999+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:48:26.498999+0000","last_peered":"2023-09-23T22:48:26.498999+0000","last_clean":"2023-09-23T22:48:26.498999+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:48:26.498999+0000","last_undegraded":"2023-09-23T22:48:26.498999+0000","last_fullsized":"2023-09-23T22:48:26.498999+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":213,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493080+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:48:26.493080+0000","last_peered":"2023-09-23T22:48:26.493080+0000","last_clean":"2023-09-23T22:48:26.493080+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:48:26.493080+0000","last_undegraded":"2023-09-23T22:48:26.493080+0000","last_fullsized":"2023-09-23T22:48:26.493080+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"249'8039","reported_seq":8085,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.791691+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:48:40.791691+0000","last_peered":"2023-09-23T22:48:40.791691+0000","last_clean":"2023-09-23T22:48:40.791691+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:48:40.791691+0000","last_undegraded":"2023-09-23T22:48:40.791691+0000","last_fullsized":"2023-09-23T22:48:40.791691+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8039,"log_dups_size":0,"ondisk_log_size":8039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295895040,"num_objects":4515,"num_object_clones":0,"num_object_copies":13545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8039,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":128,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499459+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:48:26.499459+0000","last_peered":"2023-09-23T22:48:26.499459+0000","last_clean":"2023-09-23T22:48:26.499459+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:48:26.499459+0000","last_undegraded":"2023-09-23T22:48:26.499459+0000","last_fullsized":"2023-09-23T22:48:26.499459+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":229,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498858+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:48:26.498858+0000","last_peered":"2023-09-23T22:48:26.498858+0000","last_clean":"2023-09-23T22:48:26.498858+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:48:26.498858+0000","last_undegraded":"2023-09-23T22:48:26.498858+0000","last_fullsized":"2023-09-23T22:48:26.498858+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"248'7882","reported_seq":7922,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:37.596094+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:48:37.596094+0000","last_peered":"2023-09-23T22:48:37.596094+0000","last_clean":"2023-09-23T22:48:37.596094+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:48:37.596094+0000","last_undegraded":"2023-09-23T22:48:37.596094+0000","last_fullsized":"2023-09-23T22:48:37.596094+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7882,"log_dups_size":0,"ondisk_log_size":7882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":59,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.494909+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:48:26.494909+0000","last_peered":"2023-09-23T22:48:26.494909+0000","last_clean":"2023-09-23T22:48:26.494909+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:48:26.494909+0000","last_undegraded":"2023-09-23T22:48:26.494909+0000","last_fullsized":"2023-09-23T22:48:26.494909+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":241,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493393+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:48:26.493393+0000","last_peered":"2023-09-23T22:48:26.493393+0000","last_clean":"2023-09-23T22:48:26.493393+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:48:26.493393+0000","last_undegraded":"2023-09-23T22:48:26.493393+0000","last_fullsized":"2023-09-23T22:48:26.493393+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"249'8368","reported_seq":8417,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.770364+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:48:40.770364+0000","last_peered":"2023-09-23T22:48:40.770364+0000","last_clean":"2023-09-23T22:48:40.770364+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:48:40.770364+0000","last_undegraded":"2023-09-23T22:48:40.770364+0000","last_fullsized":"2023-09-23T22:48:40.770364+0000","mapping_epoch":226,"log_start":"223'1200","ondisk_log_start":"223'1200","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7168,"log_dups_size":0,"ondisk_log_size":7168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306315264,"num_objects":4674,"num_object_clones":0,"num_object_copies":14022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8368,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482816+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:48:26.482816+0000","last_peered":"2023-09-23T22:48:26.482816+0000","last_clean":"2023-09-23T22:48:26.482816+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:48:26.482816+0000","last_undegraded":"2023-09-23T22:48:26.482816+0000","last_fullsized":"2023-09-23T22:48:26.482816+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498588+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:48:26.498588+0000","last_peered":"2023-09-23T22:48:26.498588+0000","last_clean":"2023-09-23T22:48:26.498588+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:48:26.498588+0000","last_undegraded":"2023-09-23T22:48:26.498588+0000","last_fullsized":"2023-09-23T22:48:26.498588+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"249'8096","reported_seq":8138,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.673761+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:48:40.673761+0000","last_peered":"2023-09-23T22:48:40.673761+0000","last_clean":"2023-09-23T22:48:40.673761+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:48:40.673761+0000","last_undegraded":"2023-09-23T22:48:40.673761+0000","last_fullsized":"2023-09-23T22:48:40.673761+0000","mapping_epoch":217,"log_start":"223'900","ondisk_log_start":"223'900","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7196,"log_dups_size":0,"ondisk_log_size":7196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297009152,"num_objects":4532,"num_object_clones":0,"num_object_copies":13596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8096,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":249,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.481583+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:48:26.481583+0000","last_peered":"2023-09-23T22:48:26.481583+0000","last_clean":"2023-09-23T22:48:26.481583+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:48:26.481583+0000","last_undegraded":"2023-09-23T22:48:26.481583+0000","last_fullsized":"2023-09-23T22:48:26.481583+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":214,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.501662+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:48:26.501662+0000","last_peered":"2023-09-23T22:48:26.501662+0000","last_clean":"2023-09-23T22:48:26.501662+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:48:26.501662+0000","last_undegraded":"2023-09-23T22:48:26.501662+0000","last_fullsized":"2023-09-23T22:48:26.501662+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"248'8035","reported_seq":8096,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:37.600776+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:48:37.600776+0000","last_peered":"2023-09-23T22:48:37.600776+0000","last_clean":"2023-09-23T22:48:37.600776+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:48:37.600776+0000","last_undegraded":"2023-09-23T22:48:37.600776+0000","last_fullsized":"2023-09-23T22:48:37.600776+0000","mapping_epoch":226,"log_start":"223'100","ondisk_log_start":"223'100","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7935,"log_dups_size":0,"ondisk_log_size":7935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310312960,"num_objects":4735,"num_object_clones":0,"num_object_copies":14205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8035,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":233,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493494+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:48:26.493494+0000","last_peered":"2023-09-23T22:48:26.493494+0000","last_clean":"2023-09-23T22:48:26.493494+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:48:26.493494+0000","last_undegraded":"2023-09-23T22:48:26.493494+0000","last_fullsized":"2023-09-23T22:48:26.493494+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":244,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499240+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:48:26.499240+0000","last_peered":"2023-09-23T22:48:26.499240+0000","last_clean":"2023-09-23T22:48:26.499240+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:48:26.499240+0000","last_undegraded":"2023-09-23T22:48:26.499240+0000","last_fullsized":"2023-09-23T22:48:26.499240+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"249'8252","reported_seq":8298,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.515369+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:48:40.515369+0000","last_peered":"2023-09-23T22:48:40.515369+0000","last_clean":"2023-09-23T22:48:40.515369+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:48:40.515369+0000","last_undegraded":"2023-09-23T22:48:40.515369+0000","last_fullsized":"2023-09-23T22:48:40.515369+0000","mapping_epoch":219,"log_start":"223'900","ondisk_log_start":"223'900","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7352,"log_dups_size":0,"ondisk_log_size":7352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311033856,"num_objects":4746,"num_object_clones":0,"num_object_copies":14238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8252,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":257,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498750+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:48:26.498750+0000","last_peered":"2023-09-23T22:48:26.498750+0000","last_clean":"2023-09-23T22:48:26.498750+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:48:26.498750+0000","last_undegraded":"2023-09-23T22:48:26.498750+0000","last_fullsized":"2023-09-23T22:48:26.498750+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499512+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:48:26.499512+0000","last_peered":"2023-09-23T22:48:26.499512+0000","last_clean":"2023-09-23T22:48:26.499512+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:48:26.499512+0000","last_undegraded":"2023-09-23T22:48:26.499512+0000","last_fullsized":"2023-09-23T22:48:26.499512+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"249'7925","reported_seq":7982,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.762509+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:48:40.762509+0000","last_peered":"2023-09-23T22:48:40.762509+0000","last_clean":"2023-09-23T22:48:40.762509+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:48:40.762509+0000","last_undegraded":"2023-09-23T22:48:40.762509+0000","last_fullsized":"2023-09-23T22:48:40.762509+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7925,"log_dups_size":0,"ondisk_log_size":7925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296288256,"num_objects":4521,"num_object_clones":0,"num_object_copies":13563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7925,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482946+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:48:26.482946+0000","last_peered":"2023-09-23T22:48:26.482946+0000","last_clean":"2023-09-23T22:48:26.482946+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:48:26.482946+0000","last_undegraded":"2023-09-23T22:48:26.482946+0000","last_fullsized":"2023-09-23T22:48:26.482946+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":175,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.500167+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:48:26.500167+0000","last_peered":"2023-09-23T22:48:26.500167+0000","last_clean":"2023-09-23T22:48:26.500167+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:48:26.500167+0000","last_undegraded":"2023-09-23T22:48:26.500167+0000","last_fullsized":"2023-09-23T22:48:26.500167+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"248'8084","reported_seq":8124,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:37.572835+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:48:37.572835+0000","last_peered":"2023-09-23T22:48:37.572835+0000","last_clean":"2023-09-23T22:48:37.572835+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:48:37.572835+0000","last_undegraded":"2023-09-23T22:48:37.572835+0000","last_fullsized":"2023-09-23T22:48:37.572835+0000","mapping_epoch":219,"log_start":"223'100","ondisk_log_start":"223'100","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7984,"log_dups_size":0,"ondisk_log_size":7984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311033856,"num_objects":4746,"num_object_clones":0,"num_object_copies":14238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8084,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":283,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.435234+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:48:22.435234+0000","last_peered":"2023-09-23T22:48:22.435234+0000","last_clean":"2023-09-23T22:48:22.435234+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:48:22.435234+0000","last_undegraded":"2023-09-23T22:48:22.435234+0000","last_fullsized":"2023-09-23T22:48:22.435234+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":168,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482017+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:48:26.482017+0000","last_peered":"2023-09-23T22:48:26.482017+0000","last_clean":"2023-09-23T22:48:26.482017+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:48:26.482017+0000","last_undegraded":"2023-09-23T22:48:26.482017+0000","last_fullsized":"2023-09-23T22:48:26.482017+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.501906+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:48:26.501906+0000","last_peered":"2023-09-23T22:48:26.501906+0000","last_clean":"2023-09-23T22:48:26.501906+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:48:26.501906+0000","last_undegraded":"2023-09-23T22:48:26.501906+0000","last_fullsized":"2023-09-23T22:48:26.501906+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"249'8083","reported_seq":8131,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.739263+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:48:40.739263+0000","last_peered":"2023-09-23T22:48:40.739263+0000","last_clean":"2023-09-23T22:48:40.739263+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:48:40.739263+0000","last_undegraded":"2023-09-23T22:48:40.739263+0000","last_fullsized":"2023-09-23T22:48:40.739263+0000","mapping_epoch":219,"log_start":"223'900","ondisk_log_start":"223'900","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7183,"log_dups_size":0,"ondisk_log_size":7183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304545792,"num_objects":4647,"num_object_clones":0,"num_object_copies":13941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8083,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":83,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493521+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:48:26.493521+0000","last_peered":"2023-09-23T22:48:26.493521+0000","last_clean":"2023-09-23T22:48:26.493521+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:48:26.493521+0000","last_undegraded":"2023-09-23T22:48:26.493521+0000","last_fullsized":"2023-09-23T22:48:26.493521+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":245,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499292+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:48:26.499292+0000","last_peered":"2023-09-23T22:48:26.499292+0000","last_clean":"2023-09-23T22:48:26.499292+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:48:26.499292+0000","last_undegraded":"2023-09-23T22:48:26.499292+0000","last_fullsized":"2023-09-23T22:48:26.499292+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":164,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.495249+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:48:26.495249+0000","last_peered":"2023-09-23T22:48:26.495249+0000","last_clean":"2023-09-23T22:48:26.495249+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:48:26.495249+0000","last_undegraded":"2023-09-23T22:48:26.495249+0000","last_fullsized":"2023-09-23T22:48:26.495249+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"249'8231","reported_seq":8272,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.793335+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:48:40.793335+0000","last_peered":"2023-09-23T22:48:40.793335+0000","last_clean":"2023-09-23T22:48:40.793335+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:48:40.793335+0000","last_undegraded":"2023-09-23T22:48:40.793335+0000","last_fullsized":"2023-09-23T22:48:40.793335+0000","mapping_epoch":217,"log_start":"223'1000","ondisk_log_start":"223'1000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7231,"log_dups_size":0,"ondisk_log_size":7231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307822592,"num_objects":4697,"num_object_clones":0,"num_object_copies":14091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8231,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":239,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.495041+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:48:26.495041+0000","last_peered":"2023-09-23T22:48:26.495041+0000","last_clean":"2023-09-23T22:48:26.495041+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:48:26.495041+0000","last_undegraded":"2023-09-23T22:48:26.495041+0000","last_fullsized":"2023-09-23T22:48:26.495041+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":127,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.494579+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:48:26.494579+0000","last_peered":"2023-09-23T22:48:26.494579+0000","last_clean":"2023-09-23T22:48:26.494579+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:48:26.494579+0000","last_undegraded":"2023-09-23T22:48:26.494579+0000","last_fullsized":"2023-09-23T22:48:26.494579+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":252,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440290+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:48:22.440290+0000","last_peered":"2023-09-23T22:48:22.440290+0000","last_clean":"2023-09-23T22:48:22.440290+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:48:22.440290+0000","last_undegraded":"2023-09-23T22:48:22.440290+0000","last_fullsized":"2023-09-23T22:48:22.440290+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"249'8063","reported_seq":8109,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:40.795369+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:48:40.795369+0000","last_peered":"2023-09-23T22:48:40.795369+0000","last_clean":"2023-09-23T22:48:40.795369+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:48:40.795369+0000","last_undegraded":"2023-09-23T22:48:40.795369+0000","last_fullsized":"2023-09-23T22:48:40.795369+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8063,"log_dups_size":0,"ondisk_log_size":8063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298254336,"num_objects":4551,"num_object_clones":0,"num_object_copies":13653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8063,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":249,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.494716+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:48:26.494716+0000","last_peered":"2023-09-23T22:48:26.494716+0000","last_clean":"2023-09-23T22:48:26.494716+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:48:26.494716+0000","last_undegraded":"2023-09-23T22:48:26.494716+0000","last_fullsized":"2023-09-23T22:48:26.494716+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":334,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419700+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:48:22.419700+0000","last_peered":"2023-09-23T22:48:22.419700+0000","last_clean":"2023-09-23T22:48:22.419700+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:48:22.419700+0000","last_undegraded":"2023-09-23T22:48:22.419700+0000","last_fullsized":"2023-09-23T22:48:22.419700+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":180,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:30.730634+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:48:30.730634+0000","last_peered":"2023-09-23T22:48:30.730634+0000","last_clean":"2023-09-23T22:48:30.730634+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:48:30.730634+0000","last_undegraded":"2023-09-23T22:48:30.730634+0000","last_fullsized":"2023-09-23T22:48:30.730634+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":241,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.495064+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:48:26.495064+0000","last_peered":"2023-09-23T22:48:26.495064+0000","last_clean":"2023-09-23T22:48:26.495064+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:48:26.495064+0000","last_undegraded":"2023-09-23T22:48:26.495064+0000","last_fullsized":"2023-09-23T22:48:26.495064+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"249'8180","reported_seq":8229,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:39.539535+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:48:39.539535+0000","last_peered":"2023-09-23T22:48:39.539535+0000","last_clean":"2023-09-23T22:48:39.539535+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:48:39.539535+0000","last_undegraded":"2023-09-23T22:48:39.539535+0000","last_fullsized":"2023-09-23T22:48:39.539535+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8180,"log_dups_size":0,"ondisk_log_size":8180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312475648,"num_objects":4768,"num_object_clones":0,"num_object_copies":14304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8180,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419785+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:48:22.419785+0000","last_peered":"2023-09-23T22:48:22.419785+0000","last_clean":"2023-09-23T22:48:22.419785+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:48:22.419785+0000","last_undegraded":"2023-09-23T22:48:22.419785+0000","last_fullsized":"2023-09-23T22:48:22.419785+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.419923+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:48:22.419923+0000","last_peered":"2023-09-23T22:48:22.419923+0000","last_clean":"2023-09-23T22:48:22.419923+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:48:22.419923+0000","last_undegraded":"2023-09-23T22:48:22.419923+0000","last_fullsized":"2023-09-23T22:48:22.419923+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":171,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.626308+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:48:22.626308+0000","last_peered":"2023-09-23T22:48:22.626308+0000","last_clean":"2023-09-23T22:48:22.626308+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:48:22.626308+0000","last_undegraded":"2023-09-23T22:48:22.626308+0000","last_fullsized":"2023-09-23T22:48:22.626308+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"249'8344","reported_seq":10055,"reported_epoch":249,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:48:40.793868+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:48:40.793868+0000","last_peered":"2023-09-23T22:48:40.793868+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:48:40.793868+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"223'1200","ondisk_log_start":"223'1200","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7144,"log_dups_size":0,"ondisk_log_size":7144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322699264,"num_objects":4924,"num_object_clones":0,"num_object_copies":14772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":881,"num_objects_misplaced":4907,"num_objects_unfound":0,"num_objects_dirty":4924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8344,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":830,"num_bytes_recovered":54394880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-864},{"shards":"1,3,4","objects":4907}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440211+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:48:22.440211+0000","last_peered":"2023-09-23T22:48:22.440211+0000","last_clean":"2023-09-23T22:48:22.440211+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:48:22.440211+0000","last_undegraded":"2023-09-23T22:48:22.440211+0000","last_fullsized":"2023-09-23T22:48:22.440211+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":126,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.495404+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:48:26.495404+0000","last_peered":"2023-09-23T22:48:26.495404+0000","last_clean":"2023-09-23T22:48:26.495404+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:48:26.495404+0000","last_undegraded":"2023-09-23T22:48:26.495404+0000","last_fullsized":"2023-09-23T22:48:26.495404+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.420413+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:48:22.420413+0000","last_peered":"2023-09-23T22:48:22.420413+0000","last_clean":"2023-09-23T22:48:22.420413+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:48:22.420413+0000","last_undegraded":"2023-09-23T22:48:22.420413+0000","last_fullsized":"2023-09-23T22:48:22.420413+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"248'8037","reported_seq":8077,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:37.593459+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:48:37.593459+0000","last_peered":"2023-09-23T22:48:37.593459+0000","last_clean":"2023-09-23T22:48:37.593459+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:48:37.593459+0000","last_undegraded":"2023-09-23T22:48:37.593459+0000","last_fullsized":"2023-09-23T22:48:37.593459+0000","mapping_epoch":219,"log_start":"223'100","ondisk_log_start":"223'100","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319225856,"num_objects":4871,"num_object_clones":0,"num_object_copies":14613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8037,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":288,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.500655+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:48:26.500655+0000","last_peered":"2023-09-23T22:48:26.500655+0000","last_clean":"2023-09-23T22:48:26.500655+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:48:26.500655+0000","last_undegraded":"2023-09-23T22:48:26.500655+0000","last_fullsized":"2023-09-23T22:48:26.500655+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":47,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493493+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:48:26.493493+0000","last_peered":"2023-09-23T22:48:26.493493+0000","last_clean":"2023-09-23T22:48:26.493493+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:48:26.493493+0000","last_undegraded":"2023-09-23T22:48:26.493493+0000","last_fullsized":"2023-09-23T22:48:26.493493+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":274,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.483016+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:48:26.483016+0000","last_peered":"2023-09-23T22:48:26.483016+0000","last_clean":"2023-09-23T22:48:26.483016+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:48:26.483016+0000","last_undegraded":"2023-09-23T22:48:26.483016+0000","last_fullsized":"2023-09-23T22:48:26.483016+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"249'8104","reported_seq":8147,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:39.483239+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:48:39.483239+0000","last_peered":"2023-09-23T22:48:39.483239+0000","last_clean":"2023-09-23T22:48:39.483239+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:48:39.483239+0000","last_undegraded":"2023-09-23T22:48:39.483239+0000","last_fullsized":"2023-09-23T22:48:39.483239+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8104,"log_dups_size":0,"ondisk_log_size":8104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313327640,"num_objects":4782,"num_object_clones":0,"num_object_copies":14346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4782,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8104,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":281,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.493402+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:48:26.493402+0000","last_peered":"2023-09-23T22:48:26.493402+0000","last_clean":"2023-09-23T22:48:26.493402+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:48:26.493402+0000","last_undegraded":"2023-09-23T22:48:26.493402+0000","last_fullsized":"2023-09-23T22:48:26.493402+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":143,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.626327+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:48:22.626327+0000","last_peered":"2023-09-23T22:48:22.626327+0000","last_clean":"2023-09-23T22:48:22.626327+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:48:22.626327+0000","last_undegraded":"2023-09-23T22:48:22.626327+0000","last_fullsized":"2023-09-23T22:48:22.626327+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":199,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498790+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:48:26.498790+0000","last_peered":"2023-09-23T22:48:26.498790+0000","last_clean":"2023-09-23T22:48:26.498790+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:48:26.498790+0000","last_undegraded":"2023-09-23T22:48:26.498790+0000","last_fullsized":"2023-09-23T22:48:26.498790+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":178,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.420288+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:48:22.420288+0000","last_peered":"2023-09-23T22:48:22.420288+0000","last_clean":"2023-09-23T22:48:22.420288+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:48:22.420288+0000","last_undegraded":"2023-09-23T22:48:22.420288+0000","last_fullsized":"2023-09-23T22:48:22.420288+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":21,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.495326+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:48:26.495326+0000","last_peered":"2023-09-23T22:48:26.495326+0000","last_clean":"2023-09-23T22:48:26.495326+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:48:26.495326+0000","last_undegraded":"2023-09-23T22:48:26.495326+0000","last_fullsized":"2023-09-23T22:48:26.495326+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":59,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.501571+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:48:26.501571+0000","last_peered":"2023-09-23T22:48:26.501571+0000","last_clean":"2023-09-23T22:48:26.501571+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:48:26.501571+0000","last_undegraded":"2023-09-23T22:48:26.501571+0000","last_fullsized":"2023-09-23T22:48:26.501571+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":248,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.499344+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:48:26.499344+0000","last_peered":"2023-09-23T22:48:26.499344+0000","last_clean":"2023-09-23T22:48:26.499344+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:48:26.499344+0000","last_undegraded":"2023-09-23T22:48:26.499344+0000","last_fullsized":"2023-09-23T22:48:26.499344+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":59,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.501568+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:48:26.501568+0000","last_peered":"2023-09-23T22:48:26.501568+0000","last_clean":"2023-09-23T22:48:26.501568+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:48:26.501568+0000","last_undegraded":"2023-09-23T22:48:26.501568+0000","last_fullsized":"2023-09-23T22:48:26.501568+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.502003+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:48:26.502003+0000","last_peered":"2023-09-23T22:48:26.502003+0000","last_clean":"2023-09-23T22:48:26.502003+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:48:26.502003+0000","last_undegraded":"2023-09-23T22:48:26.502003+0000","last_fullsized":"2023-09-23T22:48:26.502003+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":242,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.482410+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:48:26.482410+0000","last_peered":"2023-09-23T22:48:26.482410+0000","last_clean":"2023-09-23T22:48:26.482410+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:48:26.482410+0000","last_undegraded":"2023-09-23T22:48:26.482410+0000","last_fullsized":"2023-09-23T22:48:26.482410+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":280,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.481437+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:48:26.481437+0000","last_peered":"2023-09-23T22:48:26.481437+0000","last_clean":"2023-09-23T22:48:26.481437+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:48:26.481437+0000","last_undegraded":"2023-09-23T22:48:26.481437+0000","last_fullsized":"2023-09-23T22:48:26.481437+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T22:48:22.440466+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:48:22.440466+0000","last_peered":"2023-09-23T22:48:22.440466+0000","last_clean":"2023-09-23T22:48:22.440466+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:48:22.440466+0000","last_undegraded":"2023-09-23T22:48:22.440466+0000","last_fullsized":"2023-09-23T22:48:22.440466+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":214,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:48:26.498647+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:48:26.498647+0000","last_peered":"2023-09-23T22:48:26.498647+0000","last_clean":"2023-09-23T22:48:26.498647+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:48:26.498647+0000","last_undegraded":"2023-09-23T22:48:26.498647+0000","last_fullsized":"2023-09-23T22:48:26.498647+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4924375064,"num_objects":75141,"num_object_clones":0,"num_object_copies":225423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":881,"num_objects_misplaced":4907,"num_objects_unfound":0,"num_objects_dirty":75141,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129757,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":833,"num_bytes_recovered":54591488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14832054272,"data_stored":14832042056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122657,"ondisk_log_size":122657,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430468,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626696,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2088796,"kb_used_data":1779220,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91656356,"statfs":{"total":95995035648,"available":93856108544,"internally_reserved":0,"allocated":1821921280,"data_stored":1821419903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":4.6740000000000004,"5min":1.966,"15min":1.514},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":36.651000000000003,"5min":36.651000000000003,"15min":36.651000000000003},"last":0.65700000000000003},{"interface":"front","average":{"1min":5.5049999999999999,"5min":1.8819999999999999,"15min":1.278},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":36.558999999999997,"5min":36.558999999999997,"15min":36.558999999999997},"last":0.78900000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.044,"15min":2.2370000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.741,"5min":14.391999999999999,"15min":153.78399999999999},"last":1.3700000000000001},{"interface":"front","average":{"1min":10.359999999999999,"5min":3.7250000000000001,"15min":2.819},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":185.411,"5min":185.411,"15min":185.411},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":2.3959999999999999,"15min":3.044},"min":{"1min":0.33800000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.802,"5min":12.802,"15min":100.785},"last":3.0950000000000002},{"interface":"front","average":{"1min":11.632999999999999,"5min":4.0060000000000002,"15min":3.5270000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":185.21799999999999,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.746}]},{"osd":3,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":19.305,"5min":6.641,"15min":3.2610000000000001},"min":{"1min":0.378,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":185.19399999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":1.218},{"interface":"front","average":{"1min":11.478999999999999,"5min":5.4870000000000001,"15min":2.9390000000000001},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":128.19800000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.79300000000000004,"15min":0.66400000000000003},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":6.5309999999999997,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.31},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.89900000000000002,"15min":0.69499999999999995},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":6.9089999999999998,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.443}]},{"osd":5,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":0.93200000000000005,"15min":0.72499999999999998},"min":{"1min":0.313,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":10.148999999999999,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.92300000000000004,"15min":0.73999999999999999},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.2930000000000001,"5min":9.7899999999999991,"15min":18.625},"last":0.36599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.0600000000000001,"15min":0.73699999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":4.2640000000000002,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.034,"15min":0.71599999999999997},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":4.165,"5min":16.661999999999999,"15min":16.661999999999999},"last":0.42899999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253006,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2376724,"kb_used_data":2078348,"kb_used_omap":9,"kb_used_meta":298230,"kb_avail":91368428,"statfs":{"total":95995035648,"available":93561270272,"internally_reserved":0,"allocated":2128228352,"data_stored":2127329857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":305388521},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":9.6259999999999994,"5min":5.2590000000000003,"15min":4.5309999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":101.605,"5min":101.605,"15min":101.605},"last":3.1139999999999999},{"interface":"front","average":{"1min":10.422000000000001,"5min":5.6959999999999997,"15min":4.9080000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":101.892,"5min":101.892,"15min":101.892},"last":4.4630000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":2.9689999999999999,"15min":2.0190000000000001},"min":{"1min":0.23000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":36.456000000000003,"5min":36.456000000000003,"15min":37.889000000000003},"last":2.8220000000000001},{"interface":"front","average":{"1min":4.141,"5min":3.4209999999999998,"15min":2.04},"min":{"1min":0.315,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":36.427,"5min":36.427,"15min":37.823},"last":3.2309999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6280000000000001,"5min":2.7839999999999998,"15min":1.5529999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.5259999999999998,"5min":16.631,"15min":50.033999999999999},"last":0.308},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.4900000000000002,"15min":1.847},"min":{"1min":0.34399999999999997,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.8819999999999997,"5min":29.565000000000001,"15min":52.491999999999997},"last":8.2899999999999991}]},{"osd":4,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.5830000000000002,"5min":3.2080000000000002,"15min":1.8049999999999999},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":78.948999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":4.7999999999999998,"5min":2.4159999999999999,"15min":1.6220000000000001},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.313},"max":{"1min":78.963999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":2.7240000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.3620000000000001,"5min":2.556,"15min":1.472},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":84.254999999999995,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.37},{"interface":"front","average":{"1min":6.2939999999999996,"5min":3.2879999999999998,"15min":1.6719999999999999},"min":{"1min":0.32800000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":87.515000000000001,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0250000000000004,"5min":2.6819999999999999,"15min":1.45},"min":{"1min":0.436,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":79.311000000000007,"5min":79.311000000000007,"15min":79.311000000000007},"last":2.7629999999999999},{"interface":"front","average":{"1min":5.4189999999999996,"5min":2.7029999999999998,"15min":1.405},"min":{"1min":0.34699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":79.328999999999994,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.5910000000000002,"15min":2.0059999999999998},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":8.5449999999999999,"5min":29.539000000000001,"15min":159.018},"last":8.2279999999999998},{"interface":"front","average":{"1min":1.6519999999999999,"5min":2.0259999999999998,"15min":1.9179999999999999},"min":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":6.0810000000000004,"5min":8.6120000000000001,"15min":158.96000000000001},"last":2.7949999999999999}]}]},{"osd":0,"up_from":219,"seq":940597837854,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2598668,"kb_used_data":2442436,"kb_used_omap":31,"kb_used_meta":156192,"kb_avail":91146484,"statfs":{"total":95995035648,"available":93333999616,"internally_reserved":0,"allocated":2501054464,"data_stored":2500097367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":159941554},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,0,4,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":3.556,"15min":3.6600000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":15.611000000000001,"5min":32.344999999999999,"15min":32.344999999999999},"last":2.222},{"interface":"front","average":{"1min":3.6389999999999998,"5min":3.5750000000000002,"15min":3.5640000000000001},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.092000000000001,"5min":32.106000000000002,"15min":32.106000000000002},"last":4.266}]},{"osd":2,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":2.468,"15min":2.2210000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.170999999999999,"5min":12.170999999999999,"15min":12.170999999999999},"last":9.1709999999999994},{"interface":"front","average":{"1min":3.27,"5min":2.0590000000000002,"15min":1.857},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9180000000000001,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":3.3889999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":2.1549999999999998,"15min":2.0289999999999999},"min":{"1min":0.34699999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":4.3079999999999998},{"interface":"front","average":{"1min":2.6360000000000001,"5min":1.4259999999999999,"15min":1.224},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.9730000000000008,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":2.5350000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7280000000000002,"5min":2.2280000000000002,"15min":1.978},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":15.432,"5min":15.432,"15min":15.432},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.7509999999999999,"15min":1.7490000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9790000000000001,"5min":7.5590000000000002,"15min":7.5590000000000002},"last":4.3419999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.601,"15min":1.4239999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":2.4329999999999998},{"interface":"front","average":{"1min":3.1019999999999999,"5min":1.8620000000000001,"15min":1.6559999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":15.536,"5min":15.536,"15min":15.536},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.6699999999999999,"15min":1.4710000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":2.4750000000000001},{"interface":"front","average":{"1min":3.952,"5min":2.1389999999999998,"15min":1.837},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.489000000000001,"5min":15.489000000000001,"15min":15.489000000000001},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.9690000000000001,"15min":1.72},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":15.574,"5min":15.574,"15min":15.574},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.4139999999999999,"15min":1.256},"min":{"1min":0.46200000000000002,"5min":0.432,"15min":0.432},"max":{"1min":8.9000000000000004,"5min":8.9000000000000004,"15min":8.9000000000000004},"last":2.504}]}]},{"osd":2,"up_from":131,"seq":562640715867,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2038656,"kb_used_data":1816568,"kb_used_omap":81,"kb_used_meta":221998,"kb_avail":91706496,"statfs":{"total":95995035648,"available":93907451904,"internally_reserved":0,"allocated":1860165632,"data_stored":1859228428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":227326170},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":5.1059999999999999,"5min":5.2359999999999998,"15min":5.258},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":41.723999999999997,"5min":57.811999999999998,"15min":57.811999999999998},"last":4.5940000000000003},{"interface":"front","average":{"1min":7.7679999999999998,"5min":5.8650000000000002,"15min":5.548},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":41.454000000000001,"5min":57.387999999999998,"15min":57.387999999999998},"last":13.865}]},{"osd":1,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":3.5179999999999998,"15min":3.226},"min":{"1min":0.28299999999999997,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.759,"5min":88.316000000000003,"15min":88.316000000000003},"last":3.5259999999999998},{"interface":"front","average":{"1min":4.444,"5min":4.4329999999999998,"15min":2.8719999999999999},"min":{"1min":0.34899999999999998,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":28.596,"5min":88.436999999999998,"15min":88.436999999999998},"last":7.2590000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.976,"15min":3.056},"min":{"1min":0.30499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.994,"5min":88.653000000000006,"15min":88.653000000000006},"last":6.6429999999999998},{"interface":"front","average":{"1min":3.9369999999999998,"5min":3.5880000000000001,"15min":2.4220000000000002},"min":{"1min":0.31900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":12.401,"5min":88.474000000000004,"15min":88.474000000000004},"last":3.4689999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.188000000000001,"5min":4.4569999999999999,"15min":2.1389999999999998},"min":{"1min":0.51300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":84.778999999999996,"5min":108.878,"15min":108.878},"last":1.972},{"interface":"front","average":{"1min":11.103,"5min":5.8799999999999999,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":84.753,"5min":108.93000000000001,"15min":108.93000000000001},"last":2.1800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":7.6660000000000004,"5min":5.3339999999999996,"15min":2.7480000000000002},"min":{"1min":0.42499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":90.082999999999998,"5min":97.322999999999993,"15min":97.322999999999993},"last":2.2189999999999999},{"interface":"front","average":{"1min":7.9420000000000002,"5min":5.2549999999999999,"15min":2.3340000000000001},"min":{"1min":0.437,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":89.986000000000004,"5min":97.301000000000002,"15min":97.301000000000002},"last":6.5730000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.731,"5min":4.9109999999999996,"15min":2.2530000000000001},"min":{"1min":0.40699999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":85.034999999999997,"5min":88.606999999999999,"15min":88.606999999999999},"last":6.6139999999999999},{"interface":"front","average":{"1min":9.9339999999999993,"5min":3.3479999999999999,"15min":1.7729999999999999},"min":{"1min":0.47399999999999998,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":85.209000000000003,"5min":85.209000000000003,"15min":85.209000000000003},"last":1.9390000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.726,"15min":1.266},"min":{"1min":0.40699999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9640000000000004,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":2.032},{"interface":"front","average":{"1min":2.9329999999999998,"5min":3.4910000000000001,"15min":2.1720000000000002},"min":{"1min":0.45100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.5549999999999997,"5min":88.584000000000003,"15min":88.584000000000003},"last":2.2410000000000001}]}]},{"osd":3,"up_from":94,"seq":403726925953,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2184464,"kb_used_data":1819272,"kb_used_omap":7,"kb_used_meta":364984,"kb_avail":91560688,"statfs":{"total":95995035648,"available":93758144512,"internally_reserved":0,"allocated":1862934528,"data_stored":1862037790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":373744232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,2,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:13 2023","interfaces":[{"interface":"back","average":{"1min":6.5970000000000004,"5min":2.8069999999999999,"15min":2.1749999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":80.099000000000004,"5min":80.099000000000004,"15min":80.099000000000004},"last":1.4450000000000001},{"interface":"front","average":{"1min":6.5149999999999997,"5min":2.581,"15min":1.925},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":80.150000000000006,"5min":80.150000000000006,"15min":80.150000000000006},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.8639999999999999,"15min":1.3979999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":6.7110000000000003,"5min":22.818000000000001,"15min":22.818000000000001},"last":8.0489999999999995},{"interface":"front","average":{"1min":1.472,"5min":2,"15min":1.1160000000000001},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4160000000000004,"5min":23.917999999999999,"15min":23.917999999999999},"last":15.098000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.367,"5min":3.879,"15min":2.2509999999999999},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.431000000000001,"5min":125.446,"15min":125.446},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.476,"5min":5.7249999999999996,"15min":2.7330000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":14.670999999999999,"5min":182.483,"15min":182.483},"last":5.5469999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1150000000000002,"15min":1.2310000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":3.7959999999999998,"5min":18.757000000000001,"15min":18.757000000000001},"last":3.8279999999999998},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.5740000000000001,"15min":1.0649999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.516,"5min":12.097,"15min":12.097},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.4570000000000001,"15min":1.0229999999999999},"min":{"1min":0.30099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.4710000000000001,"5min":12.127000000000001,"15min":12.127000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.52,"5min":2.105,"15min":1.2150000000000001},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.9950000000000001,"5min":9.0090000000000003,"15min":9.0090000000000003},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.589,"15min":1.0669999999999999},"min":{"1min":0.34000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.173,"5min":12.16,"15min":12.16},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.5830000000000002,"5min":2.089,"15min":1.2869999999999999},"min":{"1min":0.47099999999999997,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.0259999999999998,"5min":9.1029999999999998,"15min":10.755000000000001},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8399999999999999,"5min":2.9940000000000002,"15min":1.5649999999999999},"min":{"1min":0.40100000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.0510000000000002,"5min":91.757999999999996,"15min":91.757999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.7170000000000001,"5min":3.234,"15min":1.639},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.7709999999999999,"5min":91.709999999999994,"15min":91.709999999999994},"last":3.9119999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019152,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2232088,"kb_used_data":1922576,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91513064,"statfs":{"total":95995035648,"available":93709377536,"internally_reserved":0,"allocated":1968717824,"data_stored":1968220017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":6.6619999999999999,"15min":7.5640000000000001},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.0190000000000001,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.2759999999999998,"5min":7.2779999999999996,"15min":8.1120000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.7200000000000006,"5min":110.33799999999999,"15min":110.33799999999999},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":2.597,"15min":1.4610000000000001},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.681,"5min":41.643000000000001,"15min":45.320999999999998},"last":4.2190000000000003},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.117,"15min":1.3160000000000001},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.8210000000000002,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":4.2560000000000002,"15min":3.2400000000000002},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.7030000000000003,"5min":84.909000000000006,"15min":84.909000000000006},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.1659999999999999,"5min":4.2590000000000003,"15min":3.1669999999999998},"min":{"1min":0.371,"5min":0.309,"15min":0.309},"max":{"1min":7.0570000000000004,"5min":83.840000000000003,"15min":83.840000000000003},"last":0.70899999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":6.5529999999999999,"15min":2.8809999999999998},"min":{"1min":0.42699999999999999,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":6.1040000000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":1.097},{"interface":"front","average":{"1min":1.7689999999999999,"5min":6.7160000000000002,"15min":2.915},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.657,"5min":129.197,"15min":129.197},"last":4.125}]},{"osd":5,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.746,"15min":0.73999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.8979999999999997,"5min":5.8979999999999997,"15min":43.189999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.78000000000000003,"15min":0.76100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26500000000000001},"max":{"1min":2.7040000000000002,"5min":9.5190000000000001,"15min":42.450000000000003},"last":0.85699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.79300000000000004,"15min":0.73099999999999998},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.7570000000000001,"5min":6.8109999999999999,"15min":27.056000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78800000000000003,"15min":0.75800000000000001},"min":{"1min":0.32500000000000001,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":2.6520000000000001,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.754,"15min":0.66400000000000003},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.8460000000000001,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.69199999999999995,"15min":0.628},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":2.3130000000000002,"5min":3.2519999999999998,"15min":5.9409999999999998},"last":0.47799999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822924,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2994052,"kb_used_data":2688636,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90751100,"statfs":{"total":95995035648,"available":92929126400,"internally_reserved":0,"allocated":2753163264,"data_stored":2752681075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":18.346,"5min":5.1150000000000002,"15min":2.9100000000000001},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":121.319,"5min":121.319,"15min":121.319},"last":2.0659999999999998},{"interface":"front","average":{"1min":18.654,"5min":5.0469999999999997,"15min":2.7789999999999999},"min":{"1min":0.32700000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":121.075,"5min":121.075,"15min":121.075},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8,"15min":1.716},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":28.658999999999999,"5min":28.658999999999999,"15min":149.203},"last":0.35399999999999998},{"interface":"front","average":{"1min":3.8969999999999998,"5min":2.3999999999999999,"15min":1.871},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":28.664999999999999,"5min":28.664999999999999,"15min":149.554},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":4.5599999999999996,"15min":7.141},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.2879999999999998,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.861,"5min":4.7949999999999999,"15min":7.1959999999999997},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3899999999999997,"5min":158.91399999999999,"15min":158.91399999999999},"last":1.8999999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":14.204000000000001,"5min":8.2560000000000002,"15min":3.9020000000000001},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":108.58799999999999,"5min":257.05000000000001,"15min":257.05000000000001},"last":1.004},{"interface":"front","average":{"1min":14.087999999999999,"5min":7.6820000000000004,"15min":3.7570000000000001},"min":{"1min":0.311,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":108.15600000000001,"5min":257.35000000000002,"15min":257.35000000000002},"last":3.8849999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.79300000000000004,"15min":0.60299999999999998},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":1.4790000000000001,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.79800000000000004,"15min":0.61299999999999999},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.619,"5min":10.535,"15min":10.535},"last":0.32000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.411,"15min":1.3879999999999999},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":12.615,"5min":48.360999999999997,"15min":140.07499999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.417,"5min":1.532,"15min":1.4399999999999999},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.254},"max":{"1min":12.74,"5min":48.292999999999999,"15min":139.739},"last":0.372}]},{"osd":7,"last update":"Sat Sep 23 22:47:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.81699999999999995,"15min":0.69599999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.3450000000000002,"5min":7.0010000000000003,"15min":7.9809999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.78500000000000003,"15min":0.66000000000000003},"min":{"1min":0.36699999999999999,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":1.9830000000000001,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.46899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2488598528,"data_stored":2488598528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2120351744,"data_stored":2120351744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1856180224,"data_stored":1856176152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1847656448,"data_stored":1847656448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1963065344,"data_stored":1963065344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2746290176,"data_stored":2746286104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1809911808,"data_stored":1809907736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:48:44.474 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:48:44.516 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:48:44.537 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:44.678 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:44 smithi052 ceph-mon[180979]: from='client.45377 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:44.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:44 smithi052 ceph-mon[187743]: from='client.45377 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:44.678 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:44 smithi151 ceph-mon[159210]: from='client.45377 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:48:44.851 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:44.916 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:44.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:44.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:44.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:45.076 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:45.161 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:45.295 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:45.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:45.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:45.410 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:45.454 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:45.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:45 smithi052 ceph-mon[180979]: pgmap v655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 881/223251 objects degraded (0.395%); 4907/223251 objects misplaced (2.198%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:45.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 881/223251 objects degraded (0.395%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:45.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:45 smithi052 ceph-mon[187743]: pgmap v655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 881/223251 objects degraded (0.395%); 4907/223251 objects misplaced (2.198%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:45.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 881/223251 objects degraded (0.395%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:45.513 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:45.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:45.618 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:45.643 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:45 smithi151 ceph-mon[159210]: pgmap v655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 881/223251 objects degraded (0.395%); 4907/223251 objects misplaced (2.198%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:45.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 881/223251 objects degraded (0.395%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:45.646 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:45.706 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:45.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:45.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:45.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:45.898 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:45.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:46.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:46.054 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:46.942 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:46.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:46.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:47.062 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:47.111 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:47.517 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:47.529 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:47.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:47.621 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:47.639 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:47.663 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:47 smithi151 ceph-mon[159210]: pgmap v656: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1004/216480 objects degraded (0.464%); 4708/216480 objects misplaced (2.175%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:47.666 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:47.716 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:47.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:47.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:47.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:47 smithi052 ceph-mon[187743]: pgmap v656: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1004/216480 objects degraded (0.464%); 4708/216480 objects misplaced (2.175%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:47.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:47 smithi052 ceph-mon[180979]: pgmap v656: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1004/216480 objects degraded (0.464%); 4708/216480 objects misplaced (2.175%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:47.822 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:47.854 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:47.913 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:47.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:48.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:48.115 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:48.322 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:48.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:48.442 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:48.555 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:48.603 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:48:48.668 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:48:48.685 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:48.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:48.723 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:48:48.768 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:48:48.802 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:48.817 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:48:48.863 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:48:48.903 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:48.915 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:48:48.960 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:48:48.979 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.979 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.979 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.980 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.980 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.980 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.980 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:48:48.980 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:48:49.437 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:48:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:49 smithi151 ceph-mon[159210]: pgmap v657: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 1004/214380 objects degraded (0.468%); 4708/214380 objects misplaced (2.196%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2226239378' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:49.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1897247203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:49 smithi052 ceph-mon[187743]: pgmap v657: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 1004/214380 objects degraded (0.468%); 4708/214380 objects misplaced (2.196%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2226239378' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1897247203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:49.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:49 smithi052 ceph-mon[180979]: pgmap v657: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 1004/214380 objects degraded (0.468%); 4708/214380 objects misplaced (2.196%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:49.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2226239378' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:48:49.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1897247203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:48:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2226239378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:50.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:50.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1004/214380 objects degraded (0.468%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2226239378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1004/214380 objects degraded (0.468%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2226239378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: failed to encode map e250 with expected crc 2023-09-23T22:48:50.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:48:50.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:48:50.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1004/214380 objects degraded (0.468%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:48:51.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:51 smithi151 ceph-mon[159210]: pgmap v659: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1004/213324 objects degraded (0.471%); 4708/213324 objects misplaced (2.207%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:51.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:51 smithi052 ceph-mon[187743]: pgmap v659: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1004/213324 objects degraded (0.471%); 4708/213324 objects misplaced (2.207%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:51.724 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:51 smithi052 ceph-mon[180979]: pgmap v659: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1004/213324 objects degraded (0.471%); 4708/213324 objects misplaced (2.207%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:51.748 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:51.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:51.864 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:51.963 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:52.269 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:52.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:52.378 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:52.410 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:52.462 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:52.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:52.569 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:52.631 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:48:52.631 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:48:52.631 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40792 6.f 6.8b4b57ef (undecoded) ondisk+retry+write+known_if_redirected e131)", 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 462.76471543700001, 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602794099999999, 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:48:52.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40792 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861822+0000", 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861820+0000", 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861828+0000", 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861841+0000", 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2709e-05 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861851+0000", 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861957+0000", 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106518 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861972+0000", 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5226e-05 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875022+0000", 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013050091999999999 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.875034+0000", 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1232e-05 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922087+0000", 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047052836 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922101+0000", 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4476e-05 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.638 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40806 6.f 6.e80b2dbf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 462.764656333, 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0602518750000001, 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40806 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.639 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861881+0000", 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861880+0000", 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.861885+0000", 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.561e-06 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862005+0000", 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120701 2023-09-23T22:48:52.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862017+0000", 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862139+0000", 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012248000000000001 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.862145+0000", 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881337+0000", 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019192607 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.881351+0000", 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3879999999999999e-05 2023-09-23T22:48:52.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922124+0000", 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0407723719999999 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922133+0000", 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.644 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40815 6.f 6.8b4fc53f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 462.74227519300001, 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037899828, 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:52.645 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40815 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884262+0000", 2023-09-23T22:48:52.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884260+0000", 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884266+0000", 2023-09-23T22:48:52.647 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6060000000000002e-06 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884276+0000", 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7180000000000008e-06 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884284+0000", 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6170000000000007e-06 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884403+0000", 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011853099999999999 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884415+0000", 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1683e-05 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922153+0000", 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037738136 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922162+0000", 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9279999999999999e-06 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.650 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40816 6.f 6.4f46a35f (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 462.74223432700001, 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378988650000001, 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40816 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.651 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884303+0000", 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884302+0000", 2023-09-23T22:48:52.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884309+0000", 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884317+0000", 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5660000000000007e-06 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884324+0000", 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0859999999999996e-06 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884441+0000", 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011671 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.884449+0000", 2023-09-23T22:48:52.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.571e-06 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922191+0000", 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.037741888 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922202+0000", 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.092e-05 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.655 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40818 6.f 6.431951cf (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 462.73631700200002, 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.032011485, 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:52.656 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40818 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890220+0000", 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890219+0000", 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890224+0000", 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7840000000000003e-06 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.658 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890235+0000", 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0899e-05 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890243+0000", 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.225e-06 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890301+0000", 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:48:52.659 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.890312+0000", 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1235e-05 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922221+0000", 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.031909277 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.660 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922232+0000", 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0465000000000001e-05 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:40826 6.f 6.3e4c7bff (undecoded) ondisk+write+known_if_redirected e131)", 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 462.72413152299998, 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.019876078, 2023-09-23T22:48:52.661 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 40826 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.662 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902406+0000", 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902404+0000", 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.663 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902413+0000", 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5449999999999992e-06 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902432+0000", 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8488999999999999e-05 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902441+0000", 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902502+0000", 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1064999999999999e-05 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:09.902511+0000", 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7350000000000002e-06 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922270+0000", 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0197598290000001 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:10.922282+0000", 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.666 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161600 7.8 7:101b9260:::benchmark_data_smithi052_277165_object69108:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 216.38010276399999, 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28372276299999999, 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161600 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.667 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.668 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246437+0000", 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.666e-06 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246443+0000", 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2940000000000002e-06 2023-09-23T22:48:52.669 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246448+0000", 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5199999999999997e-06 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251386+0000", 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00493751 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251523+0000", 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013705799999999999 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251637+0000", 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000114521 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263851+0000", 2023-09-23T22:48:52.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012213869 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263860+0000", 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.529403+0000", 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26554292099999999 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.529465+0000", 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1758000000000001e-05 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530157+0000", 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069202700000000001 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.673 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161606 7.8 7:189adb79:::benchmark_data_smithi052_277165_object69114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 216.38005898899999, 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28428864599999998, 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161606 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.674 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.675 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246480+0000", 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6749999999999998e-06 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246488+0000", 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0269999999999995e-06 2023-09-23T22:48:52.676 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246494+0000", 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6579999999999999e-06 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255454+0000", 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089600449999999998 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255609+0000", 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015489900000000001 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255761+0000", 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015231500000000001 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263866+0000", 2023-09-23T22:48:52.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081051539999999998 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.264798+0000", 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000931608 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530292+0000", 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26549436100000001 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530346+0000", 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3328000000000002e-05 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530767+0000", 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00042096800000000003 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161607 7.8 7:17f6cc07:::benchmark_data_smithi052_277165_object69115:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 216.37417575000001, 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27881840099999999, 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.681 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161607 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.682 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252360+0000", 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252365+0000", 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4920000000000001e-06 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.683 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252374+0000", 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.49e-06 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252379+0000", 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2750000000000003e-06 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259765+0000", 2023-09-23T22:48:52.684 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073862559999999999 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259893+0000", 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012820900000000001 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259999+0000", 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105535 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.685 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263874+0000", 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003875295 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.264894+0000", 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010201089999999999 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530815+0000", 2023-09-23T22:48:52.686 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26592050099999998 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530865+0000", 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9648999999999998e-05 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.531180+0000", 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031517999999999997 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.687 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162195 7.2 e205/146 7:4bcbd502:::benchmark_data_smithi052_277165_object69703:head v 205'10032, mlcod=205'10027)", 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 215.378101402, 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.263093829, 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.688 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248435+0000", 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.689 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248443+0000", 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7219999999999999e-06 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248449+0000", 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8039999999999998e-06 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.690 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248453+0000", 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4100000000000001e-06 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.252722+0000", 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004268679 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.252744+0000", 2023-09-23T22:48:52.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1648e-05 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.261335+0000", 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085910190000000001 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511529+0000", 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25019480999999999 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.692 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162201 7.c e205/197 7:3f19fea2:::benchmark_data_smithi052_277165_object69709:head v 205'9991, mlcod=164'3559)", 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 215.37096809400001, 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27094163999999998, 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.693 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255568+0000", 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255577+0000", 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1099999999999992e-06 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255589+0000", 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255597+0000", 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7239999999999992e-06 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255657+0000", 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0096000000000002e-05 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255685+0000", 2023-09-23T22:48:52.696 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7429000000000001e-05 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526472+0000", 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.270787481 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526511+0000", 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8352000000000001e-05 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.697 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:162200 7.8 7:1ffee83e:::benchmark_data_smithi052_277165_object69708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 215.37035256199999, 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28005922700000002, 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.698 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 162200 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256183+0000", 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256190+0000", 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8719999999999996e-06 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.700 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256200+0000", 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7520000000000008e-06 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256207+0000", 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4189999999999996e-06 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.701 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259499+0000", 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032920950000000001 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259640+0000", 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000141433 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259790+0000", 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014928000000000001 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.702 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516611+0000", 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25682131899999999 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516650+0000", 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8952000000000002e-05 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.535505+0000", 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018855212999999999 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.535560+0000", 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5226999999999998e-05 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.536244+0000", 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00068324599999999998 2023-09-23T22:48:52.704 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162202 7.2 e205/146 7:4736e59a:::benchmark_data_smithi052_277165_object69710:head v 205'10033, mlcod=205'10030)", 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 215.363351709, 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.255326575, 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.705 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263184+0000", 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263192+0000", 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3390000000000004e-06 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511467+0000", 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24827568999999999 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511483+0000", 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5209e-05 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511557+0000", 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4202999999999994e-05 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.708 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511576+0000", 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8870000000000001e-05 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.518481+0000", 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0069055719999999996 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.518512+0000", 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0574999999999998e-05 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.709 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162214 7.c e205/197 7:33fa1cb0:::benchmark_data_smithi052_277165_object69722:head v 205'9992, mlcod=164'3559)", 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 215.354893976, 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25489373100000001, 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.710 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271642+0000", 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.711 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271650+0000", 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1620000000000002e-06 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511482+0000", 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239832029 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511494+0000", 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511527+0000", 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2679999999999999e-05 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511557+0000", 2023-09-23T22:48:52.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0304e-05 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526516+0000", 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014958767 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526537+0000", 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.12e-05 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.714 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164704 7.8 7:160987c2:::benchmark_data_smithi052_277165_object72212:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 211.23030614199999, 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31601756800000003, 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.715 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164704 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396230+0000", 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396236+0000", 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4389999999999997e-06 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396252+0000", 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6412000000000001e-05 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396260+0000", 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1160000000000001e-06 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396306+0000", 2023-09-23T22:48:52.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6183999999999997e-05 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396424+0000", 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118237 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396585+0000", 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016076799999999999 2023-09-23T22:48:52.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.401552+0000", 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049672090000000002 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.405610+0000", 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040580670000000003 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711571+0000", 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30596056300000002 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711627+0000", 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6586999999999999e-05 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.712248+0000", 2023-09-23T22:48:52.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00062132300000000001 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168547 7.e 7:7817ab24:::benchmark_data_smithi052_277165_object76055:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 205.300613056, 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248803264, 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.722 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168547 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325922+0000", 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325931+0000", 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4120000000000007e-06 2023-09-23T22:48:52.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325949+0000", 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7567999999999998e-05 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325959+0000", 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8749999999999995e-06 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326592+0000", 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00063299999999999999 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326706+0000", 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113925 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326821+0000", 2023-09-23T22:48:52.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115184 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341186+0000", 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014364745 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574591+0000", 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.233404956 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574635+0000", 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4357000000000002e-05 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574727+0000", 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2262999999999994e-05 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168550 7.e 7:7960d031:::benchmark_data_smithi052_277165_object76058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 205.29532664300001, 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24916470700000001, 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168550 2023-09-23T22:48:52.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331209+0000", 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331217+0000", 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0609999999999996e-06 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331230+0000", 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3159000000000001e-05 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331249+0000", 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9205999999999999e-05 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332625+0000", 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001375498 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332749+0000", 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012410500000000001 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332864+0000", 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115203 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574774+0000", 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.241910027 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580246+0000", 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054718789999999998 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580287+0000", 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0991000000000001e-05 2023-09-23T22:48:52.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580375+0000", 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8368999999999994e-05 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168552 7.e 7:7af651e0:::benchmark_data_smithi052_277165_object76060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 205.295270327, 2023-09-23T22:48:52.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24921663199999999, 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168552 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:48:52.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331266+0000", 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331269+0000", 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2389999999999998e-06 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331276+0000", 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8900000000000001e-06 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331281+0000", 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0660000000000003e-06 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341213+0000", 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0099318059999999996 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341329+0000", 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115771 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341443+0000", 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113826 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574809+0000", 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23336596900000001 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580383+0000", 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055746950000000002 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580417+0000", 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3581999999999998e-05 2023-09-23T22:48:52.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580483+0000", 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6353999999999995e-05 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48295 8.d 8:ba748e4c:::benchmark_data_smithi052_337357_object48294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:48:52.743 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 121.679689279, 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29879999400000001, 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48295 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946847+0000", 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946865+0000", 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8448999999999999e-05 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946874+0000", 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3960000000000006e-06 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946879+0000", 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946921+0000", 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2422000000000001e-05 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.947042+0000", 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120188 2023-09-23T22:48:52.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.947180+0000", 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013842699999999999 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948502+0000", 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001321979 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948841+0000", 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033914399999999999 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245568+0000", 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29672640900000002 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245607+0000", 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9313000000000001e-05 2023-09-23T22:48:52.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245648+0000", 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0911999999999997e-05 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48295 8.d e228/217 ondisk, result = 0)", 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:48:52.750 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 121.677664361, 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29678497500000001, 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:48:52.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948872+0000", 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948876+0000", 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1010000000000002e-06 2023-09-23T22:48:52.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245491+0000", 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29661482700000003 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245512+0000", 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1078e-05 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245551+0000", 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9286999999999998e-05 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245561+0000", 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7019999999999996e-06 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245658+0000", 2023-09-23T22:48:52.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6485999999999997e-05 2023-09-23T22:48:52.755 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.755 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.755 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.755 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:48:52.755 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:48:52.755 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:48:52.756 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:52.890 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:52.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:53.005 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:53.036 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:53.039 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:48:53.040 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1103/207984 objects degraded (0.530%), 1 pg degraded","count":1},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":865,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":250,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":7,"osd_in_since":1695509092,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":69328,"data_bytes":4542437304,"bytes_used":15727562752,"bytes_avail":656237686784,"bytes_total":671965249536,"degraded_objects":1103,"degraded_total":207984,"degraded_ratio":0.0053032925609662286,"misplaced_objects":4520,"misplaced_total":207984,"misplaced_ratio":0.021732440956996689,"recovering_objects_per_sec":16,"recovering_bytes_per_sec":1081019,"recovering_keys_per_sec":0,"num_objects_recovered":165,"num_bytes_recovered":10813440,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":583},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":26,"modified":"2023-09-23T22:48:52.236301+0000","services":{}},"progress_events":{"3d9c7ac0-c827-48ce-9357-8c3e026dd16f":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:48:53.096 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:53.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:53.201 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:53.283 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:53.404 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:53.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:53.516 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:53.542 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:53.592 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:53.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:53.698 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:48:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:53 smithi151 ceph-mon[159210]: pgmap v660: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 1103/207984 objects degraded (0.530%); 4520/207984 objects misplaced (2.173%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3167558631' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:48:53.720 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:48:53.742 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:48:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:48:53] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-09-23T22:48:53.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:53 smithi052 ceph-mon[187743]: pgmap v660: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 1103/207984 objects degraded (0.530%); 4520/207984 objects misplaced (2.173%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:53.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3167558631' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:48:53.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:53 smithi052 ceph-mon[180979]: pgmap v660: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 1103/207984 objects degraded (0.530%); 4520/207984 objects misplaced (2.173%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:53.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3167558631' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:48:53.748 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:53.770 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:48:54.020 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:48:54.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:54.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:48:54.136 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:54.165 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:54.228 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:54.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:54.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:54.335 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:54.371 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:54.435 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:54.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:54.542 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:54.570 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:54.615 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:48:54.628 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:54.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:54.733 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:54.762 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:54.815 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:54.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:54.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:54.880 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:48:54.880 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:48:54.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:54.950 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:54.996 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:55.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:55.026 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:55.105 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:55.108 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:55.137 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:55.189 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:55.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:55.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:55.300 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:55.374 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:55.635 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:55 smithi052 ceph-mon[187743]: pgmap v661: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1103/205665 objects degraded (0.536%); 4520/205665 objects misplaced (2.198%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:55.636 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1103/205665 objects degraded (0.536%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:48:55.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:55 smithi052 ceph-mon[180979]: pgmap v661: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1103/205665 objects degraded (0.536%); 4520/205665 objects misplaced (2.198%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:55.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1103/205665 objects degraded (0.536%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:48:55.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:55 smithi151 ceph-mon[159210]: pgmap v661: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1103/205665 objects degraded (0.536%); 4520/205665 objects misplaced (2.198%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:48:55.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1103/205665 objects degraded (0.536%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:48:55.705 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:48:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:55.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:48:55.813 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:48:55.847 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:55.901 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:48:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:55.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:48:55.987 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:56.006 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:56.032 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:56.090 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:56.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:56.196 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:56.227 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:56.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:56.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:56.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:56.388 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:48:56.421 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:56.482 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:48:56.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:56.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:48:56.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:56.657 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:57 smithi151 ceph-mon[159210]: pgmap v662: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 1188/199065 objects degraded (0.597%); 4348/199065 objects misplaced (2.184%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:57.725 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:57.737 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:57 smithi052 ceph-mon[187743]: pgmap v662: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 1188/199065 objects degraded (0.597%); 4348/199065 objects misplaced (2.184%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:57.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:57.738 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:57 smithi052 ceph-mon[180979]: pgmap v662: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 1188/199065 objects degraded (0.597%); 4348/199065 objects misplaced (2.184%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:57.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:48:57.952 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:58.066 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:48:58.185 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:48:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:58.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:48:58.295 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:48:58.323 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:58.389 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:48:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:58.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:48:58.493 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:48:58.566 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:59.352 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:48:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:59.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:48:59.469 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:48:59.497 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:59.549 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:48:59.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:48:59.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:48:59.627 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:48:59 smithi052 ceph-mon[180979]: pgmap v663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 1188/196935 objects degraded (0.603%); 4348/196935 objects misplaced (2.208%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:59.627 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:48:59 smithi052 ceph-mon[187743]: pgmap v663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 1188/196935 objects degraded (0.603%); 4348/196935 objects misplaced (2.208%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:48:59.655 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:48:59.779 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:48:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:48:59 smithi151 ceph-mon[159210]: pgmap v663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 1188/196935 objects degraded (0.603%); 4348/196935 objects misplaced (2.208%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:00.101 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:49:00.150 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:00.151 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:49:00.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:00.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:00.218 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:49:00.261 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:00.290 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:49:00.326 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:49:00.377 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:00.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:49:00.442 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:49:00.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1188/196935 objects degraded (0.603%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:00.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1349337154' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:00.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2961783385' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:00.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:00.468 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1188/196935 objects degraded (0.603%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:00.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1349337154' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:00.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2961783385' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:00.469 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:00.478 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:49:00.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.497 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.497 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:49:00.513 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:00.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:00.622 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:00.707 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:00.843 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:00.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1188/196935 objects degraded (0.603%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1349337154' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2961783385' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:00.958 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:00.989 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:00.998 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:49:01.048 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:01.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:01.155 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:01.236 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:01.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[187743]: pgmap v664: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 1188/195903 objects degraded (0.606%); 4348/195903 objects misplaced (2.219%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:01.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:01.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[187743]: failed to encode map e251 with expected crc 2023-09-23T22:49:01.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:49:01.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[187743]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T22:49:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[180979]: pgmap v664: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 1188/195903 objects degraded (0.606%); 4348/195903 objects misplaced (2.219%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:01.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[180979]: failed to encode map e251 with expected crc 2023-09-23T22:49:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:49:01.498 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:01 smithi052 ceph-mon[180979]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T22:49:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:01 smithi151 ceph-mon[159210]: pgmap v664: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 1188/195903 objects degraded (0.606%); 4348/195903 objects misplaced (2.219%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:01 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:01 smithi151 ceph-mon[159210]: failed to encode map e251 with expected crc 2023-09-23T22:49:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:01 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:49:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:01 smithi151 ceph-mon[159210]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T22:49:02.509 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:49:02.510 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:49:02.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:02 smithi052 ceph-mon[187743]: failed to encode map e251 with expected crc 2023-09-23T22:49:02.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:02 smithi052 ceph-mon[180979]: failed to encode map e251 with expected crc 2023-09-23T22:49:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:02 smithi151 ceph-mon[159210]: failed to encode map e251 with expected crc 2023-09-23T22:49:02.981 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:02.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:03.089 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:03.120 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:03.170 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:03.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:03.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:03.276 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:03.306 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:03.381 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:03.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:03.487 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:03.516 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:03.574 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:03.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:03.597 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:49:03.597 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:49:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:49:03] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-09-23T22:49:03.597 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:03 smithi052 ceph-mon[187743]: pgmap v666: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1332/190233 objects degraded (0.700%); 4127/190233 objects misplaced (2.169%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:03.598 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:03 smithi052 ceph-mon[180979]: pgmap v666: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1332/190233 objects degraded (0.700%); 4127/190233 objects misplaced (2.169%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:03.680 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:03 smithi151 ceph-mon[159210]: pgmap v666: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1332/190233 objects degraded (0.700%); 4127/190233 objects misplaced (2.169%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:03.715 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:03.771 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:03.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:03.880 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:03.912 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:03.963 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:03.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:04.076 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:04.108 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:04.161 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:04.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:04.271 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:04.299 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:04.353 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:04.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:04.427 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:49:04.431 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:04.459 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:04.475 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:49:04.557 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:04.590 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:49:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:04.841 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:49:04.989 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:04.989 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:05.076 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:05.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:05.191 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:05.294 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:05.640 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:05.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:05.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:05 smithi052 ceph-mon[187743]: pgmap v667: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1332/188076 objects degraded (0.708%); 4127/188076 objects misplaced (2.194%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:05.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:05 smithi052 ceph-mon[187743]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:05.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1332/188076 objects degraded (0.708%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:05.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:05 smithi052 ceph-mon[180979]: pgmap v667: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1332/188076 objects degraded (0.708%); 4127/188076 objects misplaced (2.194%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:05.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:05 smithi052 ceph-mon[180979]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:05.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1332/188076 objects degraded (0.708%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:05.754 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:05.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:05.832 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":667,"stamp":"2023-09-23T22:49:04.239616+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4107540408,"num_objects":62692,"num_object_clones":0,"num_object_copies":188076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1332,"num_objects_misplaced":4127,"num_objects_unfound":0,"num_objects_dirty":62692,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":142490,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1194,"num_bytes_recovered":76163146,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126524,"ondisk_log_size":126524,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14198932,"kb_used_data":12253084,"kb_used_omap":129,"kb_used_meta":1945342,"kb_avail":642017132,"statfs":{"total":671965249536,"available":657425543168,"internally_reserved":0,"allocated":12547158016,"data_stored":12541958577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":1992030941},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384237568,"num_objects":-5863,"num_object_clones":0,"num_object_copies":-17589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":229,"num_objects_misplaced":-393,"num_objects_unfound":0,"num_objects_dirty":-5863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":10747904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002928"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":247,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353384+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:48:38.353384+0000","last_peered":"2023-09-23T22:48:38.353384+0000","last_clean":"2023-09-23T22:48:38.353384+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:48:38.353384+0000","last_undegraded":"2023-09-23T22:48:38.353384+0000","last_fullsized":"2023-09-23T22:48:38.353384+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":222,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346528+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:48:38.346528+0000","last_peered":"2023-09-23T22:48:38.346528+0000","last_clean":"2023-09-23T22:48:38.346528+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:48:38.346528+0000","last_undegraded":"2023-09-23T22:48:38.346528+0000","last_fullsized":"2023-09-23T22:48:38.346528+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":74,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346173+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:48:38.346173+0000","last_peered":"2023-09-23T22:48:38.346173+0000","last_clean":"2023-09-23T22:48:38.346173+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:48:38.346173+0000","last_undegraded":"2023-09-23T22:48:38.346173+0000","last_fullsized":"2023-09-23T22:48:38.346173+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483595+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:48:49.483595+0000","last_peered":"2023-09-23T22:48:49.483595+0000","last_clean":"2023-09-23T22:48:49.483595+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:48:49.483595+0000","last_undegraded":"2023-09-23T22:48:49.483595+0000","last_fullsized":"2023-09-23T22:48:49.483595+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":281,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.343497+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:48:38.343497+0000","last_peered":"2023-09-23T22:48:38.343497+0000","last_clean":"2023-09-23T22:48:38.343497+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:48:38.343497+0000","last_undegraded":"2023-09-23T22:48:38.343497+0000","last_fullsized":"2023-09-23T22:48:38.343497+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":215,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368618+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:48:38.368618+0000","last_peered":"2023-09-23T22:48:38.368618+0000","last_clean":"2023-09-23T22:48:38.368618+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:48:38.368618+0000","last_undegraded":"2023-09-23T22:48:38.368618+0000","last_fullsized":"2023-09-23T22:48:38.368618+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":150,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483610+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:48:49.483610+0000","last_peered":"2023-09-23T22:48:49.483610+0000","last_clean":"2023-09-23T22:48:49.483610+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:48:49.483610+0000","last_undegraded":"2023-09-23T22:48:49.483610+0000","last_fullsized":"2023-09-23T22:48:49.483610+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":249,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.343568+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:48:38.343568+0000","last_peered":"2023-09-23T22:48:38.343568+0000","last_clean":"2023-09-23T22:48:38.343568+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:48:38.343568+0000","last_undegraded":"2023-09-23T22:48:38.343568+0000","last_fullsized":"2023-09-23T22:48:38.343568+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346148+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:48:38.346148+0000","last_peered":"2023-09-23T22:48:38.346148+0000","last_clean":"2023-09-23T22:48:38.346148+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:48:38.346148+0000","last_undegraded":"2023-09-23T22:48:38.346148+0000","last_fullsized":"2023-09-23T22:48:38.346148+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":207,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346094+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:48:38.346094+0000","last_peered":"2023-09-23T22:48:38.346094+0000","last_clean":"2023-09-23T22:48:38.346094+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:48:38.346094+0000","last_undegraded":"2023-09-23T22:48:38.346094+0000","last_fullsized":"2023-09-23T22:48:38.346094+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":124,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346460+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:48:38.346460+0000","last_peered":"2023-09-23T22:48:38.346460+0000","last_clean":"2023-09-23T22:48:38.346460+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:48:38.346460+0000","last_undegraded":"2023-09-23T22:48:38.346460+0000","last_fullsized":"2023-09-23T22:48:38.346460+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":124,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368088+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:48:38.368088+0000","last_peered":"2023-09-23T22:48:38.368088+0000","last_clean":"2023-09-23T22:48:38.368088+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:48:38.368088+0000","last_undegraded":"2023-09-23T22:48:38.368088+0000","last_fullsized":"2023-09-23T22:48:38.368088+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":270,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353818+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:48:38.353818+0000","last_peered":"2023-09-23T22:48:38.353818+0000","last_clean":"2023-09-23T22:48:38.353818+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:48:38.353818+0000","last_undegraded":"2023-09-23T22:48:38.353818+0000","last_fullsized":"2023-09-23T22:48:38.353818+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":215,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353653+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:48:38.353653+0000","last_peered":"2023-09-23T22:48:38.353653+0000","last_clean":"2023-09-23T22:48:38.353653+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:48:38.353653+0000","last_undegraded":"2023-09-23T22:48:38.353653+0000","last_fullsized":"2023-09-23T22:48:38.353653+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":108,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353681+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:48:38.353681+0000","last_peered":"2023-09-23T22:48:38.353681+0000","last_clean":"2023-09-23T22:48:38.353681+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:48:38.353681+0000","last_undegraded":"2023-09-23T22:48:38.353681+0000","last_fullsized":"2023-09-23T22:48:38.353681+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":269,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346953+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:48:38.346953+0000","last_peered":"2023-09-23T22:48:38.346953+0000","last_clean":"2023-09-23T22:48:38.346953+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:48:38.346953+0000","last_undegraded":"2023-09-23T22:48:38.346953+0000","last_fullsized":"2023-09-23T22:48:38.346953+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":206,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483041+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:48:49.483041+0000","last_peered":"2023-09-23T22:48:49.483041+0000","last_clean":"2023-09-23T22:48:49.483041+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:48:49.483041+0000","last_undegraded":"2023-09-23T22:48:49.483041+0000","last_fullsized":"2023-09-23T22:48:49.483041+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346606+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:48:38.346606+0000","last_peered":"2023-09-23T22:48:38.346606+0000","last_clean":"2023-09-23T22:48:38.346606+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:48:38.346606+0000","last_undegraded":"2023-09-23T22:48:38.346606+0000","last_fullsized":"2023-09-23T22:48:38.346606+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":100,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353456+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:48:38.353456+0000","last_peered":"2023-09-23T22:48:38.353456+0000","last_clean":"2023-09-23T22:48:38.353456+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:48:38.353456+0000","last_undegraded":"2023-09-23T22:48:38.353456+0000","last_fullsized":"2023-09-23T22:48:38.353456+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":207,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483647+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:48:49.483647+0000","last_peered":"2023-09-23T22:48:49.483647+0000","last_clean":"2023-09-23T22:48:49.483647+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:48:49.483647+0000","last_undegraded":"2023-09-23T22:48:49.483647+0000","last_fullsized":"2023-09-23T22:48:49.483647+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":294,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.367940+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:48:38.367940+0000","last_peered":"2023-09-23T22:48:38.367940+0000","last_clean":"2023-09-23T22:48:38.367940+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:48:38.367940+0000","last_undegraded":"2023-09-23T22:48:38.367940+0000","last_fullsized":"2023-09-23T22:48:38.367940+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":249,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.354837+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:48:38.354837+0000","last_peered":"2023-09-23T22:48:38.354837+0000","last_clean":"2023-09-23T22:48:38.354837+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:48:38.354837+0000","last_undegraded":"2023-09-23T22:48:38.354837+0000","last_fullsized":"2023-09-23T22:48:38.354837+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"250'8747","reported_seq":8788,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.457262+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:49:00.457262+0000","last_peered":"2023-09-23T22:49:00.457262+0000","last_clean":"2023-09-23T22:49:00.457262+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:49:00.457262+0000","last_undegraded":"2023-09-23T22:49:00.457262+0000","last_fullsized":"2023-09-23T22:49:00.457262+0000","mapping_epoch":217,"log_start":"223'1400","ondisk_log_start":"223'1400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7347,"log_dups_size":0,"ondisk_log_size":7347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262471680,"num_objects":4005,"num_object_clones":0,"num_object_copies":12015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8747,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":119,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.354006+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:48:38.354006+0000","last_peered":"2023-09-23T22:48:38.354006+0000","last_clean":"2023-09-23T22:48:38.354006+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:48:38.354006+0000","last_undegraded":"2023-09-23T22:48:38.354006+0000","last_fullsized":"2023-09-23T22:48:38.354006+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":214,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.367934+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:48:38.367934+0000","last_peered":"2023-09-23T22:48:38.367934+0000","last_clean":"2023-09-23T22:48:38.367934+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:48:38.367934+0000","last_undegraded":"2023-09-23T22:48:38.367934+0000","last_fullsized":"2023-09-23T22:48:38.367934+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"250'8759","reported_seq":8806,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.777945+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:49:00.777945+0000","last_peered":"2023-09-23T22:49:00.777945+0000","last_clean":"2023-09-23T22:49:00.777945+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:49:00.777945+0000","last_undegraded":"2023-09-23T22:49:00.777945+0000","last_fullsized":"2023-09-23T22:49:00.777945+0000","mapping_epoch":219,"log_start":"223'100","ondisk_log_start":"223'100","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8659,"log_dups_size":0,"ondisk_log_size":8659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248709120,"num_objects":3795,"num_object_clones":0,"num_object_copies":11385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8759,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":129,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353974+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:48:38.353974+0000","last_peered":"2023-09-23T22:48:38.353974+0000","last_clean":"2023-09-23T22:48:38.353974+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:48:38.353974+0000","last_undegraded":"2023-09-23T22:48:38.353974+0000","last_fullsized":"2023-09-23T22:48:38.353974+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":230,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353398+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:48:38.353398+0000","last_peered":"2023-09-23T22:48:38.353398+0000","last_clean":"2023-09-23T22:48:38.353398+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:48:38.353398+0000","last_undegraded":"2023-09-23T22:48:38.353398+0000","last_fullsized":"2023-09-23T22:48:38.353398+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"251'8763","reported_seq":8806,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:02.696276+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:49:02.696276+0000","last_peered":"2023-09-23T22:49:02.696276+0000","last_clean":"2023-09-23T22:49:02.696276+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:49:02.696276+0000","last_undegraded":"2023-09-23T22:49:02.696276+0000","last_fullsized":"2023-09-23T22:49:02.696276+0000","mapping_epoch":219,"log_start":"223'800","ondisk_log_start":"223'800","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7963,"log_dups_size":0,"ondisk_log_size":7963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251199488,"num_objects":3833,"num_object_clones":0,"num_object_copies":11499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8763,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":60,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368516+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:48:38.368516+0000","last_peered":"2023-09-23T22:48:38.368516+0000","last_clean":"2023-09-23T22:48:38.368516+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:48:38.368516+0000","last_undegraded":"2023-09-23T22:48:38.368516+0000","last_fullsized":"2023-09-23T22:48:38.368516+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":242,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368130+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:48:38.368130+0000","last_peered":"2023-09-23T22:48:38.368130+0000","last_clean":"2023-09-23T22:48:38.368130+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:48:38.368130+0000","last_undegraded":"2023-09-23T22:48:38.368130+0000","last_fullsized":"2023-09-23T22:48:38.368130+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"250'9116","reported_seq":9167,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.820502+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:49:00.820502+0000","last_peered":"2023-09-23T22:49:00.820502+0000","last_clean":"2023-09-23T22:49:00.820502+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:49:00.820502+0000","last_undegraded":"2023-09-23T22:49:00.820502+0000","last_fullsized":"2023-09-23T22:49:00.820502+0000","mapping_epoch":226,"log_start":"224'1900","ondisk_log_start":"224'1900","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7216,"log_dups_size":0,"ondisk_log_size":7216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257294336,"num_objects":3926,"num_object_clones":0,"num_object_copies":11778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9116,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.347110+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:48:38.347110+0000","last_peered":"2023-09-23T22:48:38.347110+0000","last_clean":"2023-09-23T22:48:38.347110+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:48:38.347110+0000","last_undegraded":"2023-09-23T22:48:38.347110+0000","last_fullsized":"2023-09-23T22:48:38.347110+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":143,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483165+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:48:49.483165+0000","last_peered":"2023-09-23T22:48:49.483165+0000","last_clean":"2023-09-23T22:48:49.483165+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:48:49.483165+0000","last_undegraded":"2023-09-23T22:48:49.483165+0000","last_fullsized":"2023-09-23T22:48:49.483165+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"250'8807","reported_seq":8850,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.820140+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:49:00.820140+0000","last_peered":"2023-09-23T22:49:00.820140+0000","last_clean":"2023-09-23T22:49:00.820140+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:49:00.820140+0000","last_undegraded":"2023-09-23T22:49:00.820140+0000","last_fullsized":"2023-09-23T22:49:00.820140+0000","mapping_epoch":217,"log_start":"224'1600","ondisk_log_start":"224'1600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7207,"log_dups_size":0,"ondisk_log_size":7207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250413056,"num_objects":3821,"num_object_clones":0,"num_object_copies":11463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8807,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":250,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346611+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:48:38.346611+0000","last_peered":"2023-09-23T22:48:38.346611+0000","last_clean":"2023-09-23T22:48:38.346611+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:48:38.346611+0000","last_undegraded":"2023-09-23T22:48:38.346611+0000","last_fullsized":"2023-09-23T22:48:38.346611+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":215,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353834+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:48:38.353834+0000","last_peered":"2023-09-23T22:48:38.353834+0000","last_clean":"2023-09-23T22:48:38.353834+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:48:38.353834+0000","last_undegraded":"2023-09-23T22:48:38.353834+0000","last_fullsized":"2023-09-23T22:48:38.353834+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"251'8970","reported_seq":9034,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:02.667721+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:49:02.667721+0000","last_peered":"2023-09-23T22:49:02.667721+0000","last_clean":"2023-09-23T22:49:02.667721+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:49:02.667721+0000","last_undegraded":"2023-09-23T22:49:02.667721+0000","last_fullsized":"2023-09-23T22:49:02.667721+0000","mapping_epoch":226,"log_start":"223'1000","ondisk_log_start":"223'1000","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249036800,"num_objects":3800,"num_object_clones":0,"num_object_copies":11400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8970,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":234,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368221+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:48:38.368221+0000","last_peered":"2023-09-23T22:48:38.368221+0000","last_clean":"2023-09-23T22:48:38.368221+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:48:38.368221+0000","last_undegraded":"2023-09-23T22:48:38.368221+0000","last_fullsized":"2023-09-23T22:48:38.368221+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":245,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353774+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:48:38.353774+0000","last_peered":"2023-09-23T22:48:38.353774+0000","last_clean":"2023-09-23T22:48:38.353774+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:48:38.353774+0000","last_undegraded":"2023-09-23T22:48:38.353774+0000","last_fullsized":"2023-09-23T22:48:38.353774+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"250'8998","reported_seq":9045,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.278454+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:49:00.278454+0000","last_peered":"2023-09-23T22:49:00.278454+0000","last_clean":"2023-09-23T22:49:00.278454+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:49:00.278454+0000","last_undegraded":"2023-09-23T22:49:00.278454+0000","last_fullsized":"2023-09-23T22:49:00.278454+0000","mapping_epoch":219,"log_start":"224'1600","ondisk_log_start":"224'1600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7398,"log_dups_size":0,"ondisk_log_size":7398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144000,"num_objects":4000,"num_object_clones":0,"num_object_copies":12000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8998,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":258,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353201+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:48:38.353201+0000","last_peered":"2023-09-23T22:48:38.353201+0000","last_clean":"2023-09-23T22:48:38.353201+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:48:38.353201+0000","last_undegraded":"2023-09-23T22:48:38.353201+0000","last_fullsized":"2023-09-23T22:48:38.353201+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.354032+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:48:38.354032+0000","last_peered":"2023-09-23T22:48:38.354032+0000","last_clean":"2023-09-23T22:48:38.354032+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:48:38.354032+0000","last_undegraded":"2023-09-23T22:48:38.354032+0000","last_fullsized":"2023-09-23T22:48:38.354032+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"250'8636","reported_seq":8694,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.768425+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:49:00.768425+0000","last_peered":"2023-09-23T22:49:00.768425+0000","last_clean":"2023-09-23T22:49:00.768425+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:49:00.768425+0000","last_undegraded":"2023-09-23T22:49:00.768425+0000","last_fullsized":"2023-09-23T22:49:00.768425+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8636,"log_dups_size":0,"ondisk_log_size":8636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249692160,"num_objects":3810,"num_object_clones":0,"num_object_copies":11430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8636,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425709+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:48:49.425709+0000","last_peered":"2023-09-23T22:48:49.425709+0000","last_clean":"2023-09-23T22:48:49.425709+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:48:49.425709+0000","last_undegraded":"2023-09-23T22:48:49.425709+0000","last_fullsized":"2023-09-23T22:48:49.425709+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":176,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.361716+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:48:38.361716+0000","last_peered":"2023-09-23T22:48:38.361716+0000","last_clean":"2023-09-23T22:48:38.361716+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:48:38.361716+0000","last_undegraded":"2023-09-23T22:48:38.361716+0000","last_fullsized":"2023-09-23T22:48:38.361716+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"251'8962","reported_seq":9005,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:02.694410+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:49:02.694410+0000","last_peered":"2023-09-23T22:49:02.694410+0000","last_clean":"2023-09-23T22:49:02.694410+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:49:02.694410+0000","last_undegraded":"2023-09-23T22:49:02.694410+0000","last_fullsized":"2023-09-23T22:49:02.694410+0000","mapping_epoch":219,"log_start":"223'1000","ondisk_log_start":"223'1000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7962,"log_dups_size":0,"ondisk_log_size":7962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253493248,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8962,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":287,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:52.695437+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:48:52.695437+0000","last_peered":"2023-09-23T22:48:52.695437+0000","last_clean":"2023-09-23T22:48:52.695437+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:48:52.695437+0000","last_undegraded":"2023-09-23T22:48:52.695437+0000","last_fullsized":"2023-09-23T22:48:52.695437+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":169,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346392+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:48:38.346392+0000","last_peered":"2023-09-23T22:48:38.346392+0000","last_clean":"2023-09-23T22:48:38.346392+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:48:38.346392+0000","last_undegraded":"2023-09-23T22:48:38.346392+0000","last_fullsized":"2023-09-23T22:48:38.346392+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353281+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:48:38.353281+0000","last_peered":"2023-09-23T22:48:38.353281+0000","last_clean":"2023-09-23T22:48:38.353281+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:48:38.353281+0000","last_undegraded":"2023-09-23T22:48:38.353281+0000","last_fullsized":"2023-09-23T22:48:38.353281+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"250'8842","reported_seq":8891,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.661888+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:49:00.661888+0000","last_peered":"2023-09-23T22:49:00.661888+0000","last_clean":"2023-09-23T22:49:00.661888+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:49:00.661888+0000","last_undegraded":"2023-09-23T22:49:00.661888+0000","last_fullsized":"2023-09-23T22:49:00.661888+0000","mapping_epoch":219,"log_start":"224'1600","ondisk_log_start":"224'1600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254803968,"num_objects":3888,"num_object_clones":0,"num_object_copies":11664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8842,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":84,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368169+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:48:38.368169+0000","last_peered":"2023-09-23T22:48:38.368169+0000","last_clean":"2023-09-23T22:48:38.368169+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:48:38.368169+0000","last_undegraded":"2023-09-23T22:48:38.368169+0000","last_fullsized":"2023-09-23T22:48:38.368169+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":246,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353774+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:48:38.353774+0000","last_peered":"2023-09-23T22:48:38.353774+0000","last_clean":"2023-09-23T22:48:38.353774+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:48:38.353774+0000","last_undegraded":"2023-09-23T22:48:38.353774+0000","last_fullsized":"2023-09-23T22:48:38.353774+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":165,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.361301+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:48:38.361301+0000","last_peered":"2023-09-23T22:48:38.361301+0000","last_clean":"2023-09-23T22:48:38.361301+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:48:38.361301+0000","last_undegraded":"2023-09-23T22:48:38.361301+0000","last_fullsized":"2023-09-23T22:48:38.361301+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"250'8974","reported_seq":9017,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.756071+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:49:00.756071+0000","last_peered":"2023-09-23T22:49:00.756071+0000","last_clean":"2023-09-23T22:49:00.756071+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:49:00.756071+0000","last_undegraded":"2023-09-23T22:49:00.756071+0000","last_fullsized":"2023-09-23T22:49:00.756071+0000","mapping_epoch":217,"log_start":"224'1800","ondisk_log_start":"224'1800","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259129344,"num_objects":3954,"num_object_clones":0,"num_object_copies":11862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8974,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":240,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.362362+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:48:38.362362+0000","last_peered":"2023-09-23T22:48:38.362362+0000","last_clean":"2023-09-23T22:48:38.362362+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:48:38.362362+0000","last_undegraded":"2023-09-23T22:48:38.362362+0000","last_fullsized":"2023-09-23T22:48:38.362362+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":129,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:44.561691+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:48:44.561691+0000","last_peered":"2023-09-23T22:48:44.561691+0000","last_clean":"2023-09-23T22:48:44.561691+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:48:44.561691+0000","last_undegraded":"2023-09-23T22:48:44.561691+0000","last_fullsized":"2023-09-23T22:48:44.561691+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":256,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:57.698330+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:48:57.698330+0000","last_peered":"2023-09-23T22:48:57.698330+0000","last_clean":"2023-09-23T22:48:57.698330+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:48:57.698330+0000","last_undegraded":"2023-09-23T22:48:57.698330+0000","last_fullsized":"2023-09-23T22:48:57.698330+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"250'8778","reported_seq":8825,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.800369+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:49:00.800369+0000","last_peered":"2023-09-23T22:49:00.800369+0000","last_clean":"2023-09-23T22:49:00.800369+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:49:00.800369+0000","last_undegraded":"2023-09-23T22:49:00.800369+0000","last_fullsized":"2023-09-23T22:49:00.800369+0000","mapping_epoch":219,"log_start":"223'200","ondisk_log_start":"223'200","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8578,"log_dups_size":0,"ondisk_log_size":8578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251396096,"num_objects":3836,"num_object_clones":0,"num_object_copies":11508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8778,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":250,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.362505+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:48:38.362505+0000","last_peered":"2023-09-23T22:48:38.362505+0000","last_clean":"2023-09-23T22:48:38.362505+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:48:38.362505+0000","last_undegraded":"2023-09-23T22:48:38.362505+0000","last_fullsized":"2023-09-23T22:48:38.362505+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":337,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.482739+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:48:49.482739+0000","last_peered":"2023-09-23T22:48:49.482739+0000","last_clean":"2023-09-23T22:48:49.482739+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:48:49.482739+0000","last_undegraded":"2023-09-23T22:48:49.482739+0000","last_fullsized":"2023-09-23T22:48:49.482739+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":181,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346686+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:48:38.346686+0000","last_peered":"2023-09-23T22:48:38.346686+0000","last_clean":"2023-09-23T22:48:38.346686+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:48:38.346686+0000","last_undegraded":"2023-09-23T22:48:38.346686+0000","last_fullsized":"2023-09-23T22:48:38.346686+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368617+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:48:38.368617+0000","last_peered":"2023-09-23T22:48:38.368617+0000","last_clean":"2023-09-23T22:48:38.368617+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:48:38.368617+0000","last_undegraded":"2023-09-23T22:48:38.368617+0000","last_fullsized":"2023-09-23T22:48:38.368617+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"250'8898","reported_seq":8949,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:59.509177+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:48:59.509177+0000","last_peered":"2023-09-23T22:48:59.509177+0000","last_clean":"2023-09-23T22:48:59.509177+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:48:59.509177+0000","last_undegraded":"2023-09-23T22:48:59.509177+0000","last_fullsized":"2023-09-23T22:48:59.509177+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265420800,"num_objects":4050,"num_object_clones":0,"num_object_copies":12150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8898,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":216,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.482827+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:48:49.482827+0000","last_peered":"2023-09-23T22:48:49.482827+0000","last_clean":"2023-09-23T22:48:49.482827+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:48:49.482827+0000","last_undegraded":"2023-09-23T22:48:49.482827+0000","last_fullsized":"2023-09-23T22:48:49.482827+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":187,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.482775+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:48:49.482775+0000","last_peered":"2023-09-23T22:48:49.482775+0000","last_clean":"2023-09-23T22:48:49.482775+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:48:49.482775+0000","last_undegraded":"2023-09-23T22:48:49.482775+0000","last_fullsized":"2023-09-23T22:48:49.482775+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":174,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483449+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:48:49.483449+0000","last_peered":"2023-09-23T22:48:49.483449+0000","last_clean":"2023-09-23T22:48:49.483449+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:48:49.483449+0000","last_undegraded":"2023-09-23T22:48:49.483449+0000","last_fullsized":"2023-09-23T22:48:49.483449+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"250'9125","reported_seq":11495,"reported_epoch":250,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:49:00.824635+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:49:00.824635+0000","last_peered":"2023-09-23T22:49:00.824635+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:49:00.824635+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"224'1900","ondisk_log_start":"224'1900","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7225,"log_dups_size":0,"ondisk_log_size":7225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1332,"num_objects_misplaced":4127,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9125,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1159,"num_bytes_recovered":75956224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-1316},{"shards":"1,3,4","objects":4127}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":179,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483355+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:48:49.483355+0000","last_peered":"2023-09-23T22:48:49.483355+0000","last_clean":"2023-09-23T22:48:49.483355+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:48:49.483355+0000","last_undegraded":"2023-09-23T22:48:49.483355+0000","last_fullsized":"2023-09-23T22:48:49.483355+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449543+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:48:49.449543+0000","last_peered":"2023-09-23T22:48:49.449543+0000","last_clean":"2023-09-23T22:48:49.449543+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:48:49.449543+0000","last_undegraded":"2023-09-23T22:48:49.449543+0000","last_fullsized":"2023-09-23T22:48:49.449543+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":179,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483354+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:48:49.483354+0000","last_peered":"2023-09-23T22:48:49.483354+0000","last_clean":"2023-09-23T22:48:49.483354+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:48:49.483354+0000","last_undegraded":"2023-09-23T22:48:49.483354+0000","last_fullsized":"2023-09-23T22:48:49.483354+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"251'8993","reported_seq":9036,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:02.661216+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:49:02.661216+0000","last_peered":"2023-09-23T22:49:02.661216+0000","last_clean":"2023-09-23T22:49:02.661216+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:49:02.661216+0000","last_undegraded":"2023-09-23T22:49:02.661216+0000","last_fullsized":"2023-09-23T22:49:02.661216+0000","mapping_epoch":219,"log_start":"223'1000","ondisk_log_start":"223'1000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7993,"log_dups_size":0,"ondisk_log_size":7993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256573440,"num_objects":3915,"num_object_clones":0,"num_object_copies":11745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8993,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":290,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.440429+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:48:49.440429+0000","last_peered":"2023-09-23T22:48:49.440429+0000","last_clean":"2023-09-23T22:48:49.440429+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:48:49.440429+0000","last_undegraded":"2023-09-23T22:48:49.440429+0000","last_fullsized":"2023-09-23T22:48:49.440429+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":48,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368596+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:48:38.368596+0000","last_peered":"2023-09-23T22:48:38.368596+0000","last_clean":"2023-09-23T22:48:38.368596+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:48:38.368596+0000","last_undegraded":"2023-09-23T22:48:38.368596+0000","last_fullsized":"2023-09-23T22:48:38.368596+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":275,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.343426+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:48:38.343426+0000","last_peered":"2023-09-23T22:48:38.343426+0000","last_clean":"2023-09-23T22:48:38.343426+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:48:38.343426+0000","last_undegraded":"2023-09-23T22:48:38.343426+0000","last_fullsized":"2023-09-23T22:48:38.343426+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"250'8862","reported_seq":8906,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:59.546805+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:48:59.546805+0000","last_peered":"2023-09-23T22:48:59.546805+0000","last_clean":"2023-09-23T22:48:59.546805+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:48:59.546805+0000","last_undegraded":"2023-09-23T22:48:59.546805+0000","last_fullsized":"2023-09-23T22:48:59.546805+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8862,"log_dups_size":0,"ondisk_log_size":8862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263651352,"num_objects":4024,"num_object_clones":0,"num_object_copies":12072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4024,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8862,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":282,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.368601+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:48:38.368601+0000","last_peered":"2023-09-23T22:48:38.368601+0000","last_clean":"2023-09-23T22:48:38.368601+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:48:38.368601+0000","last_undegraded":"2023-09-23T22:48:38.368601+0000","last_fullsized":"2023-09-23T22:48:38.368601+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":146,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483531+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:48:49.483531+0000","last_peered":"2023-09-23T22:48:49.483531+0000","last_clean":"2023-09-23T22:48:49.483531+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:48:49.483531+0000","last_undegraded":"2023-09-23T22:48:49.483531+0000","last_fullsized":"2023-09-23T22:48:49.483531+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":200,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353280+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:48:38.353280+0000","last_peered":"2023-09-23T22:48:38.353280+0000","last_clean":"2023-09-23T22:48:38.353280+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:48:38.353280+0000","last_undegraded":"2023-09-23T22:48:38.353280+0000","last_fullsized":"2023-09-23T22:48:38.353280+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":182,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483267+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:48:49.483267+0000","last_peered":"2023-09-23T22:48:49.483267+0000","last_clean":"2023-09-23T22:48:49.483267+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:48:49.483267+0000","last_undegraded":"2023-09-23T22:48:49.483267+0000","last_fullsized":"2023-09-23T22:48:49.483267+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.361370+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:48:38.361370+0000","last_peered":"2023-09-23T22:48:38.361370+0000","last_clean":"2023-09-23T22:48:38.361370+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:48:38.361370+0000","last_undegraded":"2023-09-23T22:48:38.361370+0000","last_fullsized":"2023-09-23T22:48:38.361370+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":60,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353703+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:48:38.353703+0000","last_peered":"2023-09-23T22:48:38.353703+0000","last_clean":"2023-09-23T22:48:38.353703+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:48:38.353703+0000","last_undegraded":"2023-09-23T22:48:38.353703+0000","last_fullsized":"2023-09-23T22:48:38.353703+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":249,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353777+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:48:38.353777+0000","last_peered":"2023-09-23T22:48:38.353777+0000","last_clean":"2023-09-23T22:48:38.353777+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:48:38.353777+0000","last_undegraded":"2023-09-23T22:48:38.353777+0000","last_fullsized":"2023-09-23T22:48:38.353777+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":60,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353910+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:48:38.353910+0000","last_peered":"2023-09-23T22:48:38.353910+0000","last_clean":"2023-09-23T22:48:38.353910+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:48:38.353910+0000","last_undegraded":"2023-09-23T22:48:38.353910+0000","last_fullsized":"2023-09-23T22:48:38.353910+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":121,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353561+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:48:38.353561+0000","last_peered":"2023-09-23T22:48:38.353561+0000","last_clean":"2023-09-23T22:48:38.353561+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:48:38.353561+0000","last_undegraded":"2023-09-23T22:48:38.353561+0000","last_fullsized":"2023-09-23T22:48:38.353561+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":243,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346216+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:48:38.346216+0000","last_peered":"2023-09-23T22:48:38.346216+0000","last_clean":"2023-09-23T22:48:38.346216+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:48:38.346216+0000","last_undegraded":"2023-09-23T22:48:38.346216+0000","last_fullsized":"2023-09-23T22:48:38.346216+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":281,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.346002+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:48:38.346002+0000","last_peered":"2023-09-23T22:48:38.346002+0000","last_clean":"2023-09-23T22:48:38.346002+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:48:38.346002+0000","last_undegraded":"2023-09-23T22:48:38.346002+0000","last_fullsized":"2023-09-23T22:48:38.346002+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":195,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.483692+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:48:49.483692+0000","last_peered":"2023-09-23T22:48:49.483692+0000","last_clean":"2023-09-23T22:48:49.483692+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:48:49.483692+0000","last_undegraded":"2023-09-23T22:48:49.483692+0000","last_fullsized":"2023-09-23T22:48:49.483692+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":215,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:48:38.353146+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:48:38.353146+0000","last_peered":"2023-09-23T22:48:38.353146+0000","last_clean":"2023-09-23T22:48:38.353146+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:48:38.353146+0000","last_undegraded":"2023-09-23T22:48:38.353146+0000","last_fullsized":"2023-09-23T22:48:38.353146+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4106944536,"num_objects":62668,"num_object_clones":0,"num_object_copies":188004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1332,"num_objects_misplaced":4127,"num_objects_unfound":0,"num_objects_dirty":62668,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142230,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1162,"num_bytes_recovered":76152832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12481605632,"data_stored":12481593416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126330,"ondisk_log_size":126330,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430472,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626700,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810788,"kb_used_data":1501276,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91934364,"statfs":{"total":95995035648,"available":94140788736,"internally_reserved":0,"allocated":1537306624,"data_stored":1536805009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":4.6740000000000004,"5min":1.966,"15min":1.514},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":36.651000000000003,"5min":36.651000000000003,"15min":36.651000000000003},"last":0.73399999999999999},{"interface":"front","average":{"1min":5.5049999999999999,"5min":1.8819999999999999,"15min":1.278},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":36.558999999999997,"5min":36.558999999999997,"15min":36.558999999999997},"last":2.6059999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.044,"15min":2.2370000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.741,"5min":14.391999999999999,"15min":153.78399999999999},"last":0.438},{"interface":"front","average":{"1min":10.359999999999999,"5min":3.7250000000000001,"15min":2.819},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":185.411,"5min":185.411,"15min":185.411},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":2.3959999999999999,"15min":3.044},"min":{"1min":0.33800000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.802,"5min":12.802,"15min":100.785},"last":0.92200000000000004},{"interface":"front","average":{"1min":11.632999999999999,"5min":4.0060000000000002,"15min":3.5270000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":185.21799999999999,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.76800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":19.305,"5min":6.641,"15min":3.2610000000000001},"min":{"1min":0.378,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":185.19399999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":1.4279999999999999},{"interface":"front","average":{"1min":11.478999999999999,"5min":5.4870000000000001,"15min":2.9390000000000001},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":128.19800000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.79300000000000004,"15min":0.66400000000000003},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":6.5309999999999997,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.89900000000000002,"15min":0.69499999999999995},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":6.9089999999999998,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":0.93200000000000005,"15min":0.72499999999999998},"min":{"1min":0.313,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":10.148999999999999,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.92300000000000004,"15min":0.73999999999999999},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.2930000000000001,"5min":9.7899999999999991,"15min":18.625},"last":0.36199999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:48:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1,"15min":0.746},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.4049999999999998,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.83699999999999997,"15min":0.72799999999999998},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":3.7130000000000001,"5min":5.1790000000000003,"15min":16.661999999999999},"last":0.46300000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253010,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2047852,"kb_used_data":1749412,"kb_used_omap":9,"kb_used_meta":298230,"kb_avail":91697300,"statfs":{"total":95995035648,"available":93898035200,"internally_reserved":0,"allocated":1791397888,"data_stored":1790491445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":305388521},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":9.6259999999999994,"5min":5.2590000000000003,"15min":4.5309999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":101.605,"5min":101.605,"15min":101.605},"last":10.52},{"interface":"front","average":{"1min":10.422000000000001,"5min":5.6959999999999997,"15min":4.9080000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":101.892,"5min":101.892,"15min":101.892},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":2.9689999999999999,"15min":2.0190000000000001},"min":{"1min":0.23000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":36.456000000000003,"5min":36.456000000000003,"15min":37.889000000000003},"last":10.584},{"interface":"front","average":{"1min":4.141,"5min":3.4209999999999998,"15min":2.04},"min":{"1min":0.315,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":36.427,"5min":36.427,"15min":37.823},"last":10.772}]},{"osd":3,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6280000000000001,"5min":2.7839999999999998,"15min":1.5529999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.5259999999999998,"5min":16.631,"15min":50.033999999999999},"last":6.359},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.4900000000000002,"15min":1.847},"min":{"1min":0.34399999999999997,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.8819999999999997,"5min":29.565000000000001,"15min":52.491999999999997},"last":2.5950000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.5830000000000002,"5min":3.2080000000000002,"15min":1.8049999999999999},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":78.948999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":2.5720000000000001},{"interface":"front","average":{"1min":4.7999999999999998,"5min":2.4159999999999999,"15min":1.6220000000000001},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.313},"max":{"1min":78.963999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":10.619}]},{"osd":5,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.3620000000000001,"5min":2.556,"15min":1.472},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":84.254999999999995,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":6.2939999999999996,"5min":3.2879999999999998,"15min":1.6719999999999999},"min":{"1min":0.32800000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":87.515000000000001,"5min":87.515000000000001,"15min":87.515000000000001},"last":2.5289999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0250000000000004,"5min":2.6819999999999999,"15min":1.45},"min":{"1min":0.436,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":79.311000000000007,"5min":79.311000000000007,"15min":79.311000000000007},"last":10.65},{"interface":"front","average":{"1min":5.4189999999999996,"5min":2.7029999999999998,"15min":1.405},"min":{"1min":0.34699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":79.328999999999994,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.5910000000000002,"15min":2.0059999999999998},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":8.5449999999999999,"5min":29.539000000000001,"15min":159.018},"last":2.617},{"interface":"front","average":{"1min":1.6519999999999999,"5min":2.0259999999999998,"15min":1.9179999999999999},"min":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":6.0810000000000004,"5min":8.6120000000000001,"15min":158.96000000000001},"last":10.676}]}]},{"osd":0,"up_from":219,"seq":940597837859,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2115952,"kb_used_data":1967912,"kb_used_omap":31,"kb_used_meta":148000,"kb_avail":91629200,"statfs":{"total":95995035648,"available":93828300800,"internally_reserved":0,"allocated":2015141888,"data_stored":2014172869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":151552946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":3.556,"15min":3.6600000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":15.611000000000001,"5min":32.344999999999999,"15min":32.344999999999999},"last":4.0199999999999996},{"interface":"front","average":{"1min":3.6389999999999998,"5min":3.5750000000000002,"15min":3.5640000000000001},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.092000000000001,"5min":32.106000000000002,"15min":32.106000000000002},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":2.468,"15min":2.2210000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.170999999999999,"5min":12.170999999999999,"15min":12.170999999999999},"last":3.161},{"interface":"front","average":{"1min":3.27,"5min":2.0590000000000002,"15min":1.857},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9180000000000001,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":0.30299999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":2.1549999999999998,"15min":2.0289999999999999},"min":{"1min":0.34699999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":1.7050000000000001},{"interface":"front","average":{"1min":2.6360000000000001,"5min":1.4259999999999999,"15min":1.224},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.9730000000000008,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":0.34100000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7280000000000002,"5min":2.2280000000000002,"15min":1.978},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":15.432,"5min":15.432,"15min":15.432},"last":3.9399999999999999},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.7509999999999999,"15min":1.7490000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9790000000000001,"5min":7.5590000000000002,"15min":7.5590000000000002},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.601,"15min":1.4239999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.1019999999999999,"5min":1.8620000000000001,"15min":1.6559999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":15.536,"5min":15.536,"15min":15.536},"last":3.9049999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.6699999999999999,"15min":1.4710000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.39300000000000002},{"interface":"front","average":{"1min":3.952,"5min":2.1389999999999998,"15min":1.837},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.489000000000001,"5min":15.489000000000001,"15min":15.489000000000001},"last":3.9940000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.9690000000000001,"15min":1.72},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":15.574,"5min":15.574,"15min":15.574},"last":3.968},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.4139999999999999,"15min":1.256},"min":{"1min":0.46200000000000002,"5min":0.432,"15min":0.432},"max":{"1min":8.9000000000000004,"5min":8.9000000000000004,"15min":8.9000000000000004},"last":0.44500000000000001}]}]},{"osd":2,"up_from":131,"seq":562640715871,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1762252,"kb_used_data":1540036,"kb_used_omap":81,"kb_used_meta":221998,"kb_avail":91982900,"statfs":{"total":95995035648,"available":94190489600,"internally_reserved":0,"allocated":1576996864,"data_stored":1576055686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":227326170},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":5.1059999999999999,"5min":5.2359999999999998,"15min":5.258},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":41.723999999999997,"5min":57.811999999999998,"15min":57.811999999999998},"last":5.069},{"interface":"front","average":{"1min":7.7679999999999998,"5min":5.8650000000000002,"15min":5.548},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":41.454000000000001,"5min":57.387999999999998,"15min":57.387999999999998},"last":8.9619999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":3.5179999999999998,"15min":3.226},"min":{"1min":0.28299999999999997,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.759,"5min":88.316000000000003,"15min":88.316000000000003},"last":1.806},{"interface":"front","average":{"1min":4.444,"5min":4.4329999999999998,"15min":2.8719999999999999},"min":{"1min":0.34899999999999998,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":28.596,"5min":88.436999999999998,"15min":88.436999999999998},"last":7.617}]},{"osd":3,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.976,"15min":3.056},"min":{"1min":0.30499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.994,"5min":88.653000000000006,"15min":88.653000000000006},"last":7.718},{"interface":"front","average":{"1min":3.9369999999999998,"5min":3.5880000000000001,"15min":2.4220000000000002},"min":{"1min":0.31900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":12.401,"5min":88.474000000000004,"15min":88.474000000000004},"last":4.694}]},{"osd":4,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.188000000000001,"5min":4.4569999999999999,"15min":2.1389999999999998},"min":{"1min":0.51300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":84.778999999999996,"5min":108.878,"15min":108.878},"last":0.40899999999999997},{"interface":"front","average":{"1min":11.103,"5min":5.8799999999999999,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":84.753,"5min":108.93000000000001,"15min":108.93000000000001},"last":1.7090000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":7.6660000000000004,"5min":5.3339999999999996,"15min":2.7480000000000002},"min":{"1min":0.42499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":90.082999999999998,"5min":97.322999999999993,"15min":97.322999999999993},"last":1.746},{"interface":"front","average":{"1min":7.9420000000000002,"5min":5.2549999999999999,"15min":2.3340000000000001},"min":{"1min":0.437,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":89.986000000000004,"5min":97.301000000000002,"15min":97.301000000000002},"last":7.694}]},{"osd":6,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.731,"5min":4.9109999999999996,"15min":2.2530000000000001},"min":{"1min":0.40699999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":85.034999999999997,"5min":88.606999999999999,"15min":88.606999999999999},"last":7.6669999999999998},{"interface":"front","average":{"1min":9.9339999999999993,"5min":3.3479999999999999,"15min":1.7729999999999999},"min":{"1min":0.47399999999999998,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":85.209000000000003,"5min":85.209000000000003,"15min":85.209000000000003},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.726,"15min":1.266},"min":{"1min":0.40699999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9640000000000004,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.9329999999999998,"5min":3.4910000000000001,"15min":2.1720000000000002},"min":{"1min":0.45100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.5549999999999997,"5min":88.584000000000003,"15min":88.584000000000003},"last":1.7689999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925957,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1898076,"kb_used_data":1533076,"kb_used_omap":7,"kb_used_meta":364984,"kb_avail":91847076,"statfs":{"total":95995035648,"available":94051405824,"internally_reserved":0,"allocated":1569869824,"data_stored":1568969112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":373744232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:13 2023","interfaces":[{"interface":"back","average":{"1min":6.5970000000000004,"5min":2.8069999999999999,"15min":2.1749999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":80.099000000000004,"5min":80.099000000000004,"15min":80.099000000000004},"last":0.46300000000000002},{"interface":"front","average":{"1min":6.5149999999999997,"5min":2.581,"15min":1.925},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":80.150000000000006,"5min":80.150000000000006,"15min":80.150000000000006},"last":3.3330000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.8639999999999999,"15min":1.3979999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":6.7110000000000003,"5min":22.818000000000001,"15min":22.818000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.472,"5min":2,"15min":1.1160000000000001},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4160000000000004,"5min":23.917999999999999,"15min":23.917999999999999},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.367,"5min":3.879,"15min":2.2509999999999999},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.431000000000001,"5min":125.446,"15min":125.446},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.476,"5min":5.7249999999999996,"15min":2.7330000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":14.670999999999999,"5min":182.483,"15min":182.483},"last":2.7869999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1150000000000002,"15min":1.2310000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":3.7959999999999998,"5min":18.757000000000001,"15min":18.757000000000001},"last":0.437},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.5740000000000001,"15min":1.0649999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.516,"5min":12.097,"15min":12.097},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.4570000000000001,"15min":1.0229999999999999},"min":{"1min":0.30099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.4710000000000001,"5min":12.127000000000001,"15min":12.127000000000001},"last":1.2969999999999999},{"interface":"front","average":{"1min":2.52,"5min":2.105,"15min":1.2150000000000001},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.9950000000000001,"5min":9.0090000000000003,"15min":9.0090000000000003},"last":2.831}]},{"osd":6,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.589,"15min":1.0669999999999999},"min":{"1min":0.34000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.173,"5min":12.16,"15min":12.16},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.5830000000000002,"5min":2.089,"15min":1.2869999999999999},"min":{"1min":0.47099999999999997,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.0259999999999998,"5min":9.1029999999999998,"15min":10.755000000000001},"last":2.9039999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8399999999999999,"5min":2.9940000000000002,"15min":1.5649999999999999},"min":{"1min":0.40100000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.0510000000000002,"5min":91.757999999999996,"15min":91.757999999999996},"last":2.8580000000000001},{"interface":"front","average":{"1min":1.7170000000000001,"5min":3.234,"15min":1.639},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.7709999999999999,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.49099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019156,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1997600,"kb_used_data":1692184,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91747552,"statfs":{"total":95995035648,"available":93949493248,"internally_reserved":0,"allocated":1732796416,"data_stored":1732298371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":6.6619999999999999,"15min":7.5640000000000001},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.0190000000000001,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.2759999999999998,"5min":7.2779999999999996,"15min":8.1120000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.7200000000000006,"5min":110.33799999999999,"15min":110.33799999999999},"last":2.3849999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":2.597,"15min":1.4610000000000001},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.681,"5min":41.643000000000001,"15min":45.320999999999998},"last":3.5569999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.117,"15min":1.3160000000000001},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.8210000000000002,"5min":41.712000000000003,"15min":45.372999999999998},"last":6.4489999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":4.2560000000000002,"15min":3.2400000000000002},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.7030000000000003,"5min":84.909000000000006,"15min":84.909000000000006},"last":7.9189999999999996},{"interface":"front","average":{"1min":2.1659999999999999,"5min":4.2590000000000003,"15min":3.1669999999999998},"min":{"1min":0.371,"5min":0.309,"15min":0.309},"max":{"1min":7.0570000000000004,"5min":83.840000000000003,"15min":83.840000000000003},"last":2.4769999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":6.5529999999999999,"15min":2.8809999999999998},"min":{"1min":0.42699999999999999,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":6.1040000000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":5.0899999999999999},{"interface":"front","average":{"1min":1.7689999999999999,"5min":6.7160000000000002,"15min":2.915},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.657,"5min":129.197,"15min":129.197},"last":5.641}]},{"osd":5,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.746,"15min":0.73999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.8979999999999997,"5min":5.8979999999999997,"15min":43.189999999999998},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.78000000000000003,"15min":0.76100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26500000000000001},"max":{"1min":2.7040000000000002,"5min":9.5190000000000001,"15min":42.450000000000003},"last":0.27200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.79300000000000004,"15min":0.73099999999999998},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.7570000000000001,"5min":6.8109999999999999,"15min":27.056000000000001},"last":0.307},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78800000000000003,"15min":0.75800000000000001},"min":{"1min":0.32500000000000001,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":2.6520000000000001,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.754,"15min":0.66900000000000004},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.464,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.68600000000000005,"15min":0.64600000000000002},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":2.5710000000000002,"5min":3.2519999999999998,"15min":5.9409999999999998},"last":0.38800000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822928,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2566412,"kb_used_data":2269188,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91178740,"statfs":{"total":95995035648,"available":93367029760,"internally_reserved":0,"allocated":2323648512,"data_stored":2323166085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":18.346,"5min":5.1150000000000002,"15min":2.9100000000000001},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":121.319,"5min":121.319,"15min":121.319},"last":37.344000000000001},{"interface":"front","average":{"1min":18.654,"5min":5.0469999999999997,"15min":2.7789999999999999},"min":{"1min":0.32700000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":121.075,"5min":121.075,"15min":121.075},"last":37.470999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8,"15min":1.716},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":28.658999999999999,"5min":28.658999999999999,"15min":149.203},"last":0.63100000000000001},{"interface":"front","average":{"1min":3.8969999999999998,"5min":2.3999999999999999,"15min":1.871},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":28.664999999999999,"5min":28.664999999999999,"15min":149.554},"last":0.54800000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":4.5599999999999996,"15min":7.141},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.2879999999999998,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.861,"5min":4.7949999999999999,"15min":7.1959999999999997},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3899999999999997,"5min":158.91399999999999,"15min":158.91399999999999},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":14.204000000000001,"5min":8.2560000000000002,"15min":3.9020000000000001},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":108.58799999999999,"5min":257.05000000000001,"15min":257.05000000000001},"last":37.427999999999997},{"interface":"front","average":{"1min":14.087999999999999,"5min":7.6820000000000004,"15min":3.7570000000000001},"min":{"1min":0.311,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":108.15600000000001,"5min":257.35000000000002,"15min":257.35000000000002},"last":37.277999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.79300000000000004,"15min":0.60299999999999998},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":1.4790000000000001,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.79800000000000004,"15min":0.61299999999999999},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.619,"5min":10.535,"15min":10.535},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.411,"15min":1.3879999999999999},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":12.615,"5min":48.360999999999997,"15min":140.07499999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.417,"5min":1.532,"15min":1.4399999999999999},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.254},"max":{"1min":12.74,"5min":48.292999999999999,"15min":139.739},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.82399999999999995,"15min":0.70599999999999996},"min":{"1min":0.33400000000000002,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.464,"5min":7.0010000000000003,"15min":7.9809999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.74299999999999999,"15min":0.67000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":1.9650000000000001,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.443}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2009071616,"data_stored":2009071616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1782972416,"data_stored":1782972416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1570246656,"data_stored":1570242584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1554251776,"data_stored":1554251776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1725693952,"data_stored":1725693952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2316111872,"data_stored":2316107800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1523257344,"data_stored":1523253272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:49:05.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:05 smithi151 ceph-mon[159210]: pgmap v667: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 1332/188076 objects degraded (0.708%); 4127/188076 objects misplaced (2.194%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:05.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:05 smithi151 ceph-mon[159210]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:05.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1332/188076 objects degraded (0.708%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:06.162 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:06.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:06.269 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:06.311 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:06.378 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:06.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:06.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:06.564 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:06 smithi151 ceph-mon[159210]: pgmap v668: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 1427/181401 objects degraded (0.787%); 3934/181401 objects misplaced (2.169%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:06.891 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:06.892 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:06.901 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:06 smithi052 ceph-mon[180979]: pgmap v668: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 1427/181401 objects degraded (0.787%); 3934/181401 objects misplaced (2.169%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:06.906 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:06 smithi052 ceph-mon[187743]: pgmap v668: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 1427/181401 objects degraded (0.787%); 3934/181401 objects misplaced (2.169%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:06.924 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:06.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:07.047 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:07.077 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:07.130 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:07.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:07.242 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:07.274 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:07.329 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:07.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:07.437 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:07.475 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:07.532 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:07.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:07.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:07.637 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:07.723 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:08.419 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:49:08.420 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:49:08.840 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:49:09.356 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:09.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:09.470 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:09.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:09 smithi052 ceph-mon[187743]: pgmap v669: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 1427/179007 objects degraded (0.797%); 3934/179007 objects misplaced (2.198%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:09.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:09 smithi052 ceph-mon[180979]: pgmap v669: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 1427/179007 objects degraded (0.797%); 3934/179007 objects misplaced (2.198%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:09.504 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:09.577 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:09.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:09 smithi151 ceph-mon[159210]: pgmap v669: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 1427/179007 objects degraded (0.797%); 3934/179007 objects misplaced (2.198%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:09.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:09.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:09.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:09.738 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:10.447 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:10.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:10.487 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:49:10.560 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:10.606 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1427/179007 objects degraded (0.797%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:10.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1427/179007 objects degraded (0.797%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:10.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1427/179007 objects degraded (0.797%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:10.705 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:10.998 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:11.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:11.109 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:11.140 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:11.198 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:11.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:11.249 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:49:11.287 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:11.288 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:49:11.306 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:11.344 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:49:11.410 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:49:11.414 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:11.439 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:49:11.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:11 smithi151 ceph-mon[159210]: pgmap v670: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1427/177858 objects degraded (0.802%); 3934/177858 objects misplaced (2.212%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3599307217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:11.490 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:49:11.530 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:49:11.587 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:49:11.661 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:11 smithi052 ceph-mon[180979]: pgmap v670: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1427/177858 objects degraded (0.802%); 3934/177858 objects misplaced (2.212%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:11.661 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3599307217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:11.668 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:11 smithi052 ceph-mon[187743]: pgmap v670: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 1427/177858 objects degraded (0.802%); 3934/177858 objects misplaced (2.212%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:11.668 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3599307217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:11.900 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:11.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:12.014 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:12.100 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:12.384 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:49:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3137041064' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:12.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3137041064' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:12.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3137041064' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[187743]: pgmap v671: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 1525/172269 objects degraded (0.885%); 3741/172269 objects misplaced (2.172%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:49:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3137041064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:13.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[187743]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T22:49:13.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[187743]: failed to encode map e252 with expected crc 2023-09-23T22:49:13.498 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:49:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:49:13] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.33.4" 2023-09-23T22:49:13.501 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[180979]: pgmap v671: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 1525/172269 objects degraded (0.885%); 3741/172269 objects misplaced (2.172%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:13.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:49:13.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3137041064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:13.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[180979]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T22:49:13.502 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:13 smithi052 ceph-mon[180979]: failed to encode map e252 with expected crc 2023-09-23T22:49:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:13 smithi151 ceph-mon[159210]: pgmap v671: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 1525/172269 objects degraded (0.885%); 3741/172269 objects misplaced (2.172%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:49:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:13 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:49:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3137041064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:13 smithi151 ceph-mon[159210]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T22:49:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:13 smithi151 ceph-mon[159210]: failed to encode map e252 with expected crc 2023-09-23T22:49:14.162 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:14.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:14.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:14.269 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:14.318 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:14.388 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:14.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:14.397 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:14.498 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:14.525 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:14.578 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:14.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:14.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:14.684 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:14.818 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:14.977 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:14.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:15.088 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:15.197 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:15.594 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:15.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:15.641 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:49:15.669 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:49:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:15 smithi151 ceph-mon[159210]: pgmap v673: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 1525/170151 objects degraded (0.896%); 3741/170151 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1525/170151 objects degraded (0.896%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:15.706 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:15.707 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:15 smithi052 ceph-mon[187743]: pgmap v673: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 1525/170151 objects degraded (0.896%); 3741/170151 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:15.707 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1525/170151 objects degraded (0.896%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:15.707 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:15 smithi052 ceph-mon[180979]: pgmap v673: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 1525/170151 objects degraded (0.896%); 3741/170151 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:15.707 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1525/170151 objects degraded (0.896%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:15.740 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:15.785 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:15.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:15.890 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:15.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:16.241 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:49:16.468 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:49:16.566 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:16.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:16.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:16.706 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:16.759 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:16.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:16.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:16.865 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:16.940 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:17.044 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":673,"stamp":"2023-09-23T22:49:14.243824+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3715962808,"num_objects":56717,"num_object_clones":0,"num_object_copies":170151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1525,"num_objects_misplaced":3741,"num_objects_unfound":0,"num_objects_dirty":56717,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":148465,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1387,"num_bytes_recovered":88811594,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127099,"ondisk_log_size":127099,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13120968,"kb_used_data":11136592,"kb_used_omap":129,"kb_used_meta":1983870,"kb_avail":643095096,"statfs":{"total":671965249536,"available":658529378304,"internally_reserved":0,"allocated":11403870208,"data_stored":11398665605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2031483613},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-391577600,"num_objects":-5975,"num_object_clones":0,"num_object_copies":-17925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":193,"num_objects_misplaced":-386,"num_objects_unfound":0,"num_objects_dirty":-5975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":193,"num_bytes_recovered":12648448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004208"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":248,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452037+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:48:49.452037+0000","last_peered":"2023-09-23T22:48:49.452037+0000","last_clean":"2023-09-23T22:48:49.452037+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:48:49.452037+0000","last_undegraded":"2023-09-23T22:48:49.452037+0000","last_fullsized":"2023-09-23T22:48:49.452037+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426640+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:48:49.426640+0000","last_peered":"2023-09-23T22:48:49.426640+0000","last_clean":"2023-09-23T22:48:49.426640+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:48:49.426640+0000","last_undegraded":"2023-09-23T22:48:49.426640+0000","last_fullsized":"2023-09-23T22:48:49.426640+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":76,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.981210+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:49:00.981210+0000","last_peered":"2023-09-23T22:49:00.981210+0000","last_clean":"2023-09-23T22:49:00.981210+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:49:00.981210+0000","last_undegraded":"2023-09-23T22:49:00.981210+0000","last_fullsized":"2023-09-23T22:49:00.981210+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":219,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.997042+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:49:00.997042+0000","last_peered":"2023-09-23T22:49:00.997042+0000","last_clean":"2023-09-23T22:49:00.997042+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:49:00.997042+0000","last_undegraded":"2023-09-23T22:49:00.997042+0000","last_fullsized":"2023-09-23T22:49:00.997042+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":282,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425804+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:48:49.425804+0000","last_peered":"2023-09-23T22:48:49.425804+0000","last_clean":"2023-09-23T22:48:49.425804+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:48:49.425804+0000","last_undegraded":"2023-09-23T22:48:49.425804+0000","last_fullsized":"2023-09-23T22:48:49.425804+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":216,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.489025+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:48:49.489025+0000","last_peered":"2023-09-23T22:48:49.489025+0000","last_clean":"2023-09-23T22:48:49.489025+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:48:49.489025+0000","last_undegraded":"2023-09-23T22:48:49.489025+0000","last_fullsized":"2023-09-23T22:48:49.489025+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994981+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:49:00.994981+0000","last_peered":"2023-09-23T22:49:00.994981+0000","last_clean":"2023-09-23T22:49:00.994981+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:49:00.994981+0000","last_undegraded":"2023-09-23T22:49:00.994981+0000","last_fullsized":"2023-09-23T22:49:00.994981+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":250,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425835+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:48:49.425835+0000","last_peered":"2023-09-23T22:48:49.425835+0000","last_clean":"2023-09-23T22:48:49.425835+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:48:49.425835+0000","last_undegraded":"2023-09-23T22:48:49.425835+0000","last_fullsized":"2023-09-23T22:48:49.425835+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.982205+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:49:00.982205+0000","last_peered":"2023-09-23T22:49:00.982205+0000","last_clean":"2023-09-23T22:49:00.982205+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:49:00.982205+0000","last_undegraded":"2023-09-23T22:49:00.982205+0000","last_fullsized":"2023-09-23T22:49:00.982205+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425791+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:48:49.425791+0000","last_peered":"2023-09-23T22:48:49.425791+0000","last_clean":"2023-09-23T22:48:49.425791+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:48:49.425791+0000","last_undegraded":"2023-09-23T22:48:49.425791+0000","last_fullsized":"2023-09-23T22:48:49.425791+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":125,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426600+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:48:49.426600+0000","last_peered":"2023-09-23T22:48:49.426600+0000","last_clean":"2023-09-23T22:48:49.426600+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:48:49.426600+0000","last_undegraded":"2023-09-23T22:48:49.426600+0000","last_fullsized":"2023-09-23T22:48:49.426600+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488482+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:48:49.488482+0000","last_peered":"2023-09-23T22:48:49.488482+0000","last_clean":"2023-09-23T22:48:49.488482+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:48:49.488482+0000","last_undegraded":"2023-09-23T22:48:49.488482+0000","last_fullsized":"2023-09-23T22:48:49.488482+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":271,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452199+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:48:49.452199+0000","last_peered":"2023-09-23T22:48:49.452199+0000","last_clean":"2023-09-23T22:48:49.452199+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:48:49.452199+0000","last_undegraded":"2023-09-23T22:48:49.452199+0000","last_fullsized":"2023-09-23T22:48:49.452199+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":216,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452537+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:48:49.452537+0000","last_peered":"2023-09-23T22:48:49.452537+0000","last_clean":"2023-09-23T22:48:49.452537+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:48:49.452537+0000","last_undegraded":"2023-09-23T22:48:49.452537+0000","last_fullsized":"2023-09-23T22:48:49.452537+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452552+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:48:49.452552+0000","last_peered":"2023-09-23T22:48:49.452552+0000","last_clean":"2023-09-23T22:48:49.452552+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:48:49.452552+0000","last_undegraded":"2023-09-23T22:48:49.452552+0000","last_fullsized":"2023-09-23T22:48:49.452552+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":270,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426282+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:48:49.426282+0000","last_peered":"2023-09-23T22:48:49.426282+0000","last_clean":"2023-09-23T22:48:49.426282+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:48:49.426282+0000","last_undegraded":"2023-09-23T22:48:49.426282+0000","last_fullsized":"2023-09-23T22:48:49.426282+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994935+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:49:00.994935+0000","last_peered":"2023-09-23T22:49:00.994935+0000","last_clean":"2023-09-23T22:49:00.994935+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:49:00.994935+0000","last_undegraded":"2023-09-23T22:49:00.994935+0000","last_fullsized":"2023-09-23T22:49:00.994935+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":231,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426654+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:48:49.426654+0000","last_peered":"2023-09-23T22:48:49.426654+0000","last_clean":"2023-09-23T22:48:49.426654+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:48:49.426654+0000","last_undegraded":"2023-09-23T22:48:49.426654+0000","last_fullsized":"2023-09-23T22:48:49.426654+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":101,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452066+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:48:49.452066+0000","last_peered":"2023-09-23T22:48:49.452066+0000","last_clean":"2023-09-23T22:48:49.452066+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:48:49.452066+0000","last_undegraded":"2023-09-23T22:48:49.452066+0000","last_fullsized":"2023-09-23T22:48:49.452066+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":208,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.997131+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:49:00.997131+0000","last_peered":"2023-09-23T22:49:00.997131+0000","last_clean":"2023-09-23T22:49:00.997131+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:49:00.997131+0000","last_undegraded":"2023-09-23T22:49:00.997131+0000","last_fullsized":"2023-09-23T22:49:00.997131+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":295,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488367+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:48:49.488367+0000","last_peered":"2023-09-23T22:48:49.488367+0000","last_clean":"2023-09-23T22:48:49.488367+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:48:49.488367+0000","last_undegraded":"2023-09-23T22:48:49.488367+0000","last_fullsized":"2023-09-23T22:48:49.488367+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.440519+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:48:49.440519+0000","last_peered":"2023-09-23T22:48:49.440519+0000","last_clean":"2023-09-23T22:48:49.440519+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:48:49.440519+0000","last_undegraded":"2023-09-23T22:48:49.440519+0000","last_fullsized":"2023-09-23T22:48:49.440519+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"251'9128","reported_seq":9170,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.556446+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:49:10.556446+0000","last_peered":"2023-09-23T22:49:10.556446+0000","last_clean":"2023-09-23T22:49:10.556446+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:49:10.556446+0000","last_undegraded":"2023-09-23T22:49:10.556446+0000","last_fullsized":"2023-09-23T22:49:10.556446+0000","mapping_epoch":217,"log_start":"224'1800","ondisk_log_start":"224'1800","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237502464,"num_objects":3624,"num_object_clones":0,"num_object_copies":10872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9128,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":120,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452200+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:48:49.452200+0000","last_peered":"2023-09-23T22:48:49.452200+0000","last_clean":"2023-09-23T22:48:49.452200+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:48:49.452200+0000","last_undegraded":"2023-09-23T22:48:49.452200+0000","last_fullsized":"2023-09-23T22:48:49.452200+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":215,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488359+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:48:49.488359+0000","last_peered":"2023-09-23T22:48:49.488359+0000","last_clean":"2023-09-23T22:48:49.488359+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:48:49.488359+0000","last_undegraded":"2023-09-23T22:48:49.488359+0000","last_fullsized":"2023-09-23T22:48:49.488359+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"251'9116","reported_seq":9164,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.819801+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:49:10.819801+0000","last_peered":"2023-09-23T22:49:10.819801+0000","last_clean":"2023-09-23T22:49:10.819801+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:49:10.819801+0000","last_undegraded":"2023-09-23T22:49:10.819801+0000","last_fullsized":"2023-09-23T22:49:10.819801+0000","mapping_epoch":219,"log_start":"223'500","ondisk_log_start":"223'500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225312768,"num_objects":3438,"num_object_clones":0,"num_object_copies":10314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9116,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":130,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452312+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:48:49.452312+0000","last_peered":"2023-09-23T22:48:49.452312+0000","last_clean":"2023-09-23T22:48:49.452312+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:48:49.452312+0000","last_undegraded":"2023-09-23T22:48:49.452312+0000","last_fullsized":"2023-09-23T22:48:49.452312+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":231,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452076+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:48:49.452076+0000","last_peered":"2023-09-23T22:48:49.452076+0000","last_clean":"2023-09-23T22:48:49.452076+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:48:49.452076+0000","last_undegraded":"2023-09-23T22:48:49.452076+0000","last_fullsized":"2023-09-23T22:48:49.452076+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"252'9126","reported_seq":9170,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:49:12.684323+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:49:12.684323+0000","last_peered":"2023-09-23T22:49:12.684323+0000","last_clean":"2023-09-23T22:49:12.684323+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:49:12.684323+0000","last_undegraded":"2023-09-23T22:49:12.684323+0000","last_fullsized":"2023-09-23T22:49:12.684323+0000","mapping_epoch":219,"log_start":"223'1200","ondisk_log_start":"223'1200","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7926,"log_dups_size":0,"ondisk_log_size":7926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9126,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":61,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488851+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:48:49.488851+0000","last_peered":"2023-09-23T22:48:49.488851+0000","last_clean":"2023-09-23T22:48:49.488851+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:48:49.488851+0000","last_undegraded":"2023-09-23T22:48:49.488851+0000","last_fullsized":"2023-09-23T22:48:49.488851+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":243,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488627+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:48:49.488627+0000","last_peered":"2023-09-23T22:48:49.488627+0000","last_clean":"2023-09-23T22:48:49.488627+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:48:49.488627+0000","last_undegraded":"2023-09-23T22:48:49.488627+0000","last_fullsized":"2023-09-23T22:48:49.488627+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"251'9520","reported_seq":9572,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.826339+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:49:10.826339+0000","last_peered":"2023-09-23T22:49:10.826339+0000","last_clean":"2023-09-23T22:49:10.826339+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:49:10.826339+0000","last_undegraded":"2023-09-23T22:49:10.826339+0000","last_fullsized":"2023-09-23T22:49:10.826339+0000","mapping_epoch":226,"log_start":"224'2300","ondisk_log_start":"224'2300","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7220,"log_dups_size":0,"ondisk_log_size":7220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230817792,"num_objects":3522,"num_object_clones":0,"num_object_copies":10566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9520,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":103,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426444+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:48:49.426444+0000","last_peered":"2023-09-23T22:48:49.426444+0000","last_clean":"2023-09-23T22:48:49.426444+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:48:49.426444+0000","last_undegraded":"2023-09-23T22:48:49.426444+0000","last_fullsized":"2023-09-23T22:48:49.426444+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994570+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:49:00.994570+0000","last_peered":"2023-09-23T22:49:00.994570+0000","last_clean":"2023-09-23T22:49:00.994570+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:49:00.994570+0000","last_undegraded":"2023-09-23T22:49:00.994570+0000","last_fullsized":"2023-09-23T22:49:00.994570+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"251'9163","reported_seq":9207,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.675643+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:49:10.675643+0000","last_peered":"2023-09-23T22:49:10.675643+0000","last_clean":"2023-09-23T22:49:10.675643+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:49:10.675643+0000","last_undegraded":"2023-09-23T22:49:10.675643+0000","last_fullsized":"2023-09-23T22:49:10.675643+0000","mapping_epoch":217,"log_start":"224'2000","ondisk_log_start":"224'2000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7163,"log_dups_size":0,"ondisk_log_size":7163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9163,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":251,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426284+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:48:49.426284+0000","last_peered":"2023-09-23T22:48:49.426284+0000","last_clean":"2023-09-23T22:48:49.426284+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:48:49.426284+0000","last_undegraded":"2023-09-23T22:48:49.426284+0000","last_fullsized":"2023-09-23T22:48:49.426284+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":216,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452103+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:48:49.452103+0000","last_peered":"2023-09-23T22:48:49.452103+0000","last_clean":"2023-09-23T22:48:49.452103+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:48:49.452103+0000","last_undegraded":"2023-09-23T22:48:49.452103+0000","last_fullsized":"2023-09-23T22:48:49.452103+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"252'9368","reported_seq":9433,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:49:12.711109+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:49:12.711109+0000","last_peered":"2023-09-23T22:49:12.711109+0000","last_clean":"2023-09-23T22:49:12.711109+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:49:12.711109+0000","last_undegraded":"2023-09-23T22:49:12.711109+0000","last_fullsized":"2023-09-23T22:49:12.711109+0000","mapping_epoch":226,"log_start":"223'1400","ondisk_log_start":"223'1400","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7968,"log_dups_size":0,"ondisk_log_size":7968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9368,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":235,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488716+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:48:49.488716+0000","last_peered":"2023-09-23T22:48:49.488716+0000","last_clean":"2023-09-23T22:48:49.488716+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:48:49.488716+0000","last_undegraded":"2023-09-23T22:48:49.488716+0000","last_fullsized":"2023-09-23T22:48:49.488716+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":246,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452390+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:48:49.452390+0000","last_peered":"2023-09-23T22:48:49.452390+0000","last_clean":"2023-09-23T22:48:49.452390+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:48:49.452390+0000","last_undegraded":"2023-09-23T22:48:49.452390+0000","last_fullsized":"2023-09-23T22:48:49.452390+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"251'9383","reported_seq":9431,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.523774+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:49:10.523774+0000","last_peered":"2023-09-23T22:49:10.523774+0000","last_clean":"2023-09-23T22:49:10.523774+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:49:10.523774+0000","last_undegraded":"2023-09-23T22:49:10.523774+0000","last_fullsized":"2023-09-23T22:49:10.523774+0000","mapping_epoch":219,"log_start":"224'2000","ondisk_log_start":"224'2000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236912640,"num_objects":3615,"num_object_clones":0,"num_object_copies":10845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9383,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":259,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.451890+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:48:49.451890+0000","last_peered":"2023-09-23T22:48:49.451890+0000","last_clean":"2023-09-23T22:48:49.451890+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:48:49.451890+0000","last_undegraded":"2023-09-23T22:48:49.451890+0000","last_fullsized":"2023-09-23T22:48:49.451890+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452348+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:48:49.452348+0000","last_peered":"2023-09-23T22:48:49.452348+0000","last_clean":"2023-09-23T22:48:49.452348+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:48:49.452348+0000","last_undegraded":"2023-09-23T22:48:49.452348+0000","last_fullsized":"2023-09-23T22:48:49.452348+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"251'8967","reported_seq":9026,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.723695+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:49:10.723695+0000","last_peered":"2023-09-23T22:49:10.723695+0000","last_clean":"2023-09-23T22:49:10.723695+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:49:10.723695+0000","last_undegraded":"2023-09-23T22:49:10.723695+0000","last_fullsized":"2023-09-23T22:49:10.723695+0000","mapping_epoch":226,"log_start":"223'300","ondisk_log_start":"223'300","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227999744,"num_objects":3479,"num_object_clones":0,"num_object_copies":10437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8967,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:55.829563+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:48:55.829563+0000","last_peered":"2023-09-23T22:48:55.829563+0000","last_clean":"2023-09-23T22:48:55.829563+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:48:55.829563+0000","last_undegraded":"2023-09-23T22:48:55.829563+0000","last_fullsized":"2023-09-23T22:48:55.829563+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":177,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449894+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:48:49.449894+0000","last_peered":"2023-09-23T22:48:49.449894+0000","last_clean":"2023-09-23T22:48:49.449894+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:48:49.449894+0000","last_undegraded":"2023-09-23T22:48:49.449894+0000","last_fullsized":"2023-09-23T22:48:49.449894+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"252'9318","reported_seq":9362,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:49:12.645270+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:49:12.645270+0000","last_peered":"2023-09-23T22:49:12.645270+0000","last_clean":"2023-09-23T22:49:12.645270+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:49:12.645270+0000","last_undegraded":"2023-09-23T22:49:12.645270+0000","last_fullsized":"2023-09-23T22:49:12.645270+0000","mapping_epoch":219,"log_start":"223'1400","ondisk_log_start":"223'1400","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7918,"log_dups_size":0,"ondisk_log_size":7918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230162432,"num_objects":3512,"num_object_clones":0,"num_object_copies":10536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9318,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":288,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.995162+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:49:00.995162+0000","last_peered":"2023-09-23T22:49:00.995162+0000","last_clean":"2023-09-23T22:49:00.995162+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:49:00.995162+0000","last_undegraded":"2023-09-23T22:49:00.995162+0000","last_fullsized":"2023-09-23T22:49:00.995162+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":170,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426569+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:48:49.426569+0000","last_peered":"2023-09-23T22:48:49.426569+0000","last_clean":"2023-09-23T22:48:49.426569+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:48:49.426569+0000","last_undegraded":"2023-09-23T22:48:49.426569+0000","last_fullsized":"2023-09-23T22:48:49.426569+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":82,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.451991+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:48:49.451991+0000","last_peered":"2023-09-23T22:48:49.451991+0000","last_clean":"2023-09-23T22:48:49.451991+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:48:49.451991+0000","last_undegraded":"2023-09-23T22:48:49.451991+0000","last_fullsized":"2023-09-23T22:48:49.451991+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"251'9211","reported_seq":9261,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.806242+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:49:10.806242+0000","last_peered":"2023-09-23T22:49:10.806242+0000","last_clean":"2023-09-23T22:49:10.806242+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:49:10.806242+0000","last_undegraded":"2023-09-23T22:49:10.806242+0000","last_fullsized":"2023-09-23T22:49:10.806242+0000","mapping_epoch":219,"log_start":"224'2000","ondisk_log_start":"224'2000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7211,"log_dups_size":0,"ondisk_log_size":7211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230621184,"num_objects":3519,"num_object_clones":0,"num_object_copies":10557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9211,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":85,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488689+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:48:49.488689+0000","last_peered":"2023-09-23T22:48:49.488689+0000","last_clean":"2023-09-23T22:48:49.488689+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:48:49.488689+0000","last_undegraded":"2023-09-23T22:48:49.488689+0000","last_fullsized":"2023-09-23T22:48:49.488689+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":247,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452601+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:48:49.452601+0000","last_peered":"2023-09-23T22:48:49.452601+0000","last_clean":"2023-09-23T22:48:49.452601+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:48:49.452601+0000","last_undegraded":"2023-09-23T22:48:49.452601+0000","last_fullsized":"2023-09-23T22:48:49.452601+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449456+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:48:49.449456+0000","last_peered":"2023-09-23T22:48:49.449456+0000","last_clean":"2023-09-23T22:48:49.449456+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:48:49.449456+0000","last_undegraded":"2023-09-23T22:48:49.449456+0000","last_fullsized":"2023-09-23T22:48:49.449456+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"251'9361","reported_seq":9405,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.786982+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:49:10.786982+0000","last_peered":"2023-09-23T22:49:10.786982+0000","last_clean":"2023-09-23T22:49:10.786982+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:49:10.786982+0000","last_undegraded":"2023-09-23T22:49:10.786982+0000","last_fullsized":"2023-09-23T22:49:10.786982+0000","mapping_epoch":217,"log_start":"224'2200","ondisk_log_start":"224'2200","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7161,"log_dups_size":0,"ondisk_log_size":7161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233766912,"num_objects":3567,"num_object_clones":0,"num_object_copies":10701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9361,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":241,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449287+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:48:49.449287+0000","last_peered":"2023-09-23T22:48:49.449287+0000","last_clean":"2023-09-23T22:48:49.449287+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:48:49.449287+0000","last_undegraded":"2023-09-23T22:48:49.449287+0000","last_fullsized":"2023-09-23T22:48:49.449287+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449202+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:48:49.449202+0000","last_peered":"2023-09-23T22:48:49.449202+0000","last_clean":"2023-09-23T22:48:49.449202+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:48:49.449202+0000","last_undegraded":"2023-09-23T22:48:49.449202+0000","last_fullsized":"2023-09-23T22:48:49.449202+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":257,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994908+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:49:00.994908+0000","last_peered":"2023-09-23T22:49:00.994908+0000","last_clean":"2023-09-23T22:49:00.994908+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:49:00.994908+0000","last_undegraded":"2023-09-23T22:49:00.994908+0000","last_fullsized":"2023-09-23T22:49:00.994908+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"251'9126","reported_seq":9174,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:10.794089+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:49:10.794089+0000","last_peered":"2023-09-23T22:49:10.794089+0000","last_clean":"2023-09-23T22:49:10.794089+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:49:10.794089+0000","last_undegraded":"2023-09-23T22:49:10.794089+0000","last_fullsized":"2023-09-23T22:49:10.794089+0000","mapping_epoch":219,"log_start":"223'500","ondisk_log_start":"223'500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228589568,"num_objects":3488,"num_object_clones":0,"num_object_copies":10464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9126,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":251,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449177+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:48:49.449177+0000","last_peered":"2023-09-23T22:48:49.449177+0000","last_clean":"2023-09-23T22:48:49.449177+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:48:49.449177+0000","last_undegraded":"2023-09-23T22:48:49.449177+0000","last_fullsized":"2023-09-23T22:48:49.449177+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":338,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994359+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:49:00.994359+0000","last_peered":"2023-09-23T22:49:00.994359+0000","last_clean":"2023-09-23T22:49:00.994359+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:49:00.994359+0000","last_undegraded":"2023-09-23T22:49:00.994359+0000","last_fullsized":"2023-09-23T22:49:00.994359+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425854+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:48:49.425854+0000","last_peered":"2023-09-23T22:48:49.425854+0000","last_clean":"2023-09-23T22:48:49.425854+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:48:49.425854+0000","last_undegraded":"2023-09-23T22:48:49.425854+0000","last_fullsized":"2023-09-23T22:48:49.425854+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":243,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488933+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:48:49.488933+0000","last_peered":"2023-09-23T22:48:49.488933+0000","last_clean":"2023-09-23T22:48:49.488933+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:48:49.488933+0000","last_undegraded":"2023-09-23T22:48:49.488933+0000","last_fullsized":"2023-09-23T22:48:49.488933+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"251'9281","reported_seq":9333,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:09.563776+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:49:09.563776+0000","last_peered":"2023-09-23T22:49:09.563776+0000","last_clean":"2023-09-23T22:49:09.563776+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:49:09.563776+0000","last_undegraded":"2023-09-23T22:49:09.563776+0000","last_fullsized":"2023-09-23T22:49:09.563776+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9281,"log_dups_size":0,"ondisk_log_size":9281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320512,"num_objects":3667,"num_object_clones":0,"num_object_copies":11001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9281,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":217,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994487+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:49:00.994487+0000","last_peered":"2023-09-23T22:49:00.994487+0000","last_clean":"2023-09-23T22:49:00.994487+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:49:00.994487+0000","last_undegraded":"2023-09-23T22:49:00.994487+0000","last_fullsized":"2023-09-23T22:49:00.994487+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":188,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994447+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:49:00.994447+0000","last_peered":"2023-09-23T22:49:00.994447+0000","last_clean":"2023-09-23T22:49:00.994447+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:49:00.994447+0000","last_undegraded":"2023-09-23T22:49:00.994447+0000","last_fullsized":"2023-09-23T22:49:00.994447+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.996862+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:49:00.996862+0000","last_peered":"2023-09-23T22:49:00.996862+0000","last_clean":"2023-09-23T22:49:00.996862+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:49:00.996862+0000","last_undegraded":"2023-09-23T22:49:00.996862+0000","last_fullsized":"2023-09-23T22:49:00.996862+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"251'9510","reported_seq":12267,"reported_epoch":251,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:49:10.831499+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:49:10.831499+0000","last_peered":"2023-09-23T22:49:10.831499+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:49:10.831499+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"224'2300","ondisk_log_start":"224'2300","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7210,"log_dups_size":0,"ondisk_log_size":7210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246284288,"num_objects":3758,"num_object_clones":0,"num_object_copies":11274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1525,"num_objects_misplaced":3741,"num_objects_unfound":0,"num_objects_dirty":3758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9510,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1352,"num_bytes_recovered":88604672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-1508},{"shards":"1,3,4","objects":3741}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":180,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.995388+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:49:00.995388+0000","last_peered":"2023-09-23T22:49:00.995388+0000","last_clean":"2023-09-23T22:49:00.995388+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:49:00.995388+0000","last_undegraded":"2023-09-23T22:49:00.995388+0000","last_fullsized":"2023-09-23T22:49:00.995388+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":129,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.563851+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:48:49.563851+0000","last_peered":"2023-09-23T22:48:49.563851+0000","last_clean":"2023-09-23T22:48:49.563851+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:48:49.563851+0000","last_undegraded":"2023-09-23T22:48:49.563851+0000","last_fullsized":"2023-09-23T22:48:49.563851+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":180,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994822+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:49:00.994822+0000","last_peered":"2023-09-23T22:49:00.994822+0000","last_clean":"2023-09-23T22:49:00.994822+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:49:00.994822+0000","last_undegraded":"2023-09-23T22:49:00.994822+0000","last_fullsized":"2023-09-23T22:49:00.994822+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"252'9380","reported_seq":9424,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:49:12.689045+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:49:12.689045+0000","last_peered":"2023-09-23T22:49:12.689045+0000","last_clean":"2023-09-23T22:49:12.689045+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:49:12.689045+0000","last_undegraded":"2023-09-23T22:49:12.689045+0000","last_fullsized":"2023-09-23T22:49:12.689045+0000","mapping_epoch":219,"log_start":"223'1400","ondisk_log_start":"223'1400","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7980,"log_dups_size":0,"ondisk_log_size":7980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231211008,"num_objects":3528,"num_object_clones":0,"num_object_copies":10584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9380,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":291,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:55.829771+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:48:55.829771+0000","last_peered":"2023-09-23T22:48:55.829771+0000","last_clean":"2023-09-23T22:48:55.829771+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:48:55.829771+0000","last_undegraded":"2023-09-23T22:48:55.829771+0000","last_fullsized":"2023-09-23T22:48:55.829771+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":49,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488992+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:48:49.488992+0000","last_peered":"2023-09-23T22:48:49.488992+0000","last_clean":"2023-09-23T22:48:49.488992+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:48:49.488992+0000","last_undegraded":"2023-09-23T22:48:49.488992+0000","last_fullsized":"2023-09-23T22:48:49.488992+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":276,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425751+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:48:49.425751+0000","last_peered":"2023-09-23T22:48:49.425751+0000","last_clean":"2023-09-23T22:48:49.425751+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:48:49.425751+0000","last_undegraded":"2023-09-23T22:48:49.425751+0000","last_fullsized":"2023-09-23T22:48:49.425751+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"251'9247","reported_seq":9292,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:09.569879+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:49:09.569879+0000","last_peered":"2023-09-23T22:49:09.569879+0000","last_clean":"2023-09-23T22:49:09.569879+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:49:09.569879+0000","last_undegraded":"2023-09-23T22:49:09.569879+0000","last_fullsized":"2023-09-23T22:49:09.569879+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9247,"log_dups_size":0,"ondisk_log_size":9247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238419992,"num_objects":3639,"num_object_clones":0,"num_object_copies":10917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3639,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9247,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":283,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.488956+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:48:49.488956+0000","last_peered":"2023-09-23T22:48:49.488956+0000","last_clean":"2023-09-23T22:48:49.488956+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:48:49.488956+0000","last_undegraded":"2023-09-23T22:48:49.488956+0000","last_fullsized":"2023-09-23T22:48:49.488956+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":147,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.996960+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:49:00.996960+0000","last_peered":"2023-09-23T22:49:00.996960+0000","last_clean":"2023-09-23T22:49:00.996960+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:49:00.996960+0000","last_undegraded":"2023-09-23T22:49:00.996960+0000","last_fullsized":"2023-09-23T22:49:00.996960+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":201,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.451887+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:48:49.451887+0000","last_peered":"2023-09-23T22:48:49.451887+0000","last_clean":"2023-09-23T22:48:49.451887+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:48:49.451887+0000","last_undegraded":"2023-09-23T22:48:49.451887+0000","last_fullsized":"2023-09-23T22:48:49.451887+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":183,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.994706+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:49:00.994706+0000","last_peered":"2023-09-23T22:49:00.994706+0000","last_clean":"2023-09-23T22:49:00.994706+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:49:00.994706+0000","last_undegraded":"2023-09-23T22:49:00.994706+0000","last_fullsized":"2023-09-23T22:49:00.994706+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.449489+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:48:49.449489+0000","last_peered":"2023-09-23T22:48:49.449489+0000","last_clean":"2023-09-23T22:48:49.449489+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:48:49.449489+0000","last_undegraded":"2023-09-23T22:48:49.449489+0000","last_fullsized":"2023-09-23T22:48:49.449489+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":61,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452194+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:48:49.452194+0000","last_peered":"2023-09-23T22:48:49.452194+0000","last_clean":"2023-09-23T22:48:49.452194+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:48:49.452194+0000","last_undegraded":"2023-09-23T22:48:49.452194+0000","last_fullsized":"2023-09-23T22:48:49.452194+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":250,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452613+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:48:49.452613+0000","last_peered":"2023-09-23T22:48:49.452613+0000","last_clean":"2023-09-23T22:48:49.452613+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:48:49.452613+0000","last_undegraded":"2023-09-23T22:48:49.452613+0000","last_fullsized":"2023-09-23T22:48:49.452613+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":61,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452473+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:48:49.452473+0000","last_peered":"2023-09-23T22:48:49.452473+0000","last_clean":"2023-09-23T22:48:49.452473+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:48:49.452473+0000","last_undegraded":"2023-09-23T22:48:49.452473+0000","last_fullsized":"2023-09-23T22:48:49.452473+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":122,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.452148+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:48:49.452148+0000","last_peered":"2023-09-23T22:48:49.452148+0000","last_clean":"2023-09-23T22:48:49.452148+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:48:49.452148+0000","last_undegraded":"2023-09-23T22:48:49.452148+0000","last_fullsized":"2023-09-23T22:48:49.452148+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":244,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.426326+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:48:49.426326+0000","last_peered":"2023-09-23T22:48:49.426326+0000","last_clean":"2023-09-23T22:48:49.426326+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:48:49.426326+0000","last_undegraded":"2023-09-23T22:48:49.426326+0000","last_fullsized":"2023-09-23T22:48:49.426326+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":282,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.425698+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:48:49.425698+0000","last_peered":"2023-09-23T22:48:49.425698+0000","last_clean":"2023-09-23T22:48:49.425698+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:48:49.425698+0000","last_undegraded":"2023-09-23T22:48:49.425698+0000","last_fullsized":"2023-09-23T22:48:49.425698+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":196,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:49:00.995066+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:49:00.995066+0000","last_peered":"2023-09-23T22:49:00.995066+0000","last_clean":"2023-09-23T22:49:00.995066+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:49:00.995066+0000","last_undegraded":"2023-09-23T22:49:00.995066+0000","last_fullsized":"2023-09-23T22:49:00.995066+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:48:49.451840+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:48:49.451840+0000","last_peered":"2023-09-23T22:48:49.451840+0000","last_clean":"2023-09-23T22:48:49.451840+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:48:49.451840+0000","last_undegraded":"2023-09-23T22:48:49.451840+0000","last_fullsized":"2023-09-23T22:48:49.451840+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3715366936,"num_objects":56693,"num_object_clones":0,"num_object_copies":170079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1525,"num_objects_misplaced":3741,"num_objects_unfound":0,"num_objects_dirty":56693,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148205,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1355,"num_bytes_recovered":88801280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11333939200,"data_stored":11333926984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126905,"ondisk_log_size":126905,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430474,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626702,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1662444,"kb_used_data":1361124,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92082708,"statfs":{"total":95995035648,"available":94292692992,"internally_reserved":0,"allocated":1393790976,"data_stored":1393289123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":4.6740000000000004,"5min":1.966,"15min":1.514},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":36.651000000000003,"5min":36.651000000000003,"15min":36.651000000000003},"last":27.803999999999998},{"interface":"front","average":{"1min":5.5049999999999999,"5min":1.8819999999999999,"15min":1.278},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":36.558999999999997,"5min":36.558999999999997,"15min":36.558999999999997},"last":27.882000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.044,"15min":2.2370000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.741,"5min":14.391999999999999,"15min":153.78399999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":10.359999999999999,"5min":3.7250000000000001,"15min":2.819},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":185.411,"5min":185.411,"15min":185.411},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":2.3959999999999999,"15min":3.044},"min":{"1min":0.33800000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.802,"5min":12.802,"15min":100.785},"last":0.377},{"interface":"front","average":{"1min":11.632999999999999,"5min":4.0060000000000002,"15min":3.5270000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":185.21799999999999,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":19.305,"5min":6.641,"15min":3.2610000000000001},"min":{"1min":0.378,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":185.19399999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":11.478999999999999,"5min":5.4870000000000001,"15min":2.9390000000000001},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":128.19800000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.79300000000000004,"15min":0.66400000000000003},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":6.5309999999999997,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.89900000000000002,"15min":0.69499999999999995},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":6.9089999999999998,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":0.93200000000000005,"15min":0.72499999999999998},"min":{"1min":0.313,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":10.148999999999999,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.92300000000000004,"15min":0.73999999999999999},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.2930000000000001,"5min":9.7899999999999991,"15min":18.625},"last":0.56599999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:48:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1,"15min":0.746},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.4049999999999998,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.436},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.83699999999999997,"15min":0.72799999999999998},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":3.7130000000000001,"5min":5.1790000000000003,"15min":16.661999999999999},"last":0.59899999999999998}]}]},{"osd":1,"up_from":84,"seq":360777253012,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1870700,"kb_used_data":1580452,"kb_used_omap":9,"kb_used_meta":290038,"kb_avail":91874452,"statfs":{"total":95995035648,"available":94079438848,"internally_reserved":0,"allocated":1618382848,"data_stored":1617476405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":296999913},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":9.6259999999999994,"5min":5.2590000000000003,"15min":4.5309999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":101.605,"5min":101.605,"15min":101.605},"last":2.0539999999999998},{"interface":"front","average":{"1min":10.422000000000001,"5min":5.6959999999999997,"15min":4.9080000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":101.892,"5min":101.892,"15min":101.892},"last":2.0710000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":2.9689999999999999,"15min":2.0190000000000001},"min":{"1min":0.23000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":36.456000000000003,"5min":36.456000000000003,"15min":37.889000000000003},"last":3.2120000000000002},{"interface":"front","average":{"1min":4.141,"5min":3.4209999999999998,"15min":2.04},"min":{"1min":0.315,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":36.427,"5min":36.427,"15min":37.823},"last":2.222}]},{"osd":3,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6280000000000001,"5min":2.7839999999999998,"15min":1.5529999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.5259999999999998,"5min":16.631,"15min":50.033999999999999},"last":1.978},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.4900000000000002,"15min":1.847},"min":{"1min":0.34399999999999997,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.8819999999999997,"5min":29.565000000000001,"15min":52.491999999999997},"last":2.9689999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.5830000000000002,"5min":3.2080000000000002,"15min":1.8049999999999999},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":78.948999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":2.2730000000000001},{"interface":"front","average":{"1min":4.7999999999999998,"5min":2.4159999999999999,"15min":1.6220000000000001},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.313},"max":{"1min":78.963999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":2.1539999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.3620000000000001,"5min":2.556,"15min":1.472},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":84.254999999999995,"5min":84.254999999999995,"15min":84.254999999999995},"last":1.946},{"interface":"front","average":{"1min":6.2939999999999996,"5min":3.2879999999999998,"15min":1.6719999999999999},"min":{"1min":0.32800000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":87.515000000000001,"5min":87.515000000000001,"15min":87.515000000000001},"last":2.3029999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0250000000000004,"5min":2.6819999999999999,"15min":1.45},"min":{"1min":0.436,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":79.311000000000007,"5min":79.311000000000007,"15min":79.311000000000007},"last":2.1030000000000002},{"interface":"front","average":{"1min":5.4189999999999996,"5min":2.7029999999999998,"15min":1.405},"min":{"1min":0.34699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":79.328999999999994,"5min":79.328999999999994,"15min":79.328999999999994},"last":1.9119999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.5910000000000002,"15min":2.0059999999999998},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":8.5449999999999999,"5min":29.539000000000001,"15min":159.018},"last":2.3300000000000001},{"interface":"front","average":{"1min":1.6519999999999999,"5min":2.0259999999999998,"15min":1.9179999999999999},"min":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.37},"max":{"1min":6.0810000000000004,"5min":8.6120000000000001,"15min":158.96000000000001},"last":2.1909999999999998}]}]},{"osd":0,"up_from":219,"seq":940597837861,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1938872,"kb_used_data":1778544,"kb_used_omap":31,"kb_used_meta":160288,"kb_avail":91806280,"statfs":{"total":95995035648,"available":94009630720,"internally_reserved":0,"allocated":1821229056,"data_stored":1820259799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":164135858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":3.556,"15min":3.6600000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":15.611000000000001,"5min":32.344999999999999,"15min":32.344999999999999},"last":21.512},{"interface":"front","average":{"1min":3.6389999999999998,"5min":3.5750000000000002,"15min":3.5640000000000001},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.092000000000001,"5min":32.106000000000002,"15min":32.106000000000002},"last":20.064}]},{"osd":2,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":2.468,"15min":2.2210000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.170999999999999,"5min":12.170999999999999,"15min":12.170999999999999},"last":20.385000000000002},{"interface":"front","average":{"1min":3.27,"5min":2.0590000000000002,"15min":1.857},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.9180000000000001,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":1.9650000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":2.1549999999999998,"15min":2.0289999999999999},"min":{"1min":0.34699999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":20.257000000000001},{"interface":"front","average":{"1min":2.6360000000000001,"5min":1.4259999999999999,"15min":1.224},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.9730000000000008,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7280000000000002,"5min":2.2280000000000002,"15min":1.978},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":15.432,"5min":15.432,"15min":15.432},"last":13.105},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.7509999999999999,"15min":1.7490000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9790000000000001,"5min":7.5590000000000002,"15min":7.5590000000000002},"last":20.009}]},{"osd":5,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.601,"15min":1.4239999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.1019999999999999,"5min":1.8620000000000001,"15min":1.6559999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":15.536,"5min":15.536,"15min":15.536},"last":13.058999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.6699999999999999,"15min":1.4710000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.8550000000000004,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":1.891},{"interface":"front","average":{"1min":3.952,"5min":2.1389999999999998,"15min":1.837},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.489000000000001,"5min":15.489000000000001,"15min":15.489000000000001},"last":13.175000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:48:19 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.9690000000000001,"15min":1.72},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":15.574,"5min":15.574,"15min":15.574},"last":13.141999999999999},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.4139999999999999,"15min":1.256},"min":{"1min":0.46200000000000002,"5min":0.432,"15min":0.432},"max":{"1min":8.9000000000000004,"5min":8.9000000000000004,"15min":8.9000000000000004},"last":1.9339999999999999}]}]},{"osd":2,"up_from":131,"seq":562640715873,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632660,"kb_used_data":1396620,"kb_used_omap":81,"kb_used_meta":235950,"kb_avail":92112492,"statfs":{"total":95995035648,"available":94323191808,"internally_reserved":0,"allocated":1430138880,"data_stored":1429197464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":241613018},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":5.1059999999999999,"5min":5.2359999999999998,"15min":5.258},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":41.723999999999997,"5min":57.811999999999998,"15min":57.811999999999998},"last":1.9430000000000001},{"interface":"front","average":{"1min":7.7679999999999998,"5min":5.8650000000000002,"15min":5.548},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":41.454000000000001,"5min":57.387999999999998,"15min":57.387999999999998},"last":11.362}]},{"osd":1,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":3.5179999999999998,"15min":3.226},"min":{"1min":0.28299999999999997,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.759,"5min":88.316000000000003,"15min":88.316000000000003},"last":3.7090000000000001},{"interface":"front","average":{"1min":4.444,"5min":4.4329999999999998,"15min":2.8719999999999999},"min":{"1min":0.34899999999999998,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":28.596,"5min":88.436999999999998,"15min":88.436999999999998},"last":4.9960000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.976,"15min":3.056},"min":{"1min":0.30499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.994,"5min":88.653000000000006,"15min":88.653000000000006},"last":5.0780000000000003},{"interface":"front","average":{"1min":3.9369999999999998,"5min":3.5880000000000001,"15min":2.4220000000000002},"min":{"1min":0.31900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":12.401,"5min":88.474000000000004,"15min":88.474000000000004},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.188000000000001,"5min":4.4569999999999999,"15min":2.1389999999999998},"min":{"1min":0.51300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":84.778999999999996,"5min":108.878,"15min":108.878},"last":0.57099999999999995},{"interface":"front","average":{"1min":11.103,"5min":5.8799999999999999,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":84.753,"5min":108.93000000000001,"15min":108.93000000000001},"last":3.746}]},{"osd":5,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":7.6660000000000004,"5min":5.3339999999999996,"15min":2.7480000000000002},"min":{"1min":0.42499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":90.082999999999998,"5min":97.322999999999993,"15min":97.322999999999993},"last":3.8109999999999999},{"interface":"front","average":{"1min":7.9420000000000002,"5min":5.2549999999999999,"15min":2.3340000000000001},"min":{"1min":0.437,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":89.986000000000004,"5min":97.301000000000002,"15min":97.301000000000002},"last":5.0289999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.731,"5min":4.9109999999999996,"15min":2.2530000000000001},"min":{"1min":0.40699999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":85.034999999999997,"5min":88.606999999999999,"15min":88.606999999999999},"last":5.0540000000000003},{"interface":"front","average":{"1min":9.9339999999999993,"5min":3.3479999999999999,"15min":1.7729999999999999},"min":{"1min":0.47399999999999998,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":85.209000000000003,"5min":85.209000000000003,"15min":85.209000000000003},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.726,"15min":1.266},"min":{"1min":0.40699999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9640000000000004,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.9329999999999998,"5min":3.4910000000000001,"15min":2.1720000000000002},"min":{"1min":0.45100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.5549999999999997,"5min":88.584000000000003,"15min":88.584000000000003},"last":3.8319999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925959,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760296,"kb_used_data":1386976,"kb_used_omap":7,"kb_used_meta":373176,"kb_avail":91984856,"statfs":{"total":95995035648,"available":94192492544,"internally_reserved":0,"allocated":1420263424,"data_stored":1419358738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":382132840},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:13 2023","interfaces":[{"interface":"back","average":{"1min":6.5970000000000004,"5min":2.8069999999999999,"15min":2.1749999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":80.099000000000004,"5min":80.099000000000004,"15min":80.099000000000004},"last":3.7389999999999999},{"interface":"front","average":{"1min":6.5149999999999997,"5min":2.581,"15min":1.925},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":80.150000000000006,"5min":80.150000000000006,"15min":80.150000000000006},"last":1.2729999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.8639999999999999,"15min":1.3979999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":6.7110000000000003,"5min":22.818000000000001,"15min":22.818000000000001},"last":3.7989999999999999},{"interface":"front","average":{"1min":1.472,"5min":2,"15min":1.1160000000000001},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":6.4160000000000004,"5min":23.917999999999999,"15min":23.917999999999999},"last":0.67700000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.367,"5min":3.879,"15min":2.2509999999999999},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.431000000000001,"5min":125.446,"15min":125.446},"last":3.3410000000000002},{"interface":"front","average":{"1min":3.476,"5min":5.7249999999999996,"15min":2.7330000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":14.670999999999999,"5min":182.483,"15min":182.483},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":2.1150000000000002,"15min":1.2310000000000001},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":3.7959999999999998,"5min":18.757000000000001,"15min":18.757000000000001},"last":3.831},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.5740000000000001,"15min":1.0649999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.516,"5min":12.097,"15min":12.097},"last":12.131}]},{"osd":5,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.4570000000000001,"15min":1.0229999999999999},"min":{"1min":0.30099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.4710000000000001,"5min":12.127000000000001,"15min":12.127000000000001},"last":0.622},{"interface":"front","average":{"1min":2.52,"5min":2.105,"15min":1.2150000000000001},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.9950000000000001,"5min":9.0090000000000003,"15min":9.0090000000000003},"last":0.77800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.589,"15min":1.0669999999999999},"min":{"1min":0.34000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.173,"5min":12.16,"15min":12.16},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.5830000000000002,"5min":2.089,"15min":1.2869999999999999},"min":{"1min":0.47099999999999997,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.0259999999999998,"5min":9.1029999999999998,"15min":10.755000000000001},"last":0.70899999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8399999999999999,"5min":2.9940000000000002,"15min":1.5649999999999999},"min":{"1min":0.40100000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.0510000000000002,"5min":91.757999999999996,"15min":91.757999999999996},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.7170000000000001,"5min":3.234,"15min":1.639},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":3.7709999999999999,"5min":91.709999999999994,"15min":91.709999999999994},"last":3.8559999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019158,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1889448,"kb_used_data":1575840,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91855704,"statfs":{"total":95995035648,"available":94060240896,"internally_reserved":0,"allocated":1613660160,"data_stored":1613161877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":6.6619999999999999,"15min":7.5640000000000001},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.0190000000000001,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":7.2779999999999996,"15min":8.1120000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.7200000000000006,"5min":110.33799999999999,"15min":110.33799999999999},"last":1.264}]},{"osd":1,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":2.597,"15min":1.4610000000000001},"min":{"1min":0.27800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.681,"5min":41.643000000000001,"15min":45.320999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.117,"15min":1.3160000000000001},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.8210000000000002,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":4.2560000000000002,"15min":3.2400000000000002},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.7030000000000003,"5min":84.909000000000006,"15min":84.909000000000006},"last":7.8940000000000001},{"interface":"front","average":{"1min":2.1659999999999999,"5min":4.2590000000000003,"15min":3.1669999999999998},"min":{"1min":0.371,"5min":0.309,"15min":0.309},"max":{"1min":7.0570000000000004,"5min":83.840000000000003,"15min":83.840000000000003},"last":8.0609999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":6.5529999999999999,"15min":2.8809999999999998},"min":{"1min":0.42699999999999999,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":6.1040000000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":1.79},{"interface":"front","average":{"1min":1.7689999999999999,"5min":6.7160000000000002,"15min":2.915},"min":{"1min":0.33000000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.657,"5min":129.197,"15min":129.197},"last":1.97}]},{"osd":5,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.746,"15min":0.73999999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.8979999999999997,"5min":5.8979999999999997,"15min":43.189999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.78000000000000003,"15min":0.76100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26500000000000001},"max":{"1min":2.7040000000000002,"5min":9.5190000000000001,"15min":42.450000000000003},"last":1.405}]},{"osd":6,"last update":"Sat Sep 23 22:48:31 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.79300000000000004,"15min":0.73099999999999998},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":2.7570000000000001,"5min":6.8109999999999999,"15min":27.056000000000001},"last":1.4359999999999999},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78800000000000003,"15min":0.75800000000000001},"min":{"1min":0.32500000000000001,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":2.6520000000000001,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.754,"15min":0.66900000000000004},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.464,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.68600000000000005,"15min":0.64600000000000002},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":2.5710000000000002,"5min":3.2519999999999998,"15min":5.9409999999999998},"last":0.52400000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822930,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2366548,"kb_used_data":2057036,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91378604,"statfs":{"total":95995035648,"available":93571690496,"internally_reserved":0,"allocated":2106404864,"data_stored":2105922199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:48:16 2023","interfaces":[{"interface":"back","average":{"1min":18.346,"5min":5.1150000000000002,"15min":2.9100000000000001},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":121.319,"5min":121.319,"15min":121.319},"last":0.249},{"interface":"front","average":{"1min":18.654,"5min":5.0469999999999997,"15min":2.7789999999999999},"min":{"1min":0.32700000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":121.075,"5min":121.075,"15min":121.075},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8,"15min":1.716},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":28.658999999999999,"5min":28.658999999999999,"15min":149.203},"last":0.28999999999999998},{"interface":"front","average":{"1min":3.8969999999999998,"5min":2.3999999999999999,"15min":1.871},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":28.664999999999999,"5min":28.664999999999999,"15min":149.554},"last":0.38700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":4.5599999999999996,"15min":7.141},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.2879999999999998,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.442},{"interface":"front","average":{"1min":1.861,"5min":4.7949999999999999,"15min":7.1959999999999997},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3899999999999997,"5min":158.91399999999999,"15min":158.91399999999999},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":14.204000000000001,"5min":8.2560000000000002,"15min":3.9020000000000001},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":108.58799999999999,"5min":257.05000000000001,"15min":257.05000000000001},"last":3.3900000000000001},{"interface":"front","average":{"1min":14.087999999999999,"5min":7.6820000000000004,"15min":3.7570000000000001},"min":{"1min":0.311,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":108.15600000000001,"5min":257.35000000000002,"15min":257.35000000000002},"last":5.0369999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:48:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.79300000000000004,"15min":0.60299999999999998},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":1.4790000000000001,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.79800000000000004,"15min":0.61299999999999999},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.619,"5min":10.535,"15min":10.535},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:48:18 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.411,"15min":1.3879999999999999},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":12.615,"5min":48.360999999999997,"15min":140.07499999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.417,"5min":1.532,"15min":1.4399999999999999},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.254},"max":{"1min":12.74,"5min":48.292999999999999,"15min":139.739},"last":0.315}]},{"osd":7,"last update":"Sat Sep 23 22:48:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.82399999999999995,"15min":0.70599999999999996},"min":{"1min":0.33400000000000002,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.464,"5min":7.0010000000000003,"15min":7.9809999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.74299999999999999,"15min":0.67000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":1.9650000000000001,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.32700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1814167552,"data_stored":1814167552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1611923456,"data_stored":1611923456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1421742080,"data_stored":1421738008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1404239872,"data_stored":1404239872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1602945024,"data_stored":1602945024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2099122176,"data_stored":2099118104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1379799040,"data_stored":1379794968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:49:17.051 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:49:17.240 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:17.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:17.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:17.445 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:17.605 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:17.627 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:17.659 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:17 smithi052 ceph-mon[180979]: from='client.35418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:17.659 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:17 smithi052 ceph-mon[180979]: pgmap v674: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 1622/163521 objects degraded (0.992%); 3546/163521 objects misplaced (2.169%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:17.659 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:17 smithi052 ceph-mon[187743]: from='client.35418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:17.659 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:17 smithi052 ceph-mon[187743]: pgmap v674: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 1622/163521 objects degraded (0.992%); 3546/163521 objects misplaced (2.169%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:17.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:17 smithi151 ceph-mon[159210]: from='client.35418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:17 smithi151 ceph-mon[159210]: pgmap v674: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 1622/163521 objects degraded (0.992%); 3546/163521 objects misplaced (2.169%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:17.724 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:49:17.724 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:49:17.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:46237 8.1 8:8495a6ee:::benchmark_data_smithi052_337357_object46236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:49.158391+0000", 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 148.56298095899999, 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20055051300000001, 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 46237 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158391+0000", 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158391+0000", 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158389+0000", 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158428+0000", 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9335999999999997e-05 2023-09-23T22:49:17.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158448+0000", 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.039e-05 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158459+0000", 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0463e-05 2023-09-23T22:49:17.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158711+0000", 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025165300000000001 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158845+0000", 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013468700000000001 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158988+0000", 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014239499999999999 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.169176+0000", 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010188271 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.172612+0000", 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034360089999999999 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.358852+0000", 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.186239827 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.358906+0000", 2023-09-23T22:49:17.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3792999999999998e-05 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.358942+0000", 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5995999999999999e-05 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:46238 8.1 8:82498144:::benchmark_data_smithi052_337357_object46237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-09-23T22:49:17.732 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:49.158477+0000", 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 148.56289526899999, 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.200572848, 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 46238 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158477+0000", 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158477+0000", 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158476+0000", 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158512+0000", 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5746e-05 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158521+0000", 2023-09-23T22:49:17.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3519999999999996e-06 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158528+0000", 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9589999999999998e-06 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.160508+0000", 2023-09-23T22:49:17.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019802489999999999 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.160606+0000", 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7596999999999994e-05 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.160700+0000", 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4168999999999999e-05 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.172571+0000", 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011870597 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.172688+0000", 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011773 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.358977+0000", 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18628857600000001 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.359016+0000", 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8982e-05 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.359050+0000", 2023-09-23T22:49:17.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3488000000000002e-05 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:46240 8.1 8:8eb2ab80:::benchmark_data_smithi052_337357_object46239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:49.158536+0000", 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 148.56283641799999, 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20393469, 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.740 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.741 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.741 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.741 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.741 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 46240 2023-09-23T22:49:17.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.742 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158536+0000", 2023-09-23T22:49:17.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158536+0000", 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158535+0000", 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158564+0000", 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8813999999999999e-05 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158573+0000", 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0820000000000005e-06 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158579+0000", 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8279999999999997e-06 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.162509+0000", 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003929592 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.162627+0000", 2023-09-23T22:49:17.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011759100000000001 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.162750+0000", 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012305199999999999 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.172576+0000", 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098266159999999998 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.173997+0000", 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001420517 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.362400+0000", 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.188403078 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.362433+0000", 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3510999999999998e-05 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.362470+0000", 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6767999999999998e-05 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:46242 8.1 8:86f7e65c:::benchmark_data_smithi052_337357_object46241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:49.158586+0000", 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 148.56278643100001, 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20666042800000001, 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 46242 2023-09-23T22:49:17.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158586+0000", 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158586+0000", 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158585+0000", 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158608+0000", 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3394999999999999e-05 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158616+0000", 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8699999999999992e-06 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158623+0000", 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4439999999999996e-06 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.170637+0000", 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012014014999999999 2023-09-23T22:49:17.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.170781+0000", 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014411900000000001 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.170887+0000", 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010601 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.174012+0000", 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031246910000000002 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.174044+0000", 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2441999999999999e-05 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.365167+0000", 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.191123183 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.365211+0000", 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4122000000000002e-05 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.365246+0000", 2023-09-23T22:49:17.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4668000000000001e-05 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:46246 8.1 8:863d3705:::benchmark_data_smithi052_337357_object46245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:49.158629+0000", 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 148.562743004, 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20674826199999999, 2023-09-23T22:49:17.756 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 46246 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158629+0000", 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158629+0000", 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158628+0000", 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158652+0000", 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3366999999999998e-05 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158660+0000", 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.089e-06 2023-09-23T22:49:17.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.158666+0000", 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2110000000000003e-06 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.174075+0000", 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015408893 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.174205+0000", 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0001295 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.174350+0000", 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014527600000000001 2023-09-23T22:49:17.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.175902+0000", 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001552405 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.177899+0000", 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019966789999999999 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.365299+0000", 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18740000800000001 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.365344+0000", 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5011e-05 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:49.365377+0000", 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3287000000000002e-05 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48261 8.1 8:87871911:::benchmark_data_smithi052_337357_object48260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.794281947, 2023-09-23T22:49:17.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27316745399999998, 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48261 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:49:17.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927089+0000", 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927116+0000", 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7282999999999999e-05 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927128+0000", 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1987000000000001e-05 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927135+0000", 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4120000000000004e-06 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927186+0000", 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0982000000000001e-05 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927313+0000", 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012680600000000001 2023-09-23T22:49:17.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927452+0000", 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013846099999999999 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.930719+0000", 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032677600000000002 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934336+0000", 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036163720000000001 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200180+0000", 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26584448100000002 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200227+0000", 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7108999999999999e-05 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200257+0000", 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0210999999999999e-05 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.771 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48276 8.c 8:3458bc4f:::benchmark_data_smithi052_337357_object48275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.78691700600001, 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26596831100000001, 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48276 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934454+0000", 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934485+0000", 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1665999999999998e-05 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934497+0000", 2023-09-23T22:49:17.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1136e-05 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934505+0000", 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3399999999999998e-06 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934560+0000", 2023-09-23T22:49:17.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4852000000000003e-05 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934713+0000", 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000153129 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934856+0000", 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143199 2023-09-23T22:49:17.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.936652+0000", 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001796157 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.936814+0000", 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016158000000000001 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200327+0000", 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26351350400000001 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200382+0000", 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4231000000000003e-05 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200423+0000", 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1641999999999999e-05 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48281 8.c 8:3ea84d87:::benchmark_data_smithi052_337357_object48280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.78400430299999, 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26513863300000001, 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.779 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48281 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937366+0000", 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937390+0000", 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3839e-05 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937401+0000", 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1238e-05 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937408+0000", 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.148e-06 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937510+0000", 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101549 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937622+0000", 2023-09-23T22:49:17.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111784 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937748+0000", 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012662900000000001 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941281+0000", 2023-09-23T22:49:17.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003532355 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941456+0000", 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017523800000000001 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202425+0000", 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26096917400000003 2023-09-23T22:49:17.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202471+0000", 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6286999999999999e-05 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202506+0000", 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4978999999999998e-05 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.786 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48284 8.e 8:76fd02f7:::benchmark_data_smithi052_337357_object48283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.78186189799999, 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.264529664, 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.787 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48284 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939509+0000", 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939533+0000", 2023-09-23T22:49:17.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4029e-05 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939543+0000", 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0251999999999999e-05 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939550+0000", 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3560000000000004e-06 2023-09-23T22:49:17.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939596+0000", 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5862000000000002e-05 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939720+0000", 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124368 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:49:17.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939922+0000", 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020159799999999999 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201651+0000", 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.261728827 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201691+0000", 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9750999999999999e-05 2023-09-23T22:49:17.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203971+0000", 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022803329999999998 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204008+0000", 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7373000000000003e-05 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204040+0000", 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1350999999999998e-05 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48287 8.e 8:77bdb954:::benchmark_data_smithi052_337357_object48286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:49:17.794 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.77989664899999, 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26634659700000002, 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48287 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941474+0000", 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941498+0000", 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3611000000000001e-05 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941507+0000", 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8169999999999994e-06 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941519+0000", 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1724e-05 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941642+0000", 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012283800000000001 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941737+0000", 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4828999999999996e-05 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941823+0000", 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6247000000000003e-05 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201726+0000", 2023-09-23T22:49:17.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25990260700000001 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204073+0000", 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023471569999999999 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207731+0000", 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036582989999999998 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207785+0000", 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3621e-05 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207822+0000", 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7115e-05 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48288 8.6 8:6358135e:::benchmark_data_smithi052_337357_object48287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.77983937600001, 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.262402097, 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.802 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48288 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941532+0000", 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941551+0000", 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9185999999999999e-05 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941558+0000", 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6479999999999997e-06 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941563+0000", 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3430000000000003e-06 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941634+0000", 2023-09-23T22:49:17.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0407000000000005e-05 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941713+0000", 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9330000000000001e-05 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941808+0000", 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5241000000000005e-05 2023-09-23T22:49:17.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201641+0000", 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25983307300000003 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201657+0000", 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5184000000000001e-05 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203850+0000", 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021936019999999998 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203898+0000", 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8186999999999998e-05 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203935+0000", 2023-09-23T22:49:17.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6375000000000003e-05 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48276 8.c e228/219 ondisk, result = 0)", 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.779453682, 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25851532799999999, 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:49:17.809 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941922+0000", 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3780000000000003e-06 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200261+0000", 2023-09-23T22:49:17.811 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25833932399999998 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200269+0000", 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.255e-06 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.812 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200312+0000", 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3016999999999998e-05 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200321+0000", 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2790000000000004e-06 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200434+0000", 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000112623 2023-09-23T22:49:17.813 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48261 8.1 e228/226 ondisk, result = 0)", 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.77921862599999, 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25811125899999998, 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:49:17.814 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942157+0000", 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5770000000000004e-06 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200098+0000", 2023-09-23T22:49:17.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25794090400000003 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200110+0000", 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2119999999999999e-05 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200163+0000", 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2859999999999999e-05 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200175+0000", 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1765e-05 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200265+0000", 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9689000000000001e-05 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48284 8.e e228/219 ondisk, result = 0)", 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.776305944, 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25662737099999999, 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:49:17.819 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945065+0000", 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945071+0000", 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.088e-06 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200085+0000", 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25501373599999999 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200109+0000", 2023-09-23T22:49:17.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3581e-05 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201679+0000", 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015704989999999999 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201685+0000", 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.404e-06 2023-09-23T22:49:17.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201693+0000", 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9060000000000002e-06 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:53505 8.e 8:7740416d:::benchmark_data_smithi052_337357_object53504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:55.824909+0000", 2023-09-23T22:49:17.823 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 141.89646340900001, 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21468008599999999, 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 53505 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.824909+0000", 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.824909+0000", 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.824908+0000", 2023-09-23T22:49:17.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.824927+0000", 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9653e-05 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.824937+0000", 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8910000000000007e-06 2023-09-23T22:49:17.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.824944+0000", 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0450000000000003e-06 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.827006+0000", 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020618530000000002 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.827163+0000", 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015728500000000001 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.827321+0000", 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000157658 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.829832+0000", 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025110290000000001 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.830294+0000", 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00046155700000000003 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.039488+0000", 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20919453900000001 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.039531+0000", 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3164000000000002e-05 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.039589+0000", 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7218e-05 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.830 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:53523 8.1 8:895c505a:::benchmark_data_smithi052_337357_object53522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:55.837636+0000", 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 141.88373640399999, 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20963746799999999, 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 53523 2023-09-23T22:49:17.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837636+0000", 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837636+0000", 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837634+0000", 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837653+0000", 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9174999999999999e-05 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837662+0000", 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7569999999999993e-06 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837668+0000", 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5339999999999999e-06 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837707+0000", 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9368e-05 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837803+0000", 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5786999999999996e-05 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.837894+0000", 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1240000000000001e-05 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.839474+0000", 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015797579999999999 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.839695+0000", 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022092100000000001 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.047178+0000", 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20748282900000001 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.047229+0000", 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1211999999999997e-05 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.047273+0000", 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4231999999999999e-05 2023-09-23T22:49:17.837 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:53534 8.1 8:8064a8ba:::benchmark_data_smithi052_337357_object53533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e229)", 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:55.847558+0000", 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 141.873813711, 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20048379099999999, 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.838 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 53534 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847558+0000", 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847558+0000", 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847556+0000", 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847587+0000", 2023-09-23T22:49:17.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0307e-05 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847601+0000", 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4174e-05 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847610+0000", 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1689999999999998e-06 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:55.847653+0000", 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3238000000000002e-05 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.039836+0000", 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19218262899999999 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.039975+0000", 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139529 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.047313+0000", 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.007338064 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.047320+0000", 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6830000000000002e-06 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.047964+0000", 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00064419300000000004 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.048009+0000", 2023-09-23T22:49:17.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4969000000000003e-05 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:56.048042+0000", 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2833999999999998e-05 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.845 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:72878 8.c 8:30ad6d0a:::benchmark_data_smithi052_337357_object72877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:47:14.855341+0000", 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 122.866030491, 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20060967399999999, 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72878 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.855341+0000", 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.855341+0000", 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.855341+0000", 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.855362+0000", 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1149e-05 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.855375+0000", 2023-09-23T22:49:17.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2693e-05 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.855380+0000", 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.519e-06 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.858274+0000", 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002893657 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.858419+0000", 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014496999999999999 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.858582+0000", 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016269599999999999 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.867273+0000", 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0086916319999999995 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:14.867281+0000", 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.435e-06 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:15.055866+0000", 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18858496399999999 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:15.055911+0000", 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5464999999999999e-05 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:15.055951+0000", 2023-09-23T22:49:17.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9987999999999997e-05 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:88322 8.e 8:77e9f0d2:::benchmark_data_smithi052_337357_object88321:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e233)", 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:47:32.706723+0000", 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 105.01464941099999, 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20947416699999999, 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 88322 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706723+0000", 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706723+0000", 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706720+0000", 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706750+0000", 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9340999999999999e-05 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706767+0000", 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7577000000000001e-05 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706777+0000", 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6420000000000002e-06 2023-09-23T22:49:17.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706843+0000", 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5790000000000005e-05 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.706991+0000", 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014857100000000001 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.707155+0000", 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016360100000000001 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.854468+0000", 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.1473131 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.854505+0000", 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6604999999999999e-05 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.916104+0000", 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061599411 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.916157+0000", 2023-09-23T22:49:17.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2672e-05 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:32.916197+0000", 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9946e-05 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:95424 8.c 8:308751a2:::benchmark_data_smithi052_337357_object95423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e236)", 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:47:41.670881+0000", 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 96.050491262999998, 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.223511547, 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 95424 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670881+0000", 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670881+0000", 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670879+0000", 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670910+0000", 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1600000000000002e-05 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670929+0000", 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8493000000000001e-05 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670939+0000", 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0542e-05 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670989+0000", 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9926999999999998e-05 2023-09-23T22:49:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.671110+0000", 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121308 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.769466+0000", 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.098355572000000002 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.773071+0000", 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036054149999999998 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.777224+0000", 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041521400000000003 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.894287+0000", 2023-09-23T22:49:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.117063365 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.894346+0000", 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9623000000000002e-05 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.894392+0000", 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5745000000000003e-05 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:17.868 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:49:17.868 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:17.899 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:17.908 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:17.960 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:17.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:17.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:18.069 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:18.105 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:18.158 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:18.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:18.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:18.265 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:18.363 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:19.148 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:19.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:19.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:19.181 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:49:19.258 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:19.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:19.601 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:19 smithi151 ceph-mon[159210]: pgmap v675: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 1622/161280 objects degraded (1.006%); 3546/161280 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:19.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:19 smithi052 ceph-mon[180979]: pgmap v675: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 1622/161280 objects degraded (1.006%); 3546/161280 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:19.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:19 smithi052 ceph-mon[187743]: pgmap v675: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 1622/161280 objects degraded (1.006%); 3546/161280 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:19.782 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:19.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:19.886 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:19.942 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1622/161280 objects degraded (1.006%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1622/161280 objects degraded (1.006%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:20.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:20.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:20.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1622/161280 objects degraded (1.006%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:21.452 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:21.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:21.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:21.567 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:21.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:21 smithi052 ceph-mon[187743]: pgmap v676: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1622/160146 objects degraded (1.013%); 3546/160146 objects misplaced (2.214%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:21.771 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:21 smithi052 ceph-mon[180979]: pgmap v676: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1622/160146 objects degraded (1.013%); 3546/160146 objects misplaced (2.214%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:21.772 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:21.889 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:21 smithi151 ceph-mon[159210]: pgmap v676: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1622/160146 objects degraded (1.013%); 3546/160146 objects misplaced (2.214%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:22.777 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:22.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:22.886 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:22.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:22 smithi052 ceph-mon[180979]: pgmap v677: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 1737/154716 objects degraded (1.123%); 3337/154716 objects misplaced (2.157%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:22.897 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:22 smithi052 ceph-mon[187743]: pgmap v677: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 1737/154716 objects degraded (1.123%); 3337/154716 objects misplaced (2.157%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:22 smithi151 ceph-mon[159210]: pgmap v677: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 1737/154716 objects degraded (1.123%); 3337/154716 objects misplaced (2.157%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:22.999 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:23.245 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:23.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:23.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:23.304 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:49:23.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:23.365 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:49:23.387 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:49:23.433 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:49:23.450 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:49:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:49:23] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-09-23T22:49:23.459 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:23.479 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:49:23.525 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:49:23.557 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:49:23.607 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:49:23.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.640 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.641 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.641 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.641 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.641 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:49:23.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3451838538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:23.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3451838538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3451838538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:24.672 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:49:24.800 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:24.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:24.912 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:24 smithi151 ceph-mon[159210]: pgmap v678: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 1737/152562 objects degraded (1.139%); 3337/152562 objects misplaced (2.187%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1798199176' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:49:24.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:24 smithi052 ceph-mon[180979]: pgmap v678: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 1737/152562 objects degraded (1.139%); 3337/152562 objects misplaced (2.187%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:24.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1798199176' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:49:24.969 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:24 smithi052 ceph-mon[187743]: pgmap v678: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 1737/152562 objects degraded (1.139%); 3337/152562 objects misplaced (2.187%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:49:24.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1798199176' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:49:24.973 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:25.868 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:25.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[180979]: failed to encode map e253 with expected crc 2023-09-23T22:49:25.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1798199176' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:49:25.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[180979]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T22:49:25.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[180979]: failed to encode map e253 with expected crc 2023-09-23T22:49:25.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1737/152562 objects degraded (1.139%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:25.869 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:25.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[187743]: failed to encode map e253 with expected crc 2023-09-23T22:49:25.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1798199176' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:49:25.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[187743]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T22:49:25.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[187743]: failed to encode map e253 with expected crc 2023-09-23T22:49:25.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1737/152562 objects degraded (1.139%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:25.944 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:25.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:25 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:25.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:25 smithi151 ceph-mon[159210]: failed to encode map e253 with expected crc 2023-09-23T22:49:25.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1798199176' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:49:25.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:25 smithi151 ceph-mon[159210]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T22:49:25.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:25 smithi151 ceph-mon[159210]: failed to encode map e253 with expected crc 2023-09-23T22:49:25.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1737/152562 objects degraded (1.139%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:25.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:25.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:26.060 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:26.090 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:26.152 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:26.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:26.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:26.261 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:26.295 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:26.351 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:26.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:26.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:26.460 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:26.562 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:26 smithi052 ceph-mon[180979]: pgmap v680: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 1839/145734 objects degraded (1.262%); 3128/145734 objects misplaced (2.146%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:26.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:26 smithi052 ceph-mon[187743]: pgmap v680: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 1839/145734 objects degraded (1.262%); 3128/145734 objects misplaced (2.146%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:26 smithi151 ceph-mon[159210]: pgmap v680: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 1839/145734 objects degraded (1.262%); 3128/145734 objects misplaced (2.146%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:26.976 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:26.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:26.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:27.091 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:27.123 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:27.205 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:27.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:27.249 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:49:27.249 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:49:27.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:27.346 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:27.399 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:49:27.399 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1839/145734 objects degraded (1.262%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":900,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":253,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":7,"osd_in_since":1695509092,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":48578,"data_bytes":3182565304,"bytes_used":11900915712,"bytes_avail":660064333824,"bytes_total":671965249536,"degraded_objects":1839,"degraded_total":145734,"degraded_ratio":0.012618880974926921,"misplaced_objects":3128,"misplaced_total":145734,"misplaced_ratio":0.02146376274582458,"recovering_objects_per_sec":20,"recovering_bytes_per_sec":1315815,"recovering_keys_per_sec":0,"num_objects_recovered":201,"num_bytes_recovered":13172736,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":592},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":26,"modified":"2023-09-23T22:48:52.236301+0000","services":{}},"progress_events":{"3d9c7ac0-c827-48ce-9357-8c3e026dd16f":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:49:27.412 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:27.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:27.518 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:27.554 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:27.626 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:27.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:27.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:27.735 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:27.805 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/284584869' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:49:27.808 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/284584869' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:49:27.814 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:27.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/284584869' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:49:28.461 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:49:28.810 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:28.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:28.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:28.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:28 smithi052 ceph-mon[187743]: pgmap v681: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1839/143592 objects degraded (1.281%); 3128/143592 objects misplaced (2.178%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:28.824 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:28 smithi052 ceph-mon[180979]: pgmap v681: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1839/143592 objects degraded (1.281%); 3128/143592 objects misplaced (2.178%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:28.924 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:28 smithi151 ceph-mon[159210]: pgmap v681: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1839/143592 objects degraded (1.281%); 3128/143592 objects misplaced (2.178%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:28.958 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:49:28.995 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:49:29.020 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:29.305 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:29.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:29.353 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:49:29.353 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:49:29.424 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:29.452 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:29.507 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:29.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:29.575 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:29.618 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:29.646 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:29.699 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:29.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:29.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:29.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:29.822 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:29.836 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:29.886 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:29.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:29.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:29.990 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:30.020 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:30.072 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:30.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:30.178 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:30.208 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:30.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1839/143592 objects degraded (1.281%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:30.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1839/143592 objects degraded (1.281%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:30.268 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:30.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:30.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:30.276 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1839/143592 objects degraded (1.281%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:30.370 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:30.377 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:30.410 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:30.461 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:30.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:30.568 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:30.641 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:31 smithi151 ceph-mon[159210]: pgmap v682: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 1839/142569 objects degraded (1.290%); 3128/142569 objects misplaced (2.194%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:31.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:31 smithi052 ceph-mon[180979]: pgmap v682: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 1839/142569 objects degraded (1.290%); 3128/142569 objects misplaced (2.194%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:31.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:31 smithi052 ceph-mon[187743]: pgmap v682: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 1839/142569 objects degraded (1.290%); 3128/142569 objects misplaced (2.194%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:31.952 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:31.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:31.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:32.068 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:32.097 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:32.159 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:32.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:32.264 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:32.265 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:32.294 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:32.343 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:32.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:32.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:32.451 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:32.478 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:32.526 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:32.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:32.631 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:32.722 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:32.896 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:32.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:33.024 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:33.054 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:33.121 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:33.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:33.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:33.229 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:33.261 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:33.318 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:33 smithi151 ceph-mon[159210]: pgmap v683: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1913/137067 objects degraded (1.396%); 2946/137067 objects misplaced (2.149%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:49:33.324 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:33.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:33.433 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:33.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:33.692 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:33 smithi052 ceph-mon[180979]: pgmap v683: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1913/137067 objects degraded (1.396%); 2946/137067 objects misplaced (2.149%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:49:33.692 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:49:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:49:33] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-09-23T22:49:33.693 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:33 smithi052 ceph-mon[187743]: pgmap v683: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1913/137067 objects degraded (1.396%); 2946/137067 objects misplaced (2.149%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:49:34.212 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:34.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:34.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:34.283 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:49:34.325 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:34.360 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:49:34.363 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:49:34.367 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:34.398 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:49:34.426 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:34.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:34.438 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:49:34.486 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:49:34.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:34.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/106074672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:34.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/937733535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:34.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:34.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/106074672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/937733535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:34.537 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:34.542 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:49:34.606 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:49:34.627 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:34.647 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:49:34.664 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.665 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:49:34.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:49:34.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/106074672' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:34.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/937733535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:34.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: pgmap v684: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1913/134934 objects degraded (1.418%); 2946/134934 objects misplaced (2.183%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:49:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T22:49:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1913/134934 objects degraded (1.418%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: pgmap v684: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1913/134934 objects degraded (1.418%); 2946/134934 objects misplaced (2.183%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1913/134934 objects degraded (1.418%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: pgmap v684: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1913/134934 objects degraded (1.418%); 2946/134934 objects misplaced (2.183%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:35.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: failed to encode map e254 with expected crc 2023-09-23T22:49:35.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1913/134934 objects degraded (1.418%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:35.971 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:35.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:36.083 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:36.148 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:36.707 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:49:36.707 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:49:36.707 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:49:36.707 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:49:36.707 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:49:37.413 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:37.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:37.515 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:37.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:37 smithi052 ceph-mon[180979]: pgmap v686: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 1969/128241 objects degraded (1.535%); 2780/128241 objects misplaced (2.168%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:37.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:37.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:37.614 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:37 smithi052 ceph-mon[187743]: pgmap v686: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 1969/128241 objects degraded (1.535%); 2780/128241 objects misplaced (2.168%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:37.614 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:37.614 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:37.616 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:37 smithi151 ceph-mon[159210]: pgmap v686: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 1969/128241 objects degraded (1.535%); 2780/128241 objects misplaced (2.168%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:38.194 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:38.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:38.213 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:49:38.305 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:38.437 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:38.676 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:38.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:38.750 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:49:38.795 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:38.929 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:39.041 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:39.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:39.155 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:49:39.159 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:39.174 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:39.177 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:49:39.193 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:39.251 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:39.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:39.360 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:39.381 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:39 smithi052 ceph-mon[180979]: pgmap v687: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 1969/125943 objects degraded (1.563%); 2780/125943 objects misplaced (2.207%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:39.387 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:39 smithi151 ceph-mon[159210]: pgmap v687: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 1969/125943 objects degraded (1.563%); 2780/125943 objects misplaced (2.207%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:39.455 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:39.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:39.560 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:39.600 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:39.662 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:39.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:39.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:39 smithi052 ceph-mon[187743]: pgmap v687: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 1969/125943 objects degraded (1.563%); 2780/125943 objects misplaced (2.207%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:39.770 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:39.810 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:39.868 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:39.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:39.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:39.978 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:40.019 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:40.186 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:49:40.390 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:40 smithi052 ceph-mon[180979]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:40.390 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1969/125943 objects degraded (1.563%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:40.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:40 smithi151 ceph-mon[159210]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:40.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1969/125943 objects degraded (1.563%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:40.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:40 smithi052 ceph-mon[187743]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:40.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1969/125943 objects degraded (1.563%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:40.740 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":687,"stamp":"2023-09-23T22:49:38.261375+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2750224312,"num_objects":41981,"num_object_clones":0,"num_object_copies":125943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1969,"num_objects_misplaced":2780,"num_objects_unfound":0,"num_objects_dirty":41981,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":163201,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1904,"num_bytes_recovered":122693706,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127935,"ondisk_log_size":127935,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10354040,"kb_used_data":8366592,"kb_used_omap":129,"kb_used_meta":1987198,"kb_avail":645862024,"statfs":{"total":671965249536,"available":661362712576,"internally_reserved":0,"allocated":8567390208,"data_stored":8562158667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2034891485},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,2,1,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-385548288,"num_objects":-5883,"num_object_clones":0,"num_object_copies":-17649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":130,"num_objects_misplaced":-348,"num_objects_unfound":0,"num_objects_dirty":-5883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5883,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":14286848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003555"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":251,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656655+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:49:24.656655+0000","last_peered":"2023-09-23T22:49:24.656655+0000","last_clean":"2023-09-23T22:49:24.656655+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:49:24.656655+0000","last_undegraded":"2023-09-23T22:49:24.656655+0000","last_fullsized":"2023-09-23T22:49:24.656655+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":226,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648125+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:49:24.648125+0000","last_peered":"2023-09-23T22:49:24.648125+0000","last_clean":"2023-09-23T22:49:24.648125+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:49:24.648125+0000","last_undegraded":"2023-09-23T22:49:24.648125+0000","last_fullsized":"2023-09-23T22:49:24.648125+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647375+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:49:24.647375+0000","last_peered":"2023-09-23T22:49:24.647375+0000","last_clean":"2023-09-23T22:49:24.647375+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:49:24.647375+0000","last_undegraded":"2023-09-23T22:49:24.647375+0000","last_fullsized":"2023-09-23T22:49:24.647375+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":221,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669563+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:49:24.669563+0000","last_peered":"2023-09-23T22:49:24.669563+0000","last_clean":"2023-09-23T22:49:24.669563+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:49:24.669563+0000","last_undegraded":"2023-09-23T22:49:24.669563+0000","last_fullsized":"2023-09-23T22:49:24.669563+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":285,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.652058+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:49:24.652058+0000","last_peered":"2023-09-23T22:49:24.652058+0000","last_clean":"2023-09-23T22:49:24.652058+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:49:24.652058+0000","last_undegraded":"2023-09-23T22:49:24.652058+0000","last_fullsized":"2023-09-23T22:49:24.652058+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658930+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:49:24.658930+0000","last_peered":"2023-09-23T22:49:24.658930+0000","last_clean":"2023-09-23T22:49:24.658930+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:49:24.658930+0000","last_undegraded":"2023-09-23T22:49:24.658930+0000","last_fullsized":"2023-09-23T22:49:24.658930+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":153,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669446+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:49:24.669446+0000","last_peered":"2023-09-23T22:49:24.669446+0000","last_clean":"2023-09-23T22:49:24.669446+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:49:24.669446+0000","last_undegraded":"2023-09-23T22:49:24.669446+0000","last_fullsized":"2023-09-23T22:49:24.669446+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.652136+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:49:24.652136+0000","last_peered":"2023-09-23T22:49:24.652136+0000","last_clean":"2023-09-23T22:49:24.652136+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:49:24.652136+0000","last_undegraded":"2023-09-23T22:49:24.652136+0000","last_fullsized":"2023-09-23T22:49:24.652136+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":223,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647893+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:49:24.647893+0000","last_peered":"2023-09-23T22:49:24.647893+0000","last_clean":"2023-09-23T22:49:24.647893+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:49:24.647893+0000","last_undegraded":"2023-09-23T22:49:24.647893+0000","last_fullsized":"2023-09-23T22:49:24.647893+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":211,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647332+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:49:24.647332+0000","last_peered":"2023-09-23T22:49:24.647332+0000","last_clean":"2023-09-23T22:49:24.647332+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:49:24.647332+0000","last_undegraded":"2023-09-23T22:49:24.647332+0000","last_fullsized":"2023-09-23T22:49:24.647332+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":128,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648036+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:49:24.648036+0000","last_peered":"2023-09-23T22:49:24.648036+0000","last_clean":"2023-09-23T22:49:24.648036+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:49:24.648036+0000","last_undegraded":"2023-09-23T22:49:24.648036+0000","last_fullsized":"2023-09-23T22:49:24.648036+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655484+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:49:24.655484+0000","last_peered":"2023-09-23T22:49:24.655484+0000","last_clean":"2023-09-23T22:49:24.655484+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:49:24.655484+0000","last_undegraded":"2023-09-23T22:49:24.655484+0000","last_fullsized":"2023-09-23T22:49:24.655484+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":274,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.657077+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:49:24.657077+0000","last_peered":"2023-09-23T22:49:24.657077+0000","last_clean":"2023-09-23T22:49:24.657077+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:49:24.657077+0000","last_undegraded":"2023-09-23T22:49:24.657077+0000","last_fullsized":"2023-09-23T22:49:24.657077+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658530+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:49:24.658530+0000","last_peered":"2023-09-23T22:49:24.658530+0000","last_clean":"2023-09-23T22:49:24.658530+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:49:24.658530+0000","last_undegraded":"2023-09-23T22:49:24.658530+0000","last_fullsized":"2023-09-23T22:49:24.658530+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":112,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658589+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:49:24.658589+0000","last_peered":"2023-09-23T22:49:24.658589+0000","last_clean":"2023-09-23T22:49:24.658589+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:49:24.658589+0000","last_undegraded":"2023-09-23T22:49:24.658589+0000","last_fullsized":"2023-09-23T22:49:24.658589+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":273,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647589+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:49:24.647589+0000","last_peered":"2023-09-23T22:49:24.647589+0000","last_clean":"2023-09-23T22:49:24.647589+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:49:24.647589+0000","last_undegraded":"2023-09-23T22:49:24.647589+0000","last_fullsized":"2023-09-23T22:49:24.647589+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":209,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.668715+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:49:24.668715+0000","last_peered":"2023-09-23T22:49:24.668715+0000","last_clean":"2023-09-23T22:49:24.668715+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:49:24.668715+0000","last_undegraded":"2023-09-23T22:49:24.668715+0000","last_fullsized":"2023-09-23T22:49:24.668715+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":234,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648193+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:49:24.648193+0000","last_peered":"2023-09-23T22:49:24.648193+0000","last_clean":"2023-09-23T22:49:24.648193+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:49:24.648193+0000","last_undegraded":"2023-09-23T22:49:24.648193+0000","last_fullsized":"2023-09-23T22:49:24.648193+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":104,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656383+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:49:24.656383+0000","last_peered":"2023-09-23T22:49:24.656383+0000","last_clean":"2023-09-23T22:49:24.656383+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:49:24.656383+0000","last_undegraded":"2023-09-23T22:49:24.656383+0000","last_fullsized":"2023-09-23T22:49:24.656383+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":210,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669629+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:49:24.669629+0000","last_peered":"2023-09-23T22:49:24.669629+0000","last_clean":"2023-09-23T22:49:24.669629+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:49:24.669629+0000","last_undegraded":"2023-09-23T22:49:24.669629+0000","last_fullsized":"2023-09-23T22:49:24.669629+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":298,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655086+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:49:24.655086+0000","last_peered":"2023-09-23T22:49:24.655086+0000","last_clean":"2023-09-23T22:49:24.655086+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:49:24.655086+0000","last_undegraded":"2023-09-23T22:49:24.655086+0000","last_fullsized":"2023-09-23T22:49:24.655086+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.649221+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:49:24.649221+0000","last_peered":"2023-09-23T22:49:24.649221+0000","last_clean":"2023-09-23T22:49:24.649221+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:49:24.649221+0000","last_undegraded":"2023-09-23T22:49:24.649221+0000","last_fullsized":"2023-09-23T22:49:24.649221+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"254'10085","reported_seq":10130,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.455977+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:49:35.455977+0000","last_peered":"2023-09-23T22:49:35.455977+0000","last_clean":"2023-09-23T22:49:35.455977+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:49:35.455977+0000","last_undegraded":"2023-09-23T22:49:35.455977+0000","last_fullsized":"2023-09-23T22:49:35.455977+0000","mapping_epoch":217,"log_start":"227'2700","ondisk_log_start":"227'2700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7385,"log_dups_size":0,"ondisk_log_size":7385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10085,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":123,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656700+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:49:24.656700+0000","last_peered":"2023-09-23T22:49:24.656700+0000","last_clean":"2023-09-23T22:49:24.656700+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:49:24.656700+0000","last_undegraded":"2023-09-23T22:49:24.656700+0000","last_fullsized":"2023-09-23T22:49:24.656700+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655170+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:49:24.655170+0000","last_peered":"2023-09-23T22:49:24.655170+0000","last_clean":"2023-09-23T22:49:24.655170+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:49:24.655170+0000","last_undegraded":"2023-09-23T22:49:24.655170+0000","last_fullsized":"2023-09-23T22:49:24.655170+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"254'10039","reported_seq":10090,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.747616+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:49:35.747616+0000","last_peered":"2023-09-23T22:49:35.747616+0000","last_clean":"2023-09-23T22:49:35.747616+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:49:35.747616+0000","last_undegraded":"2023-09-23T22:49:35.747616+0000","last_fullsized":"2023-09-23T22:49:35.747616+0000","mapping_epoch":219,"log_start":"223'1400","ondisk_log_start":"223'1400","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8639,"log_dups_size":0,"ondisk_log_size":8639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164823040,"num_objects":2515,"num_object_clones":0,"num_object_copies":7545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10039,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":133,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656614+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:49:24.656614+0000","last_peered":"2023-09-23T22:49:24.656614+0000","last_clean":"2023-09-23T22:49:24.656614+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:49:24.656614+0000","last_undegraded":"2023-09-23T22:49:24.656614+0000","last_fullsized":"2023-09-23T22:49:24.656614+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":234,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656371+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:49:24.656371+0000","last_peered":"2023-09-23T22:49:24.656371+0000","last_clean":"2023-09-23T22:49:24.656371+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:49:24.656371+0000","last_undegraded":"2023-09-23T22:49:24.656371+0000","last_fullsized":"2023-09-23T22:49:24.656371+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"254'10067","reported_seq":10113,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:37.606110+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:49:37.606110+0000","last_peered":"2023-09-23T22:49:37.606110+0000","last_clean":"2023-09-23T22:49:37.606110+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:49:37.606110+0000","last_undegraded":"2023-09-23T22:49:37.606110+0000","last_fullsized":"2023-09-23T22:49:37.606110+0000","mapping_epoch":219,"log_start":"224'2100","ondisk_log_start":"224'2100","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7967,"log_dups_size":0,"ondisk_log_size":7967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165740544,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10067,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":64,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658207+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:49:24.658207+0000","last_peered":"2023-09-23T22:49:24.658207+0000","last_clean":"2023-09-23T22:49:24.658207+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:49:24.658207+0000","last_undegraded":"2023-09-23T22:49:24.658207+0000","last_fullsized":"2023-09-23T22:49:24.658207+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":246,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655616+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:49:24.655616+0000","last_peered":"2023-09-23T22:49:24.655616+0000","last_clean":"2023-09-23T22:49:24.655616+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:49:24.655616+0000","last_undegraded":"2023-09-23T22:49:24.655616+0000","last_fullsized":"2023-09-23T22:49:24.655616+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"254'10413","reported_seq":10466,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.876590+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:49:35.876590+0000","last_peered":"2023-09-23T22:49:35.876590+0000","last_clean":"2023-09-23T22:49:35.876590+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:49:35.876590+0000","last_undegraded":"2023-09-23T22:49:35.876590+0000","last_fullsized":"2023-09-23T22:49:35.876590+0000","mapping_epoch":226,"log_start":"229'3200","ondisk_log_start":"229'3200","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7213,"log_dups_size":0,"ondisk_log_size":7213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172294144,"num_objects":2629,"num_object_clones":0,"num_object_copies":7887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10413,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647819+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:49:24.647819+0000","last_peered":"2023-09-23T22:49:24.647819+0000","last_clean":"2023-09-23T22:49:24.647819+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:49:24.647819+0000","last_undegraded":"2023-09-23T22:49:24.647819+0000","last_fullsized":"2023-09-23T22:49:24.647819+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":146,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.668841+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:49:24.668841+0000","last_peered":"2023-09-23T22:49:24.668841+0000","last_clean":"2023-09-23T22:49:24.668841+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:49:24.668841+0000","last_undegraded":"2023-09-23T22:49:24.668841+0000","last_fullsized":"2023-09-23T22:49:24.668841+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"254'10067","reported_seq":10114,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.857127+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:49:35.857127+0000","last_peered":"2023-09-23T22:49:35.857127+0000","last_clean":"2023-09-23T22:49:35.857127+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:49:35.857127+0000","last_undegraded":"2023-09-23T22:49:35.857127+0000","last_fullsized":"2023-09-23T22:49:35.857127+0000","mapping_epoch":217,"log_start":"227'2900","ondisk_log_start":"227'2900","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7167,"log_dups_size":0,"ondisk_log_size":7167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167837696,"num_objects":2561,"num_object_clones":0,"num_object_copies":7683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10067,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":255,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.653666+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:49:24.653666+0000","last_peered":"2023-09-23T22:49:24.653666+0000","last_clean":"2023-09-23T22:49:24.653666+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:49:24.653666+0000","last_undegraded":"2023-09-23T22:49:24.653666+0000","last_fullsized":"2023-09-23T22:49:24.653666+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656995+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:49:24.656995+0000","last_peered":"2023-09-23T22:49:24.656995+0000","last_clean":"2023-09-23T22:49:24.656995+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:49:24.656995+0000","last_undegraded":"2023-09-23T22:49:24.656995+0000","last_fullsized":"2023-09-23T22:49:24.656995+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"254'10247","reported_seq":10315,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:37.708004+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:49:37.708004+0000","last_peered":"2023-09-23T22:49:37.708004+0000","last_clean":"2023-09-23T22:49:37.708004+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:49:37.708004+0000","last_undegraded":"2023-09-23T22:49:37.708004+0000","last_fullsized":"2023-09-23T22:49:37.708004+0000","mapping_epoch":226,"log_start":"224'2300","ondisk_log_start":"224'2300","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7947,"log_dups_size":0,"ondisk_log_size":7947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165347328,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10247,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":238,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655757+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:49:24.655757+0000","last_peered":"2023-09-23T22:49:24.655757+0000","last_clean":"2023-09-23T22:49:24.655757+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:49:24.655757+0000","last_undegraded":"2023-09-23T22:49:24.655757+0000","last_fullsized":"2023-09-23T22:49:24.655757+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":249,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656508+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:49:24.656508+0000","last_peered":"2023-09-23T22:49:24.656508+0000","last_clean":"2023-09-23T22:49:24.656508+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:49:24.656508+0000","last_undegraded":"2023-09-23T22:49:24.656508+0000","last_fullsized":"2023-09-23T22:49:24.656508+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"254'10337","reported_seq":10388,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.546480+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:49:35.546480+0000","last_peered":"2023-09-23T22:49:35.546480+0000","last_clean":"2023-09-23T22:49:35.546480+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:49:35.546480+0000","last_undegraded":"2023-09-23T22:49:35.546480+0000","last_fullsized":"2023-09-23T22:49:35.546480+0000","mapping_epoch":219,"log_start":"228'3000","ondisk_log_start":"228'3000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7337,"log_dups_size":0,"ondisk_log_size":7337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174391296,"num_objects":2661,"num_object_clones":0,"num_object_copies":7983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10337,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":262,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656254+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:49:24.656254+0000","last_peered":"2023-09-23T22:49:24.656254+0000","last_clean":"2023-09-23T22:49:24.656254+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:49:24.656254+0000","last_undegraded":"2023-09-23T22:49:24.656254+0000","last_fullsized":"2023-09-23T22:49:24.656254+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656693+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:49:24.656693+0000","last_peered":"2023-09-23T22:49:24.656693+0000","last_clean":"2023-09-23T22:49:24.656693+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:49:24.656693+0000","last_undegraded":"2023-09-23T22:49:24.656693+0000","last_fullsized":"2023-09-23T22:49:24.656693+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"254'9905","reported_seq":9967,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.727389+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:49:35.727389+0000","last_peered":"2023-09-23T22:49:35.727389+0000","last_clean":"2023-09-23T22:49:35.727389+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:49:35.727389+0000","last_undegraded":"2023-09-23T22:49:35.727389+0000","last_fullsized":"2023-09-23T22:49:35.727389+0000","mapping_epoch":226,"log_start":"223'1300","ondisk_log_start":"223'1300","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9905,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":255,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.651882+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:49:24.651882+0000","last_peered":"2023-09-23T22:49:24.651882+0000","last_clean":"2023-09-23T22:49:24.651882+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:49:24.651882+0000","last_undegraded":"2023-09-23T22:49:24.651882+0000","last_fullsized":"2023-09-23T22:49:24.651882+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":180,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656383+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:49:24.656383+0000","last_peered":"2023-09-23T22:49:24.656383+0000","last_clean":"2023-09-23T22:49:24.656383+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:49:24.656383+0000","last_undegraded":"2023-09-23T22:49:24.656383+0000","last_fullsized":"2023-09-23T22:49:24.656383+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"254'10222","reported_seq":10267,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:37.711039+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:49:37.711039+0000","last_peered":"2023-09-23T22:49:37.711039+0000","last_clean":"2023-09-23T22:49:37.711039+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:49:37.711039+0000","last_undegraded":"2023-09-23T22:49:37.711039+0000","last_fullsized":"2023-09-23T22:49:37.711039+0000","mapping_epoch":219,"log_start":"224'2300","ondisk_log_start":"224'2300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7922,"log_dups_size":0,"ondisk_log_size":7922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170917888,"num_objects":2608,"num_object_clones":0,"num_object_copies":7824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10222,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":290,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669200+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:49:24.669200+0000","last_peered":"2023-09-23T22:49:24.669200+0000","last_clean":"2023-09-23T22:49:24.669200+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:49:24.669200+0000","last_undegraded":"2023-09-23T22:49:24.669200+0000","last_fullsized":"2023-09-23T22:49:24.669200+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647956+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:49:24.647956+0000","last_peered":"2023-09-23T22:49:24.647956+0000","last_clean":"2023-09-23T22:49:24.647956+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:49:24.647956+0000","last_undegraded":"2023-09-23T22:49:24.647956+0000","last_fullsized":"2023-09-23T22:49:24.647956+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656561+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:49:24.656561+0000","last_peered":"2023-09-23T22:49:24.656561+0000","last_clean":"2023-09-23T22:49:24.656561+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:49:24.656561+0000","last_undegraded":"2023-09-23T22:49:24.656561+0000","last_fullsized":"2023-09-23T22:49:24.656561+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"254'10140","reported_seq":10192,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.906379+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:49:35.906379+0000","last_peered":"2023-09-23T22:49:35.906379+0000","last_clean":"2023-09-23T22:49:35.906379+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:49:35.906379+0000","last_undegraded":"2023-09-23T22:49:35.906379+0000","last_fullsized":"2023-09-23T22:49:35.906379+0000","mapping_epoch":219,"log_start":"227'2900","ondisk_log_start":"227'2900","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7240,"log_dups_size":0,"ondisk_log_size":7240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169738240,"num_objects":2590,"num_object_clones":0,"num_object_copies":7770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10140,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655686+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:49:24.655686+0000","last_peered":"2023-09-23T22:49:24.655686+0000","last_clean":"2023-09-23T22:49:24.655686+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:49:24.655686+0000","last_undegraded":"2023-09-23T22:49:24.655686+0000","last_fullsized":"2023-09-23T22:49:24.655686+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":250,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658622+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:49:24.658622+0000","last_peered":"2023-09-23T22:49:24.658622+0000","last_clean":"2023-09-23T22:49:24.658622+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:49:24.658622+0000","last_undegraded":"2023-09-23T22:49:24.658622+0000","last_fullsized":"2023-09-23T22:49:24.658622+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":169,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656205+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:49:24.656205+0000","last_peered":"2023-09-23T22:49:24.656205+0000","last_clean":"2023-09-23T22:49:24.656205+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:49:24.656205+0000","last_undegraded":"2023-09-23T22:49:24.656205+0000","last_fullsized":"2023-09-23T22:49:24.656205+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"254'10278","reported_seq":10325,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.809966+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:49:35.809966+0000","last_peered":"2023-09-23T22:49:35.809966+0000","last_clean":"2023-09-23T22:49:35.809966+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:49:35.809966+0000","last_undegraded":"2023-09-23T22:49:35.809966+0000","last_fullsized":"2023-09-23T22:49:35.809966+0000","mapping_epoch":217,"log_start":"229'3100","ondisk_log_start":"229'3100","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7178,"log_dups_size":0,"ondisk_log_size":7178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173670400,"num_objects":2650,"num_object_clones":0,"num_object_copies":7950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10278,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":245,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:29.629190+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:49:29.629190+0000","last_peered":"2023-09-23T22:49:29.629190+0000","last_clean":"2023-09-23T22:49:29.629190+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:49:29.629190+0000","last_undegraded":"2023-09-23T22:49:29.629190+0000","last_fullsized":"2023-09-23T22:49:29.629190+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655554+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:49:24.655554+0000","last_peered":"2023-09-23T22:49:24.655554+0000","last_clean":"2023-09-23T22:49:24.655554+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:49:24.655554+0000","last_undegraded":"2023-09-23T22:49:24.655554+0000","last_fullsized":"2023-09-23T22:49:24.655554+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":259,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669370+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:49:24.669370+0000","last_peered":"2023-09-23T22:49:24.669370+0000","last_clean":"2023-09-23T22:49:24.669370+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:49:24.669370+0000","last_undegraded":"2023-09-23T22:49:24.669370+0000","last_fullsized":"2023-09-23T22:49:24.669370+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"254'10036","reported_seq":10087,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:35.833012+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:49:35.833012+0000","last_peered":"2023-09-23T22:49:35.833012+0000","last_clean":"2023-09-23T22:49:35.833012+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:49:35.833012+0000","last_undegraded":"2023-09-23T22:49:35.833012+0000","last_fullsized":"2023-09-23T22:49:35.833012+0000","mapping_epoch":219,"log_start":"223'1400","ondisk_log_start":"223'1400","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8636,"log_dups_size":0,"ondisk_log_size":8636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168951808,"num_objects":2578,"num_object_clones":0,"num_object_copies":7734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10036,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":254,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655927+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:49:24.655927+0000","last_peered":"2023-09-23T22:49:24.655927+0000","last_clean":"2023-09-23T22:49:24.655927+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:49:24.655927+0000","last_undegraded":"2023-09-23T22:49:24.655927+0000","last_fullsized":"2023-09-23T22:49:24.655927+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":340,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.671467+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:49:24.671467+0000","last_peered":"2023-09-23T22:49:24.671467+0000","last_clean":"2023-09-23T22:49:24.671467+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:49:24.671467+0000","last_undegraded":"2023-09-23T22:49:24.671467+0000","last_fullsized":"2023-09-23T22:49:24.671467+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":185,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647711+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:49:24.647711+0000","last_peered":"2023-09-23T22:49:24.647711+0000","last_clean":"2023-09-23T22:49:24.647711+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:49:24.647711+0000","last_undegraded":"2023-09-23T22:49:24.647711+0000","last_fullsized":"2023-09-23T22:49:24.647711+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":246,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658369+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:49:24.658369+0000","last_peered":"2023-09-23T22:49:24.658369+0000","last_clean":"2023-09-23T22:49:24.658369+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:49:24.658369+0000","last_undegraded":"2023-09-23T22:49:24.658369+0000","last_fullsized":"2023-09-23T22:49:24.658369+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"254'10218","reported_seq":10273,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.580740+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:49:34.580740+0000","last_peered":"2023-09-23T22:49:34.580740+0000","last_clean":"2023-09-23T22:49:34.580740+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:49:34.580740+0000","last_undegraded":"2023-09-23T22:49:34.580740+0000","last_fullsized":"2023-09-23T22:49:34.580740+0000","mapping_epoch":226,"log_start":"223'500","ondisk_log_start":"223'500","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9718,"log_dups_size":0,"ondisk_log_size":9718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178913280,"num_objects":2730,"num_object_clones":0,"num_object_copies":8190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10218,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.671545+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:49:24.671545+0000","last_peered":"2023-09-23T22:49:24.671545+0000","last_clean":"2023-09-23T22:49:24.671545+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:49:24.671545+0000","last_undegraded":"2023-09-23T22:49:24.671545+0000","last_fullsized":"2023-09-23T22:49:24.671545+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":190,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.672987+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:49:24.672987+0000","last_peered":"2023-09-23T22:49:24.672987+0000","last_clean":"2023-09-23T22:49:24.672987+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:49:24.672987+0000","last_undegraded":"2023-09-23T22:49:24.672987+0000","last_fullsized":"2023-09-23T22:49:24.672987+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":177,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669413+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:49:24.669413+0000","last_peered":"2023-09-23T22:49:24.669413+0000","last_clean":"2023-09-23T22:49:24.669413+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:49:24.669413+0000","last_undegraded":"2023-09-23T22:49:24.669413+0000","last_fullsized":"2023-09-23T22:49:24.669413+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"254'10471","reported_seq":14265,"reported_epoch":254,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:49:35.915361+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:49:35.915361+0000","last_peered":"2023-09-23T22:49:35.915361+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:49:35.915361+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"229'3300","ondisk_log_start":"229'3300","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7171,"log_dups_size":0,"ondisk_log_size":7171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183304192,"num_objects":2797,"num_object_clones":0,"num_object_copies":8391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1969,"num_objects_misplaced":2780,"num_objects_unfound":0,"num_objects_dirty":2797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10471,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1869,"num_bytes_recovered":122486784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-1952},{"shards":"1,3,4","objects":2780}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":182,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669287+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:49:24.669287+0000","last_peered":"2023-09-23T22:49:24.669287+0000","last_clean":"2023-09-23T22:49:24.669287+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:49:24.669287+0000","last_undegraded":"2023-09-23T22:49:24.669287+0000","last_fullsized":"2023-09-23T22:49:24.669287+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":132,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656364+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:49:24.656364+0000","last_peered":"2023-09-23T22:49:24.656364+0000","last_clean":"2023-09-23T22:49:24.656364+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:49:24.656364+0000","last_undegraded":"2023-09-23T22:49:24.656364+0000","last_fullsized":"2023-09-23T22:49:24.656364+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":182,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.668998+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:49:24.668998+0000","last_peered":"2023-09-23T22:49:24.668998+0000","last_clean":"2023-09-23T22:49:24.668998+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:49:24.668998+0000","last_undegraded":"2023-09-23T22:49:24.668998+0000","last_fullsized":"2023-09-23T22:49:24.668998+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"254'10312","reported_seq":10358,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:37.724193+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:49:37.724193+0000","last_peered":"2023-09-23T22:49:37.724193+0000","last_clean":"2023-09-23T22:49:37.724193+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:49:37.724193+0000","last_undegraded":"2023-09-23T22:49:37.724193+0000","last_fullsized":"2023-09-23T22:49:37.724193+0000","mapping_epoch":219,"log_start":"225'2400","ondisk_log_start":"225'2400","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7912,"log_dups_size":0,"ondisk_log_size":7912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170131456,"num_objects":2596,"num_object_clones":0,"num_object_copies":7788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10312,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":294,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.649177+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:49:24.649177+0000","last_peered":"2023-09-23T22:49:24.649177+0000","last_clean":"2023-09-23T22:49:24.649177+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:49:24.649177+0000","last_undegraded":"2023-09-23T22:49:24.649177+0000","last_fullsized":"2023-09-23T22:49:24.649177+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":52,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658467+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:49:24.658467+0000","last_peered":"2023-09-23T22:49:24.658467+0000","last_clean":"2023-09-23T22:49:24.658467+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:49:24.658467+0000","last_undegraded":"2023-09-23T22:49:24.658467+0000","last_fullsized":"2023-09-23T22:49:24.658467+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":279,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.651968+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:49:24.651968+0000","last_peered":"2023-09-23T22:49:24.651968+0000","last_clean":"2023-09-23T22:49:24.651968+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:49:24.651968+0000","last_undegraded":"2023-09-23T22:49:24.651968+0000","last_fullsized":"2023-09-23T22:49:24.651968+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"254'10104","reported_seq":10152,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.538231+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:49:34.538231+0000","last_peered":"2023-09-23T22:49:34.538231+0000","last_clean":"2023-09-23T22:49:34.538231+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:49:34.538231+0000","last_undegraded":"2023-09-23T22:49:34.538231+0000","last_fullsized":"2023-09-23T22:49:34.538231+0000","mapping_epoch":217,"log_start":"223'400","ondisk_log_start":"223'400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9704,"log_dups_size":0,"ondisk_log_size":9704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255640,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10104,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":286,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658869+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:49:24.658869+0000","last_peered":"2023-09-23T22:49:24.658869+0000","last_clean":"2023-09-23T22:49:24.658869+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:49:24.658869+0000","last_undegraded":"2023-09-23T22:49:24.658869+0000","last_fullsized":"2023-09-23T22:49:24.658869+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":149,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669492+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:49:24.669492+0000","last_peered":"2023-09-23T22:49:24.669492+0000","last_clean":"2023-09-23T22:49:24.669492+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:49:24.669492+0000","last_undegraded":"2023-09-23T22:49:24.669492+0000","last_fullsized":"2023-09-23T22:49:24.669492+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":204,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656293+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:49:24.656293+0000","last_peered":"2023-09-23T22:49:24.656293+0000","last_clean":"2023-09-23T22:49:24.656293+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:49:24.656293+0000","last_undegraded":"2023-09-23T22:49:24.656293+0000","last_fullsized":"2023-09-23T22:49:24.656293+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":185,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.668929+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:49:24.668929+0000","last_peered":"2023-09-23T22:49:24.668929+0000","last_clean":"2023-09-23T22:49:24.668929+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:49:24.668929+0000","last_undegraded":"2023-09-23T22:49:24.668929+0000","last_fullsized":"2023-09-23T22:49:24.668929+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":26,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656282+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:49:24.656282+0000","last_peered":"2023-09-23T22:49:24.656282+0000","last_clean":"2023-09-23T22:49:24.656282+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:49:24.656282+0000","last_undegraded":"2023-09-23T22:49:24.656282+0000","last_fullsized":"2023-09-23T22:49:24.656282+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":64,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656924+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:49:24.656924+0000","last_peered":"2023-09-23T22:49:24.656924+0000","last_clean":"2023-09-23T22:49:24.656924+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:49:24.656924+0000","last_undegraded":"2023-09-23T22:49:24.656924+0000","last_fullsized":"2023-09-23T22:49:24.656924+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":253,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658671+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:49:24.658671+0000","last_peered":"2023-09-23T22:49:24.658671+0000","last_clean":"2023-09-23T22:49:24.658671+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:49:24.658671+0000","last_undegraded":"2023-09-23T22:49:24.658671+0000","last_fullsized":"2023-09-23T22:49:24.658671+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":64,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656916+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:49:24.656916+0000","last_peered":"2023-09-23T22:49:24.656916+0000","last_clean":"2023-09-23T22:49:24.656916+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:49:24.656916+0000","last_undegraded":"2023-09-23T22:49:24.656916+0000","last_fullsized":"2023-09-23T22:49:24.656916+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656730+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:49:24.656730+0000","last_peered":"2023-09-23T22:49:24.656730+0000","last_clean":"2023-09-23T22:49:24.656730+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:49:24.656730+0000","last_undegraded":"2023-09-23T22:49:24.656730+0000","last_fullsized":"2023-09-23T22:49:24.656730+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":247,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648374+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:49:24.648374+0000","last_peered":"2023-09-23T22:49:24.648374+0000","last_clean":"2023-09-23T22:49:24.648374+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:49:24.648374+0000","last_undegraded":"2023-09-23T22:49:24.648374+0000","last_fullsized":"2023-09-23T22:49:24.648374+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":285,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647282+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:49:24.647282+0000","last_peered":"2023-09-23T22:49:24.647282+0000","last_clean":"2023-09-23T22:49:24.647282+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:49:24.647282+0000","last_undegraded":"2023-09-23T22:49:24.647282+0000","last_fullsized":"2023-09-23T22:49:24.647282+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":198,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.669522+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:49:24.669522+0000","last_peered":"2023-09-23T22:49:24.669522+0000","last_clean":"2023-09-23T22:49:24.669522+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:49:24.669522+0000","last_undegraded":"2023-09-23T22:49:24.669522+0000","last_fullsized":"2023-09-23T22:49:24.669522+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656161+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:49:24.656161+0000","last_peered":"2023-09-23T22:49:24.656161+0000","last_clean":"2023-09-23T22:49:24.656161+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:49:24.656161+0000","last_undegraded":"2023-09-23T22:49:24.656161+0000","last_fullsized":"2023-09-23T22:49:24.656161+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2749628440,"num_objects":41957,"num_object_clones":0,"num_object_copies":125871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1969,"num_objects_misplaced":2780,"num_objects_unfound":0,"num_objects_dirty":41957,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162941,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1872,"num_bytes_recovered":122683392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8499048448,"data_stored":8499036232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127741,"ondisk_log_size":127741,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430479,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626707,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1322116,"kb_used_data":1016700,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92423036,"statfs":{"total":95995035648,"available":94641188864,"internally_reserved":0,"allocated":1041100800,"data_stored":1040598233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:17 2023","interfaces":[{"interface":"back","average":{"1min":8.891,"5min":3.4860000000000002,"15min":2.0209999999999999},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":117.45999999999999,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":8.6750000000000007,"5min":3.4220000000000002,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":117.48699999999999,"5min":117.48699999999999,"15min":117.48699999999999},"last":1.0549999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.044,"15min":2.2370000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.741,"5min":14.391999999999999,"15min":153.78399999999999},"last":2.048},{"interface":"front","average":{"1min":10.359999999999999,"5min":3.7250000000000001,"15min":2.819},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":185.411,"5min":185.411,"15min":185.411},"last":1.742}]},{"osd":2,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":2.8719999999999999,"15min":3.1640000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":33.174999999999997,"5min":33.174999999999997,"15min":100.785},"last":0.94499999999999995},{"interface":"front","average":{"1min":3.9079999999999999,"5min":4.5099999999999998,"15min":3.6059999999999999},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":33.310000000000002,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.93600000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":19.305,"5min":6.641,"15min":3.2610000000000001},"min":{"1min":0.378,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":185.19399999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":5.4889999999999999},{"interface":"front","average":{"1min":11.478999999999999,"5min":5.4870000000000001,"15min":2.9390000000000001},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":128.19800000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":3.0019999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.79300000000000004,"15min":0.66400000000000003},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.24199999999999999},"max":{"1min":6.5309999999999997,"5min":6.5309999999999997,"15min":18.085999999999999},"last":1.103},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.89900000000000002,"15min":0.69499999999999995},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":6.9089999999999998,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.38}]},{"osd":5,"last update":"Sat Sep 23 22:48:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":0.93200000000000005,"15min":0.72499999999999998},"min":{"1min":0.313,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":10.148999999999999,"5min":10.148999999999999,"15min":18.972000000000001},"last":1.133},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.92300000000000004,"15min":0.73999999999999999},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.2930000000000001,"5min":9.7899999999999991,"15min":18.625},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:48:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1,"15min":0.746},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.4049999999999998,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.83699999999999997,"15min":0.72799999999999998},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":3.7130000000000001,"5min":5.1790000000000003,"15min":16.661999999999999},"last":2.0209999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253017,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1458448,"kb_used_data":1164232,"kb_used_omap":9,"kb_used_meta":294134,"kb_avail":92286704,"statfs":{"total":95995035648,"available":94501584896,"internally_reserved":0,"allocated":1192173568,"data_stored":1191255203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":301194217},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:18 2023","interfaces":[{"interface":"back","average":{"1min":15.766999999999999,"5min":7.5789999999999997,"15min":5.3049999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":82.903999999999996,"5min":101.605,"15min":101.605},"last":5.3929999999999998},{"interface":"front","average":{"1min":13.898,"5min":7.5730000000000004,"15min":5.5339999999999998},"min":{"1min":0.441,"5min":0.214,"15min":0.214},"max":{"1min":82.748999999999995,"5min":101.892,"15min":101.892},"last":0.96599999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:49:18 2023","interfaces":[{"interface":"back","average":{"1min":5.2960000000000003,"5min":3.2799999999999998,"15min":2.3029999999999999},"min":{"1min":0.20000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":37.869999999999997,"5min":37.869999999999997,"15min":37.889000000000003},"last":12.255000000000001},{"interface":"front","average":{"1min":6.4699999999999998,"5min":3.931,"15min":2.4119999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":37.957000000000001,"5min":37.957000000000001,"15min":37.957000000000001},"last":5.1289999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":9.3239999999999998,"5min":3.8450000000000002,"15min":2.149},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":87.787999999999997,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.752},{"interface":"front","average":{"1min":10.641999999999999,"5min":4.7679999999999998,"15min":2.5299999999999998},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":89.861999999999995,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7949999999999999,"5min":3.1579999999999999,"15min":1.9490000000000001},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":11.432,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.3889999999999998,"15min":1.7649999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":10.619,"5min":78.963999999999999,"15min":78.963999999999999},"last":5.032}]},{"osd":5,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.266,"15min":1.53},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.4390000000000001,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.80500000000000005},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.2949999999999999,"15min":1.8160000000000001},"min":{"1min":0.35299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.458,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.4609999999999999,"15min":1.5960000000000001},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.317},"max":{"1min":10.65,"5min":79.311000000000007,"15min":79.311000000000007},"last":5.0759999999999996},{"interface":"front","average":{"1min":1.5,"5min":2.2829999999999999,"15min":1.4630000000000001},"min":{"1min":0.36299999999999999,"5min":0.313,"15min":0.313},"max":{"1min":7.3819999999999997,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.82999999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":3.3530000000000002,"5min":2.6560000000000001,"15min":2.1869999999999998},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":11.481999999999999,"5min":29.539000000000001,"15min":159.018},"last":0.54200000000000004},{"interface":"front","average":{"1min":3.335,"5min":2.048,"15min":2.0960000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":10.676,"5min":10.676,"15min":158.96000000000001},"last":5.1029999999999998}]}]},{"osd":0,"up_from":219,"seq":940597837866,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455816,"kb_used_data":1307776,"kb_used_omap":31,"kb_used_meta":148000,"kb_avail":92289336,"statfs":{"total":95995035648,"available":94504280064,"internally_reserved":0,"allocated":1339162624,"data_stored":1338192891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":151552946},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.319,"5min":4.0780000000000003,"15min":3.8340000000000001},"min":{"1min":0.24299999999999999,"5min":0.216,"15min":0.216},"max":{"1min":21.512,"5min":32.344999999999999,"15min":32.344999999999999},"last":0.23400000000000001},{"interface":"front","average":{"1min":5.3239999999999998,"5min":3.9279999999999999,"15min":3.6819999999999999},"min":{"1min":0.30499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.064,"5min":32.106000000000002,"15min":32.106000000000002},"last":2.3860000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.0640000000000001,"5min":3.2610000000000001,"15min":2.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":24.253,"5min":24.253,"15min":24.253},"last":5.25},{"interface":"front","average":{"1min":3.5,"5min":2.4079999999999999,"15min":1.974},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":15.106999999999999,"5min":15.106999999999999,"15min":15.106999999999999},"last":5.5999999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8760000000000003,"5min":2.9369999999999998,"15min":2.2890000000000001},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.710000000000001,"5min":20.710000000000001,"15min":20.710000000000001},"last":5.0890000000000004},{"interface":"front","average":{"1min":2.964,"5min":1.794,"15min":1.347},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":13.991,"5min":13.991,"15min":13.991},"last":6.0880000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":3.9729999999999999,"5min":2.6520000000000001,"15min":2.1200000000000001},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":18.670999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":3.585,"5min":2.1179999999999999,"15min":1.8720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":20.009,"5min":20.009,"15min":20.009},"last":2.4159999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":1.6160000000000001,"15min":1.429},"min":{"1min":0.36599999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.9260000000000002,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":5.7480000000000002},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.3559999999999999,"15min":1.8200000000000001},"min":{"1min":0.47199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":18.591999999999999,"5min":18.591999999999999,"15min":18.591999999999999},"last":1.8080000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.7,"15min":1.4810000000000001},"min":{"1min":0.38700000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.9790000000000001,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":5.6420000000000003},{"interface":"front","average":{"1min":4.048,"5min":2.6110000000000002,"15min":1.994},"min":{"1min":0.51800000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":18.704999999999998,"5min":18.704999999999998,"15min":18.704999999999998},"last":0.77300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":2.4660000000000002,"15min":1.8859999999999999},"min":{"1min":0.54900000000000004,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":18.640000000000001,"5min":18.640000000000001,"15min":18.640000000000001},"last":1.931},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.54,"15min":1.298},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":10.35,"5min":10.35,"15min":10.35},"last":5.782}]}]},{"osd":2,"up_from":131,"seq":562640715878,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289828,"kb_used_data":1049692,"kb_used_omap":81,"kb_used_meta":240046,"kb_avail":92455324,"statfs":{"total":95995035648,"available":94674251776,"internally_reserved":0,"allocated":1074884608,"data_stored":1073942716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":245807322},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7810000000000001,"5min":4.9379999999999997,"15min":5.1580000000000004},"min":{"1min":0.27300000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.692,"5min":57.811999999999998,"15min":57.811999999999998},"last":3.129},{"interface":"front","average":{"1min":5.9059999999999997,"5min":5.968,"15min":5.5819999999999999},"min":{"1min":0.80600000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":13.865,"5min":57.387999999999998,"15min":57.387999999999998},"last":3.194}]},{"osd":1,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":3.5179999999999998,"15min":3.226},"min":{"1min":0.28299999999999997,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.759,"5min":88.316000000000003,"15min":88.316000000000003},"last":2.0179999999999998},{"interface":"front","average":{"1min":4.444,"5min":4.4329999999999998,"15min":2.8719999999999999},"min":{"1min":0.34899999999999998,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":28.596,"5min":88.436999999999998,"15min":88.436999999999998},"last":0.246}]},{"osd":3,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.976,"15min":3.056},"min":{"1min":0.30499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.994,"5min":88.653000000000006,"15min":88.653000000000006},"last":2.141},{"interface":"front","average":{"1min":3.9369999999999998,"5min":3.5880000000000001,"15min":2.4220000000000002},"min":{"1min":0.31900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":12.401,"5min":88.474000000000004,"15min":88.474000000000004},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.188000000000001,"5min":4.4569999999999999,"15min":2.1389999999999998},"min":{"1min":0.51300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":84.778999999999996,"5min":108.878,"15min":108.878},"last":0.56000000000000005},{"interface":"front","average":{"1min":11.103,"5min":5.8799999999999999,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":84.753,"5min":108.93000000000001,"15min":108.93000000000001},"last":2.1110000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":7.6660000000000004,"5min":5.3339999999999996,"15min":2.7480000000000002},"min":{"1min":0.42499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":90.082999999999998,"5min":97.322999999999993,"15min":97.322999999999993},"last":2.0880000000000001},{"interface":"front","average":{"1min":7.9420000000000002,"5min":5.2549999999999999,"15min":2.3340000000000001},"min":{"1min":0.437,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":89.986000000000004,"5min":97.301000000000002,"15min":97.301000000000002},"last":0.39200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":10.731,"5min":4.9109999999999996,"15min":2.2530000000000001},"min":{"1min":0.40699999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":85.034999999999997,"5min":88.606999999999999,"15min":88.606999999999999},"last":0.433},{"interface":"front","average":{"1min":9.9339999999999993,"5min":3.3479999999999999,"15min":1.7729999999999999},"min":{"1min":0.47399999999999998,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":85.209000000000003,"5min":85.209000000000003,"15min":85.209000000000003},"last":0.5}]},{"osd":7,"last update":"Sat Sep 23 22:48:30 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.726,"15min":1.266},"min":{"1min":0.40699999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9640000000000004,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.9329999999999998,"5min":3.4910000000000001,"15min":2.1720000000000002},"min":{"1min":0.45100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.5549999999999997,"5min":88.584000000000003,"15min":88.584000000000003},"last":2.0619999999999998}]}]},{"osd":3,"up_from":94,"seq":403726925964,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1414540,"kb_used_data":1021636,"kb_used_omap":7,"kb_used_meta":392888,"kb_avail":92330612,"statfs":{"total":95995035648,"available":94546546688,"internally_reserved":0,"allocated":1046155264,"data_stored":1045238656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":402317928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":3.1699999999999999,"15min":2.2959999999999998},"min":{"1min":0.26100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.711,"5min":80.099000000000004,"15min":80.099000000000004},"last":7.2039999999999997},{"interface":"front","average":{"1min":3.698,"5min":3.0009999999999999,"15min":2.0649999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.337,"5min":80.150000000000006,"15min":80.150000000000006},"last":9.1289999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.5019999999999998,"5min":2.6970000000000001,"15min":1.6080000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20000000000000001},"max":{"1min":12.25,"5min":22.818000000000001,"15min":22.818000000000001},"last":7.149},{"interface":"front","average":{"1min":2.9700000000000002,"5min":1.8779999999999999,"15min":1.2869999999999999},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":15.098000000000001,"5min":23.917999999999999,"15min":23.917999999999999},"last":8.4339999999999993}]},{"osd":2,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2370000000000001,"5min":2.5800000000000001,"15min":2.4569999999999999},"min":{"1min":0.32900000000000001,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":12.339,"5min":20.431000000000001,"15min":125.446},"last":7.9359999999999999},{"interface":"front","average":{"1min":5.2999999999999998,"5min":4.9500000000000002,"15min":3.0310000000000001},"min":{"1min":0.40799999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.756,"5min":182.483,"15min":182.483},"last":0.26000000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2149999999999999,"15min":1.3919999999999999},"min":{"1min":0.371,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":12.289999999999999,"5min":18.757000000000001,"15min":18.757000000000001},"last":7.2320000000000002},{"interface":"front","average":{"1min":2.645,"5min":1.5149999999999999,"15min":1.196},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.131,"5min":12.131,"15min":12.131},"last":7.9829999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3520000000000001,"15min":1.127},"min":{"1min":0.39800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.949999999999999,"5min":12.127000000000001,"15min":12.127000000000001},"last":8.0370000000000008},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.085,"15min":1.3300000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":0.33900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":1.5269999999999999,"15min":1.175},"min":{"1min":0.38300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.897,"5min":12.16,"15min":12.16},"last":8.0109999999999992},{"interface":"front","average":{"1min":2.399,"5min":2.0760000000000001,"15min":1.403},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":10.755000000000001},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.9929999999999999,"15min":1.6819999999999999},"min":{"1min":0.47299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7710000000000008,"5min":91.757999999999996,"15min":91.757999999999996},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.1499999999999999,"5min":3.2869999999999999,"15min":1.8069999999999999},"min":{"1min":0.49099999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":12.32,"5min":91.709999999999994,"15min":91.709999999999994},"last":7.101}]}]},{"osd":4,"up_from":30,"seq":128849019163,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576768,"kb_used_data":1279544,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92168384,"statfs":{"total":95995035648,"available":94380425216,"internally_reserved":0,"allocated":1310253056,"data_stored":1309754059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":5.4880000000000004,"15min":7.173},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.442,"5min":110.08499999999999,"15min":110.08499999999999},"last":2.2719999999999998},{"interface":"front","average":{"1min":3.5299999999999998,"5min":6.2779999999999996,"15min":7.7779999999999996},"min":{"1min":0.35899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":13.193,"5min":110.33799999999999,"15min":110.33799999999999},"last":1.7549999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1230000000000002,"15min":1.53},"min":{"1min":0.317,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0269999999999992,"5min":41.643000000000001,"15min":45.320999999999998},"last":1.5720000000000001},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.9019999999999999,"15min":1.4279999999999999},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.4489999999999998,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:49:20 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":4.0970000000000004,"15min":3.2599999999999998},"min":{"1min":0.34300000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.9189999999999996,"5min":84.909000000000006,"15min":84.909000000000006},"last":1.851},{"interface":"front","average":{"1min":2.1600000000000001,"5min":3.9990000000000001,"15min":3.1419999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0609999999999999,"5min":83.840000000000003,"15min":83.840000000000003},"last":0.441}]},{"osd":3,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":7.3099999999999996,"15min":3.3730000000000002},"min":{"1min":0.442,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":100.51600000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":2.3700000000000001},{"interface":"front","average":{"1min":8.1150000000000002,"5min":7.3609999999999998,"15min":3.4169999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":100.637,"5min":129.197,"15min":129.197},"last":1.6080000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.77800000000000002},"min":{"1min":0.26600000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.1550000000000002,"5min":5.8979999999999997,"15min":43.189999999999998},"last":1.6819999999999999},{"interface":"front","average":{"1min":1.2769999999999999,"5min":0.83699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":6.9199999999999999,"5min":9.5190000000000001,"15min":42.450000000000003},"last":1.79}]},{"osd":6,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.91400000000000003,"15min":0.78500000000000003},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":10.952999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":1.823},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.72199999999999998,"15min":0.76700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":4.5359999999999996,"5min":5.8689999999999998,"15min":26.940999999999999},"last":1.657}]},{"osd":7,"last update":"Sat Sep 23 22:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.754,"15min":0.66900000000000004},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.464,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":1.635},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.68600000000000005,"15min":0.64600000000000002},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":2.5710000000000002,"5min":3.2519999999999998,"15min":5.9409999999999998},"last":0.59499999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822935,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1836524,"kb_used_data":1527012,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91908628,"statfs":{"total":95995035648,"available":94114435072,"internally_reserved":0,"allocated":1563660288,"data_stored":1563176909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":5.4729999999999999,"15min":3.0289999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.344000000000001,"5min":121.319,"15min":121.319},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.8860000000000001,"5min":5.4950000000000001,"15min":2.9289999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":37.470999999999997,"5min":121.075,"15min":121.075},"last":3.3420000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.7290000000000001,"15min":1.762},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":6.7779999999999996,"5min":28.658999999999999,"15min":149.203},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.165,"15min":1.9339999999999999},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.1310000000000002,"5min":28.664999999999999,"15min":149.554},"last":7.0119999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":3.4550000000000001,"15min":6.7249999999999996},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":7.9960000000000004,"5min":158.92599999999999,"15min":158.92599999999999},"last":3.5840000000000001},{"interface":"front","average":{"1min":2.0369999999999999,"5min":3.5169999999999999,"15min":6.7199999999999998},"min":{"1min":0.36199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0430000000000001,"5min":158.91399999999999,"15min":158.91399999999999},"last":2.956}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.4409999999999998,"5min":8.7390000000000008,"15min":4.2290000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":37.427999999999997,"5min":257.05000000000001,"15min":257.05000000000001},"last":5.5670000000000002},{"interface":"front","average":{"1min":5.4939999999999998,"5min":8.484,"15min":4.0819999999999999},"min":{"1min":0.45800000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":37.277999999999999,"5min":257.35000000000002,"15min":257.35000000000002},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:49:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.70099999999999996,"15min":0.60999999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":1.893,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":5.0309999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.77000000000000002,"15min":0.61899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.9590000000000001,"5min":10.535,"15min":10.535},"last":2.9870000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.89400000000000002,"15min":1.391},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":1.9410000000000001,"5min":12.615,"15min":140.07499999999999},"last":4.367},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.016,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":2.4609999999999999,"5min":12.74,"15min":139.739},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:48:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.82399999999999995,"15min":0.70599999999999996},"min":{"1min":0.33400000000000002,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.464,"5min":7.0010000000000003,"15min":7.9809999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.74299999999999999,"15min":0.67000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":1.9650000000000001,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":3.5539999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1330380800,"data_stored":1330380800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1185415168,"data_stored":1185415168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1065881600,"data_stored":1065877528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1041825792,"data_stored":1041825792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1299447808,"data_stored":1299447808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1552617472,"data_stored":1552613400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1023479808,"data_stored":1023475736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:49:40.801 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:40.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:40.909 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:40.997 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:41.192 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:41.394 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:41.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:41.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:41.502 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:41.536 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:41.603 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:41.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:41.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:41 smithi151 ceph-mon[159210]: pgmap v688: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 1969/124782 objects degraded (1.578%); 2780/124782 objects misplaced (2.228%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:41.710 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:41.736 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:41.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:41 smithi052 ceph-mon[187743]: pgmap v688: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 1969/124782 objects degraded (1.578%); 2780/124782 objects misplaced (2.228%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:41.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:41 smithi052 ceph-mon[180979]: pgmap v688: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 1969/124782 objects degraded (1.578%); 2780/124782 objects misplaced (2.228%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:41.777 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:41.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:41.884 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:41.990 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:42.240 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:42.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:42.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:42.256 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:42.352 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:42.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:42.434 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:42.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:42.539 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:42.578 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:42.636 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:42.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:42.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:42.769 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:42.831 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:42.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:42.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:43.048 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:43.321 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:43.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:43.360 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:49:43.360 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10170 6.1 6:8aa77601:::benchmark_data_smithi052_234456_object10169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 529.15722000400001, 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.264625532, 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.361 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10170 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199127+0000", 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199126+0000", 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199153+0000", 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7100000000000001e-05 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199170+0000", 2023-09-23T22:49:43.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6382e-05 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199177+0000", 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7449999999999995e-06 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199217+0000", 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9523000000000001e-05 2023-09-23T22:49:43.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199406+0000", 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018866 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199606+0000", 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000200278 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201347+0000", 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001740969 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463581+0000", 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26223432499999999 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463651+0000", 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9529999999999993e-05 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.463753+0000", 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000102389 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.368 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10180 6.1 6:8dfddb97:::benchmark_data_smithi052_234456_object10179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 529.15710622500001, 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26492964299999999, 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:49:43.369 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10180 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199241+0000", 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199272+0000", 2023-09-23T22:49:43.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1467999999999998e-05 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199288+0000", 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5582000000000001e-05 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199294+0000", 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200146+0000", 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00085146099999999997 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200281+0000", 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013534300000000001 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200378+0000", 2023-09-23T22:49:43.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6699000000000004e-05 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201357+0000", 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097968399999999998 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464013+0000", 2023-09-23T22:49:43.374 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26265580399999999 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464077+0000", 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4029999999999995e-05 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464171+0000", 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3728999999999997e-05 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.375 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:10183 6.1 6:8243606a:::benchmark_data_smithi052_234456_object10182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 529.15704690300004, 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265208887, 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:49:43.376 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 10183 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199301+0000", 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199300+0000", 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199322+0000", 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1863999999999999e-05 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199329+0000", 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4529999999999996e-06 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.199335+0000", 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6749999999999999e-06 2023-09-23T22:49:43.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200743+0000", 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001407569 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200839+0000", 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5946999999999995e-05 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 6", 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.200957+0000", 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011885200000000001 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.201525+0000", 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00056793000000000002 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464362+0000", 2023-09-23T22:49:43.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26283688599999999 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464416+0000", 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4191999999999999e-05 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:40:54.464509+0000", 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3066000000000006e-05 2023-09-23T22:49:43.382 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.34680.0:36160 6.4 6:2dcd31ff:::benchmark_data_smithi052_234456_object14016:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 516.50807705800003, 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32418800800000003, 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.383 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.34680", 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 36160 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.384 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848270+0000", 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848269+0000", 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.385 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848277+0000", 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2050000000000002e-06 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848301+0000", 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3801000000000001e-05 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848311+0000", 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0621e-05 2023-09-23T22:49:43.386 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848367+0000", 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5881999999999998e-05 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848522+0000", 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000154544 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.387 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.848651+0000", 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000129157 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850169+0000", 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015178349999999999 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:06.850319+0000", 2023-09-23T22:49:43.388 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015018400000000001 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172387+0000", 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32206747699999999 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.389 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172420+0000", 2023-09-23T22:49:43.390 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3751999999999998e-05 2023-09-23T22:49:43.390 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.390 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.390 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.390 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:41:07.172458+0000", 2023-09-23T22:49:43.390 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8139000000000002e-05 2023-09-23T22:49:43.391 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.391 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.391 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.391 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.391 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.391 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184397 7.0 7:0017d177:::benchmark_data_smithi052_277165_object91905:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.86114630399999, 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0558009070000001, 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.392 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184397 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.393 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495200+0000", 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495205+0000", 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9799999999999998e-06 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.394 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495218+0000", 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3602e-05 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495227+0000", 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1780000000000003e-06 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495336+0000", 2023-09-23T22:49:43.395 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000109238 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495345+0000", 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5910000000000002e-06 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500507+0000", 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051612130000000004 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.396 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500517+0000", 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0794e-05 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.550859+0000", 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0503421209999999 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.397 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551002+0000", 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014262000000000001 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184406 7.0 7:01b010b6:::benchmark_data_smithi052_277165_object91914:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.86110323099999, 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0749326939999999, 2023-09-23T22:49:43.398 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184406 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.399 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.400 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495247+0000", 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.853e-06 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495261+0000", 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4344e-05 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495266+0000", 2023-09-23T22:49:43.401 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3090000000000003e-06 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495397+0000", 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000130827 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:49:43.402 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495403+0000", 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.682e-06 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500537+0000", 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051338649999999996 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500541+0000", 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3510000000000004e-06 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.403 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551486+0000", 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.050945196 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551618+0000", 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013216700000000001 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.404 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570084+0000", 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01846515 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570177+0000", 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3393999999999994e-05 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.405 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184408 7.0 7:04875276:::benchmark_data_smithi052_277165_object91916:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.861020142, 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0894791740000001, 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184408 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.406 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.407 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495331+0000", 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6959999999999996e-06 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495343+0000", 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.201e-05 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.408 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495351+0000", 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0190000000000006e-06 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495428+0000", 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6434999999999996e-05 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:49:43.409 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495433+0000", 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3870000000000003e-06 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500552+0000", 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051191750000000001 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500556+0000", 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9040000000000002e-06 2023-09-23T22:49:43.410 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551667+0000", 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.051111047 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551765+0000", 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8115999999999994e-05 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.411 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584729+0000", 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.032963237999999999 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584807+0000", 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7855999999999995e-05 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.412 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184411 7.0 7:04cf78da:::benchmark_data_smithi052_277165_object91919:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.860990749, 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0680791730000001, 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.413 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184411 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.414 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495356+0000", 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495359+0000", 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4090000000000001e-06 2023-09-23T22:49:43.415 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495365+0000", 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0310000000000004e-06 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495370+0000", 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3120000000000001e-06 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.416 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.496599+0000", 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012286339999999999 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.496605+0000", 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1870000000000002e-06 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500568+0000", 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003962559 2023-09-23T22:49:43.417 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500571+0000", 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.698e-06 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551817+0000", 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0512461209999999 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.418 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551945+0000", 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012785800000000001 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.552407+0000", 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000461896 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.561382+0000", 2023-09-23T22:49:43.419 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089752110000000003 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563179+0000", 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017967440000000001 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563267+0000", 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7570000000000004e-05 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.420 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563330+0000", 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2973999999999997e-05 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563436+0000", 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106332 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.421 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184468 7.10 7:092ecb73:::benchmark_data_smithi052_277165_object91976:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.80816613799999, 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0031411560000001, 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.422 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184468 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.423 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548184+0000", 2023-09-23T22:49:43.424 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3390000000000001e-06 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548194+0000", 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6420000000000002e-06 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548199+0000", 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6160000000000001e-06 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.425 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548264+0000", 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4523999999999994e-05 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548276+0000", 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1939e-05 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.426 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534263+0000", 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98598719300000004 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534418+0000", 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015466899999999999 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534800+0000", 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00038215800000000001 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.427 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536718+0000", 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019180110000000001 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551075+0000", 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014356733 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.428 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551223+0000", 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014790699999999999 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551276+0000", 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3312000000000001e-05 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551323+0000", 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6589999999999999e-05 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.429 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184490 7.10 7:0fb33c7c:::benchmark_data_smithi052_277165_object91998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.78603994700001, 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98115489099999997, 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.430 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184490 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:49:43.431 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570305+0000", 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570316+0000", 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0579e-05 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570338+0000", 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2246999999999999e-05 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570347+0000", 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8189999999999998e-06 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570426+0000", 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9136999999999998e-05 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570441+0000", 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4269e-05 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.434 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536742+0000", 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.96630105200000005 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536853+0000", 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111598 2023-09-23T22:49:43.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.537044+0000", 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019121900000000001 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.436 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551089+0000", 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014044137 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551122+0000", 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3930000000000002e-05 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551393+0000", 2023-09-23T22:49:43.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027049300000000001 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551428+0000", 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4764999999999997e-05 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551462+0000", 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4730000000000001e-05 2023-09-23T22:49:43.438 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184514 7.10 7:08068522:::benchmark_data_smithi052_277165_object92022:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.76428315300001, 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99227452999999999, 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.439 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184514 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592062+0000", 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592072+0000", 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7999999999999993e-06 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592091+0000", 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8834e-05 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592102+0000", 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.079e-05 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592169+0000", 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7201999999999998e-05 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592178+0000", 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.312e-06 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570201+0000", 2023-09-23T22:49:43.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.97802318300000002 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570317+0000", 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115149 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570714+0000", 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039780000000000002 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579033+0000", 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0083185269999999992 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579064+0000", 2023-09-23T22:49:43.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1152999999999998e-05 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584277+0000", 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052129189999999999 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584310+0000", 2023-09-23T22:49:43.446 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3500000000000001e-05 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584339+0000", 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8464999999999999e-05 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184527 7.10 7:0f7ed45d:::benchmark_data_smithi052_277165_object92035:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.447 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.752928685, 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98101475999999999, 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184527 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.448 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603417+0000", 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603425+0000", 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.294e-06 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603441+0000", 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6146000000000001e-05 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603450+0000", 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1949999999999992e-06 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603514+0000", 2023-09-23T22:49:43.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4456999999999996e-05 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603524+0000", 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0066e-05 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.573937+0000", 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.97041304399999995 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574065+0000", 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000127327 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574459+0000", 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039406800000000002 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579109+0000", 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046502430000000001 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584224+0000", 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051145239999999996 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.454 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584362+0000", 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138436 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584392+0000", 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9927999999999999e-05 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584434+0000", 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1569999999999997e-05 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184549 7.10 7:0d0c97e5:::benchmark_data_smithi052_277165_object92057:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.72213759900001, 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.95037141400000003, 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.456 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184549 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634208+0000", 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634216+0000", 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.123e-06 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634229+0000", 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2289999999999999e-05 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634235+0000", 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6270000000000002e-06 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634296+0000", 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1091000000000002e-05 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634308+0000", 2023-09-23T22:49:43.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1647999999999999e-05 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576583+0000", 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.94227499800000003 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576682+0000", 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9207000000000001e-05 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576998+0000", 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031594400000000002 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579138+0000", 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021399700000000002 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584232+0000", 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050934769999999999 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584502+0000", 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027061099999999997 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584546+0000", 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4187999999999998e-05 2023-09-23T22:49:43.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584581+0000", 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4848999999999998e-05 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184550 7.0 7:012735c8:::benchmark_data_smithi052_277165_object92058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:49:43.464 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.72209520999999, 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.93521286599999998, 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184550 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.465 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634256+0000", 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9559999999999999e-06 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634264+0000", 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9179999999999994e-06 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634269+0000", 2023-09-23T22:49:43.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5249999999999996e-06 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634322+0000", 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3062000000000001e-05 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634332+0000", 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8339999999999994e-06 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.554914+0000", 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.92058223699999997 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.555025+0000", 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110911 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.555443+0000", 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041786699999999999 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563188+0000", 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0077454409999999996 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563504+0000", 2023-09-23T22:49:43.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031558900000000002 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569298+0000", 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057944709999999998 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569357+0000", 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8242000000000002e-05 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569465+0000", 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010841299999999999 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184557 7.10 7:0c22b33f:::benchmark_data_smithi052_277165_object92065:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.71199245099999, 2023-09-23T22:49:43.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.94057564299999996, 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184557 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644354+0000", 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644361+0000", 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3660000000000003e-06 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644376+0000", 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4778e-05 2023-09-23T22:49:43.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644384+0000", 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7519999999999996e-06 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644433+0000", 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9512999999999998e-05 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644441+0000", 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2390000000000002e-06 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579160+0000", 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.93471871500000003 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579275+0000", 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011521799999999999 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579546+0000", 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027091600000000001 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584451+0000", 2023-09-23T22:49:43.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049044199999999996 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584842+0000", 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039113700000000001 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584870+0000", 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8340999999999999e-05 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584897+0000", 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6838999999999998e-05 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584931+0000", 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3798999999999999e-05 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184560 7.0 7:00bb5b7f:::benchmark_data_smithi052_277165_object92068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.643188342, 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85650294199999999, 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.482 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184560 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713158+0000", 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713165+0000", 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.165e-06 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713180+0000", 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5169e-05 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713187+0000", 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5560000000000002e-06 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713588+0000", 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040024400000000001 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713601+0000", 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3078000000000001e-05 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557158+0000", 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84355768200000003 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557274+0000", 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115092 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557528+0000", 2023-09-23T22:49:43.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025477200000000001 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563196+0000", 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056677539999999997 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563652+0000", 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00045571700000000001 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569510+0000", 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058584919999999999 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569559+0000", 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8903000000000003e-05 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569662+0000", 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000102871 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184562 7.0 7:02f4adfb:::benchmark_data_smithi052_277165_object92070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.64097983600001, 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85882703400000004, 2023-09-23T22:49:43.490 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184562 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715366+0000", 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715373+0000", 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0190000000000001e-06 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715389+0000", 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6200000000000001e-05 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715398+0000", 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8440000000000004e-06 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715448+0000", 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9747000000000003e-05 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715458+0000", 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378e-05 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563719+0000", 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84826098699999997 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563811+0000", 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1298999999999994e-05 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.564148+0000", 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000337757 2023-09-23T22:49:43.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569476+0000", 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0053272930000000003 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.573915+0000", 2023-09-23T22:49:43.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004439386 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574045+0000", 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012963800000000001 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574102+0000", 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6929999999999999e-05 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574195+0000", 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3134000000000006e-05 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184566 7.0 7:034423e1:::benchmark_data_smithi052_277165_object92074:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.637174485, 2023-09-23T22:49:43.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.86009980500000005, 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184566 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719172+0000", 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:49:43.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719176+0000", 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5370000000000002e-06 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719184+0000", 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8159999999999994e-06 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719190+0000", 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4980000000000003e-06 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719260+0000", 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0012000000000007e-05 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719271+0000", 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0862999999999999e-05 2023-09-23T22:49:43.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.566596+0000", 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84732440099999995 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.566722+0000", 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012632999999999999 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.567133+0000", 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041073900000000003 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574211+0000", 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070784969999999996 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576559+0000", 2023-09-23T22:49:43.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023475700000000002 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579125+0000", 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025667670000000002 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579184+0000", 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8736999999999997e-05 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579273+0000", 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8580999999999998e-05 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184606 7.0 7:0419327f:::benchmark_data_smithi052_277165_object92114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.528426276, 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.75649038599999996, 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184606 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827920+0000", 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827925+0000", 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0579999999999997e-06 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827937+0000", 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2271e-05 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827943+0000", 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5840000000000002e-06 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.828005+0000", 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2479999999999998e-05 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.828019+0000", 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3534e-05 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571101+0000", 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.74308152100000002 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571233+0000", 2023-09-23T22:49:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013277300000000001 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571648+0000", 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041445300000000001 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579015+0000", 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073670360000000004 2023-09-23T22:49:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579081+0000", 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5988000000000006e-05 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584275+0000", 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051946539999999999 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584318+0000", 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3046000000000001e-05 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584412+0000", 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3121999999999994e-05 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184620 7.10 7:0f84e4b0:::benchmark_data_smithi052_277165_object92128:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.464636504, 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.69549317099999997, 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:49:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184620 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:49:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:49:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:49:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891709+0000", 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891720+0000", 2023-09-23T22:49:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0986e-05 2023-09-23T22:49:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:49:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891747+0000", 2023-09-23T22:49:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7750999999999999e-05 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891758+0000", 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1043000000000001e-05 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891835+0000", 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6794999999999999e-05 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891850+0000", 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4929999999999999e-05 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:49:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.581707+0000", 2023-09-23T22:49:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.68985722999999999 2023-09-23T22:49:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:49:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.581829+0000", 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012162700000000001 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.582150+0000", 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032150899999999999 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584591+0000", 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024402400000000002 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584964+0000", 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00037371099999999999 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587111+0000", 2023-09-23T22:49:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021470249999999999 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587159+0000", 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7295e-05 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587204+0000", 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5411000000000003e-05 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:49:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:49:43.526 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:49:43.526 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:43.617 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:43 smithi052 ceph-mon[187743]: pgmap v689: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 2074/119235 objects degraded (1.739%); 2582/119235 objects misplaced (2.165%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:43.622 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:49:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:49:43] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.33.4" 2023-09-23T22:49:43.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:43 smithi052 ceph-mon[180979]: pgmap v689: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 2074/119235 objects degraded (1.739%); 2582/119235 objects misplaced (2.165%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:43.623 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:43 smithi151 ceph-mon[159210]: pgmap v689: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 2074/119235 objects degraded (1.739%); 2582/119235 objects misplaced (2.165%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:43.747 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:49:44.734 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:44.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:44.843 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:44.879 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:44.947 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:44.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:44.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:45.053 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:45.090 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:45.139 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:45.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:45.249 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:45.329 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:45.465 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:49:45.577 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:45.589 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:45 smithi052 ceph-mon[180979]: pgmap v690: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 2074/117228 objects degraded (1.769%); 2582/117228 objects misplaced (2.203%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:45.589 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2074/117228 objects degraded (1.769%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:45.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:45.602 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:45 smithi052 ceph-mon[187743]: pgmap v690: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 2074/117228 objects degraded (1.769%); 2582/117228 objects misplaced (2.203%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:45.602 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2074/117228 objects degraded (1.769%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:45.692 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:45.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:45 smithi151 ceph-mon[159210]: pgmap v690: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 2074/117228 objects degraded (1.769%); 2582/117228 objects misplaced (2.203%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:49:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2074/117228 objects degraded (1.769%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:45.720 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:45.773 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:45.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:45.879 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:45.888 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:45.991 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:46.387 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:46.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:46.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:46.403 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:49:46.444 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:49:46.491 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:49:46.499 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:46.526 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:46.550 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:49:46.579 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:46.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:46.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:46.586 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:49:46.640 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:49:46.685 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:46.696 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:49:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2960908082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:46.746 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:49:46.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2960908082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.768 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:49:46.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2960908082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:47.491 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:49:47.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:47 smithi052 ceph-mon[180979]: pgmap v691: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 2159/110910 objects degraded (1.947%); 2390/110910 objects misplaced (2.155%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:49:47.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3868317770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:47.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:47.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:47 smithi052 ceph-mon[187743]: pgmap v691: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 2159/110910 objects degraded (1.947%); 2390/110910 objects misplaced (2.155%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:49:47.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3868317770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:47.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:47 smithi151 ceph-mon[159210]: pgmap v691: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 2159/110910 objects degraded (1.947%); 2390/110910 objects misplaced (2.155%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:49:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3868317770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:48 smithi151 ceph-mon[159210]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[180979]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: failed to encode map e255 with expected crc 2023-09-23T22:49:48.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:48 smithi052 ceph-mon[187743]: failed to encode map e255 with expected crc 2023-09-23T22:49:49.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:49.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:49.338 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:49.372 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:49.431 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:49.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:49.473 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:49.537 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:49:49.625 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:49 smithi151 ceph-mon[159210]: failed to encode map e255 with expected crc 2023-09-23T22:49:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:49 smithi151 ceph-mon[159210]: pgmap v693: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 2159/108702 objects degraded (1.986%); 2390/108702 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:49.815 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:49:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:49.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:49:49.822 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:49 smithi052 ceph-mon[187743]: failed to encode map e255 with expected crc 2023-09-23T22:49:49.822 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:49 smithi052 ceph-mon[187743]: pgmap v693: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 2159/108702 objects degraded (1.986%); 2390/108702 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:49.822 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:49.822 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:49 smithi052 ceph-mon[180979]: failed to encode map e255 with expected crc 2023-09-23T22:49:49.822 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:49 smithi052 ceph-mon[180979]: pgmap v693: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 2159/108702 objects degraded (1.986%); 2390/108702 objects misplaced (2.199%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:49.822 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:49:49.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:49.954 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:50.025 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:50.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:50.133 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:49:50.161 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:50.215 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:49:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:50.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:49:50.324 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:50.435 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:50.520 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2159/108702 objects degraded (1.986%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:50.668 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2159/108702 objects degraded (1.986%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:50.668 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2159/108702 objects degraded (1.986%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:50.692 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:50.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:50.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:50.885 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:50.928 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:49:50.961 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:49:50.984 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:49:51.479 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:49:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:51 smithi151 ceph-mon[159210]: pgmap v694: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2159/107499 objects degraded (2.008%); 2390/107499 objects misplaced (2.223%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:51.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:51 smithi052 ceph-mon[180979]: pgmap v694: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2159/107499 objects degraded (2.008%); 2390/107499 objects misplaced (2.223%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:52.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:51 smithi052 ceph-mon[187743]: pgmap v694: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2159/107499 objects degraded (2.008%); 2390/107499 objects misplaced (2.223%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:49:52.881 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:52 smithi052 ceph-mon[187743]: from='client.45455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:52.881 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:52 smithi052 ceph-mon[180979]: from='client.45455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:52 smithi151 ceph-mon[159210]: from='client.45455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:49:52.950 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:52.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:52.975 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":694,"stamp":"2023-09-23T22:49:50.266680+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2347308984,"num_objects":35833,"num_object_clones":0,"num_object_copies":107499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2159,"num_objects_misplaced":2390,"num_objects_unfound":0,"num_objects_dirty":35833,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":169349,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2104,"num_bytes_recovered":135800906,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127883,"ondisk_log_size":127883,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9226452,"kb_used_data":7214044,"kb_used_omap":129,"kb_used_meta":2011774,"kb_avail":646989612,"statfs":{"total":671965249536,"available":662517362688,"internally_reserved":0,"allocated":7387181056,"data_stored":7381945065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2060057309},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,4,2,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-377552896,"num_objects":-5761,"num_object_clones":0,"num_object_copies":-17283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":190,"num_objects_misplaced":-390,"num_objects_unfound":0,"num_objects_dirty":-5761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5761,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":200,"num_bytes_recovered":13107200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004758"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":251,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656655+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:49:24.656655+0000","last_peered":"2023-09-23T22:49:24.656655+0000","last_clean":"2023-09-23T22:49:24.656655+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:49:24.656655+0000","last_undegraded":"2023-09-23T22:49:24.656655+0000","last_fullsized":"2023-09-23T22:49:24.656655+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":226,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648125+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:49:24.648125+0000","last_peered":"2023-09-23T22:49:24.648125+0000","last_clean":"2023-09-23T22:49:24.648125+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:49:24.648125+0000","last_undegraded":"2023-09-23T22:49:24.648125+0000","last_fullsized":"2023-09-23T22:49:24.648125+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647375+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:49:24.647375+0000","last_peered":"2023-09-23T22:49:24.647375+0000","last_clean":"2023-09-23T22:49:24.647375+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:49:24.647375+0000","last_undegraded":"2023-09-23T22:49:24.647375+0000","last_fullsized":"2023-09-23T22:49:24.647375+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355859+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:49:34.355859+0000","last_peered":"2023-09-23T22:49:34.355859+0000","last_clean":"2023-09-23T22:49:34.355859+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:49:34.355859+0000","last_undegraded":"2023-09-23T22:49:34.355859+0000","last_fullsized":"2023-09-23T22:49:34.355859+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":285,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.652058+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:49:24.652058+0000","last_peered":"2023-09-23T22:49:24.652058+0000","last_clean":"2023-09-23T22:49:24.652058+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:49:24.652058+0000","last_undegraded":"2023-09-23T22:49:24.652058+0000","last_fullsized":"2023-09-23T22:49:24.652058+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658930+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:49:24.658930+0000","last_peered":"2023-09-23T22:49:24.658930+0000","last_clean":"2023-09-23T22:49:24.658930+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:49:24.658930+0000","last_undegraded":"2023-09-23T22:49:24.658930+0000","last_fullsized":"2023-09-23T22:49:24.658930+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":154,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355497+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:49:34.355497+0000","last_peered":"2023-09-23T22:49:34.355497+0000","last_clean":"2023-09-23T22:49:34.355497+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:49:34.355497+0000","last_undegraded":"2023-09-23T22:49:34.355497+0000","last_fullsized":"2023-09-23T22:49:34.355497+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.343078+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:49:34.343078+0000","last_peered":"2023-09-23T22:49:34.343078+0000","last_clean":"2023-09-23T22:49:34.343078+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:49:34.343078+0000","last_undegraded":"2023-09-23T22:49:34.343078+0000","last_fullsized":"2023-09-23T22:49:34.343078+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":223,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647893+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:49:24.647893+0000","last_peered":"2023-09-23T22:49:24.647893+0000","last_clean":"2023-09-23T22:49:24.647893+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:49:24.647893+0000","last_undegraded":"2023-09-23T22:49:24.647893+0000","last_fullsized":"2023-09-23T22:49:24.647893+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":212,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.343104+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:49:34.343104+0000","last_peered":"2023-09-23T22:49:34.343104+0000","last_clean":"2023-09-23T22:49:34.343104+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:49:34.343104+0000","last_undegraded":"2023-09-23T22:49:34.343104+0000","last_fullsized":"2023-09-23T22:49:34.343104+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":128,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648036+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:49:24.648036+0000","last_peered":"2023-09-23T22:49:24.648036+0000","last_clean":"2023-09-23T22:49:24.648036+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:49:24.648036+0000","last_undegraded":"2023-09-23T22:49:24.648036+0000","last_fullsized":"2023-09-23T22:49:24.648036+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655484+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:49:24.655484+0000","last_peered":"2023-09-23T22:49:24.655484+0000","last_clean":"2023-09-23T22:49:24.655484+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:49:24.655484+0000","last_undegraded":"2023-09-23T22:49:24.655484+0000","last_fullsized":"2023-09-23T22:49:24.655484+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":274,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.657077+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:49:24.657077+0000","last_peered":"2023-09-23T22:49:24.657077+0000","last_clean":"2023-09-23T22:49:24.657077+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:49:24.657077+0000","last_undegraded":"2023-09-23T22:49:24.657077+0000","last_fullsized":"2023-09-23T22:49:24.657077+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658530+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:49:24.658530+0000","last_peered":"2023-09-23T22:49:24.658530+0000","last_clean":"2023-09-23T22:49:24.658530+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:49:24.658530+0000","last_undegraded":"2023-09-23T22:49:24.658530+0000","last_fullsized":"2023-09-23T22:49:24.658530+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":112,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658589+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:49:24.658589+0000","last_peered":"2023-09-23T22:49:24.658589+0000","last_clean":"2023-09-23T22:49:24.658589+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:49:24.658589+0000","last_undegraded":"2023-09-23T22:49:24.658589+0000","last_fullsized":"2023-09-23T22:49:24.658589+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":274,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.346594+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:49:34.346594+0000","last_peered":"2023-09-23T22:49:34.346594+0000","last_clean":"2023-09-23T22:49:34.346594+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:49:34.346594+0000","last_undegraded":"2023-09-23T22:49:34.346594+0000","last_fullsized":"2023-09-23T22:49:34.346594+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":210,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355887+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:49:34.355887+0000","last_peered":"2023-09-23T22:49:34.355887+0000","last_clean":"2023-09-23T22:49:34.355887+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:49:34.355887+0000","last_undegraded":"2023-09-23T22:49:34.355887+0000","last_fullsized":"2023-09-23T22:49:34.355887+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":234,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648193+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:49:24.648193+0000","last_peered":"2023-09-23T22:49:24.648193+0000","last_clean":"2023-09-23T22:49:24.648193+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:49:24.648193+0000","last_undegraded":"2023-09-23T22:49:24.648193+0000","last_fullsized":"2023-09-23T22:49:24.648193+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":104,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656383+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:49:24.656383+0000","last_peered":"2023-09-23T22:49:24.656383+0000","last_clean":"2023-09-23T22:49:24.656383+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:49:24.656383+0000","last_undegraded":"2023-09-23T22:49:24.656383+0000","last_fullsized":"2023-09-23T22:49:24.656383+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":211,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355940+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:49:34.355940+0000","last_peered":"2023-09-23T22:49:34.355940+0000","last_clean":"2023-09-23T22:49:34.355940+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:49:34.355940+0000","last_undegraded":"2023-09-23T22:49:34.355940+0000","last_fullsized":"2023-09-23T22:49:34.355940+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":298,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655086+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:49:24.655086+0000","last_peered":"2023-09-23T22:49:24.655086+0000","last_clean":"2023-09-23T22:49:24.655086+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:49:24.655086+0000","last_undegraded":"2023-09-23T22:49:24.655086+0000","last_fullsized":"2023-09-23T22:49:24.655086+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.649221+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:49:24.649221+0000","last_peered":"2023-09-23T22:49:24.649221+0000","last_clean":"2023-09-23T22:49:24.649221+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:49:24.649221+0000","last_undegraded":"2023-09-23T22:49:24.649221+0000","last_fullsized":"2023-09-23T22:49:24.649221+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"254'10459","reported_seq":10504,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.531765+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:49:45.531765+0000","last_peered":"2023-09-23T22:49:45.531765+0000","last_clean":"2023-09-23T22:49:45.531765+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:49:45.531765+0000","last_undegraded":"2023-09-23T22:49:45.531765+0000","last_fullsized":"2023-09-23T22:49:45.531765+0000","mapping_epoch":217,"log_start":"229'3100","ondisk_log_start":"229'3100","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150274048,"num_objects":2293,"num_object_clones":0,"num_object_copies":6879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10459,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":123,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656700+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:49:24.656700+0000","last_peered":"2023-09-23T22:49:24.656700+0000","last_clean":"2023-09-23T22:49:24.656700+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:49:24.656700+0000","last_undegraded":"2023-09-23T22:49:24.656700+0000","last_fullsized":"2023-09-23T22:49:24.656700+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655170+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:49:24.655170+0000","last_peered":"2023-09-23T22:49:24.655170+0000","last_clean":"2023-09-23T22:49:24.655170+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:49:24.655170+0000","last_undegraded":"2023-09-23T22:49:24.655170+0000","last_fullsized":"2023-09-23T22:49:24.655170+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"254'10397","reported_seq":10447,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.812376+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:49:45.812376+0000","last_peered":"2023-09-23T22:49:45.812376+0000","last_clean":"2023-09-23T22:49:45.812376+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:49:45.812376+0000","last_undegraded":"2023-09-23T22:49:45.812376+0000","last_fullsized":"2023-09-23T22:49:45.812376+0000","mapping_epoch":219,"log_start":"224'1800","ondisk_log_start":"224'1800","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10397,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":133,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656614+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:49:24.656614+0000","last_peered":"2023-09-23T22:49:24.656614+0000","last_clean":"2023-09-23T22:49:24.656614+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:49:24.656614+0000","last_undegraded":"2023-09-23T22:49:24.656614+0000","last_fullsized":"2023-09-23T22:49:24.656614+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":234,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656371+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:49:24.656371+0000","last_peered":"2023-09-23T22:49:24.656371+0000","last_clean":"2023-09-23T22:49:24.656371+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:49:24.656371+0000","last_undegraded":"2023-09-23T22:49:24.656371+0000","last_fullsized":"2023-09-23T22:49:24.656371+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"255'10419","reported_seq":10466,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.710283+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:49:47.710283+0000","last_peered":"2023-09-23T22:49:47.710283+0000","last_clean":"2023-09-23T22:49:47.710283+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:49:47.710283+0000","last_undegraded":"2023-09-23T22:49:47.710283+0000","last_fullsized":"2023-09-23T22:49:47.710283+0000","mapping_epoch":219,"log_start":"227'2500","ondisk_log_start":"227'2500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7919,"log_dups_size":0,"ondisk_log_size":7919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142671872,"num_objects":2177,"num_object_clones":0,"num_object_copies":6531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10419,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":64,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658207+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:49:24.658207+0000","last_peered":"2023-09-23T22:49:24.658207+0000","last_clean":"2023-09-23T22:49:24.658207+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:49:24.658207+0000","last_undegraded":"2023-09-23T22:49:24.658207+0000","last_fullsized":"2023-09-23T22:49:24.658207+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":246,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655616+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:49:24.655616+0000","last_peered":"2023-09-23T22:49:24.655616+0000","last_clean":"2023-09-23T22:49:24.655616+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:49:24.655616+0000","last_undegraded":"2023-09-23T22:49:24.655616+0000","last_fullsized":"2023-09-23T22:49:24.655616+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"254'10755","reported_seq":10810,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.888494+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:49:45.888494+0000","last_peered":"2023-09-23T22:49:45.888494+0000","last_clean":"2023-09-23T22:49:45.888494+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:49:45.888494+0000","last_undegraded":"2023-09-23T22:49:45.888494+0000","last_fullsized":"2023-09-23T22:49:45.888494+0000","mapping_epoch":226,"log_start":"229'3600","ondisk_log_start":"229'3600","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149880832,"num_objects":2287,"num_object_clones":0,"num_object_copies":6861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10755,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647819+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:49:24.647819+0000","last_peered":"2023-09-23T22:49:24.647819+0000","last_clean":"2023-09-23T22:49:24.647819+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:49:24.647819+0000","last_undegraded":"2023-09-23T22:49:24.647819+0000","last_fullsized":"2023-09-23T22:49:24.647819+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":147,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355085+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:49:34.355085+0000","last_peered":"2023-09-23T22:49:34.355085+0000","last_clean":"2023-09-23T22:49:34.355085+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:49:34.355085+0000","last_undegraded":"2023-09-23T22:49:34.355085+0000","last_fullsized":"2023-09-23T22:49:34.355085+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"254'10411","reported_seq":10458,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.846454+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:49:45.846454+0000","last_peered":"2023-09-23T22:49:45.846454+0000","last_clean":"2023-09-23T22:49:45.846454+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:49:45.846454+0000","last_undegraded":"2023-09-23T22:49:45.846454+0000","last_fullsized":"2023-09-23T22:49:45.846454+0000","mapping_epoch":217,"log_start":"229'3200","ondisk_log_start":"229'3200","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7211,"log_dups_size":0,"ondisk_log_size":7211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10411,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":255,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.653666+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:49:24.653666+0000","last_peered":"2023-09-23T22:49:24.653666+0000","last_clean":"2023-09-23T22:49:24.653666+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:49:24.653666+0000","last_undegraded":"2023-09-23T22:49:24.653666+0000","last_fullsized":"2023-09-23T22:49:24.653666+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656995+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:49:24.656995+0000","last_peered":"2023-09-23T22:49:24.656995+0000","last_clean":"2023-09-23T22:49:24.656995+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:49:24.656995+0000","last_undegraded":"2023-09-23T22:49:24.656995+0000","last_fullsized":"2023-09-23T22:49:24.656995+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"255'10576","reported_seq":10643,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.751398+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:49:47.751398+0000","last_peered":"2023-09-23T22:49:47.751398+0000","last_clean":"2023-09-23T22:49:47.751398+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:49:47.751398+0000","last_undegraded":"2023-09-23T22:49:47.751398+0000","last_fullsized":"2023-09-23T22:49:47.751398+0000","mapping_epoch":226,"log_start":"227'2600","ondisk_log_start":"227'2600","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7976,"log_dups_size":0,"ondisk_log_size":7976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143785984,"num_objects":2194,"num_object_clones":0,"num_object_copies":6582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10576,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":238,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655757+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:49:24.655757+0000","last_peered":"2023-09-23T22:49:24.655757+0000","last_clean":"2023-09-23T22:49:24.655757+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:49:24.655757+0000","last_undegraded":"2023-09-23T22:49:24.655757+0000","last_fullsized":"2023-09-23T22:49:24.655757+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":249,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656508+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:49:24.656508+0000","last_peered":"2023-09-23T22:49:24.656508+0000","last_clean":"2023-09-23T22:49:24.656508+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:49:24.656508+0000","last_undegraded":"2023-09-23T22:49:24.656508+0000","last_fullsized":"2023-09-23T22:49:24.656508+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"254'10687","reported_seq":10738,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.535946+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:49:45.535946+0000","last_peered":"2023-09-23T22:49:45.535946+0000","last_clean":"2023-09-23T22:49:45.535946+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:49:45.535946+0000","last_undegraded":"2023-09-23T22:49:45.535946+0000","last_fullsized":"2023-09-23T22:49:45.535946+0000","mapping_epoch":219,"log_start":"229'3300","ondisk_log_start":"229'3300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7387,"log_dups_size":0,"ondisk_log_size":7387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10687,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":262,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656254+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:49:24.656254+0000","last_peered":"2023-09-23T22:49:24.656254+0000","last_clean":"2023-09-23T22:49:24.656254+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:49:24.656254+0000","last_undegraded":"2023-09-23T22:49:24.656254+0000","last_fullsized":"2023-09-23T22:49:24.656254+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656693+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:49:24.656693+0000","last_peered":"2023-09-23T22:49:24.656693+0000","last_clean":"2023-09-23T22:49:24.656693+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:49:24.656693+0000","last_undegraded":"2023-09-23T22:49:24.656693+0000","last_fullsized":"2023-09-23T22:49:24.656693+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"254'10285","reported_seq":10347,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.828470+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:49:45.828470+0000","last_peered":"2023-09-23T22:49:45.828470+0000","last_clean":"2023-09-23T22:49:45.828470+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:49:45.828470+0000","last_undegraded":"2023-09-23T22:49:45.828470+0000","last_fullsized":"2023-09-23T22:49:45.828470+0000","mapping_epoch":226,"log_start":"224'1700","ondisk_log_start":"224'1700","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8585,"log_dups_size":0,"ondisk_log_size":8585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141623296,"num_objects":2161,"num_object_clones":0,"num_object_copies":6483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10285,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":255,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.651882+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:49:24.651882+0000","last_peered":"2023-09-23T22:49:24.651882+0000","last_clean":"2023-09-23T22:49:24.651882+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:49:24.651882+0000","last_undegraded":"2023-09-23T22:49:24.651882+0000","last_fullsized":"2023-09-23T22:49:24.651882+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":181,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.363578+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:49:34.363578+0000","last_peered":"2023-09-23T22:49:34.363578+0000","last_clean":"2023-09-23T22:49:34.363578+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:49:34.363578+0000","last_undegraded":"2023-09-23T22:49:34.363578+0000","last_fullsized":"2023-09-23T22:49:34.363578+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"255'10567","reported_seq":10612,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.799104+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:49:47.799104+0000","last_peered":"2023-09-23T22:49:47.799104+0000","last_clean":"2023-09-23T22:49:47.799104+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:49:47.799104+0000","last_undegraded":"2023-09-23T22:49:47.799104+0000","last_fullsized":"2023-09-23T22:49:47.799104+0000","mapping_epoch":219,"log_start":"227'2600","ondisk_log_start":"227'2600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7967,"log_dups_size":0,"ondisk_log_size":7967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148307968,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10567,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":291,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.356207+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:49:34.356207+0000","last_peered":"2023-09-23T22:49:34.356207+0000","last_clean":"2023-09-23T22:49:34.356207+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:49:34.356207+0000","last_undegraded":"2023-09-23T22:49:34.356207+0000","last_fullsized":"2023-09-23T22:49:34.356207+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647956+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:49:24.647956+0000","last_peered":"2023-09-23T22:49:24.647956+0000","last_clean":"2023-09-23T22:49:24.647956+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:49:24.647956+0000","last_undegraded":"2023-09-23T22:49:24.647956+0000","last_fullsized":"2023-09-23T22:49:24.647956+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656561+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:49:24.656561+0000","last_peered":"2023-09-23T22:49:24.656561+0000","last_clean":"2023-09-23T22:49:24.656561+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:49:24.656561+0000","last_undegraded":"2023-09-23T22:49:24.656561+0000","last_fullsized":"2023-09-23T22:49:24.656561+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"254'10466","reported_seq":10519,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.789088+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:49:45.789088+0000","last_peered":"2023-09-23T22:49:45.789088+0000","last_clean":"2023-09-23T22:49:45.789088+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:49:45.789088+0000","last_undegraded":"2023-09-23T22:49:45.789088+0000","last_fullsized":"2023-09-23T22:49:45.789088+0000","mapping_epoch":219,"log_start":"229'3300","ondisk_log_start":"229'3300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7166,"log_dups_size":0,"ondisk_log_size":7166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10466,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.655686+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:49:24.655686+0000","last_peered":"2023-09-23T22:49:24.655686+0000","last_clean":"2023-09-23T22:49:24.655686+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:49:24.655686+0000","last_undegraded":"2023-09-23T22:49:24.655686+0000","last_fullsized":"2023-09-23T22:49:24.655686+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":250,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658622+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:49:24.658622+0000","last_peered":"2023-09-23T22:49:24.658622+0000","last_clean":"2023-09-23T22:49:24.658622+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:49:24.658622+0000","last_undegraded":"2023-09-23T22:49:24.658622+0000","last_fullsized":"2023-09-23T22:49:24.658622+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":170,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.362020+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:49:34.362020+0000","last_peered":"2023-09-23T22:49:34.362020+0000","last_clean":"2023-09-23T22:49:34.362020+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:49:34.362020+0000","last_undegraded":"2023-09-23T22:49:34.362020+0000","last_fullsized":"2023-09-23T22:49:34.362020+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"254'10629","reported_seq":10676,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.891745+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:49:45.891745+0000","last_peered":"2023-09-23T22:49:45.891745+0000","last_clean":"2023-09-23T22:49:45.891745+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:49:45.891745+0000","last_undegraded":"2023-09-23T22:49:45.891745+0000","last_fullsized":"2023-09-23T22:49:45.891745+0000","mapping_epoch":217,"log_start":"229'3400","ondisk_log_start":"229'3400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7229,"log_dups_size":0,"ondisk_log_size":7229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150667264,"num_objects":2299,"num_object_clones":0,"num_object_copies":6897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10629,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":246,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.360486+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:49:34.360486+0000","last_peered":"2023-09-23T22:49:34.360486+0000","last_clean":"2023-09-23T22:49:34.360486+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:49:34.360486+0000","last_undegraded":"2023-09-23T22:49:34.360486+0000","last_fullsized":"2023-09-23T22:49:34.360486+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":134,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355201+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:49:34.355201+0000","last_peered":"2023-09-23T22:49:34.355201+0000","last_clean":"2023-09-23T22:49:34.355201+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:49:34.355201+0000","last_undegraded":"2023-09-23T22:49:34.355201+0000","last_fullsized":"2023-09-23T22:49:34.355201+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":260,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355415+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:49:34.355415+0000","last_peered":"2023-09-23T22:49:34.355415+0000","last_clean":"2023-09-23T22:49:34.355415+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:49:34.355415+0000","last_undegraded":"2023-09-23T22:49:34.355415+0000","last_fullsized":"2023-09-23T22:49:34.355415+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"254'10401","reported_seq":10452,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:45.724677+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:49:45.724677+0000","last_peered":"2023-09-23T22:49:45.724677+0000","last_clean":"2023-09-23T22:49:45.724677+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:49:45.724677+0000","last_undegraded":"2023-09-23T22:49:45.724677+0000","last_fullsized":"2023-09-23T22:49:45.724677+0000","mapping_epoch":219,"log_start":"224'1800","ondisk_log_start":"224'1800","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8601,"log_dups_size":0,"ondisk_log_size":8601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10401,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":255,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.357840+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:49:34.357840+0000","last_peered":"2023-09-23T22:49:34.357840+0000","last_clean":"2023-09-23T22:49:34.357840+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:49:34.357840+0000","last_undegraded":"2023-09-23T22:49:34.357840+0000","last_fullsized":"2023-09-23T22:49:34.357840+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":341,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.362036+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:49:34.362036+0000","last_peered":"2023-09-23T22:49:34.362036+0000","last_clean":"2023-09-23T22:49:34.362036+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:49:34.362036+0000","last_undegraded":"2023-09-23T22:49:34.362036+0000","last_fullsized":"2023-09-23T22:49:34.362036+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":185,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647711+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:49:24.647711+0000","last_peered":"2023-09-23T22:49:24.647711+0000","last_clean":"2023-09-23T22:49:24.647711+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:49:24.647711+0000","last_undegraded":"2023-09-23T22:49:24.647711+0000","last_fullsized":"2023-09-23T22:49:24.647711+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":246,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658369+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:49:24.658369+0000","last_peered":"2023-09-23T22:49:24.658369+0000","last_clean":"2023-09-23T22:49:24.658369+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:49:24.658369+0000","last_undegraded":"2023-09-23T22:49:24.658369+0000","last_fullsized":"2023-09-23T22:49:24.658369+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"255'10791","reported_seq":10847,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:49.623152+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:49:49.623152+0000","last_peered":"2023-09-23T22:49:49.623152+0000","last_clean":"2023-09-23T22:49:49.623152+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:49:49.623152+0000","last_undegraded":"2023-09-23T22:49:49.623152+0000","last_fullsized":"2023-09-23T22:49:49.623152+0000","mapping_epoch":226,"log_start":"223'1100","ondisk_log_start":"223'1100","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9691,"log_dups_size":0,"ondisk_log_size":9691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10791,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":220,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.362140+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:49:34.362140+0000","last_peered":"2023-09-23T22:49:34.362140+0000","last_clean":"2023-09-23T22:49:34.362140+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:49:34.362140+0000","last_undegraded":"2023-09-23T22:49:34.362140+0000","last_fullsized":"2023-09-23T22:49:34.362140+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":191,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.363086+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:49:34.363086+0000","last_peered":"2023-09-23T22:49:34.363086+0000","last_clean":"2023-09-23T22:49:34.363086+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:49:34.363086+0000","last_undegraded":"2023-09-23T22:49:34.363086+0000","last_fullsized":"2023-09-23T22:49:34.363086+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":178,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355674+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:49:34.355674+0000","last_peered":"2023-09-23T22:49:34.355674+0000","last_clean":"2023-09-23T22:49:34.355674+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:49:34.355674+0000","last_undegraded":"2023-09-23T22:49:34.355674+0000","last_fullsized":"2023-09-23T22:49:34.355674+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"254'10862","reported_seq":15055,"reported_epoch":254,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:49:45.936353+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:49:45.936353+0000","last_peered":"2023-09-23T22:49:45.936353+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:49:45.936353+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"229'3700","ondisk_log_start":"229'3700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7162,"log_dups_size":0,"ondisk_log_size":7162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2159,"num_objects_misplaced":2390,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10862,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2069,"num_bytes_recovered":135593984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-2143},{"shards":"1,3,4","objects":2390}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":183,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.366368+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:49:34.366368+0000","last_peered":"2023-09-23T22:49:34.366368+0000","last_clean":"2023-09-23T22:49:34.366368+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:49:34.366368+0000","last_undegraded":"2023-09-23T22:49:34.366368+0000","last_fullsized":"2023-09-23T22:49:34.366368+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.362202+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:49:34.362202+0000","last_peered":"2023-09-23T22:49:34.362202+0000","last_clean":"2023-09-23T22:49:34.362202+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:49:34.362202+0000","last_undegraded":"2023-09-23T22:49:34.362202+0000","last_fullsized":"2023-09-23T22:49:34.362202+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":183,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355325+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:49:34.355325+0000","last_peered":"2023-09-23T22:49:34.355325+0000","last_clean":"2023-09-23T22:49:34.355325+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:49:34.355325+0000","last_undegraded":"2023-09-23T22:49:34.355325+0000","last_fullsized":"2023-09-23T22:49:34.355325+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"255'10691","reported_seq":10738,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.740658+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:49:47.740658+0000","last_peered":"2023-09-23T22:49:47.740658+0000","last_clean":"2023-09-23T22:49:47.740658+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:49:47.740658+0000","last_undegraded":"2023-09-23T22:49:47.740658+0000","last_fullsized":"2023-09-23T22:49:47.740658+0000","mapping_epoch":219,"log_start":"227'2700","ondisk_log_start":"227'2700","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7991,"log_dups_size":0,"ondisk_log_size":7991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10691,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":294,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.649177+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:49:24.649177+0000","last_peered":"2023-09-23T22:49:24.649177+0000","last_clean":"2023-09-23T22:49:24.649177+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:49:24.649177+0000","last_undegraded":"2023-09-23T22:49:24.649177+0000","last_fullsized":"2023-09-23T22:49:24.649177+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":52,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658467+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:49:24.658467+0000","last_peered":"2023-09-23T22:49:24.658467+0000","last_clean":"2023-09-23T22:49:24.658467+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:49:24.658467+0000","last_undegraded":"2023-09-23T22:49:24.658467+0000","last_fullsized":"2023-09-23T22:49:24.658467+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":279,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.651968+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:49:24.651968+0000","last_peered":"2023-09-23T22:49:24.651968+0000","last_clean":"2023-09-23T22:49:24.651968+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:49:24.651968+0000","last_undegraded":"2023-09-23T22:49:24.651968+0000","last_fullsized":"2023-09-23T22:49:24.651968+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"255'10693","reported_seq":10742,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:49.654539+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:49:49.654539+0000","last_peered":"2023-09-23T22:49:49.654539+0000","last_clean":"2023-09-23T22:49:49.654539+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:49:49.654539+0000","last_undegraded":"2023-09-23T22:49:49.654539+0000","last_fullsized":"2023-09-23T22:49:49.654539+0000","mapping_epoch":217,"log_start":"223'1000","ondisk_log_start":"223'1000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9693,"log_dups_size":0,"ondisk_log_size":9693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143654936,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10693,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":286,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658869+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:49:24.658869+0000","last_peered":"2023-09-23T22:49:24.658869+0000","last_clean":"2023-09-23T22:49:24.658869+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:49:24.658869+0000","last_undegraded":"2023-09-23T22:49:24.658869+0000","last_fullsized":"2023-09-23T22:49:24.658869+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":150,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355779+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:49:34.355779+0000","last_peered":"2023-09-23T22:49:34.355779+0000","last_clean":"2023-09-23T22:49:34.355779+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:49:34.355779+0000","last_undegraded":"2023-09-23T22:49:34.355779+0000","last_fullsized":"2023-09-23T22:49:34.355779+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":204,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656293+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:49:24.656293+0000","last_peered":"2023-09-23T22:49:24.656293+0000","last_clean":"2023-09-23T22:49:24.656293+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:49:24.656293+0000","last_undegraded":"2023-09-23T22:49:24.656293+0000","last_fullsized":"2023-09-23T22:49:24.656293+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":186,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355189+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:49:34.355189+0000","last_peered":"2023-09-23T22:49:34.355189+0000","last_clean":"2023-09-23T22:49:34.355189+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:49:34.355189+0000","last_undegraded":"2023-09-23T22:49:34.355189+0000","last_fullsized":"2023-09-23T22:49:34.355189+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":27,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.362119+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:49:34.362119+0000","last_peered":"2023-09-23T22:49:34.362119+0000","last_clean":"2023-09-23T22:49:34.362119+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:49:34.362119+0000","last_undegraded":"2023-09-23T22:49:34.362119+0000","last_fullsized":"2023-09-23T22:49:34.362119+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":64,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656924+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:49:24.656924+0000","last_peered":"2023-09-23T22:49:24.656924+0000","last_clean":"2023-09-23T22:49:24.656924+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:49:24.656924+0000","last_undegraded":"2023-09-23T22:49:24.656924+0000","last_fullsized":"2023-09-23T22:49:24.656924+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":253,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.658671+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:49:24.658671+0000","last_peered":"2023-09-23T22:49:24.658671+0000","last_clean":"2023-09-23T22:49:24.658671+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:49:24.658671+0000","last_undegraded":"2023-09-23T22:49:24.658671+0000","last_fullsized":"2023-09-23T22:49:24.658671+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":64,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656916+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:49:24.656916+0000","last_peered":"2023-09-23T22:49:24.656916+0000","last_clean":"2023-09-23T22:49:24.656916+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:49:24.656916+0000","last_undegraded":"2023-09-23T22:49:24.656916+0000","last_fullsized":"2023-09-23T22:49:24.656916+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656730+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:49:24.656730+0000","last_peered":"2023-09-23T22:49:24.656730+0000","last_clean":"2023-09-23T22:49:24.656730+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:49:24.656730+0000","last_undegraded":"2023-09-23T22:49:24.656730+0000","last_fullsized":"2023-09-23T22:49:24.656730+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":247,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.648374+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:49:24.648374+0000","last_peered":"2023-09-23T22:49:24.648374+0000","last_clean":"2023-09-23T22:49:24.648374+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:49:24.648374+0000","last_undegraded":"2023-09-23T22:49:24.648374+0000","last_fullsized":"2023-09-23T22:49:24.648374+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":285,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.647282+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:49:24.647282+0000","last_peered":"2023-09-23T22:49:24.647282+0000","last_clean":"2023-09-23T22:49:24.647282+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:49:24.647282+0000","last_undegraded":"2023-09-23T22:49:24.647282+0000","last_fullsized":"2023-09-23T22:49:24.647282+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":199,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:49:34.355585+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:49:34.355585+0000","last_peered":"2023-09-23T22:49:34.355585+0000","last_clean":"2023-09-23T22:49:34.355585+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:49:34.355585+0000","last_undegraded":"2023-09-23T22:49:34.355585+0000","last_fullsized":"2023-09-23T22:49:34.355585+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":219,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:49:24.656161+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:49:24.656161+0000","last_peered":"2023-09-23T22:49:24.656161+0000","last_clean":"2023-09-23T22:49:24.656161+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:49:24.656161+0000","last_undegraded":"2023-09-23T22:49:24.656161+0000","last_fullsized":"2023-09-23T22:49:24.656161+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2346713112,"num_objects":35809,"num_object_clones":0,"num_object_copies":107427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2159,"num_objects_misplaced":2390,"num_objects_unfound":0,"num_objects_dirty":35809,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169089,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2072,"num_bytes_recovered":135790592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7326543872,"data_stored":7326531656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127689,"ondisk_log_size":127689,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430481,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626709,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1172484,"kb_used_data":875196,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92572668,"statfs":{"total":95995035648,"available":94794412032,"internally_reserved":0,"allocated":896200704,"data_stored":895698137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:17 2023","interfaces":[{"interface":"back","average":{"1min":8.891,"5min":3.4860000000000002,"15min":2.0209999999999999},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":117.45999999999999,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":8.6750000000000007,"5min":3.4220000000000002,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":117.48699999999999,"5min":117.48699999999999,"15min":117.48699999999999},"last":7.9800000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":4.2599999999999998,"5min":2.577,"15min":2.4870000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":31.286000000000001,"5min":31.286000000000001,"15min":153.78399999999999},"last":0.31},{"interface":"front","average":{"1min":4.5350000000000001,"5min":4.3220000000000001,"15min":3.0830000000000002},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":31.512,"5min":185.411,"15min":185.411},"last":0.247}]},{"osd":2,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":2.8719999999999999,"15min":3.1640000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":33.174999999999997,"5min":33.174999999999997,"15min":100.785},"last":0.44900000000000001},{"interface":"front","average":{"1min":3.9079999999999999,"5min":4.5099999999999998,"15min":3.6059999999999999},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":33.310000000000002,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.377}]},{"osd":3,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":5.9370000000000003,"15min":3.4489999999999998},"min":{"1min":0.32800000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.742999999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":1.734},{"interface":"front","average":{"1min":2.5230000000000001,"5min":4.4880000000000004,"15min":3.0670000000000002},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":10.773999999999999,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.85099999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.79800000000000004,"15min":0.67600000000000005},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6070000000000002,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.85399999999999998,"15min":0.68600000000000005},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":0.69199999999999995,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.34100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.73999999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.5620000000000003,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.93400000000000005,"15min":0.754},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.4429999999999996,"5min":9.7899999999999991,"15min":18.625},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:48:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1,"15min":0.746},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.4049999999999998,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.83699999999999997,"15min":0.72799999999999998},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":3.7130000000000001,"5min":5.1790000000000003,"15min":16.661999999999999},"last":0.45800000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253019,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310672,"kb_used_data":1008328,"kb_used_omap":9,"kb_used_meta":302326,"kb_avail":92434480,"statfs":{"total":95995035648,"available":94652907520,"internally_reserved":0,"allocated":1032527872,"data_stored":1031609507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":309582825},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:18 2023","interfaces":[{"interface":"back","average":{"1min":15.766999999999999,"5min":7.5789999999999997,"15min":5.3049999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":82.903999999999996,"5min":101.605,"15min":101.605},"last":2.9260000000000002},{"interface":"front","average":{"1min":13.898,"5min":7.5730000000000004,"15min":5.5339999999999998},"min":{"1min":0.441,"5min":0.214,"15min":0.214},"max":{"1min":82.748999999999995,"5min":101.892,"15min":101.892},"last":4.7510000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:49:18 2023","interfaces":[{"interface":"back","average":{"1min":5.2960000000000003,"5min":3.2799999999999998,"15min":2.3029999999999999},"min":{"1min":0.20000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":37.869999999999997,"5min":37.869999999999997,"15min":37.889000000000003},"last":3.0019999999999998},{"interface":"front","average":{"1min":6.4699999999999998,"5min":3.931,"15min":2.4119999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":37.957000000000001,"5min":37.957000000000001,"15min":37.957000000000001},"last":2.976}]},{"osd":3,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":9.3239999999999998,"5min":3.8450000000000002,"15min":2.149},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":87.787999999999997,"5min":87.787999999999997,"15min":87.787999999999997},"last":1.7090000000000001},{"interface":"front","average":{"1min":10.641999999999999,"5min":4.7679999999999998,"15min":2.5299999999999998},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":89.861999999999995,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.80400000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7949999999999999,"5min":3.1579999999999999,"15min":1.9490000000000001},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":11.432,"5min":78.948999999999998,"15min":78.948999999999998},"last":3.2869999999999999},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.3889999999999998,"15min":1.7649999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":10.619,"5min":78.963999999999999,"15min":78.963999999999999},"last":3.3069999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.266,"15min":1.53},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.4390000000000001,"5min":84.254999999999995,"15min":84.254999999999995},"last":4.3230000000000004},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.2949999999999999,"15min":1.8160000000000001},"min":{"1min":0.35299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.458,"5min":87.515000000000001,"15min":87.515000000000001},"last":2.883}]},{"osd":6,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.4609999999999999,"15min":1.5960000000000001},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.317},"max":{"1min":10.65,"5min":79.311000000000007,"15min":79.311000000000007},"last":3.4569999999999999},{"interface":"front","average":{"1min":1.5,"5min":2.2829999999999999,"15min":1.4630000000000001},"min":{"1min":0.36299999999999999,"5min":0.313,"15min":0.313},"max":{"1min":7.3819999999999997,"5min":79.328999999999994,"15min":79.328999999999994},"last":2.984}]},{"osd":7,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":3.3530000000000002,"5min":2.6560000000000001,"15min":2.1869999999999998},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":11.481999999999999,"5min":29.539000000000001,"15min":159.018},"last":1.794},{"interface":"front","average":{"1min":3.335,"5min":2.048,"15min":2.0960000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":10.676,"5min":10.676,"15min":158.96000000000001},"last":3.2719999999999998}]}]},{"osd":0,"up_from":219,"seq":940597837868,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289748,"kb_used_data":1129420,"kb_used_omap":31,"kb_used_meta":160288,"kb_avail":92455404,"statfs":{"total":95995035648,"available":94674333696,"internally_reserved":0,"allocated":1156526080,"data_stored":1155552373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":164135858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.319,"5min":4.0780000000000003,"15min":3.8340000000000001},"min":{"1min":0.24299999999999999,"5min":0.216,"15min":0.216},"max":{"1min":21.512,"5min":32.344999999999999,"15min":32.344999999999999},"last":6.2670000000000003},{"interface":"front","average":{"1min":5.3239999999999998,"5min":3.9279999999999999,"15min":3.6819999999999999},"min":{"1min":0.30499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.064,"5min":32.106000000000002,"15min":32.106000000000002},"last":0.54900000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.0640000000000001,"5min":3.2610000000000001,"15min":2.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":24.253,"5min":24.253,"15min":24.253},"last":0.71099999999999997},{"interface":"front","average":{"1min":3.5,"5min":2.4079999999999999,"15min":1.974},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":15.106999999999999,"5min":15.106999999999999,"15min":15.106999999999999},"last":3.2069999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8760000000000003,"5min":2.9369999999999998,"15min":2.2890000000000001},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.710000000000001,"5min":20.710000000000001,"15min":20.710000000000001},"last":1.7},{"interface":"front","average":{"1min":2.964,"5min":1.794,"15min":1.347},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":13.991,"5min":13.991,"15min":13.991},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":3.9729999999999999,"5min":2.6520000000000001,"15min":2.1200000000000001},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":18.670999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":6.2030000000000003},{"interface":"front","average":{"1min":3.585,"5min":2.1179999999999999,"15min":1.8720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":20.009,"5min":20.009,"15min":20.009},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":1.6160000000000001,"15min":1.429},"min":{"1min":0.36599999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.9260000000000002,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.3559999999999999,"15min":1.8200000000000001},"min":{"1min":0.47199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":18.591999999999999,"5min":18.591999999999999,"15min":18.591999999999999},"last":6.1619999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.7,"15min":1.4810000000000001},"min":{"1min":0.38700000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.9790000000000001,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.64100000000000001},{"interface":"front","average":{"1min":4.048,"5min":2.6110000000000002,"15min":1.994},"min":{"1min":0.51800000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":18.704999999999998,"5min":18.704999999999998,"15min":18.704999999999998},"last":6.1189999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":2.4660000000000002,"15min":1.8859999999999999},"min":{"1min":0.54900000000000004,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":18.640000000000001,"5min":18.640000000000001,"15min":18.640000000000001},"last":6.2350000000000003},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.54,"15min":1.298},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":10.35,"5min":10.35,"15min":10.35},"last":0.621}]}]},{"osd":2,"up_from":131,"seq":562640715881,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1075636,"kb_used_data":835500,"kb_used_omap":81,"kb_used_meta":240046,"kb_avail":92669516,"statfs":{"total":95995035648,"available":94893584384,"internally_reserved":0,"allocated":855552000,"data_stored":854609632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":245807322},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7810000000000001,"5min":4.9379999999999997,"15min":5.1580000000000004},"min":{"1min":0.27300000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.692,"5min":57.811999999999998,"15min":57.811999999999998},"last":10.56},{"interface":"front","average":{"1min":5.9059999999999997,"5min":5.968,"15min":5.5819999999999999},"min":{"1min":0.80600000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":13.865,"5min":57.387999999999998,"15min":57.387999999999998},"last":9.7799999999999994}]},{"osd":1,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3079999999999998,"15min":3.1850000000000001},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.3650000000000002,"5min":11.759,"15min":88.316000000000003},"last":2.0910000000000002},{"interface":"front","average":{"1min":3.8130000000000002,"5min":3.3490000000000002,"15min":2.9870000000000001},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.167,"5min":28.596,"15min":88.436999999999998},"last":8.4510000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":5.2830000000000004,"5min":3.992,"15min":3.3370000000000002},"min":{"1min":0.39800000000000002,"5min":0.223,"15min":0.223},"max":{"1min":20.41,"5min":21.210999999999999,"15min":88.653000000000006},"last":8.0709999999999997},{"interface":"front","average":{"1min":3.7839999999999998,"5min":2.734,"15min":2.6040000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":13.836,"5min":13.836,"15min":88.474000000000004},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":4.6219999999999999,"15min":2.2389999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.965,"5min":108.878,"15min":108.878},"last":10.433},{"interface":"front","average":{"1min":2.2690000000000001,"5min":4.8079999999999998,"15min":2.9969999999999999},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.2539999999999996,"5min":108.93000000000001,"15min":108.93000000000001},"last":2.0299999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":4.2210000000000001,"15min":2.8519999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.0869999999999997,"5min":97.322999999999993,"15min":97.322999999999993},"last":1.9910000000000001},{"interface":"front","average":{"1min":3.0720000000000001,"5min":4.4569999999999999,"15min":2.4889999999999999},"min":{"1min":0.34499999999999997,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":7.694,"5min":97.301000000000002,"15min":97.301000000000002},"last":7.9900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":3.8780000000000001,"15min":2.4049999999999998},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6669999999999998,"5min":85.034999999999997,"15min":88.606999999999999},"last":8.0370000000000008},{"interface":"front","average":{"1min":2.2349999999999999,"5min":3.395,"15min":1.863},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":8.1479999999999997,"5min":85.209000000000003,"15min":85.209000000000003},"last":0.42799999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.8,"15min":1.389},"min":{"1min":0.46200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":10.477},{"interface":"front","average":{"1min":2.9159999999999999,"5min":2.3570000000000002,"15min":2.2759999999999998},"min":{"1min":0.434,"5min":0.37,"15min":0.37},"max":{"1min":9.0419999999999998,"5min":9.5549999999999997,"15min":88.584000000000003},"last":2.0619999999999998}]}]},{"osd":3,"up_from":94,"seq":403726925966,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285132,"kb_used_data":883716,"kb_used_omap":7,"kb_used_meta":401080,"kb_avail":92460020,"statfs":{"total":95995035648,"available":94679060480,"internally_reserved":0,"allocated":904925184,"data_stored":904008576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":410706536},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":3.1699999999999999,"15min":2.2959999999999998},"min":{"1min":0.26100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.711,"5min":80.099000000000004,"15min":80.099000000000004},"last":8.3000000000000007},{"interface":"front","average":{"1min":3.698,"5min":3.0009999999999999,"15min":2.0649999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.337,"5min":80.150000000000006,"15min":80.150000000000006},"last":7.8079999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.5019999999999998,"5min":2.6970000000000001,"15min":1.6080000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20000000000000001},"max":{"1min":12.25,"5min":22.818000000000001,"15min":22.818000000000001},"last":8.4109999999999996},{"interface":"front","average":{"1min":2.9700000000000002,"5min":1.8779999999999999,"15min":1.2869999999999999},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":15.098000000000001,"5min":23.917999999999999,"15min":23.917999999999999},"last":0.91300000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2370000000000001,"5min":2.5800000000000001,"15min":2.4569999999999999},"min":{"1min":0.32900000000000001,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":12.339,"5min":20.431000000000001,"15min":125.446},"last":7.8339999999999996},{"interface":"front","average":{"1min":5.2999999999999998,"5min":4.9500000000000002,"15min":3.0310000000000001},"min":{"1min":0.40799999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.756,"5min":182.483,"15min":182.483},"last":2.1890000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2149999999999999,"15min":1.3919999999999999},"min":{"1min":0.371,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":12.289999999999999,"5min":18.757000000000001,"15min":18.757000000000001},"last":8.1509999999999998},{"interface":"front","average":{"1min":2.645,"5min":1.5149999999999999,"15min":1.196},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.131,"5min":12.131,"15min":12.131},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3520000000000001,"15min":1.127},"min":{"1min":0.39800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.949999999999999,"5min":12.127000000000001,"15min":12.127000000000001},"last":0.625},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.085,"15min":1.3300000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":2.1549999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":1.5269999999999999,"15min":1.175},"min":{"1min":0.38300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.897,"5min":12.16,"15min":12.16},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.399,"5min":2.0760000000000001,"15min":1.403},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":10.755000000000001},"last":2.1120000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.9929999999999999,"15min":1.6819999999999999},"min":{"1min":0.47299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7710000000000008,"5min":91.757999999999996,"15min":91.757999999999996},"last":2.2189999999999999},{"interface":"front","average":{"1min":3.1499999999999999,"5min":3.2869999999999999,"15min":1.8069999999999999},"min":{"1min":0.49099999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":12.32,"5min":91.709999999999994,"15min":91.709999999999994},"last":8.2010000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019165,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1468864,"kb_used_data":1163384,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92276288,"statfs":{"total":95995035648,"available":94490918912,"internally_reserved":0,"allocated":1191305216,"data_stored":1190806219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":5.4880000000000004,"15min":7.173},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.442,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.5299999999999998,"5min":6.2779999999999996,"15min":7.7779999999999996},"min":{"1min":0.35899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":13.193,"5min":110.33799999999999,"15min":110.33799999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1230000000000002,"15min":1.53},"min":{"1min":0.317,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0269999999999992,"5min":41.643000000000001,"15min":45.320999999999998},"last":0.316},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.9019999999999999,"15min":1.4279999999999999},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.4489999999999998,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:49:20 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":4.0970000000000004,"15min":3.2599999999999998},"min":{"1min":0.34300000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.9189999999999996,"5min":84.909000000000006,"15min":84.909000000000006},"last":17.093},{"interface":"front","average":{"1min":2.1600000000000001,"5min":3.9990000000000001,"15min":3.1419999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0609999999999999,"5min":83.840000000000003,"15min":83.840000000000003},"last":21.704000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":7.3099999999999996,"15min":3.3730000000000002},"min":{"1min":0.442,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":100.51600000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":8.1150000000000002,"5min":7.3609999999999998,"15min":3.4169999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":100.637,"5min":129.197,"15min":129.197},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.77800000000000002},"min":{"1min":0.26600000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.1550000000000002,"5min":5.8979999999999997,"15min":43.189999999999998},"last":0.379},{"interface":"front","average":{"1min":1.2769999999999999,"5min":0.83699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":6.9199999999999999,"5min":9.5190000000000001,"15min":42.450000000000003},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.91400000000000003,"15min":0.78500000000000003},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":10.952999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.72199999999999998,"15min":0.76700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":4.5359999999999996,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.432}]},{"osd":7,"last update":"Sat Sep 23 22:48:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.754,"15min":0.66900000000000004},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.464,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.68600000000000005,"15min":0.64600000000000002},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":2.5710000000000002,"5min":3.2519999999999998,"15min":5.9409999999999998},"last":0.64400000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822937,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1623916,"kb_used_data":1318500,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92121236,"statfs":{"total":95995035648,"available":94332145664,"internally_reserved":0,"allocated":1350144000,"data_stored":1349660621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":5.4729999999999999,"15min":3.0289999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.344000000000001,"5min":121.319,"15min":121.319},"last":60.084000000000003},{"interface":"front","average":{"1min":3.8860000000000001,"5min":5.4950000000000001,"15min":2.9289999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":37.470999999999997,"5min":121.075,"15min":121.075},"last":59.997}]},{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.7290000000000001,"15min":1.762},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":6.7779999999999996,"5min":28.658999999999999,"15min":149.203},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.165,"15min":1.9339999999999999},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.1310000000000002,"5min":28.664999999999999,"15min":149.554},"last":0.52500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":3.4550000000000001,"15min":6.7249999999999996},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":7.9960000000000004,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":3.5169999999999999,"15min":6.7199999999999998},"min":{"1min":0.36199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0430000000000001,"5min":158.91399999999999,"15min":158.91399999999999},"last":0.441}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.4409999999999998,"5min":8.7390000000000008,"15min":4.2290000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":37.427999999999997,"5min":257.05000000000001,"15min":257.05000000000001},"last":90.212000000000003},{"interface":"front","average":{"1min":5.4939999999999998,"5min":8.484,"15min":4.0819999999999999},"min":{"1min":0.45800000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":37.277999999999999,"5min":257.35000000000002,"15min":257.35000000000002},"last":90.472999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:49:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.70099999999999996,"15min":0.60999999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":1.893,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.375},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.77000000000000002,"15min":0.61899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.9590000000000001,"5min":10.535,"15min":10.535},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.89400000000000002,"15min":1.391},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":1.9410000000000001,"5min":12.615,"15min":140.07499999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.016,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":2.4609999999999999,"5min":12.74,"15min":139.739},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:48:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.82399999999999995,"15min":0.70599999999999996},"min":{"1min":0.33400000000000002,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.464,"5min":7.0010000000000003,"15min":7.9809999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.74299999999999999,"15min":0.67000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":1.9650000000000001,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.498}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1149829120,"data_stored":1149829120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1026752512,"data_stored":1026752512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":848498688,"data_stored":848494616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":901709824,"data_stored":901709824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1181745152,"data_stored":1181745152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1338445824,"data_stored":1338441752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":879562752,"data_stored":879558680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:49:52.981 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:49:53.059 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:53.089 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:53.140 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:53.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:53.251 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:53.371 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:53.526 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:53.527 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:53.529 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:49:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:49:53] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T22:49:53.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:53 smithi052 ceph-mon[180979]: pgmap v695: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 2225/102177 objects degraded (2.178%); 2205/102177 objects misplaced (2.158%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:49:53.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:53.533 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:53.533 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:53 smithi052 ceph-mon[187743]: pgmap v695: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 2225/102177 objects degraded (2.178%); 2205/102177 objects misplaced (2.158%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:49:53.533 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:53.533 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:53.549 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:53.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:53.655 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:53.658 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:49:53.658 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:49:53.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:53 smithi151 ceph-mon[159210]: pgmap v695: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 2225/102177 objects degraded (2.178%); 2205/102177 objects misplaced (2.158%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:49:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:54.102 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:54.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:54.211 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:49:54.241 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:54.291 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:49:54.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:54.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:49:54.399 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:54.469 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:54.517 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:49:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:54.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:49:54.620 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:54.661 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:54.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:54.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:54.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:54.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:54.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:54.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:54.721 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:54.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:54.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:54.831 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:49:54.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:49:54.864 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:49:54.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:49:54.867 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:54.924 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:49:54.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:54.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:49:55.031 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:55.151 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:55.235 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:49:55.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:55.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:55.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:55.608 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:55 smithi052 ceph-mon[180979]: pgmap v696: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 2225/99993 objects degraded (2.225%); 2205/99993 objects misplaced (2.205%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:49:55.608 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2225/99993 objects degraded (2.225%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:55.608 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:55 smithi052 ceph-mon[187743]: pgmap v696: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 2225/99993 objects degraded (2.225%); 2205/99993 objects misplaced (2.205%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:49:55.608 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2225/99993 objects degraded (2.225%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:55.616 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:55 smithi151 ceph-mon[159210]: pgmap v696: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 2.0 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 2225/99993 objects degraded (2.225%); 2205/99993 objects misplaced (2.205%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:49:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2225/99993 objects degraded (2.225%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:49:56.222 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:56.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:56.332 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:49:56.409 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:56.990 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:57.241 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:49:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:57.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:49:57.359 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:57.412 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:57.473 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:57.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:57.545 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:49:57.583 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:49:57.660 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:57.681 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:49:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:57 smithi151 ceph-mon[159210]: pgmap v697: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 2292/93939 objects degraded (2.440%); 2022/93939 objects misplaced (2.152%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:49:57.735 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:49:57.791 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:49:57.825 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:57 smithi052 ceph-mon[180979]: pgmap v697: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 2292/93939 objects degraded (2.440%); 2022/93939 objects misplaced (2.152%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:49:57.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:57 smithi052 ceph-mon[187743]: pgmap v697: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 2292/93939 objects degraded (2.440%); 2022/93939 objects misplaced (2.152%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:49:57.844 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:49:57.871 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:49:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:57.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:49:57.884 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:49:57.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:49:57.952 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:49:57.981 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:49:58.008 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:58.013 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:49:58.030 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.030 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.030 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.031 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.031 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.031 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.031 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.031 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:49:58.062 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:49:58.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:49:58.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:49:58.168 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:49:58.304 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:49:58.600 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:49:58.646 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3918317831' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:58.646 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2472207680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:58.646 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:58 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:58.646 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3918317831' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:58.646 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2472207680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:58.646 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:58 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:58.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3918317831' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:49:58.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2472207680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:58.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:58 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:49:59.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[187743]: pgmap v698: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2292/91887 objects degraded (2.494%); 2022/91887 objects misplaced (2.201%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[187743]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[187743]: failed to encode map e256 with expected crc 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[187743]: failed to encode map e256 with expected crc 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[180979]: pgmap v698: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2292/91887 objects degraded (2.494%); 2022/91887 objects misplaced (2.201%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:59.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:49:59.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:59.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[180979]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T22:49:59.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[180979]: failed to encode map e256 with expected crc 2023-09-23T22:49:59.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:49:59 smithi052 ceph-mon[180979]: failed to encode map e256 with expected crc 2023-09-23T22:49:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:59 smithi151 ceph-mon[159210]: pgmap v698: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2292/91887 objects degraded (2.494%); 2022/91887 objects misplaced (2.201%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:49:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:59 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:49:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:49:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:59 smithi151 ceph-mon[159210]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T22:49:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:59 smithi151 ceph-mon[159210]: failed to encode map e256 with expected crc 2023-09-23T22:49:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:49:59 smithi151 ceph-mon[159210]: failed to encode map e256 with expected crc 2023-09-23T22:50:00.209 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:00.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:00.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:00.356 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:00.409 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:00.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:00.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:00.543 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:00.607 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:00.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:00.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:00.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:00 smithi151 ceph-mon[159210]: failed to encode map e256 with expected crc 2023-09-23T22:50:00.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:00.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:00 smithi151 ceph-mon[159210]: Health detail: HEALTH_WARN Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized 2023-09-23T22:50:00.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:00 smithi151 ceph-mon[159210]: [WRN] PG_DEGRADED: Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized 2023-09-23T22:50:00.615 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:00 smithi151 ceph-mon[159210]: pg 8.f is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,1] 2023-09-23T22:50:00.715 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:00.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:00.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[180979]: failed to encode map e256 with expected crc 2023-09-23T22:50:00.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:00.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[180979]: Health detail: HEALTH_WARN Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized 2023-09-23T22:50:00.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[180979]: [WRN] PG_DEGRADED: Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized 2023-09-23T22:50:00.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[180979]: pg 8.f is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,1] 2023-09-23T22:50:00.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[187743]: failed to encode map e256 with expected crc 2023-09-23T22:50:00.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:00.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[187743]: Health detail: HEALTH_WARN Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized 2023-09-23T22:50:00.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[187743]: [WRN] PG_DEGRADED: Degraded data redundancy: 2292/91887 objects degraded (2.494%), 1 pg degraded, 1 pg undersized 2023-09-23T22:50:00.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:00 smithi052 ceph-mon[187743]: pg 8.f is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,1] 2023-09-23T22:50:01.560 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:01.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:01.677 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:01.689 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:01 smithi052 ceph-mon[180979]: pgmap v700: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 2292/90846 objects degraded (2.523%); 2022/90846 objects misplaced (2.226%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:50:01.690 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:01 smithi052 ceph-mon[187743]: pgmap v700: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 2292/90846 objects degraded (2.523%); 2022/90846 objects misplaced (2.226%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:50:01.713 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:01.770 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:01.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:01.878 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:01.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:01 smithi151 ceph-mon[159210]: pgmap v700: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 2292/90846 objects degraded (2.523%); 2022/90846 objects misplaced (2.226%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:50:01.920 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:01.977 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:01.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:02.085 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:02.114 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:02.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:50:02.176 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:02.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:02.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:02.283 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:02.321 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:02.368 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:02.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:02.439 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:50:02.439 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:50:02.439 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:50:02.439 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:50:02.439 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:50:02.474 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:02.577 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:02 smithi052 ceph-mon[187743]: pgmap v701: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 2381/85194 objects degraded (2.795%); 1825/85194 objects misplaced (2.142%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:50:02.579 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:02 smithi052 ceph-mon[180979]: pgmap v701: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 2381/85194 objects degraded (2.795%); 1825/85194 objects misplaced (2.142%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:50:02.608 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:02.746 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:02.747 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2381/85194 objects degraded (2.795%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":935,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":256,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":7,"osd_in_since":1695509092,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":28398,"data_bytes":1860048824,"bytes_used":7966031872,"bytes_avail":663999217664,"bytes_total":671965249536,"degraded_objects":2381,"degraded_total":85194,"degraded_ratio":0.027947977557104965,"misplaced_objects":1825,"misplaced_total":85194,"misplaced_ratio":0.02142169636359368,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1467299,"recovering_keys_per_sec":0,"num_objects_recovered":224,"num_bytes_recovered":14680064,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":565},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":26,"modified":"2023-09-23T22:48:52.236301+0000","services":{}},"progress_events":{"3d9c7ac0-c827-48ce-9357-8c3e026dd16f":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:50:02.752 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:02.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:02.864 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:02.902 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:02 smithi151 ceph-mon[159210]: pgmap v701: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 2381/85194 objects degraded (2.795%); 1825/85194 objects misplaced (2.142%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:50:02.909 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:02.965 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:02.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:03.073 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:03.108 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:03.153 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:03.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:03.259 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:03.286 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:03.354 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:03.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:03.462 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:03.549 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:03.607 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:50:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:50:03] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.33.4" 2023-09-23T22:50:03.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/327847160' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:03.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/327847160' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:03.726 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:50:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/327847160' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:04.348 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:50:04.382 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:50:04.383 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:50:04.383 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:04.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:04.491 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:04.583 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:04.683 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:04.797 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:04.825 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:04.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:04.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:04.944 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:04.972 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:05.021 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:05.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:05.128 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:05.157 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:05.208 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:05.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:05.320 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:05.415 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[187743]: pgmap v702: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 2381/82971 objects degraded (2.870%); 1825/82971 objects misplaced (2.200%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:50:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2381/82971 objects degraded (2.870%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:05.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[180979]: pgmap v702: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 2381/82971 objects degraded (2.870%); 1825/82971 objects misplaced (2.200%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:50:05.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:05.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:05.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2381/82971 objects degraded (2.870%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:05 smithi151 ceph-mon[159210]: pgmap v702: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 2381/82971 objects degraded (2.870%); 1825/82971 objects misplaced (2.200%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:50:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2381/82971 objects degraded (2.870%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:06.552 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:06.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:06.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:06.736 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:07.210 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:07.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:07.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:07.318 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:07.353 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:07.423 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:07.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:07.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:07.447 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:07.531 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:07.645 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:07 smithi151 ceph-mon[159210]: pgmap v703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 2457/76461 objects degraded (3.213%); 1639/76461 objects misplaced (2.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:07.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:07 smithi052 ceph-mon[187743]: pgmap v703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 2457/76461 objects degraded (3.213%); 1639/76461 objects misplaced (2.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:07.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:07 smithi052 ceph-mon[180979]: pgmap v703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 2457/76461 objects degraded (3.213%); 1639/76461 objects misplaced (2.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:07.832 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:07.994 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:50:08.002 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:08.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:08.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:08.043 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:50:08.088 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:50:08.121 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:08.147 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:50:08.150 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:08.197 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:50:08.198 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:08.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:08.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:08.247 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:50:08.305 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:08.310 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:50:08.333 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:08.339 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:50:08.350 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.351 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:50:08.379 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:08.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:08.484 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:08.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:08.567 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:08.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:08.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:08.661 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1676598491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:08.666 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1676598491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:08.671 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:08.700 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:08.759 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:08.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1676598491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:08.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:08.875 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:08.944 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:09.642 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:50:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:09 smithi151 ceph-mon[159210]: pgmap v704: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2457/74232 objects degraded (3.310%); 1639/74232 objects misplaced (2.208%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3137806817' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:09.991 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:09 smithi052 ceph-mon[180979]: pgmap v704: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2457/74232 objects degraded (3.310%); 1639/74232 objects misplaced (2.208%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:09.991 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3137806817' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:09.991 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:09.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:09 smithi052 ceph-mon[187743]: pgmap v704: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2457/74232 objects degraded (3.310%); 1639/74232 objects misplaced (2.208%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:09.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3137806817' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:09.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:10.497 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:10.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:10.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:10.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:10.686 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: failed to encode map e257 with expected crc 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: failed to encode map e257 with expected crc 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2457/74232 objects degraded (3.310%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:10 smithi151 ceph-mon[159210]: pgmap v706: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 2457/73065 objects degraded (3.363%); 1639/73065 objects misplaced (2.243%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: failed to encode map e257 with expected crc 2023-09-23T22:50:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: failed to encode map e257 with expected crc 2023-09-23T22:50:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:50:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T22:50:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2457/74232 objects degraded (3.310%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[180979]: pgmap v706: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 2457/73065 objects degraded (3.363%); 1639/73065 objects misplaced (2.243%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: failed to encode map e257 with expected crc 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: failed to encode map e257 with expected crc 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2457/74232 objects degraded (3.310%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:10.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:10 smithi052 ceph-mon[187743]: pgmap v706: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 2457/73065 objects degraded (3.363%); 1639/73065 objects misplaced (2.243%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:50:11.489 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:11.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:11.610 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:11.677 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:12.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:12.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:12.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:12.144 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:12.170 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:12.174 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:50:12.174 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:50:12.230 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:12.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:12.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:12.336 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:12.440 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:12.539 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:12.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:12.643 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:12.681 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:12.716 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:50:12.756 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:12.882 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:12.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:12.988 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:13.024 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:13.097 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:13.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:13.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:13.203 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:13.240 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:13.302 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:13.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:13.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:13.410 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:13.441 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:13.502 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:13.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:13.523 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:50:13.533 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:50:13.603 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:50:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:50:13] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-09-23T22:50:13.603 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:13 smithi052 ceph-mon[187743]: pgmap v707: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 2528/67482 objects degraded (3.746%); 1446/67482 objects misplaced (2.143%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:50:13.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:13 smithi052 ceph-mon[180979]: pgmap v707: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 2528/67482 objects degraded (3.746%); 1446/67482 objects misplaced (2.143%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:50:13.609 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:13.650 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:13 smithi151 ceph-mon[159210]: pgmap v707: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 2528/67482 objects degraded (3.746%); 1446/67482 objects misplaced (2.143%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:50:13.907 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:50:14.191 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:14.201 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":707,"stamp":"2023-09-23T22:50:12.281247+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1473124280,"num_objects":22494,"num_object_clones":0,"num_object_copies":67482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2528,"num_objects_misplaced":1446,"num_objects_unfound":0,"num_objects_dirty":22494,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":182688,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2679,"num_bytes_recovered":173484106,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128022,"ondisk_log_size":128022,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6699508,"kb_used_data":4669436,"kb_used_omap":129,"kb_used_meta":2029822,"kb_avail":649516556,"statfs":{"total":671965249536,"available":665104953344,"internally_reserved":0,"allocated":4781502464,"data_stored":4776240011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2078538461},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-386924544,"num_objects":-5904,"num_object_clones":0,"num_object_copies":-17712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":147,"num_objects_misplaced":-379,"num_objects_unfound":0,"num_objects_dirty":-5904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5904,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":15204352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009218"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":254,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573078+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:49:58.573078+0000","last_peered":"2023-09-23T22:49:58.573078+0000","last_clean":"2023-09-23T22:49:58.573078+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:49:58.573078+0000","last_undegraded":"2023-09-23T22:49:58.573078+0000","last_fullsized":"2023-09-23T22:49:58.573078+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":229,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.566319+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:49:58.566319+0000","last_peered":"2023-09-23T22:49:58.566319+0000","last_clean":"2023-09-23T22:49:58.566319+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:49:58.566319+0000","last_undegraded":"2023-09-23T22:49:58.566319+0000","last_fullsized":"2023-09-23T22:49:58.566319+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.565740+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:49:58.565740+0000","last_peered":"2023-09-23T22:49:58.565740+0000","last_clean":"2023-09-23T22:49:58.565740+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:49:58.565740+0000","last_undegraded":"2023-09-23T22:49:58.565740+0000","last_fullsized":"2023-09-23T22:49:58.565740+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":223,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.482245+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:49:47.482245+0000","last_peered":"2023-09-23T22:49:47.482245+0000","last_clean":"2023-09-23T22:49:47.482245+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:49:47.482245+0000","last_undegraded":"2023-09-23T22:49:47.482245+0000","last_fullsized":"2023-09-23T22:49:47.482245+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":289,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625988+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:50:09.625988+0000","last_peered":"2023-09-23T22:50:09.625988+0000","last_clean":"2023-09-23T22:50:09.625988+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:50:09.625988+0000","last_undegraded":"2023-09-23T22:50:09.625988+0000","last_fullsized":"2023-09-23T22:50:09.625988+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.575300+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:49:58.575300+0000","last_peered":"2023-09-23T22:49:58.575300+0000","last_clean":"2023-09-23T22:49:58.575300+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:49:58.575300+0000","last_undegraded":"2023-09-23T22:49:58.575300+0000","last_fullsized":"2023-09-23T22:49:58.575300+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":155,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.478210+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:49:47.478210+0000","last_peered":"2023-09-23T22:49:47.478210+0000","last_clean":"2023-09-23T22:49:47.478210+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:49:47.478210+0000","last_undegraded":"2023-09-23T22:49:47.478210+0000","last_fullsized":"2023-09-23T22:49:47.478210+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":257,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.567184+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:49:58.567184+0000","last_peered":"2023-09-23T22:49:58.567184+0000","last_clean":"2023-09-23T22:49:58.567184+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:49:58.567184+0000","last_undegraded":"2023-09-23T22:49:58.567184+0000","last_fullsized":"2023-09-23T22:49:58.567184+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.566474+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:49:58.566474+0000","last_peered":"2023-09-23T22:49:58.566474+0000","last_clean":"2023-09-23T22:49:58.566474+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:49:58.566474+0000","last_undegraded":"2023-09-23T22:49:58.566474+0000","last_fullsized":"2023-09-23T22:49:58.566474+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":215,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.565667+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:49:58.565667+0000","last_peered":"2023-09-23T22:49:58.565667+0000","last_clean":"2023-09-23T22:49:58.565667+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:49:58.565667+0000","last_undegraded":"2023-09-23T22:49:58.565667+0000","last_fullsized":"2023-09-23T22:49:58.565667+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":132,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:06.001723+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:50:06.001723+0000","last_peered":"2023-09-23T22:50:06.001723+0000","last_clean":"2023-09-23T22:50:06.001723+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:50:06.001723+0000","last_undegraded":"2023-09-23T22:50:06.001723+0000","last_fullsized":"2023-09-23T22:50:06.001723+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.574845+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:49:58.574845+0000","last_peered":"2023-09-23T22:49:58.574845+0000","last_clean":"2023-09-23T22:49:58.574845+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:49:58.574845+0000","last_undegraded":"2023-09-23T22:49:58.574845+0000","last_fullsized":"2023-09-23T22:49:58.574845+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":277,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573639+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:49:58.573639+0000","last_peered":"2023-09-23T22:49:58.573639+0000","last_clean":"2023-09-23T22:49:58.573639+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:49:58.573639+0000","last_undegraded":"2023-09-23T22:49:58.573639+0000","last_fullsized":"2023-09-23T22:49:58.573639+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573283+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:49:58.573283+0000","last_peered":"2023-09-23T22:49:58.573283+0000","last_clean":"2023-09-23T22:49:58.573283+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:49:58.573283+0000","last_undegraded":"2023-09-23T22:49:58.573283+0000","last_fullsized":"2023-09-23T22:49:58.573283+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":115,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573342+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:49:58.573342+0000","last_peered":"2023-09-23T22:49:58.573342+0000","last_clean":"2023-09-23T22:49:58.573342+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:49:58.573342+0000","last_undegraded":"2023-09-23T22:49:58.573342+0000","last_fullsized":"2023-09-23T22:49:58.573342+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":277,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.567213+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:49:58.567213+0000","last_peered":"2023-09-23T22:49:58.567213+0000","last_clean":"2023-09-23T22:49:58.567213+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:49:58.567213+0000","last_undegraded":"2023-09-23T22:49:58.567213+0000","last_fullsized":"2023-09-23T22:49:58.567213+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":211,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.480658+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:49:47.480658+0000","last_peered":"2023-09-23T22:49:47.480658+0000","last_clean":"2023-09-23T22:49:47.480658+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:49:47.480658+0000","last_undegraded":"2023-09-23T22:49:47.480658+0000","last_fullsized":"2023-09-23T22:49:47.480658+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":237,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.566385+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:49:58.566385+0000","last_peered":"2023-09-23T22:49:58.566385+0000","last_clean":"2023-09-23T22:49:58.566385+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:49:58.566385+0000","last_undegraded":"2023-09-23T22:49:58.566385+0000","last_fullsized":"2023-09-23T22:49:58.566385+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572925+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:49:58.572925+0000","last_peered":"2023-09-23T22:49:58.572925+0000","last_clean":"2023-09-23T22:49:58.572925+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:49:58.572925+0000","last_undegraded":"2023-09-23T22:49:58.572925+0000","last_fullsized":"2023-09-23T22:49:58.572925+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":212,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.482317+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:49:47.482317+0000","last_peered":"2023-09-23T22:49:47.482317+0000","last_clean":"2023-09-23T22:49:47.482317+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:49:47.482317+0000","last_undegraded":"2023-09-23T22:49:47.482317+0000","last_fullsized":"2023-09-23T22:49:47.482317+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":301,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.574737+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:49:58.574737+0000","last_peered":"2023-09-23T22:49:58.574737+0000","last_clean":"2023-09-23T22:49:58.574737+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:49:58.574737+0000","last_undegraded":"2023-09-23T22:49:58.574737+0000","last_fullsized":"2023-09-23T22:49:58.574737+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":257,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.632817+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:50:09.632817+0000","last_peered":"2023-09-23T22:50:09.632817+0000","last_clean":"2023-09-23T22:50:09.632817+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:50:09.632817+0000","last_undegraded":"2023-09-23T22:50:09.632817+0000","last_fullsized":"2023-09-23T22:50:09.632817+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"257'11373","reported_seq":11421,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.534129+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:50:10.534129+0000","last_peered":"2023-09-23T22:50:10.534129+0000","last_clean":"2023-09-23T22:50:10.534129+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:50:10.534129+0000","last_undegraded":"2023-09-23T22:50:10.534129+0000","last_fullsized":"2023-09-23T22:50:10.534129+0000","mapping_epoch":217,"log_start":"231'4000","ondisk_log_start":"231'4000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90374144,"num_objects":1379,"num_object_clones":0,"num_object_copies":4137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11373,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":126,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573707+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:49:58.573707+0000","last_peered":"2023-09-23T22:49:58.573707+0000","last_clean":"2023-09-23T22:49:58.573707+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:49:58.573707+0000","last_undegraded":"2023-09-23T22:49:58.573707+0000","last_fullsized":"2023-09-23T22:49:58.573707+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":221,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.574741+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:49:58.574741+0000","last_peered":"2023-09-23T22:49:58.574741+0000","last_clean":"2023-09-23T22:49:58.574741+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:49:58.574741+0000","last_undegraded":"2023-09-23T22:49:58.574741+0000","last_fullsized":"2023-09-23T22:49:58.574741+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"257'11232","reported_seq":11285,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.830243+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:50:10.830243+0000","last_peered":"2023-09-23T22:50:10.830243+0000","last_clean":"2023-09-23T22:50:10.830243+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:50:10.830243+0000","last_undegraded":"2023-09-23T22:50:10.830243+0000","last_fullsized":"2023-09-23T22:50:10.830243+0000","mapping_epoch":219,"log_start":"227'2600","ondisk_log_start":"227'2600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8632,"log_dups_size":0,"ondisk_log_size":8632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86638592,"num_objects":1322,"num_object_clones":0,"num_object_copies":3966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11232,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":136,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573737+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:49:58.573737+0000","last_peered":"2023-09-23T22:49:58.573737+0000","last_clean":"2023-09-23T22:49:58.573737+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:49:58.573737+0000","last_undegraded":"2023-09-23T22:49:58.573737+0000","last_fullsized":"2023-09-23T22:49:58.573737+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":237,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572884+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:49:58.572884+0000","last_peered":"2023-09-23T22:49:58.572884+0000","last_clean":"2023-09-23T22:49:58.572884+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:49:58.572884+0000","last_undegraded":"2023-09-23T22:49:58.572884+0000","last_fullsized":"2023-09-23T22:49:58.572884+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"256'11103","reported_seq":11151,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:07.791744+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:50:07.791744+0000","last_peered":"2023-09-23T22:50:07.791744+0000","last_clean":"2023-09-23T22:50:07.791744+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:50:07.791744+0000","last_undegraded":"2023-09-23T22:50:07.791744+0000","last_fullsized":"2023-09-23T22:50:07.791744+0000","mapping_epoch":219,"log_start":"229'3200","ondisk_log_start":"229'3200","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7903,"log_dups_size":0,"ondisk_log_size":7903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97845248,"num_objects":1493,"num_object_clones":0,"num_object_copies":4479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11103,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":67,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.575242+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:49:58.575242+0000","last_peered":"2023-09-23T22:49:58.575242+0000","last_clean":"2023-09-23T22:49:58.575242+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:49:58.575242+0000","last_undegraded":"2023-09-23T22:49:58.575242+0000","last_fullsized":"2023-09-23T22:49:58.575242+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.574923+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:49:58.574923+0000","last_peered":"2023-09-23T22:49:58.574923+0000","last_clean":"2023-09-23T22:49:58.574923+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:49:58.574923+0000","last_undegraded":"2023-09-23T22:49:58.574923+0000","last_fullsized":"2023-09-23T22:49:58.574923+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"257'11672","reported_seq":11730,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.967163+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:50:10.967163+0000","last_peered":"2023-09-23T22:50:10.967163+0000","last_clean":"2023-09-23T22:50:10.967163+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:50:10.967163+0000","last_undegraded":"2023-09-23T22:50:10.967163+0000","last_fullsized":"2023-09-23T22:50:10.967163+0000","mapping_epoch":226,"log_start":"231'4500","ondisk_log_start":"231'4500","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7172,"log_dups_size":0,"ondisk_log_size":7172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11672,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":109,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.567349+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:49:58.567349+0000","last_peered":"2023-09-23T22:49:58.567349+0000","last_clean":"2023-09-23T22:49:58.567349+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:49:58.567349+0000","last_undegraded":"2023-09-23T22:49:58.567349+0000","last_fullsized":"2023-09-23T22:49:58.567349+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":148,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.477777+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:49:47.477777+0000","last_peered":"2023-09-23T22:49:47.477777+0000","last_clean":"2023-09-23T22:49:47.477777+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:49:47.477777+0000","last_undegraded":"2023-09-23T22:49:47.477777+0000","last_fullsized":"2023-09-23T22:49:47.477777+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"257'11315","reported_seq":11365,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.982968+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:50:10.982968+0000","last_peered":"2023-09-23T22:50:10.982968+0000","last_clean":"2023-09-23T22:50:10.982968+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:50:10.982968+0000","last_undegraded":"2023-09-23T22:50:10.982968+0000","last_fullsized":"2023-09-23T22:50:10.982968+0000","mapping_epoch":217,"log_start":"231'4100","ondisk_log_start":"231'4100","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7215,"log_dups_size":0,"ondisk_log_size":7215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11315,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.568943+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:49:58.568943+0000","last_peered":"2023-09-23T22:49:58.568943+0000","last_clean":"2023-09-23T22:49:58.568943+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:49:58.568943+0000","last_undegraded":"2023-09-23T22:49:58.568943+0000","last_fullsized":"2023-09-23T22:49:58.568943+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573542+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:49:58.573542+0000","last_peered":"2023-09-23T22:49:58.573542+0000","last_clean":"2023-09-23T22:49:58.573542+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:49:58.573542+0000","last_undegraded":"2023-09-23T22:49:58.573542+0000","last_fullsized":"2023-09-23T22:49:58.573542+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"256'11324","reported_seq":11394,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:07.789856+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:50:07.789856+0000","last_peered":"2023-09-23T22:50:07.789856+0000","last_clean":"2023-09-23T22:50:07.789856+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:50:07.789856+0000","last_undegraded":"2023-09-23T22:50:07.789856+0000","last_fullsized":"2023-09-23T22:50:07.789856+0000","mapping_epoch":226,"log_start":"229'3400","ondisk_log_start":"229'3400","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7924,"log_dups_size":0,"ondisk_log_size":7924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94765056,"num_objects":1446,"num_object_clones":0,"num_object_copies":4338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":241,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.575048+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:49:58.575048+0000","last_peered":"2023-09-23T22:49:58.575048+0000","last_clean":"2023-09-23T22:49:58.575048+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:49:58.575048+0000","last_undegraded":"2023-09-23T22:49:58.575048+0000","last_fullsized":"2023-09-23T22:49:58.575048+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":252,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573809+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:49:58.573809+0000","last_peered":"2023-09-23T22:49:58.573809+0000","last_clean":"2023-09-23T22:49:58.573809+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:49:58.573809+0000","last_undegraded":"2023-09-23T22:49:58.573809+0000","last_fullsized":"2023-09-23T22:49:58.573809+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"257'11600","reported_seq":11654,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.502492+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:50:10.502492+0000","last_peered":"2023-09-23T22:50:10.502492+0000","last_clean":"2023-09-23T22:50:10.502492+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:50:10.502492+0000","last_undegraded":"2023-09-23T22:50:10.502492+0000","last_fullsized":"2023-09-23T22:50:10.502492+0000","mapping_epoch":219,"log_start":"231'4200","ondisk_log_start":"231'4200","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7400,"log_dups_size":0,"ondisk_log_size":7400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91619328,"num_objects":1398,"num_object_clones":0,"num_object_copies":4194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11600,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":265,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572769+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:49:58.572769+0000","last_peered":"2023-09-23T22:49:58.572769+0000","last_clean":"2023-09-23T22:49:58.572769+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:49:58.572769+0000","last_undegraded":"2023-09-23T22:49:58.572769+0000","last_fullsized":"2023-09-23T22:49:58.572769+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":240,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573791+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:49:58.573791+0000","last_peered":"2023-09-23T22:49:58.573791+0000","last_clean":"2023-09-23T22:49:58.573791+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:49:58.573791+0000","last_undegraded":"2023-09-23T22:49:58.573791+0000","last_fullsized":"2023-09-23T22:49:58.573791+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"257'11130","reported_seq":11195,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.791507+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:50:10.791507+0000","last_peered":"2023-09-23T22:50:10.791507+0000","last_clean":"2023-09-23T22:50:10.791507+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:50:10.791507+0000","last_undegraded":"2023-09-23T22:50:10.791507+0000","last_fullsized":"2023-09-23T22:50:10.791507+0000","mapping_epoch":226,"log_start":"227'2500","ondisk_log_start":"227'2500","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8630,"log_dups_size":0,"ondisk_log_size":8630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86245376,"num_objects":1316,"num_object_clones":0,"num_object_copies":3948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11130,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.566960+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:49:58.566960+0000","last_peered":"2023-09-23T22:49:58.566960+0000","last_clean":"2023-09-23T22:49:58.566960+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:49:58.566960+0000","last_undegraded":"2023-09-23T22:49:58.566960+0000","last_fullsized":"2023-09-23T22:49:58.566960+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":184,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.624243+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:50:09.624243+0000","last_peered":"2023-09-23T22:50:09.624243+0000","last_clean":"2023-09-23T22:50:09.624243+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:50:09.624243+0000","last_undegraded":"2023-09-23T22:50:09.624243+0000","last_fullsized":"2023-09-23T22:50:09.624243+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"256'11287","reported_seq":11335,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:07.809299+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:50:07.809299+0000","last_peered":"2023-09-23T22:50:07.809299+0000","last_clean":"2023-09-23T22:50:07.809299+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:50:07.809299+0000","last_undegraded":"2023-09-23T22:50:07.809299+0000","last_fullsized":"2023-09-23T22:50:07.809299+0000","mapping_epoch":219,"log_start":"229'3300","ondisk_log_start":"229'3300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7987,"log_dups_size":0,"ondisk_log_size":7987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101122048,"num_objects":1543,"num_object_clones":0,"num_object_copies":4629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11287,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":292,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.480590+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:49:47.480590+0000","last_peered":"2023-09-23T22:49:47.480590+0000","last_clean":"2023-09-23T22:49:47.480590+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:49:47.480590+0000","last_undegraded":"2023-09-23T22:49:47.480590+0000","last_fullsized":"2023-09-23T22:49:47.480590+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":177,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:06.001704+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:50:06.001704+0000","last_peered":"2023-09-23T22:50:06.001704+0000","last_clean":"2023-09-23T22:50:06.001704+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:50:06.001704+0000","last_undegraded":"2023-09-23T22:50:06.001704+0000","last_fullsized":"2023-09-23T22:50:06.001704+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":88,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573033+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:49:58.573033+0000","last_peered":"2023-09-23T22:49:58.573033+0000","last_clean":"2023-09-23T22:49:58.573033+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:49:58.573033+0000","last_undegraded":"2023-09-23T22:49:58.573033+0000","last_fullsized":"2023-09-23T22:49:58.573033+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"257'11394","reported_seq":11450,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.956710+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:50:10.956710+0000","last_peered":"2023-09-23T22:50:10.956710+0000","last_clean":"2023-09-23T22:50:10.956710+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:50:10.956710+0000","last_undegraded":"2023-09-23T22:50:10.956710+0000","last_fullsized":"2023-09-23T22:50:10.956710+0000","mapping_epoch":219,"log_start":"231'4200","ondisk_log_start":"231'4200","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7194,"log_dups_size":0,"ondisk_log_size":7194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87556096,"num_objects":1336,"num_object_clones":0,"num_object_copies":4008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11394,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":91,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.574937+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:49:58.574937+0000","last_peered":"2023-09-23T22:49:58.574937+0000","last_clean":"2023-09-23T22:49:58.574937+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:49:58.574937+0000","last_undegraded":"2023-09-23T22:49:58.574937+0000","last_fullsized":"2023-09-23T22:49:58.574937+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":253,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573329+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:49:58.573329+0000","last_peered":"2023-09-23T22:49:58.573329+0000","last_clean":"2023-09-23T22:49:58.573329+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:49:58.573329+0000","last_undegraded":"2023-09-23T22:49:58.573329+0000","last_fullsized":"2023-09-23T22:49:58.573329+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:04.683467+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:50:04.683467+0000","last_peered":"2023-09-23T22:50:04.683467+0000","last_clean":"2023-09-23T22:50:04.683467+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:50:04.683467+0000","last_undegraded":"2023-09-23T22:50:04.683467+0000","last_fullsized":"2023-09-23T22:50:04.683467+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"257'11555","reported_seq":11605,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.995946+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:50:10.995946+0000","last_peered":"2023-09-23T22:50:10.995946+0000","last_clean":"2023-09-23T22:50:10.995946+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:50:10.995946+0000","last_undegraded":"2023-09-23T22:50:10.995946+0000","last_fullsized":"2023-09-23T22:50:10.995946+0000","mapping_epoch":217,"log_start":"231'4400","ondisk_log_start":"231'4400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89980928,"num_objects":1373,"num_object_clones":0,"num_object_copies":4119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11555,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":248,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.587793+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:49:58.587793+0000","last_peered":"2023-09-23T22:49:58.587793+0000","last_clean":"2023-09-23T22:49:58.587793+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:49:58.587793+0000","last_undegraded":"2023-09-23T22:49:58.587793+0000","last_fullsized":"2023-09-23T22:49:58.587793+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.583851+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:49:58.583851+0000","last_peered":"2023-09-23T22:49:58.583851+0000","last_clean":"2023-09-23T22:49:58.583851+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:49:58.583851+0000","last_undegraded":"2023-09-23T22:49:58.583851+0000","last_fullsized":"2023-09-23T22:49:58.583851+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":261,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.478138+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:49:47.478138+0000","last_peered":"2023-09-23T22:49:47.478138+0000","last_clean":"2023-09-23T22:49:47.478138+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:49:47.478138+0000","last_undegraded":"2023-09-23T22:49:47.478138+0000","last_fullsized":"2023-09-23T22:49:47.478138+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"257'11261","reported_seq":11315,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:10.838615+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:50:10.838615+0000","last_peered":"2023-09-23T22:50:10.838615+0000","last_clean":"2023-09-23T22:50:10.838615+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:50:10.838615+0000","last_undegraded":"2023-09-23T22:50:10.838615+0000","last_fullsized":"2023-09-23T22:50:10.838615+0000","mapping_epoch":219,"log_start":"227'2600","ondisk_log_start":"227'2600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11261,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:04.683446+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:50:04.683446+0000","last_peered":"2023-09-23T22:50:04.683446+0000","last_clean":"2023-09-23T22:50:04.683446+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:50:04.683446+0000","last_undegraded":"2023-09-23T22:50:04.683446+0000","last_fullsized":"2023-09-23T22:50:04.683446+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":342,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.477299+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:49:47.477299+0000","last_peered":"2023-09-23T22:49:47.477299+0000","last_clean":"2023-09-23T22:49:47.477299+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:49:47.477299+0000","last_undegraded":"2023-09-23T22:49:47.477299+0000","last_fullsized":"2023-09-23T22:49:47.477299+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":188,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.565924+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:49:58.565924+0000","last_peered":"2023-09-23T22:49:58.565924+0000","last_clean":"2023-09-23T22:49:58.565924+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:49:58.565924+0000","last_undegraded":"2023-09-23T22:49:58.565924+0000","last_fullsized":"2023-09-23T22:49:58.565924+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.575300+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:49:58.575300+0000","last_peered":"2023-09-23T22:49:58.575300+0000","last_clean":"2023-09-23T22:49:58.575300+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:49:58.575300+0000","last_undegraded":"2023-09-23T22:49:58.575300+0000","last_fullsized":"2023-09-23T22:49:58.575300+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"257'11524","reported_seq":11582,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.651639+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:50:09.651639+0000","last_peered":"2023-09-23T22:50:09.651639+0000","last_clean":"2023-09-23T22:50:09.651639+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:50:09.651639+0000","last_undegraded":"2023-09-23T22:50:09.651639+0000","last_fullsized":"2023-09-23T22:50:09.651639+0000","mapping_epoch":226,"log_start":"224'1800","ondisk_log_start":"224'1800","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9724,"log_dups_size":0,"ondisk_log_size":9724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93323264,"num_objects":1424,"num_object_clones":0,"num_object_copies":4272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11524,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":221,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.477436+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:49:47.477436+0000","last_peered":"2023-09-23T22:49:47.477436+0000","last_clean":"2023-09-23T22:49:47.477436+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:49:47.477436+0000","last_undegraded":"2023-09-23T22:49:47.477436+0000","last_fullsized":"2023-09-23T22:49:47.477436+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":192,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.478499+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:49:47.478499+0000","last_peered":"2023-09-23T22:49:47.478499+0000","last_clean":"2023-09-23T22:49:47.478499+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:49:47.478499+0000","last_undegraded":"2023-09-23T22:49:47.478499+0000","last_fullsized":"2023-09-23T22:49:47.478499+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":179,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.482043+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:49:47.482043+0000","last_peered":"2023-09-23T22:49:47.482043+0000","last_clean":"2023-09-23T22:49:47.482043+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:49:47.482043+0000","last_undegraded":"2023-09-23T22:49:47.482043+0000","last_fullsized":"2023-09-23T22:49:47.482043+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"257'11805","reported_seq":17149,"reported_epoch":257,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:50:11.022125+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:50:11.022125+0000","last_peered":"2023-09-23T22:50:11.022125+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:50:11.022125+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"231'4600","ondisk_log_start":"231'4600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7205,"log_dups_size":0,"ondisk_log_size":7205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95879168,"num_objects":1463,"num_object_clones":0,"num_object_copies":4389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2528,"num_objects_misplaced":1446,"num_objects_unfound":0,"num_objects_dirty":1463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11805,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2644,"num_bytes_recovered":173277184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-2511},{"shards":"1,3,4","objects":1446}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":184,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.484390+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:49:47.484390+0000","last_peered":"2023-09-23T22:49:47.484390+0000","last_clean":"2023-09-23T22:49:47.484390+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:49:47.484390+0000","last_undegraded":"2023-09-23T22:49:47.484390+0000","last_fullsized":"2023-09-23T22:49:47.484390+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.584986+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:49:58.584986+0000","last_peered":"2023-09-23T22:49:58.584986+0000","last_clean":"2023-09-23T22:49:58.584986+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:49:58.584986+0000","last_undegraded":"2023-09-23T22:49:58.584986+0000","last_fullsized":"2023-09-23T22:49:58.584986+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":184,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.478042+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:49:47.478042+0000","last_peered":"2023-09-23T22:49:47.478042+0000","last_clean":"2023-09-23T22:49:47.478042+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:49:47.478042+0000","last_undegraded":"2023-09-23T22:49:47.478042+0000","last_fullsized":"2023-09-23T22:49:47.478042+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"256'11435","reported_seq":11483,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:07.763580+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:50:07.763580+0000","last_peered":"2023-09-23T22:50:07.763580+0000","last_clean":"2023-09-23T22:50:07.763580+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:50:07.763580+0000","last_undegraded":"2023-09-23T22:50:07.763580+0000","last_fullsized":"2023-09-23T22:50:07.763580+0000","mapping_epoch":219,"log_start":"229'3500","ondisk_log_start":"229'3500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7935,"log_dups_size":0,"ondisk_log_size":7935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11435,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":297,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.579847+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:49:58.579847+0000","last_peered":"2023-09-23T22:49:58.579847+0000","last_clean":"2023-09-23T22:49:58.579847+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:49:58.579847+0000","last_undegraded":"2023-09-23T22:49:58.579847+0000","last_fullsized":"2023-09-23T22:49:58.579847+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":55,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.575205+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:49:58.575205+0000","last_peered":"2023-09-23T22:49:58.575205+0000","last_clean":"2023-09-23T22:49:58.575205+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:49:58.575205+0000","last_undegraded":"2023-09-23T22:49:58.575205+0000","last_fullsized":"2023-09-23T22:49:58.575205+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":283,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:05.997424+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:50:05.997424+0000","last_peered":"2023-09-23T22:50:05.997424+0000","last_clean":"2023-09-23T22:50:05.997424+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:50:05.997424+0000","last_undegraded":"2023-09-23T22:50:05.997424+0000","last_fullsized":"2023-09-23T22:50:05.997424+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"257'11418","reported_seq":11468,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.698968+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:50:09.698968+0000","last_peered":"2023-09-23T22:50:09.698968+0000","last_clean":"2023-09-23T22:50:09.698968+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:50:09.698968+0000","last_undegraded":"2023-09-23T22:50:09.698968+0000","last_fullsized":"2023-09-23T22:50:09.698968+0000","mapping_epoch":217,"log_start":"224'1700","ondisk_log_start":"224'1700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9718,"log_dups_size":0,"ondisk_log_size":9718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96141336,"num_objects":1468,"num_object_clones":0,"num_object_copies":4404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1468,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11418,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":289,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.575290+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:49:58.575290+0000","last_peered":"2023-09-23T22:49:58.575290+0000","last_clean":"2023-09-23T22:49:58.575290+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:49:58.575290+0000","last_undegraded":"2023-09-23T22:49:58.575290+0000","last_fullsized":"2023-09-23T22:49:58.575290+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":151,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.482147+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:49:47.482147+0000","last_peered":"2023-09-23T22:49:47.482147+0000","last_clean":"2023-09-23T22:49:47.482147+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:49:47.482147+0000","last_undegraded":"2023-09-23T22:49:47.482147+0000","last_fullsized":"2023-09-23T22:49:47.482147+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":207,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572810+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:49:58.572810+0000","last_peered":"2023-09-23T22:49:58.572810+0000","last_clean":"2023-09-23T22:49:58.572810+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:49:58.572810+0000","last_undegraded":"2023-09-23T22:49:58.572810+0000","last_fullsized":"2023-09-23T22:49:58.572810+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":187,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.477923+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:49:47.477923+0000","last_peered":"2023-09-23T22:49:47.477923+0000","last_clean":"2023-09-23T22:49:47.477923+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:49:47.477923+0000","last_undegraded":"2023-09-23T22:49:47.477923+0000","last_fullsized":"2023-09-23T22:49:47.477923+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.584980+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:49:58.584980+0000","last_peered":"2023-09-23T22:49:58.584980+0000","last_clean":"2023-09-23T22:49:58.584980+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:49:58.584980+0000","last_undegraded":"2023-09-23T22:49:58.584980+0000","last_fullsized":"2023-09-23T22:49:58.584980+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":67,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573553+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:49:58.573553+0000","last_peered":"2023-09-23T22:49:58.573553+0000","last_clean":"2023-09-23T22:49:58.573553+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:49:58.573553+0000","last_undegraded":"2023-09-23T22:49:58.573553+0000","last_fullsized":"2023-09-23T22:49:58.573553+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573385+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:49:58.573385+0000","last_peered":"2023-09-23T22:49:58.573385+0000","last_clean":"2023-09-23T22:49:58.573385+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:49:58.573385+0000","last_undegraded":"2023-09-23T22:49:58.573385+0000","last_fullsized":"2023-09-23T22:49:58.573385+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":67,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573861+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:49:58.573861+0000","last_peered":"2023-09-23T22:49:58.573861+0000","last_clean":"2023-09-23T22:49:58.573861+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:49:58.573861+0000","last_undegraded":"2023-09-23T22:49:58.573861+0000","last_fullsized":"2023-09-23T22:49:58.573861+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573118+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:49:58.573118+0000","last_peered":"2023-09-23T22:49:58.573118+0000","last_clean":"2023-09-23T22:49:58.573118+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:49:58.573118+0000","last_undegraded":"2023-09-23T22:49:58.573118+0000","last_fullsized":"2023-09-23T22:49:58.573118+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":251,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:50:06.001767+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:50:06.001767+0000","last_peered":"2023-09-23T22:50:06.001767+0000","last_clean":"2023-09-23T22:50:06.001767+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:50:06.001767+0000","last_undegraded":"2023-09-23T22:50:06.001767+0000","last_fullsized":"2023-09-23T22:50:06.001767+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":289,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625660+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:50:09.625660+0000","last_peered":"2023-09-23T22:50:09.625660+0000","last_clean":"2023-09-23T22:50:09.625660+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:50:09.625660+0000","last_undegraded":"2023-09-23T22:50:09.625660+0000","last_fullsized":"2023-09-23T22:50:09.625660+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":200,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:49:47.478290+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:49:47.478290+0000","last_peered":"2023-09-23T22:49:47.478290+0000","last_clean":"2023-09-23T22:49:47.478290+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:49:47.478290+0000","last_undegraded":"2023-09-23T22:49:47.478290+0000","last_fullsized":"2023-09-23T22:49:47.478290+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572680+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:49:58.572680+0000","last_peered":"2023-09-23T22:49:58.572680+0000","last_clean":"2023-09-23T22:49:58.572680+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:49:58.572680+0000","last_undegraded":"2023-09-23T22:49:58.572680+0000","last_fullsized":"2023-09-23T22:49:58.572680+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1472528408,"num_objects":22470,"num_object_clones":0,"num_object_copies":67410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2528,"num_objects_misplaced":1446,"num_objects_unfound":0,"num_objects_dirty":22470,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182428,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2647,"num_bytes_recovered":173473792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4705628160,"data_stored":4705615944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127828,"ondisk_log_size":127828,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430486,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626714,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":832604,"kb_used_data":527188,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92912548,"statfs":{"total":95995035648,"available":95142449152,"internally_reserved":0,"allocated":539840512,"data_stored":539337231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:17 2023","interfaces":[{"interface":"back","average":{"1min":8.891,"5min":3.4860000000000002,"15min":2.0209999999999999},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":117.45999999999999,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.318},{"interface":"front","average":{"1min":8.6750000000000007,"5min":3.4220000000000002,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":117.48699999999999,"5min":117.48699999999999,"15min":117.48699999999999},"last":1.7649999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":4.2599999999999998,"5min":2.577,"15min":2.4870000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":31.286000000000001,"5min":31.286000000000001,"15min":153.78399999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.5350000000000001,"5min":4.3220000000000001,"15min":3.0830000000000002},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":31.512,"5min":185.411,"15min":185.411},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":2.8719999999999999,"15min":3.1640000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":33.174999999999997,"5min":33.174999999999997,"15min":100.785},"last":0.64800000000000002},{"interface":"front","average":{"1min":3.9079999999999999,"5min":4.5099999999999998,"15min":3.6059999999999999},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":33.310000000000002,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":5.9370000000000003,"15min":3.4489999999999998},"min":{"1min":0.32800000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.742999999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.753},{"interface":"front","average":{"1min":2.5230000000000001,"5min":4.4880000000000004,"15min":3.0670000000000002},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":10.773999999999999,"5min":128.19800000000001,"15min":128.19800000000001},"last":1.5009999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.79800000000000004,"15min":0.67600000000000005},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6070000000000002,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.85399999999999998,"15min":0.68600000000000005},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":0.69199999999999995,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.73999999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.5620000000000003,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.93400000000000005,"15min":0.754},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.4429999999999996,"5min":9.7899999999999991,"15min":18.625},"last":0.58999999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.98799999999999999,"15min":0.748},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":1.9990000000000001,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.75600000000000001,"15min":0.73399999999999999},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":2.8650000000000002,"5min":5.1790000000000003,"15min":16.661999999999999},"last":0.91900000000000004}]}]},{"osd":1,"up_from":84,"seq":360777253024,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":890992,"kb_used_data":596840,"kb_used_omap":9,"kb_used_meta":294134,"kb_avail":92854160,"statfs":{"total":95995035648,"available":95082659840,"internally_reserved":0,"allocated":611164160,"data_stored":610237609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":301194217},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:18 2023","interfaces":[{"interface":"back","average":{"1min":15.766999999999999,"5min":7.5789999999999997,"15min":5.3049999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":82.903999999999996,"5min":101.605,"15min":101.605},"last":2.4009999999999998},{"interface":"front","average":{"1min":13.898,"5min":7.5730000000000004,"15min":5.5339999999999998},"min":{"1min":0.441,"5min":0.214,"15min":0.214},"max":{"1min":82.748999999999995,"5min":101.892,"15min":101.892},"last":0.81000000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:49:18 2023","interfaces":[{"interface":"back","average":{"1min":5.2960000000000003,"5min":3.2799999999999998,"15min":2.3029999999999999},"min":{"1min":0.20000000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":37.869999999999997,"5min":37.869999999999997,"15min":37.889000000000003},"last":7.3289999999999997},{"interface":"front","average":{"1min":6.4699999999999998,"5min":3.931,"15min":2.4119999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":37.957000000000001,"5min":37.957000000000001,"15min":37.957000000000001},"last":17.890999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":9.3239999999999998,"5min":3.8450000000000002,"15min":2.149},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":87.787999999999997,"5min":87.787999999999997,"15min":87.787999999999997},"last":2.1469999999999998},{"interface":"front","average":{"1min":10.641999999999999,"5min":4.7679999999999998,"15min":2.5299999999999998},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":89.861999999999995,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.89700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7949999999999999,"5min":3.1579999999999999,"15min":1.9490000000000001},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":11.432,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.91900000000000004},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.3889999999999998,"15min":1.7649999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":10.619,"5min":78.963999999999999,"15min":78.963999999999999},"last":2.1070000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.266,"15min":1.53},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.4390000000000001,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.86499999999999999},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.2949999999999999,"15min":1.8160000000000001},"min":{"1min":0.35299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.458,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.93999999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.4609999999999999,"15min":1.5960000000000001},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.317},"max":{"1min":10.65,"5min":79.311000000000007,"15min":79.311000000000007},"last":2.3210000000000002},{"interface":"front","average":{"1min":1.5,"5min":2.2829999999999999,"15min":1.4630000000000001},"min":{"1min":0.36299999999999999,"5min":0.313,"15min":0.313},"max":{"1min":7.3819999999999997,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.84199999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":3.3530000000000002,"5min":2.6560000000000001,"15min":2.1869999999999998},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":11.481999999999999,"5min":29.539000000000001,"15min":159.018},"last":0.96599999999999997},{"interface":"front","average":{"1min":3.335,"5min":2.048,"15min":2.0960000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":10.676,"5min":10.676,"15min":158.96000000000001},"last":2.2730000000000001}]}]},{"osd":0,"up_from":219,"seq":940597837872,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951520,"kb_used_data":773144,"kb_used_omap":31,"kb_used_meta":178336,"kb_avail":92793632,"statfs":{"total":95995035648,"available":95020679168,"internally_reserved":0,"allocated":791699456,"data_stored":790721775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":182617010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.319,"5min":4.0780000000000003,"15min":3.8340000000000001},"min":{"1min":0.24299999999999999,"5min":0.216,"15min":0.216},"max":{"1min":21.512,"5min":32.344999999999999,"15min":32.344999999999999},"last":6.181},{"interface":"front","average":{"1min":5.3239999999999998,"5min":3.9279999999999999,"15min":3.6819999999999999},"min":{"1min":0.30499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.064,"5min":32.106000000000002,"15min":32.106000000000002},"last":4.3369999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.0640000000000001,"5min":3.2610000000000001,"15min":2.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":24.253,"5min":24.253,"15min":24.253},"last":4.4480000000000004},{"interface":"front","average":{"1min":3.5,"5min":2.4079999999999999,"15min":1.974},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":15.106999999999999,"5min":15.106999999999999,"15min":15.106999999999999},"last":8.1820000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8760000000000003,"5min":2.9369999999999998,"15min":2.2890000000000001},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.710000000000001,"5min":20.710000000000001,"15min":20.710000000000001},"last":4.3739999999999997},{"interface":"front","average":{"1min":2.964,"5min":1.794,"15min":1.347},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":13.991,"5min":13.991,"15min":13.991},"last":7.7359999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":3.9729999999999999,"5min":2.6520000000000001,"15min":2.1200000000000001},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":18.670999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":1.3520000000000001},{"interface":"front","average":{"1min":3.585,"5min":2.1179999999999999,"15min":1.8720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":20.009,"5min":20.009,"15min":20.009},"last":4.4000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":1.6160000000000001,"15min":1.429},"min":{"1min":0.36599999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.9260000000000002,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":0.95399999999999996},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.3559999999999999,"15min":1.8200000000000001},"min":{"1min":0.47199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":18.591999999999999,"5min":18.591999999999999,"15min":18.591999999999999},"last":1.429}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.7,"15min":1.4810000000000001},"min":{"1min":0.38700000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.9790000000000001,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":0.91900000000000004},{"interface":"front","average":{"1min":4.048,"5min":2.6110000000000002,"15min":1.994},"min":{"1min":0.51800000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":18.704999999999998,"5min":18.704999999999998,"15min":18.704999999999998},"last":1.3069999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":2.4660000000000002,"15min":1.8859999999999999},"min":{"1min":0.54900000000000004,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":18.640000000000001,"5min":18.640000000000001,"15min":18.640000000000001},"last":1.4650000000000001},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.54,"15min":1.298},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":10.35,"5min":10.35,"15min":10.35},"last":0.97799999999999998}]}]},{"osd":2,"up_from":131,"seq":562640715885,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796736,"kb_used_data":556600,"kb_used_omap":81,"kb_used_meta":240046,"kb_avail":92948416,"statfs":{"total":95995035648,"available":95179177984,"internally_reserved":0,"allocated":569958400,"data_stored":569012058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":245807322},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7810000000000001,"5min":4.9379999999999997,"15min":5.1580000000000004},"min":{"1min":0.27300000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.692,"5min":57.811999999999998,"15min":57.811999999999998},"last":4.3070000000000004},{"interface":"front","average":{"1min":5.9059999999999997,"5min":5.968,"15min":5.5819999999999999},"min":{"1min":0.80600000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":13.865,"5min":57.387999999999998,"15min":57.387999999999998},"last":4.2240000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3079999999999998,"15min":3.1850000000000001},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.3650000000000002,"5min":11.759,"15min":88.316000000000003},"last":1.4730000000000001},{"interface":"front","average":{"1min":3.8130000000000002,"5min":3.3490000000000002,"15min":2.9870000000000001},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.167,"5min":28.596,"15min":88.436999999999998},"last":0.312}]},{"osd":3,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":5.2830000000000004,"5min":3.992,"15min":3.3370000000000002},"min":{"1min":0.39800000000000002,"5min":0.223,"15min":0.223},"max":{"1min":20.41,"5min":21.210999999999999,"15min":88.653000000000006},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.7839999999999998,"5min":2.734,"15min":2.6040000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":13.836,"5min":13.836,"15min":88.474000000000004},"last":4.1589999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":4.6219999999999999,"15min":2.2389999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.965,"5min":108.878,"15min":108.878},"last":4.2460000000000004},{"interface":"front","average":{"1min":2.2690000000000001,"5min":4.8079999999999998,"15min":2.9969999999999999},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.2539999999999996,"5min":108.93000000000001,"15min":108.93000000000001},"last":1.5349999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":4.2210000000000001,"15min":2.8519999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.0869999999999997,"5min":97.322999999999993,"15min":97.322999999999993},"last":1.5109999999999999},{"interface":"front","average":{"1min":3.0720000000000001,"5min":4.4569999999999999,"15min":2.4889999999999999},"min":{"1min":0.34499999999999997,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":7.694,"5min":97.301000000000002,"15min":97.301000000000002},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":3.8780000000000001,"15min":2.4049999999999998},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6669999999999998,"5min":85.034999999999997,"15min":88.606999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.2349999999999999,"5min":3.395,"15min":1.863},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":8.1479999999999997,"5min":85.209000000000003,"15min":85.209000000000003},"last":4.2069999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.8,"15min":1.389},"min":{"1min":0.46200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":4.2779999999999996},{"interface":"front","average":{"1min":2.9159999999999999,"5min":2.3570000000000002,"15min":2.2759999999999998},"min":{"1min":0.434,"5min":0.37,"15min":0.37},"max":{"1min":9.0419999999999998,"5min":9.5549999999999997,"15min":88.584000000000003},"last":1.5589999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925971,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":932652,"kb_used_data":531492,"kb_used_omap":7,"kb_used_meta":401080,"kb_avail":92812500,"statfs":{"total":95995035648,"available":95040000000,"internally_reserved":0,"allocated":544247808,"data_stored":543323014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":410706536},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":3.1699999999999999,"15min":2.2959999999999998},"min":{"1min":0.26100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.711,"5min":80.099000000000004,"15min":80.099000000000004},"last":1.171},{"interface":"front","average":{"1min":3.698,"5min":3.0009999999999999,"15min":2.0649999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":20.337,"5min":80.150000000000006,"15min":80.150000000000006},"last":1.4750000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.5019999999999998,"5min":2.6970000000000001,"15min":1.6080000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20000000000000001},"max":{"1min":12.25,"5min":22.818000000000001,"15min":22.818000000000001},"last":0.877},{"interface":"front","average":{"1min":2.9700000000000002,"5min":1.8779999999999999,"15min":1.2869999999999999},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":15.098000000000001,"5min":23.917999999999999,"15min":23.917999999999999},"last":0.48499999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2370000000000001,"5min":2.5800000000000001,"15min":2.4569999999999999},"min":{"1min":0.32900000000000001,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":12.339,"5min":20.431000000000001,"15min":125.446},"last":6.2290000000000001},{"interface":"front","average":{"1min":5.2999999999999998,"5min":4.9500000000000002,"15min":3.0310000000000001},"min":{"1min":0.40799999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.756,"5min":182.483,"15min":182.483},"last":1.3220000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2149999999999999,"15min":1.3919999999999999},"min":{"1min":0.371,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":12.289999999999999,"5min":18.757000000000001,"15min":18.757000000000001},"last":0.96299999999999997},{"interface":"front","average":{"1min":2.645,"5min":1.5149999999999999,"15min":1.196},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.131,"5min":12.131,"15min":12.131},"last":0.54000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3520000000000001,"15min":1.127},"min":{"1min":0.39800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.949999999999999,"5min":12.127000000000001,"15min":12.127000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.085,"15min":1.3300000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":1.109}]},{"osd":6,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":1.5269999999999999,"15min":1.175},"min":{"1min":0.38300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.897,"5min":12.16,"15min":12.16},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.399,"5min":2.0760000000000001,"15min":1.403},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":10.755000000000001},"last":1.1299999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.9929999999999999,"15min":1.6819999999999999},"min":{"1min":0.47299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7710000000000008,"5min":91.757999999999996,"15min":91.757999999999996},"last":1.1599999999999999},{"interface":"front","average":{"1min":3.1499999999999999,"5min":3.2869999999999999,"15min":1.8069999999999999},"min":{"1min":0.49099999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":12.32,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.93300000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019170,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185880,"kb_used_data":880464,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92559272,"statfs":{"total":95995035648,"available":94780694528,"internally_reserved":0,"allocated":901595136,"data_stored":901095425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":5.4880000000000004,"15min":7.173},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.442,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.312},{"interface":"front","average":{"1min":3.5299999999999998,"5min":6.2779999999999996,"15min":7.7779999999999996},"min":{"1min":0.35899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":13.193,"5min":110.33799999999999,"15min":110.33799999999999},"last":4.8310000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1230000000000002,"15min":1.53},"min":{"1min":0.317,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0269999999999992,"5min":41.643000000000001,"15min":45.320999999999998},"last":12.192},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.9019999999999999,"15min":1.4279999999999999},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.4489999999999998,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.27800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:49:20 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":4.0970000000000004,"15min":3.2599999999999998},"min":{"1min":0.34300000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.9189999999999996,"5min":84.909000000000006,"15min":84.909000000000006},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.1600000000000001,"5min":3.9990000000000001,"15min":3.1419999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0609999999999999,"5min":83.840000000000003,"15min":83.840000000000003},"last":2.8679999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":7.3099999999999996,"15min":3.3730000000000002},"min":{"1min":0.442,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":100.51600000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":1.2350000000000001},{"interface":"front","average":{"1min":8.1150000000000002,"5min":7.3609999999999998,"15min":3.4169999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":100.637,"5min":129.197,"15min":129.197},"last":12.345000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.77800000000000002},"min":{"1min":0.26600000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.1550000000000002,"5min":5.8979999999999997,"15min":43.189999999999998},"last":12.238},{"interface":"front","average":{"1min":1.2769999999999999,"5min":0.83699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":6.9199999999999999,"5min":9.5190000000000001,"15min":42.450000000000003},"last":7.5460000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.91400000000000003,"15min":0.78500000000000003},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":10.952999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.378},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.72199999999999998,"15min":0.76700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":4.5359999999999996,"5min":5.8689999999999998,"15min":26.940999999999999},"last":12.269}]},{"osd":7,"last update":"Sat Sep 23 22:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.69799999999999995,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.181,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":12.311},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.75900000000000001,"15min":0.68400000000000005},"min":{"1min":0.38800000000000001,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":8.6739999999999995,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.47199999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822942,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1109124,"kb_used_data":803708,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92636028,"statfs":{"total":95995035648,"available":94859292672,"internally_reserved":0,"allocated":822996992,"data_stored":822512899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5950000000000002,"5min":5.4729999999999999,"15min":3.0289999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.344000000000001,"5min":121.319,"15min":121.319},"last":84.367999999999995},{"interface":"front","average":{"1min":3.8860000000000001,"5min":5.4950000000000001,"15min":2.9289999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":37.470999999999997,"5min":121.075,"15min":121.075},"last":84.311999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.7290000000000001,"15min":1.762},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":6.7779999999999996,"5min":28.658999999999999,"15min":149.203},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.165,"15min":1.9339999999999999},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.1310000000000002,"5min":28.664999999999999,"15min":149.554},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":3.4550000000000001,"15min":6.7249999999999996},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":7.9960000000000004,"5min":158.92599999999999,"15min":158.92599999999999},"last":63.156999999999996},{"interface":"front","average":{"1min":2.0369999999999999,"5min":3.5169999999999999,"15min":6.7199999999999998},"min":{"1min":0.36199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0430000000000001,"5min":158.91399999999999,"15min":158.91399999999999},"last":63.277999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.4409999999999998,"5min":8.7390000000000008,"15min":4.2290000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":37.427999999999997,"5min":257.05000000000001,"15min":257.05000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":5.4939999999999998,"5min":8.484,"15min":4.0819999999999999},"min":{"1min":0.45800000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":37.277999999999999,"5min":257.35000000000002,"15min":257.35000000000002},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.70099999999999996,"15min":0.60999999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":1.893,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.309},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.77000000000000002,"15min":0.61899999999999999},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":1.9590000000000001,"5min":10.535,"15min":10.535},"last":0.32300000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.89400000000000002,"15min":1.391},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":1.9410000000000001,"5min":12.615,"15min":140.07499999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.016,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":2.4609999999999999,"5min":12.74,"15min":139.739},"last":0.34899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.74399999999999999,"15min":0.71999999999999997},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.738,"5min":4.2510000000000003,"15min":7.9809999999999999},"last":0.376},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.73599999999999999,"15min":0.68000000000000005},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":3.5539999999999998,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.42099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":769130496,"data_stored":769130496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":605356032,"data_stored":605356032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":564727808,"data_stored":564723736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":537198592,"data_stored":537198592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":889520128,"data_stored":889520128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":803868672,"data_stored":803864600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":535826432,"data_stored":535822360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:50:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:14.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:14.309 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:14.351 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:14.398 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:14.421 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:14.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:14.465 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:14.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:14 smithi052 ceph-mon[180979]: from='client.45476 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:14.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:14 smithi052 ceph-mon[187743]: from='client.45476 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:14.527 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:14.555 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:14.591 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:14.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:14 smithi151 ceph-mon[159210]: from='client.45476 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:14.728 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:14.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:14.832 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:14.907 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:15.164 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:15.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:15.272 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:15.349 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:15.609 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:15 smithi052 ceph-mon[187743]: pgmap v708: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 2528/65310 objects degraded (3.871%); 1446/65310 objects misplaced (2.214%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:50:15.609 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2528/65310 objects degraded (3.871%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:15.609 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:15 smithi052 ceph-mon[180979]: pgmap v708: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 2528/65310 objects degraded (3.871%); 1446/65310 objects misplaced (2.214%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:50:15.610 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2528/65310 objects degraded (3.871%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:15 smithi151 ceph-mon[159210]: pgmap v708: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 2528/65310 objects degraded (3.871%); 1446/65310 objects misplaced (2.214%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:50:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2528/65310 objects degraded (3.871%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:15.916 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:15.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:16.021 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:16.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:16.914 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:16.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:17.019 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:17.087 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:17.141 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:50:17.141 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:50:17.208 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:50:17.414 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:17.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:17.436 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:17 smithi052 ceph-mon[180979]: pgmap v709: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 2594/58500 objects degraded (4.434%); 1250/58500 objects misplaced (2.137%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:17.437 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:17 smithi052 ceph-mon[187743]: pgmap v709: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 2594/58500 objects degraded (4.434%); 1250/58500 objects misplaced (2.137%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:17 smithi151 ceph-mon[159210]: pgmap v709: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 2594/58500 objects degraded (4.434%); 1250/58500 objects misplaced (2.137%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:17.536 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:17.585 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:17.798 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:50:17.856 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:50:17.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:50:17.874 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:17.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:17.902 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:50:17.960 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:50:17.986 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:18.015 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:50:18.037 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:18.062 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:50:18.109 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:18.111 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:50:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:18.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:18.148 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:50:18.215 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:18.218 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.219 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.219 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.219 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.219 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.220 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.220 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.220 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:50:18.232 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:18.252 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:18.314 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:18.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:18.341 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:18.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2140658510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:18.425 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:18.515 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2140658510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:18.516 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2140658510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:18.523 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:19.471 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:19.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:19.485 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:50:19.577 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:19.619 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:19 smithi151 ceph-mon[159210]: pgmap v710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2594/56211 objects degraded (4.615%); 1250/56211 objects misplaced (2.224%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/932720198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:19 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:19.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:19 smithi052 ceph-mon[187743]: pgmap v710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2594/56211 objects degraded (4.615%); 1250/56211 objects misplaced (2.224%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:19.709 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/932720198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:19.709 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:19 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:19.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:19 smithi052 ceph-mon[180979]: pgmap v710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2594/56211 objects degraded (4.615%); 1250/56211 objects misplaced (2.224%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:19.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/932720198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:19.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:19 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:19.782 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:19.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:19.888 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:19.924 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:20.002 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:20.002 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:50:20.002 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:50:20.002 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:50:20.002 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:20.005 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:20.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:20.113 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:20.200 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:20.261 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:20.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:20.365 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:20.396 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:20.452 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:20.452 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:50:20.452 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T22:50:20.452 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: failed to encode map e258 with expected crc 2023-09-23T22:50:20.452 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: failed to encode map e258 with expected crc 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2594/56211 objects degraded (4.615%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: failed to encode map e258 with expected crc 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: failed to encode map e258 with expected crc 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:20.453 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2594/56211 objects degraded (4.615%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:20.454 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:20.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:20.557 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:20.596 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: failed to encode map e258 with expected crc 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: failed to encode map e258 with expected crc 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2594/56211 objects degraded (4.615%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:20.866 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:50:20.873 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:20.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:20.983 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:21.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:21.184 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:21.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:21.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:21.297 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:21.367 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:21.373 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:21.677 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:50:21.709 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:21 smithi052 ceph-mon[187743]: pgmap v712: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 2594/54969 objects degraded (4.719%); 1250/54969 objects misplaced (2.274%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:21.710 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:21 smithi052 ceph-mon[180979]: pgmap v712: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 2594/54969 objects degraded (4.719%); 1250/54969 objects misplaced (2.274%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:21.735 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:21.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:21.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:21.762 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:21 smithi151 ceph-mon[159210]: pgmap v712: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.1 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 2594/54969 objects degraded (4.719%); 1250/54969 objects misplaced (2.274%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:50:21.851 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:21.963 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:22.155 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:22.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:22.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:22.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:22.301 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:22.525 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:22.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:22.564 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:22.631 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:22.665 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:22.784 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:50:22.784 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:22.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:22.823 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:22.860 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:50:22.889 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:22.979 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:23.013 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:23.013 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:50:23.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:50:23.013 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:50:23.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40802 6.e 6:7b05ccce:::benchmark_data_smithi052_234456_object18658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 553.14682127100002, 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533, 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:50:23.014 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40802 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862706+0000", 2023-09-23T22:50:23.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862704+0000", 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0949999999999996e-06 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862712+0000", 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2490000000000001e-05 2023-09-23T22:50:23.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862725+0000", 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.041e-06 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.862732+0000", 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036689000000000001 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866401+0000", 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0216e-05 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866411+0000", 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99412088600000004 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860532+0000", 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3456e-05 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860545+0000", 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2107602909999999 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071306+0000", 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014411400000000001 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071450+0000", 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017796199999999999 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071628+0000", 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036812979999999999 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075309+0000", 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001368813 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076678+0000", 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064926749999999998 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083170+0000", 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9931999999999994e-05 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083240+0000", 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034446200000000002 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083585+0000", 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.220878533 2023-09-23T22:50:23.022 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40804 6.e 6:7187807f:::benchmark_data_smithi052_234456_object18660:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 553.14407570200001, 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001, 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.023 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40804 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865452+0000", 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865450+0000", 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4629999999999998e-06 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865457+0000", 2023-09-23T22:50:23.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1907e-05 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865469+0000", 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1515000000000001e-05 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865480+0000", 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096401899999999999 2023-09-23T22:50:23.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866444+0000", 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8879999999999997e-06 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866452+0000", 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413119599999999 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860583+0000", 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7849999999999997e-06 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860592+0000", 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.212291711 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072884+0000", 2023-09-23T22:50:23.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011587900000000001 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073000+0000", 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000105 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073105+0000", 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002208621 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.075313+0000", 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020589470000000002 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.077372+0000", 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062710200000000004 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083643+0000", 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9950000000000002e-05 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083683+0000", 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000162177 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083845+0000", 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2183934650000001 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40805 6.e 6:798273ba:::benchmark_data_smithi052_234456_object18661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 553.14403876200004, 2023-09-23T22:50:23.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999, 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40805 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:50:23.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865489+0000", 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865488+0000", 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6610000000000002e-06 2023-09-23T22:50:23.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865492+0000", 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7939999999999997e-06 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865499+0000", 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7509999999999999e-06 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.865504+0000", 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097241299999999999 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866476+0000", 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6120000000000001e-06 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866484+0000", 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99413304199999997 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860617+0000", 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3080000000000002e-06 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.860624+0000", 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.213177682 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073802+0000", 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8152000000000005e-05 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073900+0000", 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106062 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.074006+0000", 2023-09-23T22:50:23.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022883959999999998 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.076294+0000", 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002385603 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.078680+0000", 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052328410000000002 2023-09-23T22:50:23.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083913+0000", 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2397000000000005e-05 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083985+0000", 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026911300000000002 2023-09-23T22:50:23.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.084254+0000", 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2187652629999999 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.34680.0:40809 6.2 6:400fe117:::benchmark_data_smithi052_234456_object18665:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:50:23.041 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 553.14315944099997, 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002, 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.34680", 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/128772971", 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 40809 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.042 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866368+0000", 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866367+0000", 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8289999999999996e-06 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866373+0000", 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1122e-05 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866384+0000", 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8669999999999996e-06 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866391+0000", 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000202492 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866594+0000", 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7580000000000004e-06 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:09.866603+0000", 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.994511967 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861115+0000", 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6169999999999996e-06 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:10.861125+0000", 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.209770244 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.047 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.070895+0000", 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013600900000000001 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071031+0000", 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121377 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.071153+0000", 2023-09-23T22:50:23.048 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016472430000000001 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.072800+0000", 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000377547 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.073177+0000", 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0096328860000000002 2023-09-23T22:50:23.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082810+0000", 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6335999999999996e-05 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.082877+0000", 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000259743 2023-09-23T22:50:23.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:41:12.083136+0000", 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2167681140000002 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39271 7.1 7:803b19ab:::benchmark_data_smithi052_277165_object39270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.051 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.253937775, 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002, 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39271 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755590+0000", 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755588+0000", 2023-09-23T22:50:23.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100054 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755689+0000", 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755701+0000", 2023-09-23T22:50:23.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3199999999999999e-06 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755707+0000", 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9419000000000002e-05 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755756+0000", 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5290000000000008e-06 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.755765+0000", 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99249923500000004 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748264+0000", 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6199999999999994e-06 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.056 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748273+0000", 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.538374364 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286648+0000", 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000173475 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.286821+0000", 2023-09-23T22:50:23.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024065800000000001 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.287062+0000", 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003676038 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290738+0000", 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3826999999999998e-05 2023-09-23T22:50:23.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290792+0000", 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2153e-05 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290804+0000", 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5701000000000002e-05 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.290850+0000", 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093742900000000004 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291787+0000", 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5361971680000002 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39291 7.1 7:8a0225f8:::benchmark_data_smithi052_277165_object39290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.23922900100001, 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001, 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39291 2023-09-23T22:50:23.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770299+0000", 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:23.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770297+0000", 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5182e-05 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770362+0000", 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498999999999998e-05 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770378+0000", 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0129999999999996e-06 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770385+0000", 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2045000000000007e-05 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770447+0000", 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1433999999999999e-05 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.770458+0000", 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97785107400000004 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748309+0000", 2023-09-23T22:50:23.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8459999999999996e-06 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748315+0000", 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5410234169999999 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289338+0000", 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117113 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289455+0000", 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000150991 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.289606+0000", 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023599210000000001 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291966+0000", 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4410000000000004e-06 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.291974+0000", 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8967999999999994e-05 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292043+0000", 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7804000000000003e-05 2023-09-23T22:50:23.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292111+0000", 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00075040600000000001 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292861+0000", 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5225623690000001 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.069 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39293 7.1 7:8d7e8869:::benchmark_data_smithi052_277165_object39292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.23581085699999, 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908, 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.070 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39293 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773717+0000", 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773716+0000", 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2105000000000003e-05 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773778+0000", 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2449000000000001e-05 2023-09-23T22:50:23.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773790+0000", 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6099999999999997e-06 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773796+0000", 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9072000000000001e-05 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773835+0000", 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.773843+0000", 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97449834000000002 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748341+0000", 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5279999999999997e-06 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748346+0000", 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.544637525 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.292983+0000", 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016144799999999999 2023-09-23T22:50:23.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293145+0000", 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015888799999999999 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.293304+0000", 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039379849999999997 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.076 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297242+0000", 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6013000000000001e-05 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297288+0000", 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003665958 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.077 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300954+0000", 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7888999999999999e-05 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.301032+0000", 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012321579999999999 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.078 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302264+0000", 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.528546908 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39318 7.1 7:887a21fd:::benchmark_data_smithi052_277165_object39317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.21427723300002, 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997, 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.079 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39318 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.080 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795250+0000", 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795249+0000", 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1755000000000002e-05 2023-09-23T22:50:23.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795301+0000", 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3233e-05 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795314+0000", 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1940000000000001e-06 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795321+0000", 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1218999999999999e-05 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795372+0000", 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0147e-05 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.795383+0000", 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.95298535699999998 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748368+0000", 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2440000000000004e-06 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748372+0000", 2023-09-23T22:50:23.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5462721529999999 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294644+0000", 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112796 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294757+0000", 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018578900000000001 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.085 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.294943+0000", 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023897250000000001 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297333+0000", 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8390000000000001e-06 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.086 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297340+0000", 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050090029999999997 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302349+0000", 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3208999999999999e-05 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.087 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302392+0000", 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00080982299999999999 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303202+0000", 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5079511759999997 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.088 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39334 7.1 7:83eed48f:::benchmark_data_smithi052_277165_object39333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.20311764600001, 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001, 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39334 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.089 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806410+0000", 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.090 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806408+0000", 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8466e-05 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806457+0000", 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3062e-05 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.091 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806470+0000", 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2000000000000002e-06 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806475+0000", 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9246999999999998e-05 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806524+0000", 2023-09-23T22:50:23.092 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0499999999999997e-06 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.806533+0000", 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94188278000000003 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748416+0000", 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.52e-06 2023-09-23T22:50:23.093 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748423+0000", 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5475971120000001 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296021+0000", 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015821300000000001 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.094 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296179+0000", 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013869899999999999 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.296317+0000", 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045890180000000003 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.300906+0000", 2023-09-23T22:50:23.095 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023854169999999999 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303292+0000", 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028062899999999999 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306098+0000", 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2596999999999999e-05 2023-09-23T22:50:23.096 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306141+0000", 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068390599999999997 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306825+0000", 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5004146970000001 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.097 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39346 7.1 7:8a73189a:::benchmark_data_smithi052_277165_object39345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.19816193899999, 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001, 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.098 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39346 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.099 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811366+0000", 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811364+0000", 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012578900000000001 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811490+0000", 2023-09-23T22:50:23.100 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8567999999999999e-05 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811508+0000", 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9329999999999996e-06 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811518+0000", 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8912000000000001e-05 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.101 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811577+0000", 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1018e-05 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.811588+0000", 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93686738199999997 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.102 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748455+0000", 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0320000000000003e-06 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748460+0000", 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5488962179999999 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297357+0000", 2023-09-23T22:50:23.103 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010246 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297459+0000", 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012592199999999999 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.297585+0000", 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004696787 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.104 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302282+0000", 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001097813 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303380+0000", 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034811529999999999 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.105 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306861+0000", 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9079000000000001e-05 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.306920+0000", 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074140899999999995 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.106 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.307661+0000", 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4962955170000001 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39360 7.1 7:8f91ddcd:::benchmark_data_smithi052_277165_object39359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.18676954400001, 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001, 2023-09-23T22:50:23.107 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39360 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:50:23.108 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822758+0000", 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822756+0000", 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0686999999999998e-05 2023-09-23T22:50:23.109 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822807+0000", 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4600000000000001e-05 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822821+0000", 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2920000000000001e-06 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.110 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822828+0000", 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2878000000000003e-05 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822901+0000", 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9390000000000003e-06 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.111 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.822909+0000", 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92559491500000002 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748504+0000", 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4299999999999997e-06 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748510+0000", 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5498821280000001 2023-09-23T22:50:23.112 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298392+0000", 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1963e-05 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298484+0000", 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011828300000000001 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.113 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.298602+0000", 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036917579999999998 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.302294+0000", 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011534850000000001 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.114 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303447+0000", 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049678559999999997 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308415+0000", 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0241000000000001e-05 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308476+0000", 2023-09-23T22:50:23.115 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00084742900000000002 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309323+0000", 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4865648550000001 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39366 7.1 7:86dcbc64:::benchmark_data_smithi052_277165_object39365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.116 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.18314034700001, 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001, 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39366 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.117 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826387+0000", 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.118 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826386+0000", 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4935999999999997e-05 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826441+0000", 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2864000000000001e-05 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826454+0000", 2023-09-23T22:50:23.119 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7969999999999997e-06 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826460+0000", 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5853000000000003e-05 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826505+0000", 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0890000000000006e-06 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.120 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.826514+0000", 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92203426200000005 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748549+0000", 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9800000000000003e-06 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.121 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748555+0000", 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.55081775 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299372+0000", 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155606 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299528+0000", 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021651799999999999 2023-09-23T22:50:23.122 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.299745+0000", 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035645 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303309+0000", 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021267000000000001 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.123 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303522+0000", 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0058789619999999997 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309401+0000", 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0636999999999995e-05 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.124 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.309471+0000", 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00099162699999999992 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310463+0000", 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4840756910000001 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.125 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39373 7.1 7:85f7eb6c:::benchmark_data_smithi052_277165_object39372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.18006282599998, 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812, 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39373 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.126 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829465+0000", 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.127 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829464+0000", 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1137000000000002e-05 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829515+0000", 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3482000000000001e-05 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.128 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829528+0000", 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.959e-06 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829534+0000", 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5408999999999999e-05 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829570+0000", 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2763e-05 2023-09-23T22:50:23.129 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.829582+0000", 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.91900465799999997 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748587+0000", 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7450000000000003e-06 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.130 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748592+0000", 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.554972391 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303564+0000", 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012435699999999999 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.131 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303689+0000", 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000131853 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.303820+0000", 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004451455 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.132 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308272+0000", 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0795000000000003e-05 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.308333+0000", 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022271669999999999 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.133 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310560+0000", 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7315000000000001e-05 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.310637+0000", 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083538299999999998 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.134 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311473+0000", 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.482007812 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39414 7.1 7:8075013a:::benchmark_data_smithi052_277165_object39413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.14790215099998, 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001, 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.135 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39414 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.136 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861625+0000", 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861624+0000", 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4787000000000002e-05 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861678+0000", 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3568e-05 2023-09-23T22:50:23.137 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861692+0000", 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1750000000000002e-06 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.138 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861698+0000", 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9217e-05 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861747+0000", 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0080000000000002e-06 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.141 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.861755+0000", 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88686667100000005 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748622+0000", 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0063e-05 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748632+0000", 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.562879379 2023-09-23T22:50:23.142 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311512+0000", 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018041500000000001 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311692+0000", 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020370699999999999 2023-09-23T22:50:23.143 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.311896+0000", 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077409310000000004 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319637+0000", 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1062000000000006e-05 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.144 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319698+0000", 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103898 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319802+0000", 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7586e-05 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319869+0000", 2023-09-23T22:50:23.145 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017220569999999999 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321591+0000", 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4599656940000001 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.146 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39416 7.1 7:83076ecc:::benchmark_data_smithi052_277165_object39415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.14651288800002, 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074, 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39416 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.147 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863015+0000", 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.148 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863014+0000", 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106535 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863120+0000", 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1805e-05 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.149 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863132+0000", 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4430000000000002e-06 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863138+0000", 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0098999999999997e-05 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863178+0000", 2023-09-23T22:50:23.150 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0545e-05 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.863188+0000", 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88547513300000003 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748663+0000", 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.507e-06 2023-09-23T22:50:23.151 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748669+0000", 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5646240410000001 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313293+0000", 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110319 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.152 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313403+0000", 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196856 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.313600+0000", 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060436789999999997 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.153 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319644+0000", 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011535 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319759+0000", 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019084060000000001 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.154 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321668+0000", 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2410999999999996e-05 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.321730+0000", 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0011838980000000001 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322914+0000", 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.459899074 2023-09-23T22:50:23.155 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39440 7.1 7:846ccd4b:::benchmark_data_smithi052_277165_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.12603758199998, 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002, 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.156 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39440 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.157 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883490+0000", 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883489+0000", 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0798999999999997e-05 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883569+0000", 2023-09-23T22:50:23.158 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5083999999999999e-05 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883594+0000", 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1099999999999999e-06 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883601+0000", 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5002e-05 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883646+0000", 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5539999999999998e-06 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.883654+0000", 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86504454099999994 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748699+0000", 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0530000000000003e-06 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748705+0000", 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.566234793 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.314939+0000", 2023-09-23T22:50:23.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014568900000000001 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315085+0000", 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019602700000000001 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.315281+0000", 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004426915 2023-09-23T22:50:23.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319708+0000", 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032665569999999998 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322975+0000", 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5554e-05 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323070+0000", 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1078999999999998e-05 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323111+0000", 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095274200000000002 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324064+0000", 2023-09-23T22:50:23.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4405739940000002 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39442 7.1 7:821661b1:::benchmark_data_smithi052_277165_object39441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.12299998499998, 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999, 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.165 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39442 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.166 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886528+0000", 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999895 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886517+0000", 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0640000000000002e-05 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.167 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886578+0000", 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4535999999999999e-05 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886592+0000", 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.024e-06 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886599+0000", 2023-09-23T22:50:23.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5583e-05 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886655+0000", 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9129999999999992e-06 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.886664+0000", 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86206608799999995 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.169 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748730+0000", 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6200000000000001e-06 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748737+0000", 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.567891876 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316628+0000", 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014458900000000001 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316773+0000", 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000198579 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.316972+0000", 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027449810000000001 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.319717+0000", 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033197000000000001 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.323036+0000", 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001091297 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324128+0000", 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0636999999999997e-05 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324168+0000", 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071410600000000005 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324882+0000", 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4383546279999999 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39446 7.1 7:8475cdd4:::benchmark_data_smithi052_277165_object39445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:50:23.174 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.11972075800003, 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001, 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39446 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.175 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889807+0000", 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889806+0000", 2023-09-23T22:50:23.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7126000000000002e-05 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889853+0000", 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2593e-05 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889865+0000", 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889871+0000", 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7049999999999998e-05 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889918+0000", 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5479999999999998e-06 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.889924+0000", 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.858839715 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748764+0000", 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5000000000000001e-06 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748768+0000", 2023-09-23T22:50:23.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5693916369999998 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318160+0000", 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016406600000000001 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318324+0000", 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000199247 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.318523+0000", 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044604579999999996 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.322984+0000", 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019935700000000001 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.324977+0000", 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4416e-05 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325052+0000", 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9657000000000003e-05 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325092+0000", 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00089603700000000003 2023-09-23T22:50:23.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.325988+0000", 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4361807010000001 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39458 7.1 7:846e660e:::benchmark_data_smithi052_277165_object39457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.10364408999999, 2023-09-23T22:50:23.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002, 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39458 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:50:23.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905884+0000", 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905882+0000", 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7948000000000002e-05 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905940+0000", 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1592e-05 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905952+0000", 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4750000000000001e-06 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.186 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.905957+0000", 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2416000000000001e-05 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906010+0000", 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4849999999999992e-06 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.906018+0000", 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.84277395099999997 2023-09-23T22:50:23.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748792+0000", 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748796+0000", 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5772432570000001 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326040+0000", 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015276800000000001 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326193+0000", 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018607500000000001 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.326379+0000", 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038150480000000001 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.330194+0000", 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001090159 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331284+0000", 2023-09-23T22:50:23.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0065274460000000001 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337811+0000", 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1335999999999999e-05 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.337853+0000", 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014175889999999999 2023-09-23T22:50:23.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339270+0000", 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4333866300000002 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:39462 7.1 7:8cf1a796:::benchmark_data_smithi052_277165_object39461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:50:23.192 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 492.09308337800002, 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001, 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 39462 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:23.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916444+0000", 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916441+0000", 2023-09-23T22:50:23.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5293000000000001e-05 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916497+0000", 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3080000000000002e-05 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916530+0000", 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4319e-05 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916544+0000", 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018227799999999999 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916726+0000", 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3705999999999999e-05 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:10.916760+0000", 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.83205244499999997 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748813+0000", 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5040000000000004e-06 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:11.748817+0000", 2023-09-23T22:50:23.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.578915206 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327732+0000", 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115865 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327848+0000", 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000123075 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.327971+0000", 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033208170000000002 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.331292+0000", 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0080585329999999997 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339351+0000", 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1529e-05 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339412+0000", 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7031000000000002e-05 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.339479+0000", 2023-09-23T22:50:23.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010848730000000001 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:13.340564+0000", 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4241197240000001 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:23.201 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:23.202 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:23.204 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:23.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:23.309 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:23.386 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:23.485 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:23 smithi052 ceph-mon[180979]: pgmap v713: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.0 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 2667/49323 objects degraded (5.407%); 1035/49323 objects misplaced (2.098%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:50:23.485 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:50:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:50:23] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:50:23.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:23 smithi052 ceph-mon[187743]: pgmap v713: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.0 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 2667/49323 objects degraded (5.407%); 1035/49323 objects misplaced (2.098%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:50:23.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:23 smithi151 ceph-mon[159210]: pgmap v713: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.0 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 2667/49323 objects degraded (5.407%); 1035/49323 objects misplaced (2.098%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:50:23.837 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:50:24.327 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:24.643 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:24.657 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:24 smithi052 ceph-mon[187743]: from='client.55381 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:24.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:24.658 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:24 smithi052 ceph-mon[180979]: from='client.55381 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:24.758 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:24.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:24 smithi151 ceph-mon[159210]: from='client.55381 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:24.852 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:24.897 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":713,"stamp":"2023-09-23T22:50:22.295687+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1076434872,"num_objects":16441,"num_object_clones":0,"num_object_copies":49323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2667,"num_objects_misplaced":1035,"num_objects_unfound":0,"num_objects_dirty":16441,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":188741,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2951,"num_bytes_recovered":191309898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127975,"ondisk_log_size":127975,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5578676,"kb_used_data":3528508,"kb_used_omap":129,"kb_used_meta":2049534,"kb_avail":650637388,"statfs":{"total":671965249536,"available":666252685312,"internally_reserved":0,"allocated":3613192192,"data_stored":3607920601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2098723549},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,4,0,2,0,0,8],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-396689408,"num_objects":-6053,"num_object_clones":0,"num_object_copies":-18159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":139,"num_objects_misplaced":-411,"num_objects_unfound":0,"num_objects_dirty":-6053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6053,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":272,"num_bytes_recovered":17825792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.014440"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":255,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613439+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:50:09.613439+0000","last_peered":"2023-09-23T22:50:09.613439+0000","last_clean":"2023-09-23T22:50:09.613439+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:50:09.613439+0000","last_undegraded":"2023-09-23T22:50:09.613439+0000","last_fullsized":"2023-09-23T22:50:09.613439+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":230,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626674+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:50:09.626674+0000","last_peered":"2023-09-23T22:50:09.626674+0000","last_clean":"2023-09-23T22:50:09.626674+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:50:09.626674+0000","last_undegraded":"2023-09-23T22:50:09.626674+0000","last_fullsized":"2023-09-23T22:50:09.626674+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625810+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:50:09.625810+0000","last_peered":"2023-09-23T22:50:09.625810+0000","last_clean":"2023-09-23T22:50:09.625810+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:50:09.625810+0000","last_undegraded":"2023-09-23T22:50:09.625810+0000","last_fullsized":"2023-09-23T22:50:09.625810+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":225,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.631451+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:50:09.631451+0000","last_peered":"2023-09-23T22:50:09.631451+0000","last_clean":"2023-09-23T22:50:09.631451+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:50:09.631451+0000","last_undegraded":"2023-09-23T22:50:09.631451+0000","last_fullsized":"2023-09-23T22:50:09.631451+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":290,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:11.026927+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:50:11.026927+0000","last_peered":"2023-09-23T22:50:11.026927+0000","last_clean":"2023-09-23T22:50:11.026927+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:50:11.026927+0000","last_undegraded":"2023-09-23T22:50:11.026927+0000","last_fullsized":"2023-09-23T22:50:11.026927+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":223,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618912+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:50:09.618912+0000","last_peered":"2023-09-23T22:50:09.618912+0000","last_clean":"2023-09-23T22:50:09.618912+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:50:09.618912+0000","last_undegraded":"2023-09-23T22:50:09.618912+0000","last_fullsized":"2023-09-23T22:50:09.618912+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":156,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572550+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:49:58.572550+0000","last_peered":"2023-09-23T22:49:58.572550+0000","last_clean":"2023-09-23T22:49:58.572550+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:49:58.572550+0000","last_undegraded":"2023-09-23T22:49:58.572550+0000","last_fullsized":"2023-09-23T22:49:58.572550+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":258,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626078+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:50:09.626078+0000","last_peered":"2023-09-23T22:50:09.626078+0000","last_clean":"2023-09-23T22:50:09.626078+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:50:09.626078+0000","last_undegraded":"2023-09-23T22:50:09.626078+0000","last_fullsized":"2023-09-23T22:50:09.626078+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":227,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626499+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:50:09.626499+0000","last_peered":"2023-09-23T22:50:09.626499+0000","last_clean":"2023-09-23T22:50:09.626499+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:50:09.626499+0000","last_undegraded":"2023-09-23T22:50:09.626499+0000","last_fullsized":"2023-09-23T22:50:09.626499+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":216,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625732+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:50:09.625732+0000","last_peered":"2023-09-23T22:50:09.625732+0000","last_clean":"2023-09-23T22:50:09.625732+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:50:09.625732+0000","last_undegraded":"2023-09-23T22:50:09.625732+0000","last_fullsized":"2023-09-23T22:50:09.625732+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":133,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626351+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:50:09.626351+0000","last_peered":"2023-09-23T22:50:09.626351+0000","last_clean":"2023-09-23T22:50:09.626351+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:50:09.626351+0000","last_undegraded":"2023-09-23T22:50:09.626351+0000","last_fullsized":"2023-09-23T22:50:09.626351+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.617931+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:50:09.617931+0000","last_peered":"2023-09-23T22:50:09.617931+0000","last_clean":"2023-09-23T22:50:09.617931+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:50:09.617931+0000","last_undegraded":"2023-09-23T22:50:09.617931+0000","last_fullsized":"2023-09-23T22:50:09.617931+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":278,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.616791+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:50:09.616791+0000","last_peered":"2023-09-23T22:50:09.616791+0000","last_clean":"2023-09-23T22:50:09.616791+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:50:09.616791+0000","last_undegraded":"2023-09-23T22:50:09.616791+0000","last_fullsized":"2023-09-23T22:50:09.616791+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":223,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.614672+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:50:09.614672+0000","last_peered":"2023-09-23T22:50:09.614672+0000","last_clean":"2023-09-23T22:50:09.614672+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:50:09.614672+0000","last_undegraded":"2023-09-23T22:50:09.614672+0000","last_fullsized":"2023-09-23T22:50:09.614672+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":116,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.614758+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:50:09.614758+0000","last_peered":"2023-09-23T22:50:09.614758+0000","last_clean":"2023-09-23T22:50:09.614758+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:50:09.614758+0000","last_undegraded":"2023-09-23T22:50:09.614758+0000","last_fullsized":"2023-09-23T22:50:09.614758+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":278,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.629771+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:50:09.629771+0000","last_peered":"2023-09-23T22:50:09.629771+0000","last_clean":"2023-09-23T22:50:09.629771+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:50:09.629771+0000","last_undegraded":"2023-09-23T22:50:09.629771+0000","last_fullsized":"2023-09-23T22:50:09.629771+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.576643+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:49:58.576643+0000","last_peered":"2023-09-23T22:49:58.576643+0000","last_clean":"2023-09-23T22:49:58.576643+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:49:58.576643+0000","last_undegraded":"2023-09-23T22:49:58.576643+0000","last_fullsized":"2023-09-23T22:49:58.576643+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626743+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:50:09.626743+0000","last_peered":"2023-09-23T22:50:09.626743+0000","last_clean":"2023-09-23T22:50:09.626743+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:50:09.626743+0000","last_undegraded":"2023-09-23T22:50:09.626743+0000","last_fullsized":"2023-09-23T22:50:09.626743+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":108,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613033+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:50:09.613033+0000","last_peered":"2023-09-23T22:50:09.613033+0000","last_clean":"2023-09-23T22:50:09.613033+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:50:09.613033+0000","last_undegraded":"2023-09-23T22:50:09.613033+0000","last_fullsized":"2023-09-23T22:50:09.613033+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":213,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.583128+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:49:58.583128+0000","last_peered":"2023-09-23T22:49:58.583128+0000","last_clean":"2023-09-23T22:49:58.583128+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:49:58.583128+0000","last_undegraded":"2023-09-23T22:49:58.583128+0000","last_fullsized":"2023-09-23T22:49:58.583128+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":302,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.617894+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:50:09.617894+0000","last_peered":"2023-09-23T22:50:09.617894+0000","last_clean":"2023-09-23T22:50:09.617894+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:50:09.617894+0000","last_undegraded":"2023-09-23T22:50:09.617894+0000","last_fullsized":"2023-09-23T22:50:09.617894+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":258,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:11.039567+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:50:11.039567+0000","last_peered":"2023-09-23T22:50:11.039567+0000","last_clean":"2023-09-23T22:50:11.039567+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:50:11.039567+0000","last_undegraded":"2023-09-23T22:50:11.039567+0000","last_fullsized":"2023-09-23T22:50:11.039567+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"258'11762","reported_seq":11811,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.594678+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:50:20.594678+0000","last_peered":"2023-09-23T22:50:20.594678+0000","last_clean":"2023-09-23T22:50:20.594678+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:50:20.594678+0000","last_undegraded":"2023-09-23T22:50:20.594678+0000","last_fullsized":"2023-09-23T22:50:20.594678+0000","mapping_epoch":217,"log_start":"231'4400","ondisk_log_start":"231'4400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11762,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":127,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613263+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:50:09.613263+0000","last_peered":"2023-09-23T22:50:09.613263+0000","last_clean":"2023-09-23T22:50:09.613263+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:50:09.613263+0000","last_undegraded":"2023-09-23T22:50:09.613263+0000","last_fullsized":"2023-09-23T22:50:09.613263+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":222,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.617977+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:50:09.617977+0000","last_peered":"2023-09-23T22:50:09.617977+0000","last_clean":"2023-09-23T22:50:09.617977+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:50:09.617977+0000","last_undegraded":"2023-09-23T22:50:09.617977+0000","last_fullsized":"2023-09-23T22:50:09.617977+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"258'11598","reported_seq":11653,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.800476+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:50:20.800476+0000","last_peered":"2023-09-23T22:50:20.800476+0000","last_clean":"2023-09-23T22:50:20.800476+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:50:20.800476+0000","last_undegraded":"2023-09-23T22:50:20.800476+0000","last_fullsized":"2023-09-23T22:50:20.800476+0000","mapping_epoch":219,"log_start":"228'3000","ondisk_log_start":"228'3000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8598,"log_dups_size":0,"ondisk_log_size":8598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652416,"num_objects":956,"num_object_clones":0,"num_object_copies":2868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11598,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":137,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613397+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:50:09.613397+0000","last_peered":"2023-09-23T22:50:09.613397+0000","last_clean":"2023-09-23T22:50:09.613397+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:50:09.613397+0000","last_undegraded":"2023-09-23T22:50:09.613397+0000","last_fullsized":"2023-09-23T22:50:09.613397+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613007+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:50:09.613007+0000","last_peered":"2023-09-23T22:50:09.613007+0000","last_clean":"2023-09-23T22:50:09.613007+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:50:09.613007+0000","last_undegraded":"2023-09-23T22:50:09.613007+0000","last_fullsized":"2023-09-23T22:50:09.613007+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"257'11476","reported_seq":11525,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:17.793706+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:50:17.793706+0000","last_peered":"2023-09-23T22:50:17.793706+0000","last_clean":"2023-09-23T22:50:17.793706+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:50:17.793706+0000","last_undegraded":"2023-09-23T22:50:17.793706+0000","last_fullsized":"2023-09-23T22:50:17.793706+0000","mapping_epoch":219,"log_start":"229'3500","ondisk_log_start":"229'3500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7976,"log_dups_size":0,"ondisk_log_size":7976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73400320,"num_objects":1120,"num_object_clones":0,"num_object_copies":3360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11476,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":68,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618567+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:50:09.618567+0000","last_peered":"2023-09-23T22:50:09.618567+0000","last_clean":"2023-09-23T22:50:09.618567+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:50:09.618567+0000","last_undegraded":"2023-09-23T22:50:09.618567+0000","last_fullsized":"2023-09-23T22:50:09.618567+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618072+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:50:09.618072+0000","last_peered":"2023-09-23T22:50:09.618072+0000","last_clean":"2023-09-23T22:50:09.618072+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:50:09.618072+0000","last_undegraded":"2023-09-23T22:50:09.618072+0000","last_fullsized":"2023-09-23T22:50:09.618072+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"258'12039","reported_seq":12098,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:21.028277+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:50:21.028277+0000","last_peered":"2023-09-23T22:50:21.028277+0000","last_clean":"2023-09-23T22:50:21.028277+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:50:21.028277+0000","last_undegraded":"2023-09-23T22:50:21.028277+0000","last_fullsized":"2023-09-23T22:50:21.028277+0000","mapping_epoch":226,"log_start":"232'4800","ondisk_log_start":"232'4800","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12039,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.629941+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:50:09.629941+0000","last_peered":"2023-09-23T22:50:09.629941+0000","last_clean":"2023-09-23T22:50:09.629941+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:50:09.629941+0000","last_undegraded":"2023-09-23T22:50:09.629941+0000","last_fullsized":"2023-09-23T22:50:09.629941+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":149,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572043+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:49:58.572043+0000","last_peered":"2023-09-23T22:49:58.572043+0000","last_clean":"2023-09-23T22:49:58.572043+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:49:58.572043+0000","last_undegraded":"2023-09-23T22:49:58.572043+0000","last_fullsized":"2023-09-23T22:49:58.572043+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"258'11677","reported_seq":11729,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.996918+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:50:20.996918+0000","last_peered":"2023-09-23T22:50:20.996918+0000","last_clean":"2023-09-23T22:50:20.996918+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:50:20.996918+0000","last_undegraded":"2023-09-23T22:50:20.996918+0000","last_fullsized":"2023-09-23T22:50:20.996918+0000","mapping_epoch":217,"log_start":"232'4500","ondisk_log_start":"232'4500","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7177,"log_dups_size":0,"ondisk_log_size":7177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62324736,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11677,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":259,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625782+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:50:09.625782+0000","last_peered":"2023-09-23T22:50:09.625782+0000","last_clean":"2023-09-23T22:50:09.625782+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:50:09.625782+0000","last_undegraded":"2023-09-23T22:50:09.625782+0000","last_fullsized":"2023-09-23T22:50:09.625782+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":223,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.616693+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:50:09.616693+0000","last_peered":"2023-09-23T22:50:09.616693+0000","last_clean":"2023-09-23T22:50:09.616693+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:50:09.616693+0000","last_undegraded":"2023-09-23T22:50:09.616693+0000","last_fullsized":"2023-09-23T22:50:09.616693+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"257'11691","reported_seq":11762,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:17.826450+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:50:17.826450+0000","last_peered":"2023-09-23T22:50:17.826450+0000","last_clean":"2023-09-23T22:50:17.826450+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:50:17.826450+0000","last_undegraded":"2023-09-23T22:50:17.826450+0000","last_fullsized":"2023-09-23T22:50:17.826450+0000","mapping_epoch":226,"log_start":"229'3700","ondisk_log_start":"229'3700","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7991,"log_dups_size":0,"ondisk_log_size":7991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70713344,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11691,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":242,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618215+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:50:09.618215+0000","last_peered":"2023-09-23T22:50:09.618215+0000","last_clean":"2023-09-23T22:50:09.618215+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:50:09.618215+0000","last_undegraded":"2023-09-23T22:50:09.618215+0000","last_fullsized":"2023-09-23T22:50:09.618215+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":253,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.612924+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:50:09.612924+0000","last_peered":"2023-09-23T22:50:09.612924+0000","last_clean":"2023-09-23T22:50:09.612924+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:50:09.612924+0000","last_undegraded":"2023-09-23T22:50:09.612924+0000","last_fullsized":"2023-09-23T22:50:09.612924+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"258'11957","reported_seq":12012,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.482017+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:50:20.482017+0000","last_peered":"2023-09-23T22:50:20.482017+0000","last_clean":"2023-09-23T22:50:20.482017+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:50:20.482017+0000","last_undegraded":"2023-09-23T22:50:20.482017+0000","last_fullsized":"2023-09-23T22:50:20.482017+0000","mapping_epoch":219,"log_start":"231'4600","ondisk_log_start":"231'4600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7357,"log_dups_size":0,"ondisk_log_size":7357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68222976,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11957,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":266,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.612905+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:50:09.612905+0000","last_peered":"2023-09-23T22:50:09.612905+0000","last_clean":"2023-09-23T22:50:09.612905+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:50:09.612905+0000","last_undegraded":"2023-09-23T22:50:09.612905+0000","last_fullsized":"2023-09-23T22:50:09.612905+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":241,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613477+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:50:09.613477+0000","last_peered":"2023-09-23T22:50:09.613477+0000","last_clean":"2023-09-23T22:50:09.613477+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:50:09.613477+0000","last_undegraded":"2023-09-23T22:50:09.613477+0000","last_fullsized":"2023-09-23T22:50:09.613477+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"258'11488","reported_seq":11554,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.870109+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:50:20.870109+0000","last_peered":"2023-09-23T22:50:20.870109+0000","last_clean":"2023-09-23T22:50:20.870109+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:50:20.870109+0000","last_undegraded":"2023-09-23T22:50:20.870109+0000","last_fullsized":"2023-09-23T22:50:20.870109+0000","mapping_epoch":226,"log_start":"228'2900","ondisk_log_start":"228'2900","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62783488,"num_objects":958,"num_object_clones":0,"num_object_copies":2874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11488,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":259,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625835+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:50:09.625835+0000","last_peered":"2023-09-23T22:50:09.625835+0000","last_clean":"2023-09-23T22:50:09.625835+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:50:09.625835+0000","last_undegraded":"2023-09-23T22:50:09.625835+0000","last_fullsized":"2023-09-23T22:50:09.625835+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":185,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.704301+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:50:09.704301+0000","last_peered":"2023-09-23T22:50:09.704301+0000","last_clean":"2023-09-23T22:50:09.704301+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:50:09.704301+0000","last_undegraded":"2023-09-23T22:50:09.704301+0000","last_fullsized":"2023-09-23T22:50:09.704301+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"257'11651","reported_seq":11700,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:17.760292+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:50:17.760292+0000","last_peered":"2023-09-23T22:50:17.760292+0000","last_clean":"2023-09-23T22:50:17.760292+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:50:17.760292+0000","last_undegraded":"2023-09-23T22:50:17.760292+0000","last_fullsized":"2023-09-23T22:50:17.760292+0000","mapping_epoch":219,"log_start":"229'3700","ondisk_log_start":"229'3700","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77266944,"num_objects":1179,"num_object_clones":0,"num_object_copies":3537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11651,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":293,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572047+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:49:58.572047+0000","last_peered":"2023-09-23T22:49:58.572047+0000","last_clean":"2023-09-23T22:49:58.572047+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:49:58.572047+0000","last_undegraded":"2023-09-23T22:49:58.572047+0000","last_fullsized":"2023-09-23T22:49:58.572047+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626272+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:50:09.626272+0000","last_peered":"2023-09-23T22:50:09.626272+0000","last_clean":"2023-09-23T22:50:09.626272+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:50:09.626272+0000","last_undegraded":"2023-09-23T22:50:09.626272+0000","last_fullsized":"2023-09-23T22:50:09.626272+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613356+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:50:09.613356+0000","last_peered":"2023-09-23T22:50:09.613356+0000","last_clean":"2023-09-23T22:50:09.613356+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:50:09.613356+0000","last_undegraded":"2023-09-23T22:50:09.613356+0000","last_fullsized":"2023-09-23T22:50:09.613356+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"258'11763","reported_seq":11820,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:21.010868+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:50:21.010868+0000","last_peered":"2023-09-23T22:50:21.010868+0000","last_clean":"2023-09-23T22:50:21.010868+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:50:21.010868+0000","last_undegraded":"2023-09-23T22:50:21.010868+0000","last_fullsized":"2023-09-23T22:50:21.010868+0000","mapping_epoch":219,"log_start":"232'4600","ondisk_log_start":"232'4600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7163,"log_dups_size":0,"ondisk_log_size":7163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63373312,"num_objects":967,"num_object_clones":0,"num_object_copies":2901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11763,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":92,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618143+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:50:09.618143+0000","last_peered":"2023-09-23T22:50:09.618143+0000","last_clean":"2023-09-23T22:50:09.618143+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:50:09.618143+0000","last_undegraded":"2023-09-23T22:50:09.618143+0000","last_fullsized":"2023-09-23T22:50:09.618143+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.614831+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:50:09.614831+0000","last_peered":"2023-09-23T22:50:09.614831+0000","last_clean":"2023-09-23T22:50:09.614831+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:50:09.614831+0000","last_undegraded":"2023-09-23T22:50:09.614831+0000","last_fullsized":"2023-09-23T22:50:09.614831+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":174,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.622739+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:50:09.622739+0000","last_peered":"2023-09-23T22:50:09.622739+0000","last_clean":"2023-09-23T22:50:09.622739+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:50:09.622739+0000","last_undegraded":"2023-09-23T22:50:09.622739+0000","last_fullsized":"2023-09-23T22:50:09.622739+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"258'11950","reported_seq":12001,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.930855+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:50:20.930855+0000","last_peered":"2023-09-23T22:50:20.930855+0000","last_clean":"2023-09-23T22:50:20.930855+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:50:20.930855+0000","last_undegraded":"2023-09-23T22:50:20.930855+0000","last_fullsized":"2023-09-23T22:50:20.930855+0000","mapping_epoch":217,"log_start":"232'4800","ondisk_log_start":"232'4800","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7150,"log_dups_size":0,"ondisk_log_size":7150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64094208,"num_objects":978,"num_object_clones":0,"num_object_copies":2934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11950,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":249,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618643+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:50:09.618643+0000","last_peered":"2023-09-23T22:50:09.618643+0000","last_clean":"2023-09-23T22:50:09.618643+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:50:09.618643+0000","last_undegraded":"2023-09-23T22:50:09.618643+0000","last_fullsized":"2023-09-23T22:50:09.618643+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":137,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.617599+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:50:09.617599+0000","last_peered":"2023-09-23T22:50:09.617599+0000","last_clean":"2023-09-23T22:50:09.617599+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:50:09.617599+0000","last_undegraded":"2023-09-23T22:50:09.617599+0000","last_fullsized":"2023-09-23T22:50:09.617599+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":262,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572471+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:49:58.572471+0000","last_peered":"2023-09-23T22:49:58.572471+0000","last_clean":"2023-09-23T22:49:58.572471+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:49:58.572471+0000","last_undegraded":"2023-09-23T22:49:58.572471+0000","last_fullsized":"2023-09-23T22:49:58.572471+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"258'11652","reported_seq":11707,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:20.856375+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:50:20.856375+0000","last_peered":"2023-09-23T22:50:20.856375+0000","last_clean":"2023-09-23T22:50:20.856375+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:50:20.856375+0000","last_undegraded":"2023-09-23T22:50:20.856375+0000","last_fullsized":"2023-09-23T22:50:20.856375+0000","mapping_epoch":219,"log_start":"229'3000","ondisk_log_start":"229'3000","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8652,"log_dups_size":0,"ondisk_log_size":8652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63045632,"num_objects":962,"num_object_clones":0,"num_object_copies":2886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11652,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":259,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.617654+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:50:09.617654+0000","last_peered":"2023-09-23T22:50:09.617654+0000","last_clean":"2023-09-23T22:50:09.617654+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:50:09.617654+0000","last_undegraded":"2023-09-23T22:50:09.617654+0000","last_fullsized":"2023-09-23T22:50:09.617654+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":344,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.631455+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:50:09.631455+0000","last_peered":"2023-09-23T22:50:09.631455+0000","last_clean":"2023-09-23T22:50:09.631455+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:50:09.631455+0000","last_undegraded":"2023-09-23T22:50:09.631455+0000","last_fullsized":"2023-09-23T22:50:09.631455+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":189,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.626013+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:50:09.626013+0000","last_peered":"2023-09-23T22:50:09.626013+0000","last_clean":"2023-09-23T22:50:09.626013+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:50:09.626013+0000","last_undegraded":"2023-09-23T22:50:09.626013+0000","last_fullsized":"2023-09-23T22:50:09.626013+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":250,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618623+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:50:09.618623+0000","last_peered":"2023-09-23T22:50:09.618623+0000","last_clean":"2023-09-23T22:50:09.618623+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:50:09.618623+0000","last_undegraded":"2023-09-23T22:50:09.618623+0000","last_fullsized":"2023-09-23T22:50:09.618623+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"258'11925","reported_seq":11984,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.716435+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:50:19.716435+0000","last_peered":"2023-09-23T22:50:19.716435+0000","last_clean":"2023-09-23T22:50:19.716435+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:50:19.716435+0000","last_undegraded":"2023-09-23T22:50:19.716435+0000","last_fullsized":"2023-09-23T22:50:19.716435+0000","mapping_epoch":226,"log_start":"224'2200","ondisk_log_start":"224'2200","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9725,"log_dups_size":0,"ondisk_log_size":9725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67043328,"num_objects":1023,"num_object_clones":0,"num_object_copies":3069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11925,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.573062+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:49:58.573062+0000","last_peered":"2023-09-23T22:49:58.573062+0000","last_clean":"2023-09-23T22:49:58.573062+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:49:58.573062+0000","last_undegraded":"2023-09-23T22:49:58.573062+0000","last_fullsized":"2023-09-23T22:49:58.573062+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":193,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572244+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:49:58.572244+0000","last_peered":"2023-09-23T22:49:58.572244+0000","last_clean":"2023-09-23T22:49:58.572244+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:49:58.572244+0000","last_undegraded":"2023-09-23T22:49:58.572244+0000","last_fullsized":"2023-09-23T22:49:58.572244+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":180,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.582886+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:49:58.582886+0000","last_peered":"2023-09-23T22:49:58.582886+0000","last_clean":"2023-09-23T22:49:58.582886+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:49:58.582886+0000","last_undegraded":"2023-09-23T22:49:58.582886+0000","last_fullsized":"2023-09-23T22:49:58.582886+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"258'12217","reported_seq":18105,"reported_epoch":258,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:50:21.058465+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:50:21.058465+0000","last_peered":"2023-09-23T22:50:21.058465+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:50:21.058465+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"232'5000","ondisk_log_start":"232'5000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7217,"log_dups_size":0,"ondisk_log_size":7217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68878336,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2667,"num_objects_misplaced":1035,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12217,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2916,"num_bytes_recovered":191102976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-2651},{"shards":"1,3,4","objects":1035}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":185,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.578493+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:49:58.578493+0000","last_peered":"2023-09-23T22:49:58.578493+0000","last_clean":"2023-09-23T22:49:58.578493+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:49:58.578493+0000","last_undegraded":"2023-09-23T22:49:58.578493+0000","last_fullsized":"2023-09-23T22:49:58.578493+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.622886+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:50:09.622886+0000","last_peered":"2023-09-23T22:50:09.622886+0000","last_clean":"2023-09-23T22:50:09.622886+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:50:09.622886+0000","last_undegraded":"2023-09-23T22:50:09.622886+0000","last_fullsized":"2023-09-23T22:50:09.622886+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":185,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572377+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:49:58.572377+0000","last_peered":"2023-09-23T22:49:58.572377+0000","last_clean":"2023-09-23T22:49:58.572377+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:49:58.572377+0000","last_undegraded":"2023-09-23T22:49:58.572377+0000","last_fullsized":"2023-09-23T22:49:58.572377+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"257'11818","reported_seq":11867,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:17.860247+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:50:17.860247+0000","last_peered":"2023-09-23T22:50:17.860247+0000","last_clean":"2023-09-23T22:50:17.860247+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:50:17.860247+0000","last_undegraded":"2023-09-23T22:50:17.860247+0000","last_fullsized":"2023-09-23T22:50:17.860247+0000","mapping_epoch":219,"log_start":"230'3900","ondisk_log_start":"230'3900","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7918,"log_dups_size":0,"ondisk_log_size":7918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71434240,"num_objects":1090,"num_object_clones":0,"num_object_copies":3270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11818,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":298,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.632738+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:50:09.632738+0000","last_peered":"2023-09-23T22:50:09.632738+0000","last_clean":"2023-09-23T22:50:09.632738+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:50:09.632738+0000","last_undegraded":"2023-09-23T22:50:09.632738+0000","last_fullsized":"2023-09-23T22:50:09.632738+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":56,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.621871+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:50:09.621871+0000","last_peered":"2023-09-23T22:50:09.621871+0000","last_clean":"2023-09-23T22:50:09.621871+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:50:09.621871+0000","last_undegraded":"2023-09-23T22:50:09.621871+0000","last_fullsized":"2023-09-23T22:50:09.621871+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":284,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.625914+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:50:09.625914+0000","last_peered":"2023-09-23T22:50:09.625914+0000","last_clean":"2023-09-23T22:50:09.625914+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:50:09.625914+0000","last_undegraded":"2023-09-23T22:50:09.625914+0000","last_fullsized":"2023-09-23T22:50:09.625914+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"258'11817","reported_seq":11869,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.704185+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:50:19.704185+0000","last_peered":"2023-09-23T22:50:19.704185+0000","last_clean":"2023-09-23T22:50:19.704185+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:50:19.704185+0000","last_undegraded":"2023-09-23T22:50:19.704185+0000","last_fullsized":"2023-09-23T22:50:19.704185+0000","mapping_epoch":217,"log_start":"224'2100","ondisk_log_start":"224'2100","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9717,"log_dups_size":0,"ondisk_log_size":9717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69992472,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11817,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":290,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.618842+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:50:09.618842+0000","last_peered":"2023-09-23T22:50:09.618842+0000","last_clean":"2023-09-23T22:50:09.618842+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:50:09.618842+0000","last_undegraded":"2023-09-23T22:50:09.618842+0000","last_fullsized":"2023-09-23T22:50:09.618842+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":152,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.582978+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:49:58.582978+0000","last_peered":"2023-09-23T22:49:58.582978+0000","last_clean":"2023-09-23T22:49:58.582978+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:49:58.582978+0000","last_undegraded":"2023-09-23T22:49:58.582978+0000","last_fullsized":"2023-09-23T22:49:58.582978+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.612929+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:50:09.612929+0000","last_peered":"2023-09-23T22:50:09.612929+0000","last_clean":"2023-09-23T22:50:09.612929+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:50:09.612929+0000","last_undegraded":"2023-09-23T22:50:09.612929+0000","last_fullsized":"2023-09-23T22:50:09.612929+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":188,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572236+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:49:58.572236+0000","last_peered":"2023-09-23T22:49:58.572236+0000","last_clean":"2023-09-23T22:49:58.572236+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:49:58.572236+0000","last_undegraded":"2023-09-23T22:49:58.572236+0000","last_fullsized":"2023-09-23T22:49:58.572236+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":30,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.622805+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:50:09.622805+0000","last_peered":"2023-09-23T22:50:09.622805+0000","last_clean":"2023-09-23T22:50:09.622805+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:50:09.622805+0000","last_undegraded":"2023-09-23T22:50:09.622805+0000","last_fullsized":"2023-09-23T22:50:09.622805+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":68,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.616609+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:50:09.616609+0000","last_peered":"2023-09-23T22:50:09.616609+0000","last_clean":"2023-09-23T22:50:09.616609+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:50:09.616609+0000","last_undegraded":"2023-09-23T22:50:09.616609+0000","last_fullsized":"2023-09-23T22:50:09.616609+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":257,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.614896+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:50:09.614896+0000","last_peered":"2023-09-23T22:50:09.614896+0000","last_clean":"2023-09-23T22:50:09.614896+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:50:09.614896+0000","last_undegraded":"2023-09-23T22:50:09.614896+0000","last_fullsized":"2023-09-23T22:50:09.614896+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":68,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.621152+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:50:09.621152+0000","last_peered":"2023-09-23T22:50:09.621152+0000","last_clean":"2023-09-23T22:50:09.621152+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:50:09.621152+0000","last_undegraded":"2023-09-23T22:50:09.621152+0000","last_fullsized":"2023-09-23T22:50:09.621152+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.613512+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:50:09.613512+0000","last_peered":"2023-09-23T22:50:09.613512+0000","last_clean":"2023-09-23T22:50:09.613512+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:50:09.613512+0000","last_undegraded":"2023-09-23T22:50:09.613512+0000","last_fullsized":"2023-09-23T22:50:09.613512+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":252,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.635593+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:50:09.635593+0000","last_peered":"2023-09-23T22:50:09.635593+0000","last_clean":"2023-09-23T22:50:09.635593+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:50:09.635593+0000","last_undegraded":"2023-09-23T22:50:09.635593+0000","last_fullsized":"2023-09-23T22:50:09.635593+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:11.026953+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:50:11.026953+0000","last_peered":"2023-09-23T22:50:11.026953+0000","last_clean":"2023-09-23T22:50:11.026953+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:50:11.026953+0000","last_undegraded":"2023-09-23T22:50:11.026953+0000","last_fullsized":"2023-09-23T22:50:11.026953+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":201,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:49:58.572642+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:49:58.572642+0000","last_peered":"2023-09-23T22:49:58.572642+0000","last_clean":"2023-09-23T22:49:58.572642+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:49:58.572642+0000","last_undegraded":"2023-09-23T22:49:58.572642+0000","last_fullsized":"2023-09-23T22:49:58.572642+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":223,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:50:09.612813+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:50:09.612813+0000","last_peered":"2023-09-23T22:50:09.612813+0000","last_clean":"2023-09-23T22:50:09.612813+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:50:09.612813+0000","last_undegraded":"2023-09-23T22:50:09.612813+0000","last_fullsized":"2023-09-23T22:50:09.612813+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1075839000,"num_objects":16417,"num_object_clones":0,"num_object_copies":49251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2667,"num_objects_misplaced":1035,"num_objects_unfound":0,"num_objects_dirty":16417,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188481,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2919,"num_bytes_recovered":191299584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3545182208,"data_stored":3545169992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127781,"ondisk_log_size":127781,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430488,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626716,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":696164,"kb_used_data":382556,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":93048988,"statfs":{"total":95995035648,"available":95282163712,"internally_reserved":0,"allocated":391737344,"data_stored":391233825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":3.5,"15min":2.0259999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3890000000000002,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7360000000000002,"15min":1.8959999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.9800000000000004,"5min":117.48699999999999,"15min":117.48699999999999},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":4.2599999999999998,"5min":2.577,"15min":2.4870000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":31.286000000000001,"5min":31.286000000000001,"15min":153.78399999999999},"last":1.5600000000000001},{"interface":"front","average":{"1min":4.5350000000000001,"5min":4.3220000000000001,"15min":3.0830000000000002},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":31.512,"5min":185.411,"15min":185.411},"last":1.748}]},{"osd":2,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":2.8719999999999999,"15min":3.1640000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":33.174999999999997,"5min":33.174999999999997,"15min":100.785},"last":1.375},{"interface":"front","average":{"1min":3.9079999999999999,"5min":4.5099999999999998,"15min":3.6059999999999999},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":33.310000000000002,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":5.9370000000000003,"15min":3.4489999999999998},"min":{"1min":0.32800000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.742999999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.307},{"interface":"front","average":{"1min":2.5230000000000001,"5min":4.4880000000000004,"15min":3.0670000000000002},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":10.773999999999999,"5min":128.19800000000001,"15min":128.19800000000001},"last":1.224}]},{"osd":4,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.79800000000000004,"15min":0.67600000000000005},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6070000000000002,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.85399999999999998,"15min":0.68600000000000005},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":0.69199999999999995,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.31900000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.73999999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.5620000000000003,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.93400000000000005,"15min":0.754},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.4429999999999996,"5min":9.7899999999999991,"15min":18.625},"last":0.371}]},{"osd":7,"last update":"Sat Sep 23 22:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.98799999999999999,"15min":0.748},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":1.9990000000000001,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.75600000000000001,"15min":0.73399999999999999},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":2.8650000000000002,"5min":5.1790000000000003,"15min":16.661999999999999},"last":0.34499999999999997}]}]},{"osd":1,"up_from":84,"seq":360777253026,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":745276,"kb_used_data":424884,"kb_used_omap":9,"kb_used_meta":320374,"kb_avail":92999876,"statfs":{"total":95995035648,"available":95231873024,"internally_reserved":0,"allocated":435081216,"data_stored":434150691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":328063977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.367,"5min":8.0190000000000001,"15min":5.4509999999999996},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":26.042000000000002,"5min":101.605,"15min":101.605},"last":1.7949999999999999},{"interface":"front","average":{"1min":4.7430000000000003,"5min":7.6180000000000003,"15min":5.5490000000000004},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":25.379999999999999,"5min":101.892,"15min":101.892},"last":5.3769999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":3.7850000000000001,"15min":2.6589999999999998},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":26.129000000000001,"5min":37.869999999999997,"15min":37.889000000000003},"last":0.28499999999999998},{"interface":"front","average":{"1min":6.0439999999999996,"5min":4.0890000000000004,"15min":2.7549999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":26.001999999999999,"5min":37.957000000000001,"15min":37.957000000000001},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":9.3239999999999998,"5min":3.8450000000000002,"15min":2.149},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":87.787999999999997,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":10.641999999999999,"5min":4.7679999999999998,"15min":2.5299999999999998},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":89.861999999999995,"5min":89.861999999999995,"15min":89.861999999999995},"last":1.6950000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7949999999999999,"5min":3.1579999999999999,"15min":1.9490000000000001},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":11.432,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.3889999999999998,"15min":1.7649999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":10.619,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.266,"15min":1.53},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.4390000000000001,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.2949999999999999,"15min":1.8160000000000001},"min":{"1min":0.35299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.458,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.4609999999999999,"15min":1.5960000000000001},"min":{"1min":0.39000000000000001,"5min":0.34799999999999998,"15min":0.317},"max":{"1min":10.65,"5min":79.311000000000007,"15min":79.311000000000007},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.5,"5min":2.2829999999999999,"15min":1.4630000000000001},"min":{"1min":0.36299999999999999,"5min":0.313,"15min":0.313},"max":{"1min":7.3819999999999997,"5min":79.328999999999994,"15min":79.328999999999994},"last":1.484}]},{"osd":7,"last update":"Sat Sep 23 22:49:25 2023","interfaces":[{"interface":"back","average":{"1min":3.3530000000000002,"5min":2.6560000000000001,"15min":2.1869999999999998},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.32800000000000001},"max":{"1min":11.481999999999999,"5min":29.539000000000001,"15min":159.018},"last":0.435},{"interface":"front","average":{"1min":3.335,"5min":2.048,"15min":2.0960000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":10.676,"5min":10.676,"15min":158.96000000000001},"last":0.48899999999999999}]}]},{"osd":0,"up_from":219,"seq":940597837874,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":754856,"kb_used_data":584544,"kb_used_omap":31,"kb_used_meta":170144,"kb_avail":92990296,"statfs":{"total":95995035648,"available":95222063104,"internally_reserved":0,"allocated":598573056,"data_stored":597595137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":174228402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.319,"5min":4.0780000000000003,"15min":3.8340000000000001},"min":{"1min":0.24299999999999999,"5min":0.216,"15min":0.216},"max":{"1min":21.512,"5min":32.344999999999999,"15min":32.344999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":5.3239999999999998,"5min":3.9279999999999999,"15min":3.6819999999999999},"min":{"1min":0.30499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.064,"5min":32.106000000000002,"15min":32.106000000000002},"last":0.318}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":6.0640000000000001,"5min":3.2610000000000001,"15min":2.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":24.253,"5min":24.253,"15min":24.253},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.5,"5min":2.4079999999999999,"15min":1.974},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":15.106999999999999,"5min":15.106999999999999,"15min":15.106999999999999},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8760000000000003,"5min":2.9369999999999998,"15min":2.2890000000000001},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.710000000000001,"5min":20.710000000000001,"15min":20.710000000000001},"last":0.44},{"interface":"front","average":{"1min":2.964,"5min":1.794,"15min":1.347},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":13.991,"5min":13.991,"15min":13.991},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":3.9729999999999999,"5min":2.6520000000000001,"15min":2.1200000000000001},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":18.670999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":3.585,"5min":2.1179999999999999,"15min":1.8720000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":20.009,"5min":20.009,"15min":20.009},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":1.6160000000000001,"15min":1.429},"min":{"1min":0.36599999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.9260000000000002,"5min":8.9359999999999999,"15min":8.9359999999999999},"last":23.635000000000002},{"interface":"front","average":{"1min":4.0199999999999996,"5min":2.3559999999999999,"15min":1.8200000000000001},"min":{"1min":0.47199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":18.591999999999999,"5min":18.591999999999999,"15min":18.591999999999999},"last":23.614000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":1.7,"15min":1.4810000000000001},"min":{"1min":0.38700000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.9790000000000001,"5min":8.8550000000000004,"15min":8.8550000000000004},"last":59.146999999999998},{"interface":"front","average":{"1min":4.048,"5min":2.6110000000000002,"15min":1.994},"min":{"1min":0.51800000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":18.704999999999998,"5min":18.704999999999998,"15min":18.704999999999998},"last":59.164000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":2.4660000000000002,"15min":1.8859999999999999},"min":{"1min":0.54900000000000004,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":18.640000000000001,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.54,"15min":1.298},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":10.35,"5min":10.35,"15min":10.35},"last":0.56399999999999995}]}]},{"osd":2,"up_from":131,"seq":562640715887,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665544,"kb_used_data":411392,"kb_used_omap":81,"kb_used_meta":253998,"kb_avail":93079608,"statfs":{"total":95995035648,"available":95313518592,"internally_reserved":0,"allocated":421265408,"data_stored":420318828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":260094170},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:49:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7810000000000001,"5min":4.9379999999999997,"15min":5.1580000000000004},"min":{"1min":0.27300000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.692,"5min":57.811999999999998,"15min":57.811999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":5.9059999999999997,"5min":5.968,"15min":5.5819999999999999},"min":{"1min":0.80600000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":13.865,"5min":57.387999999999998,"15min":57.387999999999998},"last":0.47999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3079999999999998,"15min":3.1850000000000001},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.3650000000000002,"5min":11.759,"15min":88.316000000000003},"last":2.3580000000000001},{"interface":"front","average":{"1min":3.8130000000000002,"5min":3.3490000000000002,"15min":2.9870000000000001},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.167,"5min":28.596,"15min":88.436999999999998},"last":2.3109999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":5.2830000000000004,"5min":3.992,"15min":3.3370000000000002},"min":{"1min":0.39800000000000002,"5min":0.223,"15min":0.223},"max":{"1min":20.41,"5min":21.210999999999999,"15min":88.653000000000006},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.7839999999999998,"5min":2.734,"15min":2.6040000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":13.836,"5min":13.836,"15min":88.474000000000004},"last":1.0549999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":4.6219999999999999,"15min":2.2389999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.965,"5min":108.878,"15min":108.878},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.2690000000000001,"5min":4.8079999999999998,"15min":2.9969999999999999},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.2539999999999996,"5min":108.93000000000001,"15min":108.93000000000001},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":4.2210000000000001,"15min":2.8519999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.0869999999999997,"5min":97.322999999999993,"15min":97.322999999999993},"last":0.45500000000000002},{"interface":"front","average":{"1min":3.0720000000000001,"5min":4.4569999999999999,"15min":2.4889999999999999},"min":{"1min":0.34499999999999997,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":7.694,"5min":97.301000000000002,"15min":97.301000000000002},"last":2.1070000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":3.8780000000000001,"15min":2.4049999999999998},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6669999999999998,"5min":85.034999999999997,"15min":88.606999999999999},"last":2.1400000000000001},{"interface":"front","average":{"1min":2.2349999999999999,"5min":3.395,"15min":1.863},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":8.1479999999999997,"5min":85.209000000000003,"15min":85.209000000000003},"last":1.964}]},{"osd":7,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.8,"15min":1.389},"min":{"1min":0.46200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":2.0019999999999998},{"interface":"front","average":{"1min":2.9159999999999999,"5min":2.3570000000000002,"15min":2.2759999999999998},"min":{"1min":0.434,"5min":0.37,"15min":0.37},"max":{"1min":9.0419999999999998,"5min":9.5549999999999997,"15min":88.584000000000003},"last":0.67800000000000005}]}]},{"osd":3,"up_from":94,"seq":403726925973,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":778040,"kb_used_data":384880,"kb_used_omap":7,"kb_used_meta":392888,"kb_avail":92967112,"statfs":{"total":95995035648,"available":95198322688,"internally_reserved":0,"allocated":394117120,"data_stored":393188352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":402317928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,8],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":2.931,"5min":3.3849999999999998,"15min":2.3679999999999999},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.662000000000001,"5min":80.099000000000004,"15min":80.099000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":3.2810000000000001,"5min":3.3380000000000001,"15min":2.1779999999999999},"min":{"1min":0.374,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.128,"5min":80.150000000000006,"15min":80.150000000000006},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.5019999999999998,"5min":2.6970000000000001,"15min":1.6080000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20000000000000001},"max":{"1min":12.25,"5min":22.818000000000001,"15min":22.818000000000001},"last":0.79600000000000004},{"interface":"front","average":{"1min":2.9700000000000002,"5min":1.8779999999999999,"15min":1.2869999999999999},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":15.098000000000001,"5min":23.917999999999999,"15min":23.917999999999999},"last":0.84299999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":4.2370000000000001,"5min":2.5800000000000001,"15min":2.4569999999999999},"min":{"1min":0.32900000000000001,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":12.339,"5min":20.431000000000001,"15min":125.446},"last":0.45100000000000001},{"interface":"front","average":{"1min":5.2999999999999998,"5min":4.9500000000000002,"15min":3.0310000000000001},"min":{"1min":0.40799999999999997,"5min":0.217,"15min":0.217},"max":{"1min":15.756,"5min":182.483,"15min":182.483},"last":5.4489999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2149999999999999,"15min":1.3919999999999999},"min":{"1min":0.371,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":12.289999999999999,"5min":18.757000000000001,"15min":18.757000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.645,"5min":1.5149999999999999,"15min":1.196},"min":{"1min":0.38100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.131,"5min":12.131,"15min":12.131},"last":0.81200000000000006}]},{"osd":5,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3520000000000001,"15min":1.127},"min":{"1min":0.39800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.949999999999999,"5min":12.127000000000001,"15min":12.127000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.085,"15min":1.3300000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":4.4589999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":1.5269999999999999,"15min":1.175},"min":{"1min":0.38300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.897,"5min":12.16,"15min":12.16},"last":0.504},{"interface":"front","average":{"1min":2.399,"5min":2.0760000000000001,"15min":1.403},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":10.755000000000001},"last":4.5359999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:49:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.9929999999999999,"15min":1.6819999999999999},"min":{"1min":0.47299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7710000000000008,"5min":91.757999999999996,"15min":91.757999999999996},"last":4.5060000000000002},{"interface":"front","average":{"1min":3.1499999999999999,"5min":3.2869999999999999,"15min":1.8069999999999999},"min":{"1min":0.49099999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":12.32,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.73499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019172,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055136,"kb_used_data":757912,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92690016,"statfs":{"total":95995035648,"available":94914576384,"internally_reserved":0,"allocated":776101888,"data_stored":775601939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":4.1849999999999996,"15min":6.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.9770000000000003,"5min":110.08499999999999,"15min":110.08499999999999},"last":4.024},{"interface":"front","average":{"1min":2.5230000000000001,"5min":5.077,"15min":7.3780000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.464,"5min":110.33799999999999,"15min":110.33799999999999},"last":4.141}]},{"osd":1,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1230000000000002,"15min":1.53},"min":{"1min":0.317,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0269999999999992,"5min":41.643000000000001,"15min":45.320999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.9019999999999999,"15min":1.4279999999999999},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.4489999999999998,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:49:20 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":4.0970000000000004,"15min":3.2599999999999998},"min":{"1min":0.34300000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.9189999999999996,"5min":84.909000000000006,"15min":84.909000000000006},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.1600000000000001,"5min":3.9990000000000001,"15min":3.1419999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.0609999999999999,"5min":83.840000000000003,"15min":83.840000000000003},"last":0.54300000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":7.3099999999999996,"15min":3.3730000000000002},"min":{"1min":0.442,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":100.51600000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":8.1150000000000002,"5min":7.3609999999999998,"15min":3.4169999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":100.637,"5min":129.197,"15min":129.197},"last":1.986}]},{"osd":5,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":0.81000000000000005,"15min":0.82399999999999995},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":12.238,"5min":12.238,"15min":43.189999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.83499999999999996,"15min":0.85199999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":7.5460000000000003,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.91400000000000003,"15min":0.78500000000000003},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":10.952999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.72199999999999998,"15min":0.76700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":4.5359999999999996,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.69799999999999995,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.181,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.443},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.75900000000000001,"15min":0.68400000000000005},"min":{"1min":0.38800000000000001,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":8.6739999999999995,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.70099999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822944,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":883660,"kb_used_data":582340,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92861492,"statfs":{"total":95995035648,"available":95090167808,"internally_reserved":0,"allocated":596316160,"data_stored":595831829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":10.109999999999999,"5min":7.133,"15min":3.5830000000000002},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":84.367999999999995,"5min":121.319,"15min":121.319},"last":0.34399999999999997},{"interface":"front","average":{"1min":10.358000000000001,"5min":7.2380000000000004,"15min":3.5099999999999998},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":84.311999999999998,"5min":121.075,"15min":121.075},"last":11.028}]},{"osd":1,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.7290000000000001,"15min":1.762},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":6.7779999999999996,"5min":28.658999999999999,"15min":149.203},"last":0.38},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.165,"15min":1.9339999999999999},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.1310000000000002,"5min":28.664999999999999,"15min":149.554},"last":4.6529999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":3.4550000000000001,"15min":6.7249999999999996},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":7.9960000000000004,"5min":158.92599999999999,"15min":158.92599999999999},"last":1.014},{"interface":"front","average":{"1min":2.0369999999999999,"5min":3.5169999999999999,"15min":6.7199999999999998},"min":{"1min":0.36199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0430000000000001,"5min":158.91399999999999,"15min":158.91399999999999},"last":1.595}]},{"osd":3,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":5.4409999999999998,"5min":8.7390000000000008,"15min":4.2290000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":37.427999999999997,"5min":257.05000000000001,"15min":257.05000000000001},"last":0.31},{"interface":"front","average":{"1min":5.4939999999999998,"5min":8.484,"15min":4.0819999999999999},"min":{"1min":0.45800000000000002,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":37.277999999999999,"5min":257.35000000000002,"15min":257.35000000000002},"last":0.61599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66300000000000003,"15min":0.62},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":5.0309999999999997,"5min":5.0309999999999997,"15min":5.0309999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.75700000000000001,"15min":0.624},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":3.4409999999999998,"5min":10.535,"15min":10.535},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:49:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.89400000000000002,"15min":1.391},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":1.9410000000000001,"5min":12.615,"15min":140.07499999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.016,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":2.4609999999999999,"5min":12.74,"15min":139.739},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.74399999999999999,"15min":0.71999999999999997},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.738,"5min":4.2510000000000003,"15min":7.9809999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.73599999999999999,"15min":0.68000000000000005},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":3.5539999999999998,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.37}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":578748416,"data_stored":578748416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":431357952,"data_stored":431357952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":413011968,"data_stored":413007896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":388431872,"data_stored":388431872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":765198336,"data_stored":765198336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":581636096,"data_stored":581632024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":386797568,"data_stored":386793496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:50:24.904 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:50:25.021 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:25.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:25.090 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:25.141 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:25.185 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:25.454 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:25.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:25.561 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:25.592 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:25.624 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:25 smithi052 ceph-mon[187743]: pgmap v714: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 980 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 2667/47091 objects degraded (5.664%); 1035/47091 objects misplaced (2.198%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:50:25.625 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2667/47091 objects degraded (5.664%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:25.628 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:25 smithi052 ceph-mon[180979]: pgmap v714: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 980 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 2667/47091 objects degraded (5.664%); 1035/47091 objects misplaced (2.198%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:50:25.628 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2667/47091 objects degraded (5.664%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:25 smithi151 ceph-mon[159210]: pgmap v714: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 980 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 2667/47091 objects degraded (5.664%); 1035/47091 objects misplaced (2.198%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:50:25.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2667/47091 objects degraded (5.664%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:25.748 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:25.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:25.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:25.819 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:50:25.853 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:25.874 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:50:25.889 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:25.923 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:50:25.950 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:25.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:25.956 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:25.956 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:50:25.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.956 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45206.0:191946 8.a e258/226 8:5da3e9c6:::benchmark_data_smithi052_337357_object89495:head v 258'12152, mlcod=258'12149)", 2023-09-23T22:50:25.956 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:50:25.952146+0000", 2023-09-23T22:50:25.956 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.00296123, 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029910929999999998, 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952146+0000", 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:25.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952146+0000", 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952145+0000", 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2079000000000001e-05 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952157+0000", 2023-09-23T22:50:25.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7028999999999999e-05 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952174+0000", 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2840000000000001e-06 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952182+0000", 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8386000000000001e-05 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952230+0000", 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6550000000000002e-05 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952257+0000", 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110185 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:25.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45206.0:191948 8.9 e258/219 8:9a73f974:::benchmark_data_smithi052_337357_object89497:head v 258'11827, mlcod=258'11825)", 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:50:25.952432+0000", 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0026760040000000001, 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027766119999999999, 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952432+0000", 2023-09-23T22:50:25.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952432+0000", 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952431+0000", 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8870000000000004e-06 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952441+0000", 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6158999999999999e-05 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952457+0000", 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0620000000000003e-06 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:25.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952464+0000", 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6950000000000003e-05 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952511+0000", 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4159999999999999e-05 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:25.952535+0000", 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103371 2023-09-23T22:50:25.964 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:25.965 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:25.965 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:25.965 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:25.965 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:50:25.965 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 2 2023-09-23T22:50:25.965 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:25.971 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:50:26.031 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:50:26.056 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:26.072 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:50:26.087 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:26.102 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:50:26.151 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:50:26.152 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.163 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:50:26.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:26.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:26.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:26.296 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:26.357 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:26.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:26.462 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:26.496 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:26.642 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:26.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:26.674 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:50:26.752 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3667078982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:26.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:26.809 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:26.836 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3667078982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:26.839 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3667078982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:26.892 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:26.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:26.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:27.002 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:27.030 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:27.164 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:27.479 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:27.640 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:27.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:27.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:27.654 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:27 smithi151 ceph-mon[159210]: pgmap v715: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 842 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2704/40455 objects degraded (6.684%); 850/40455 objects misplaced (2.101%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:27.750 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:27.837 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:27 smithi052 ceph-mon[180979]: pgmap v715: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 842 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2704/40455 objects degraded (6.684%); 850/40455 objects misplaced (2.101%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:27.841 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:27 smithi052 ceph-mon[187743]: pgmap v715: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 842 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 2704/40455 objects degraded (6.684%); 850/40455 objects misplaced (2.101%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:27.843 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:27.903 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:27.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:28.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:28.109 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:28.326 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:28.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:28.437 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:28.477 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:28.676 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:28.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:28.764 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:50:28.764 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:50:28.780 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:28.816 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:28.900 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:28.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:28.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:29.005 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:29.042 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:29.098 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:29.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:29.201 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:29.276 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:29.399 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:29.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:29.507 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:29.515 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:50:29.552 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:29.578 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:50:29.617 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:50:29.628 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:29.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:29.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:29.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:29 smithi052 ceph-mon[180979]: pgmap v716: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 796 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 2704/38232 objects degraded (7.073%); 850/38232 objects misplaced (2.223%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:29.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1676606107' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:50:29.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:29 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:50:29.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:29 smithi052 ceph-mon[187743]: pgmap v716: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 796 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 2704/38232 objects degraded (7.073%); 850/38232 objects misplaced (2.223%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:29.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1676606107' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:50:29.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:29 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:50:29.841 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:29 smithi151 ceph-mon[159210]: pgmap v716: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 796 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 2704/38232 objects degraded (7.073%); 850/38232 objects misplaced (2.223%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1676606107' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:50:29.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:29 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:50:30.008 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:30.413 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:30.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:30.518 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:30.569 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:30.678 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:30.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:30.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:30.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[180979]: failed to encode map e259 with expected crc 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[180979]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[180979]: failed to encode map e259 with expected crc 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2704/38232 objects degraded (7.073%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[187743]: failed to encode map e259 with expected crc 2023-09-23T22:50:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:50:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[187743]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T22:50:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[187743]: failed to encode map e259 with expected crc 2023-09-23T22:50:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2704/38232 objects degraded (7.073%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:30.781 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:30.811 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:30.824 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2704/38232 objects degraded (7.073%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":963,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":259,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":7,"osd_in_since":1695509092,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":12744,"data_bytes":834148280,"bytes_used":5077405696,"bytes_avail":666887843840,"bytes_total":671965249536,"degraded_objects":2704,"degraded_total":38232,"degraded_ratio":0.070726093324963379,"misplaced_objects":850,"misplaced_total":38232,"misplaced_ratio":0.022232684662063195,"recovering_objects_per_sec":28,"recovering_bytes_per_sec":1898025,"recovering_keys_per_sec":0,"num_objects_recovered":290,"num_bytes_recovered":19005440,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":598},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":27,"modified":"2023-09-23T22:50:10.280818+0000","services":{}},"progress_events":{"3d9c7ac0-c827-48ce-9357-8c3e026dd16f":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:50:30.853 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:30.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:30 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:30.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:30 smithi151 ceph-mon[159210]: failed to encode map e259 with expected crc 2023-09-23T22:50:30.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:30 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:50:30.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:30 smithi151 ceph-mon[159210]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T22:50:30.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:30 smithi151 ceph-mon[159210]: failed to encode map e259 with expected crc 2023-09-23T22:50:30.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2704/38232 objects degraded (7.073%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:30.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:30.957 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:30.990 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:31.255 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:31.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:31.371 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:31.473 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:31.734 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:50:31.734 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:50:31.739 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:31 smithi052 ceph-mon[187743]: pgmap v718: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 775 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 2704/37227 objects degraded (7.264%); 850/37227 objects misplaced (2.283%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1910603141' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:31.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:31 smithi052 ceph-mon[180979]: pgmap v718: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 775 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 2704/37227 objects degraded (7.264%); 850/37227 objects misplaced (2.283%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:31.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1910603141' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:31.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:31.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:31.757 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:31.765 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:31 smithi151 ceph-mon[159210]: pgmap v718: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 775 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 2704/37227 objects degraded (7.264%); 850/37227 objects misplaced (2.283%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:50:31.765 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1910603141' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:31.783 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:50:31.784 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:50:31.856 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:31.970 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:32.188 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:32.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:32.292 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:32.326 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:32.466 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:50:32.467 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:32.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:32.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:32.574 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:32.613 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:32.681 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:32.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:32.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:32.787 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:32.814 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:32.890 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:32.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:32.910 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:50:32.967 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:32.995 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:33.092 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:33.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:33.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:33.556 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:33.659 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:33.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:33 smithi052 ceph-mon[180979]: pgmap v719: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 662 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 2730/31839 objects degraded (8.574%); 663/31839 objects misplaced (2.082%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:33.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:33 smithi052 ceph-mon[187743]: pgmap v719: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 662 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 2730/31839 objects degraded (8.574%); 663/31839 objects misplaced (2.082%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:33.675 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:50:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:50:33] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:50:33.692 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:33.797 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:33.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:33.908 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:33 smithi151 ceph-mon[159210]: pgmap v719: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 662 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 2730/31839 objects degraded (8.574%); 663/31839 objects misplaced (2.082%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:33.972 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:33.995 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:34.163 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:50:34.170 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:34.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:34.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:34.218 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:50:34.263 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:50:34.281 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:34.305 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:50:34.337 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:50:34.358 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:34.406 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:50:34.443 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:50:34.460 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:34.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:34.499 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:50:34.545 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.546 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:50:34.577 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:34.606 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1374995764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:34.608 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:34.608 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:197062 8.5 8:a8d63909:::benchmark_data_smithi052_337357_object94611:head [delete] snapc 0=[] ondisk+write+known_if_redirected e259)", 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:50:34.603480+0000", 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0047549250000000001, 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0047873170000000001, 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:50:34.609 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 197062 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603480+0000", 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603480+0000", 2023-09-23T22:50:34.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603479+0000", 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0420000000000002e-06 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603484+0000", 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6209999999999999e-06 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603493+0000", 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2980000000000002e-06 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603498+0000", 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1238000000000001e-05 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:34.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603529+0000", 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9598999999999994e-05 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603618+0000", 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9915999999999998e-05 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.603708+0000", 2023-09-23T22:50:34.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00022833599999999999 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:197054 8.3 8:c6cf493c:::benchmark_data_smithi052_337357_object94603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e259)", 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:50:34.594764+0000", 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.013470947, 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.01359489, 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:50:34.614 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 197054 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594764+0000", 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594764+0000", 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594762+0000", 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4319999999999999e-06 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594771+0000", 2023-09-23T22:50:34.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3322999999999999e-05 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594784+0000", 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7469999999999999e-06 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594791+0000", 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3640000000000001e-05 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594844+0000", 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013181099999999999 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.594976+0000", 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013553600000000001 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:50:34.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.595112+0000", 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0042822590000000001 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.599394+0000", 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034176749999999998 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:50:34.602812+0000", 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0080475970000000001 2023-09-23T22:50:34.619 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:34.620 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:34.620 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:34.620 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:34.620 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:50:34.620 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 2 2023-09-23T22:50:34.620 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:34.692 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:34.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1374995764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:34.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1374995764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:34.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:34.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:35.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:50:35.366 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:35.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:35.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:35.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:35.505 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:35.693 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:35 smithi052 ceph-mon[187743]: pgmap v720: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 615 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2730/29562 objects degraded (9.235%); 663/29562 objects misplaced (2.243%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:35.693 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2730/29562 objects degraded (9.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:35 smithi052 ceph-mon[180979]: pgmap v720: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 615 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2730/29562 objects degraded (9.235%); 663/29562 objects misplaced (2.243%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2730/29562 objects degraded (9.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:35.763 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:35 smithi151 ceph-mon[159210]: pgmap v720: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 615 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2730/29562 objects degraded (9.235%); 663/29562 objects misplaced (2.243%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2730/29562 objects degraded (9.235%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:36.390 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:36.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:36.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:36.501 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:36.543 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:36.603 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:36.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:36.707 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:36.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:36 smithi052 ceph-mon[187743]: pgmap v721: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 476 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2790/22905 objects degraded (12.181%); 454/22905 objects misplaced (1.982%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:36 smithi052 ceph-mon[180979]: pgmap v721: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 476 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2790/22905 objects degraded (12.181%); 454/22905 objects misplaced (1.982%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:36.763 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:36.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:36 smithi151 ceph-mon[159210]: pgmap v721: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 476 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2790/22905 objects degraded (12.181%); 454/22905 objects misplaced (1.982%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:36.921 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:36.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:36.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:37.032 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:37.072 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:37.201 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:37.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:37.309 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:37.406 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:37.407 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:50:37.407 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:50:37.407 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:50:37.407 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:37.436 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:37.737 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:37.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:37.846 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:37.898 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:38.081 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:38.083 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:50:38.178 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:50:38.184 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:38.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:38.290 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:38.346 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:38.419 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:38.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:38.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:38.435 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:50:38.525 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:38.611 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:38.657 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:39.271 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:39.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:39.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:39.377 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:39.404 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:50:39.408 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:39.413 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":721,"stamp":"2023-09-23T22:50:36.300552+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":499324856,"num_objects":7635,"num_object_clones":0,"num_object_copies":22905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2790,"num_objects_misplaced":454,"num_objects_unfound":0,"num_objects_dirty":7635,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":197547,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3409,"num_bytes_recovered":221325386,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127981,"ondisk_log_size":127981,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4060796,"kb_used_data":1980548,"kb_used_omap":129,"kb_used_meta":2079870,"kb_avail":652155268,"statfs":{"total":671965249536,"available":667806994432,"internally_reserved":0,"allocated":2028081152,"data_stored":2022807419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2129787613},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,4,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-383385600,"num_objects":-5850,"num_object_clones":0,"num_object_copies":-17550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":86,"num_objects_misplaced":-396,"num_objects_unfound":0,"num_objects_dirty":-5850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5850,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":20316160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003723"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":256,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456043+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:50:19.456043+0000","last_peered":"2023-09-23T22:50:19.456043+0000","last_clean":"2023-09-23T22:50:19.456043+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:50:19.456043+0000","last_undegraded":"2023-09-23T22:50:19.456043+0000","last_fullsized":"2023-09-23T22:50:19.456043+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":232,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580876+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:50:29.580876+0000","last_peered":"2023-09-23T22:50:29.580876+0000","last_clean":"2023-09-23T22:50:29.580876+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:50:29.580876+0000","last_undegraded":"2023-09-23T22:50:29.580876+0000","last_fullsized":"2023-09-23T22:50:29.580876+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.448928+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:50:19.448928+0000","last_peered":"2023-09-23T22:50:19.448928+0000","last_clean":"2023-09-23T22:50:19.448928+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:50:19.448928+0000","last_undegraded":"2023-09-23T22:50:19.448928+0000","last_fullsized":"2023-09-23T22:50:19.448928+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":227,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.474023+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:50:19.474023+0000","last_peered":"2023-09-23T22:50:19.474023+0000","last_clean":"2023-09-23T22:50:19.474023+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:50:19.474023+0000","last_undegraded":"2023-09-23T22:50:19.474023+0000","last_fullsized":"2023-09-23T22:50:19.474023+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":291,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.453667+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:50:19.453667+0000","last_peered":"2023-09-23T22:50:19.453667+0000","last_clean":"2023-09-23T22:50:19.453667+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:50:19.453667+0000","last_undegraded":"2023-09-23T22:50:19.453667+0000","last_fullsized":"2023-09-23T22:50:19.453667+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":224,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468411+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:50:19.468411+0000","last_peered":"2023-09-23T22:50:19.468411+0000","last_clean":"2023-09-23T22:50:19.468411+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:50:19.468411+0000","last_undegraded":"2023-09-23T22:50:19.468411+0000","last_fullsized":"2023-09-23T22:50:19.468411+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":158,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472762+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:50:19.472762+0000","last_peered":"2023-09-23T22:50:19.472762+0000","last_clean":"2023-09-23T22:50:19.472762+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:50:19.472762+0000","last_undegraded":"2023-09-23T22:50:19.472762+0000","last_fullsized":"2023-09-23T22:50:19.472762+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":259,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.453741+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:50:19.453741+0000","last_peered":"2023-09-23T22:50:19.453741+0000","last_clean":"2023-09-23T22:50:19.453741+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:50:19.453741+0000","last_undegraded":"2023-09-23T22:50:19.453741+0000","last_fullsized":"2023-09-23T22:50:19.453741+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.452845+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:50:19.452845+0000","last_peered":"2023-09-23T22:50:19.452845+0000","last_clean":"2023-09-23T22:50:19.452845+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:50:19.452845+0000","last_undegraded":"2023-09-23T22:50:19.452845+0000","last_fullsized":"2023-09-23T22:50:19.452845+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":217,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.448833+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:50:19.448833+0000","last_peered":"2023-09-23T22:50:19.448833+0000","last_clean":"2023-09-23T22:50:19.448833+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:50:19.448833+0000","last_undegraded":"2023-09-23T22:50:19.448833+0000","last_fullsized":"2023-09-23T22:50:19.448833+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":134,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.449313+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:50:19.449313+0000","last_peered":"2023-09-23T22:50:19.449313+0000","last_clean":"2023-09-23T22:50:19.449313+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:50:19.449313+0000","last_undegraded":"2023-09-23T22:50:19.449313+0000","last_fullsized":"2023-09-23T22:50:19.449313+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":133,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468864+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:50:19.468864+0000","last_peered":"2023-09-23T22:50:19.468864+0000","last_clean":"2023-09-23T22:50:19.468864+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:50:19.468864+0000","last_undegraded":"2023-09-23T22:50:19.468864+0000","last_fullsized":"2023-09-23T22:50:19.468864+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":279,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456363+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:50:19.456363+0000","last_peered":"2023-09-23T22:50:19.456363+0000","last_clean":"2023-09-23T22:50:19.456363+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:50:19.456363+0000","last_undegraded":"2023-09-23T22:50:19.456363+0000","last_fullsized":"2023-09-23T22:50:19.456363+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":224,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456086+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:50:19.456086+0000","last_peered":"2023-09-23T22:50:19.456086+0000","last_clean":"2023-09-23T22:50:19.456086+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:50:19.456086+0000","last_undegraded":"2023-09-23T22:50:19.456086+0000","last_fullsized":"2023-09-23T22:50:19.456086+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":117,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456118+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:50:19.456118+0000","last_peered":"2023-09-23T22:50:19.456118+0000","last_clean":"2023-09-23T22:50:19.456118+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:50:19.456118+0000","last_undegraded":"2023-09-23T22:50:19.456118+0000","last_fullsized":"2023-09-23T22:50:19.456118+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":279,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.452618+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:50:19.452618+0000","last_peered":"2023-09-23T22:50:19.452618+0000","last_clean":"2023-09-23T22:50:19.452618+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:50:19.452618+0000","last_undegraded":"2023-09-23T22:50:19.452618+0000","last_fullsized":"2023-09-23T22:50:19.452618+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":214,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.473516+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:50:19.473516+0000","last_peered":"2023-09-23T22:50:19.473516+0000","last_clean":"2023-09-23T22:50:19.473516+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:50:19.473516+0000","last_undegraded":"2023-09-23T22:50:19.473516+0000","last_fullsized":"2023-09-23T22:50:19.473516+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":239,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.449470+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:50:19.449470+0000","last_peered":"2023-09-23T22:50:19.449470+0000","last_clean":"2023-09-23T22:50:19.449470+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:50:19.449470+0000","last_undegraded":"2023-09-23T22:50:19.449470+0000","last_fullsized":"2023-09-23T22:50:19.449470+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456115+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:50:19.456115+0000","last_peered":"2023-09-23T22:50:19.456115+0000","last_clean":"2023-09-23T22:50:19.456115+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:50:19.456115+0000","last_undegraded":"2023-09-23T22:50:19.456115+0000","last_fullsized":"2023-09-23T22:50:19.456115+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":215,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.474121+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:50:19.474121+0000","last_peered":"2023-09-23T22:50:19.474121+0000","last_clean":"2023-09-23T22:50:19.474121+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:50:19.474121+0000","last_undegraded":"2023-09-23T22:50:19.474121+0000","last_fullsized":"2023-09-23T22:50:19.474121+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":303,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468245+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:50:19.468245+0000","last_peered":"2023-09-23T22:50:19.468245+0000","last_clean":"2023-09-23T22:50:19.468245+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:50:19.468245+0000","last_undegraded":"2023-09-23T22:50:19.468245+0000","last_fullsized":"2023-09-23T22:50:19.468245+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":259,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.452531+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:50:19.452531+0000","last_peered":"2023-09-23T22:50:19.452531+0000","last_clean":"2023-09-23T22:50:19.452531+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:50:19.452531+0000","last_undegraded":"2023-09-23T22:50:19.452531+0000","last_fullsized":"2023-09-23T22:50:19.452531+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"259'12304","reported_seq":12354,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:35.395036+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:50:35.395036+0000","last_peered":"2023-09-23T22:50:35.395036+0000","last_clean":"2023-09-23T22:50:35.395036+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:50:35.395036+0000","last_undegraded":"2023-09-23T22:50:35.395036+0000","last_fullsized":"2023-09-23T22:50:35.395036+0000","mapping_epoch":217,"log_start":"232'4900","ondisk_log_start":"232'4900","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12304,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":128,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456555+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:50:19.456555+0000","last_peered":"2023-09-23T22:50:19.456555+0000","last_clean":"2023-09-23T22:50:19.456555+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:50:19.456555+0000","last_undegraded":"2023-09-23T22:50:19.456555+0000","last_fullsized":"2023-09-23T22:50:19.456555+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":223,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468329+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:50:19.468329+0000","last_peered":"2023-09-23T22:50:19.468329+0000","last_clean":"2023-09-23T22:50:19.468329+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:50:19.468329+0000","last_undegraded":"2023-09-23T22:50:19.468329+0000","last_fullsized":"2023-09-23T22:50:19.468329+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"259'12151","reported_seq":12207,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:35.879101+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:50:35.879101+0000","last_peered":"2023-09-23T22:50:35.879101+0000","last_clean":"2023-09-23T22:50:35.879101+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:50:35.879101+0000","last_undegraded":"2023-09-23T22:50:35.879101+0000","last_fullsized":"2023-09-23T22:50:35.879101+0000","mapping_epoch":219,"log_start":"229'3500","ondisk_log_start":"229'3500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8651,"log_dups_size":0,"ondisk_log_size":8651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26411008,"num_objects":403,"num_object_clones":0,"num_object_copies":1209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12151,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":138,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456357+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:50:19.456357+0000","last_peered":"2023-09-23T22:50:19.456357+0000","last_clean":"2023-09-23T22:50:19.456357+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:50:19.456357+0000","last_undegraded":"2023-09-23T22:50:19.456357+0000","last_fullsized":"2023-09-23T22:50:19.456357+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":239,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456026+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:50:19.456026+0000","last_peered":"2023-09-23T22:50:19.456026+0000","last_clean":"2023-09-23T22:50:19.456026+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:50:19.456026+0000","last_undegraded":"2023-09-23T22:50:19.456026+0000","last_fullsized":"2023-09-23T22:50:19.456026+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"259'12057","reported_seq":12108,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:32.892924+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:50:32.892924+0000","last_peered":"2023-09-23T22:50:32.892924+0000","last_clean":"2023-09-23T22:50:32.892924+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:50:32.892924+0000","last_undegraded":"2023-09-23T22:50:32.892924+0000","last_fullsized":"2023-09-23T22:50:32.892924+0000","mapping_epoch":219,"log_start":"231'4100","ondisk_log_start":"231'4100","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7957,"log_dups_size":0,"ondisk_log_size":7957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35323904,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12057,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":69,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.471979+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:50:19.471979+0000","last_peered":"2023-09-23T22:50:19.471979+0000","last_clean":"2023-09-23T22:50:19.471979+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:50:19.471979+0000","last_undegraded":"2023-09-23T22:50:19.471979+0000","last_fullsized":"2023-09-23T22:50:19.471979+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":251,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468982+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:50:19.468982+0000","last_peered":"2023-09-23T22:50:19.468982+0000","last_clean":"2023-09-23T22:50:19.468982+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:50:19.468982+0000","last_undegraded":"2023-09-23T22:50:19.468982+0000","last_fullsized":"2023-09-23T22:50:19.468982+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"259'12594","reported_seq":12654,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:36.041362+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:50:36.041362+0000","last_peered":"2023-09-23T22:50:36.041362+0000","last_clean":"2023-09-23T22:50:36.041362+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:50:36.041362+0000","last_undegraded":"2023-09-23T22:50:36.041362+0000","last_fullsized":"2023-09-23T22:50:36.041362+0000","mapping_epoch":226,"log_start":"233'5400","ondisk_log_start":"233'5400","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7194,"log_dups_size":0,"ondisk_log_size":7194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12594,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":112,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:26.063836+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:50:26.063836+0000","last_peered":"2023-09-23T22:50:26.063836+0000","last_clean":"2023-09-23T22:50:26.063836+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:50:26.063836+0000","last_undegraded":"2023-09-23T22:50:26.063836+0000","last_fullsized":"2023-09-23T22:50:26.063836+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":151,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472405+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:50:19.472405+0000","last_peered":"2023-09-23T22:50:19.472405+0000","last_clean":"2023-09-23T22:50:19.472405+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:50:19.472405+0000","last_undegraded":"2023-09-23T22:50:19.472405+0000","last_fullsized":"2023-09-23T22:50:19.472405+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"259'12200","reported_seq":12253,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:36.038180+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:50:36.038180+0000","last_peered":"2023-09-23T22:50:36.038180+0000","last_clean":"2023-09-23T22:50:36.038180+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:50:36.038180+0000","last_undegraded":"2023-09-23T22:50:36.038180+0000","last_fullsized":"2023-09-23T22:50:36.038180+0000","mapping_epoch":217,"log_start":"232'5000","ondisk_log_start":"232'5000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12200,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":260,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.452637+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:50:19.452637+0000","last_peered":"2023-09-23T22:50:19.452637+0000","last_clean":"2023-09-23T22:50:19.452637+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:50:19.452637+0000","last_undegraded":"2023-09-23T22:50:19.452637+0000","last_fullsized":"2023-09-23T22:50:19.452637+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":224,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456326+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:50:19.456326+0000","last_peered":"2023-09-23T22:50:19.456326+0000","last_clean":"2023-09-23T22:50:19.456326+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:50:19.456326+0000","last_undegraded":"2023-09-23T22:50:19.456326+0000","last_fullsized":"2023-09-23T22:50:19.456326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"259'12242","reported_seq":12315,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:32.814333+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:50:32.814333+0000","last_peered":"2023-09-23T22:50:32.814333+0000","last_clean":"2023-09-23T22:50:32.814333+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:50:32.814333+0000","last_undegraded":"2023-09-23T22:50:32.814333+0000","last_fullsized":"2023-09-23T22:50:32.814333+0000","mapping_epoch":226,"log_start":"231'4300","ondisk_log_start":"231'4300","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7942,"log_dups_size":0,"ondisk_log_size":7942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34603008,"num_objects":528,"num_object_clones":0,"num_object_copies":1584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12242,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":243,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.469056+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:50:19.469056+0000","last_peered":"2023-09-23T22:50:19.469056+0000","last_clean":"2023-09-23T22:50:19.469056+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:50:19.469056+0000","last_undegraded":"2023-09-23T22:50:19.469056+0000","last_fullsized":"2023-09-23T22:50:19.469056+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":254,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456267+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:50:19.456267+0000","last_peered":"2023-09-23T22:50:19.456267+0000","last_clean":"2023-09-23T22:50:19.456267+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:50:19.456267+0000","last_undegraded":"2023-09-23T22:50:19.456267+0000","last_fullsized":"2023-09-23T22:50:19.456267+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"259'12529","reported_seq":12585,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:35.592197+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:50:35.592197+0000","last_peered":"2023-09-23T22:50:35.592197+0000","last_clean":"2023-09-23T22:50:35.592197+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:50:35.592197+0000","last_undegraded":"2023-09-23T22:50:35.592197+0000","last_fullsized":"2023-09-23T22:50:35.592197+0000","mapping_epoch":219,"log_start":"232'5200","ondisk_log_start":"232'5200","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30736384,"num_objects":469,"num_object_clones":0,"num_object_copies":1407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12529,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":267,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.455888+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:50:19.455888+0000","last_peered":"2023-09-23T22:50:19.455888+0000","last_clean":"2023-09-23T22:50:19.455888+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:50:19.455888+0000","last_undegraded":"2023-09-23T22:50:19.455888+0000","last_fullsized":"2023-09-23T22:50:19.455888+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":242,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456393+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:50:19.456393+0000","last_peered":"2023-09-23T22:50:19.456393+0000","last_clean":"2023-09-23T22:50:19.456393+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:50:19.456393+0000","last_undegraded":"2023-09-23T22:50:19.456393+0000","last_fullsized":"2023-09-23T22:50:19.456393+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"259'12006","reported_seq":12073,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:35.745104+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:50:35.745104+0000","last_peered":"2023-09-23T22:50:35.745104+0000","last_clean":"2023-09-23T22:50:35.745104+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:50:35.745104+0000","last_undegraded":"2023-09-23T22:50:35.745104+0000","last_fullsized":"2023-09-23T22:50:35.745104+0000","mapping_epoch":226,"log_start":"229'3400","ondisk_log_start":"229'3400","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8606,"log_dups_size":0,"ondisk_log_size":8606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28835840,"num_objects":440,"num_object_clones":0,"num_object_copies":1320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12006,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":260,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.453522+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:50:19.453522+0000","last_peered":"2023-09-23T22:50:19.453522+0000","last_clean":"2023-09-23T22:50:19.453522+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:50:19.453522+0000","last_undegraded":"2023-09-23T22:50:19.453522+0000","last_fullsized":"2023-09-23T22:50:19.453522+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":186,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.463897+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:50:19.463897+0000","last_peered":"2023-09-23T22:50:19.463897+0000","last_clean":"2023-09-23T22:50:19.463897+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:50:19.463897+0000","last_undegraded":"2023-09-23T22:50:19.463897+0000","last_fullsized":"2023-09-23T22:50:19.463897+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"259'12235","reported_seq":12286,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:32.877029+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:50:32.877029+0000","last_peered":"2023-09-23T22:50:32.877029+0000","last_clean":"2023-09-23T22:50:32.877029+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:50:32.877029+0000","last_undegraded":"2023-09-23T22:50:32.877029+0000","last_fullsized":"2023-09-23T22:50:32.877029+0000","mapping_epoch":219,"log_start":"231'4300","ondisk_log_start":"231'4300","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7935,"log_dups_size":0,"ondisk_log_size":7935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38993920,"num_objects":595,"num_object_clones":0,"num_object_copies":1785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12235,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":295,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472683+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:50:19.472683+0000","last_peered":"2023-09-23T22:50:19.472683+0000","last_clean":"2023-09-23T22:50:19.472683+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:50:19.472683+0000","last_undegraded":"2023-09-23T22:50:19.472683+0000","last_fullsized":"2023-09-23T22:50:19.472683+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":179,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.449222+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:50:19.449222+0000","last_peered":"2023-09-23T22:50:19.449222+0000","last_clean":"2023-09-23T22:50:19.449222+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:50:19.449222+0000","last_undegraded":"2023-09-23T22:50:19.449222+0000","last_fullsized":"2023-09-23T22:50:19.449222+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.455972+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:50:19.455972+0000","last_peered":"2023-09-23T22:50:19.455972+0000","last_clean":"2023-09-23T22:50:19.455972+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:50:19.455972+0000","last_undegraded":"2023-09-23T22:50:19.455972+0000","last_fullsized":"2023-09-23T22:50:19.455972+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"259'12286","reported_seq":12345,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:36.088266+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:50:36.088266+0000","last_peered":"2023-09-23T22:50:36.088266+0000","last_clean":"2023-09-23T22:50:36.088266+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:50:36.088266+0000","last_undegraded":"2023-09-23T22:50:36.088266+0000","last_fullsized":"2023-09-23T22:50:36.088266+0000","mapping_epoch":219,"log_start":"232'5100","ondisk_log_start":"232'5100","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7186,"log_dups_size":0,"ondisk_log_size":7186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12286,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":93,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468983+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:50:19.468983+0000","last_peered":"2023-09-23T22:50:19.468983+0000","last_clean":"2023-09-23T22:50:19.468983+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:50:19.468983+0000","last_undegraded":"2023-09-23T22:50:19.468983+0000","last_fullsized":"2023-09-23T22:50:19.468983+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":255,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456173+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:50:19.456173+0000","last_peered":"2023-09-23T22:50:19.456173+0000","last_clean":"2023-09-23T22:50:19.456173+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:50:19.456173+0000","last_undegraded":"2023-09-23T22:50:19.456173+0000","last_fullsized":"2023-09-23T22:50:19.456173+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":175,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.462710+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:50:19.462710+0000","last_peered":"2023-09-23T22:50:19.462710+0000","last_clean":"2023-09-23T22:50:19.462710+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:50:19.462710+0000","last_undegraded":"2023-09-23T22:50:19.462710+0000","last_fullsized":"2023-09-23T22:50:19.462710+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"259'12520","reported_seq":12572,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:36.040375+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:50:36.040375+0000","last_peered":"2023-09-23T22:50:36.040375+0000","last_clean":"2023-09-23T22:50:36.040375+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:50:36.040375+0000","last_undegraded":"2023-09-23T22:50:36.040375+0000","last_fullsized":"2023-09-23T22:50:36.040375+0000","mapping_epoch":217,"log_start":"233'5300","ondisk_log_start":"233'5300","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7220,"log_dups_size":0,"ondisk_log_size":7220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26738688,"num_objects":408,"num_object_clones":0,"num_object_copies":1224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12520,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":250,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.462250+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:50:19.462250+0000","last_peered":"2023-09-23T22:50:19.462250+0000","last_clean":"2023-09-23T22:50:19.462250+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:50:19.462250+0000","last_undegraded":"2023-09-23T22:50:19.462250+0000","last_fullsized":"2023-09-23T22:50:19.462250+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":138,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.461903+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:50:19.461903+0000","last_peered":"2023-09-23T22:50:19.461903+0000","last_clean":"2023-09-23T22:50:19.461903+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:50:19.461903+0000","last_undegraded":"2023-09-23T22:50:19.461903+0000","last_fullsized":"2023-09-23T22:50:19.461903+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":264,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472678+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:50:19.472678+0000","last_peered":"2023-09-23T22:50:19.472678+0000","last_clean":"2023-09-23T22:50:19.472678+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:50:19.472678+0000","last_undegraded":"2023-09-23T22:50:19.472678+0000","last_fullsized":"2023-09-23T22:50:19.472678+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"259'12183","reported_seq":12239,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:35.672811+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:50:35.672811+0000","last_peered":"2023-09-23T22:50:35.672811+0000","last_clean":"2023-09-23T22:50:35.672811+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:50:35.672811+0000","last_undegraded":"2023-09-23T22:50:35.672811+0000","last_fullsized":"2023-09-23T22:50:35.672811+0000","mapping_epoch":219,"log_start":"229'3600","ondisk_log_start":"229'3600","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8583,"log_dups_size":0,"ondisk_log_size":8583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28246016,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12183,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":260,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.463774+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:50:19.463774+0000","last_peered":"2023-09-23T22:50:19.463774+0000","last_clean":"2023-09-23T22:50:19.463774+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:50:19.463774+0000","last_undegraded":"2023-09-23T22:50:19.463774+0000","last_fullsized":"2023-09-23T22:50:19.463774+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":346,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472335+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:50:19.472335+0000","last_peered":"2023-09-23T22:50:19.472335+0000","last_clean":"2023-09-23T22:50:19.472335+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:50:19.472335+0000","last_undegraded":"2023-09-23T22:50:19.472335+0000","last_fullsized":"2023-09-23T22:50:19.472335+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":191,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580187+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:50:29.580187+0000","last_peered":"2023-09-23T22:50:29.580187+0000","last_clean":"2023-09-23T22:50:29.580187+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:50:29.580187+0000","last_undegraded":"2023-09-23T22:50:29.580187+0000","last_fullsized":"2023-09-23T22:50:29.580187+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":251,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472029+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:50:19.472029+0000","last_peered":"2023-09-23T22:50:19.472029+0000","last_clean":"2023-09-23T22:50:19.472029+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:50:19.472029+0000","last_undegraded":"2023-09-23T22:50:19.472029+0000","last_fullsized":"2023-09-23T22:50:19.472029+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"259'12441","reported_seq":12501,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:34.691870+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:50:34.691870+0000","last_peered":"2023-09-23T22:50:34.691870+0000","last_clean":"2023-09-23T22:50:34.691870+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:50:34.691870+0000","last_undegraded":"2023-09-23T22:50:34.691870+0000","last_fullsized":"2023-09-23T22:50:34.691870+0000","mapping_epoch":226,"log_start":"227'2700","ondisk_log_start":"227'2700","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9741,"log_dups_size":0,"ondisk_log_size":9741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12441,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":224,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472490+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:50:19.472490+0000","last_peered":"2023-09-23T22:50:19.472490+0000","last_clean":"2023-09-23T22:50:19.472490+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:50:19.472490+0000","last_undegraded":"2023-09-23T22:50:19.472490+0000","last_fullsized":"2023-09-23T22:50:19.472490+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":196,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:22.894450+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:50:22.894450+0000","last_peered":"2023-09-23T22:50:22.894450+0000","last_clean":"2023-09-23T22:50:22.894450+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:50:22.894450+0000","last_undegraded":"2023-09-23T22:50:22.894450+0000","last_fullsized":"2023-09-23T22:50:22.894450+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":183,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580674+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:50:29.580674+0000","last_peered":"2023-09-23T22:50:29.580674+0000","last_clean":"2023-09-23T22:50:29.580674+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:50:29.580674+0000","last_undegraded":"2023-09-23T22:50:29.580674+0000","last_fullsized":"2023-09-23T22:50:29.580674+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"259'12798","reported_seq":19605,"reported_epoch":259,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:50:36.081953+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:50:36.081953+0000","last_peered":"2023-09-23T22:50:36.081953+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:50:36.081953+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"233'5600","ondisk_log_start":"233'5600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7198,"log_dups_size":0,"ondisk_log_size":7198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30801920,"num_objects":470,"num_object_clones":0,"num_object_copies":1410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2790,"num_objects_misplaced":454,"num_objects_unfound":0,"num_objects_dirty":470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12798,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3374,"num_bytes_recovered":221118464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-2774},{"shards":"1,3,4","objects":454}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":188,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:22.894533+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:50:22.894533+0000","last_peered":"2023-09-23T22:50:22.894533+0000","last_clean":"2023-09-23T22:50:22.894533+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:50:22.894533+0000","last_undegraded":"2023-09-23T22:50:22.894533+0000","last_fullsized":"2023-09-23T22:50:22.894533+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":137,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.462787+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:50:19.462787+0000","last_peered":"2023-09-23T22:50:19.462787+0000","last_clean":"2023-09-23T22:50:19.462787+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:50:19.462787+0000","last_undegraded":"2023-09-23T22:50:19.462787+0000","last_fullsized":"2023-09-23T22:50:19.462787+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":188,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:22.894497+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:50:22.894497+0000","last_peered":"2023-09-23T22:50:22.894497+0000","last_clean":"2023-09-23T22:50:22.894497+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:50:22.894497+0000","last_undegraded":"2023-09-23T22:50:22.894497+0000","last_fullsized":"2023-09-23T22:50:22.894497+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"259'12346","reported_seq":12397,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:32.832604+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:50:32.832604+0000","last_peered":"2023-09-23T22:50:32.832604+0000","last_clean":"2023-09-23T22:50:32.832604+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:50:32.832604+0000","last_undegraded":"2023-09-23T22:50:32.832604+0000","last_fullsized":"2023-09-23T22:50:32.832604+0000","mapping_epoch":219,"log_start":"231'4400","ondisk_log_start":"231'4400","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12346,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":299,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.452447+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:50:19.452447+0000","last_peered":"2023-09-23T22:50:19.452447+0000","last_clean":"2023-09-23T22:50:19.452447+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:50:19.452447+0000","last_undegraded":"2023-09-23T22:50:19.452447+0000","last_fullsized":"2023-09-23T22:50:19.452447+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":57,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468796+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:50:19.468796+0000","last_peered":"2023-09-23T22:50:19.468796+0000","last_clean":"2023-09-23T22:50:19.468796+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:50:19.468796+0000","last_undegraded":"2023-09-23T22:50:19.468796+0000","last_fullsized":"2023-09-23T22:50:19.468796+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":285,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.453595+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:50:19.453595+0000","last_peered":"2023-09-23T22:50:19.453595+0000","last_clean":"2023-09-23T22:50:19.453595+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:50:19.453595+0000","last_undegraded":"2023-09-23T22:50:19.453595+0000","last_fullsized":"2023-09-23T22:50:19.453595+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"259'12395","reported_seq":12448,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:34.659198+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:50:34.659198+0000","last_peered":"2023-09-23T22:50:34.659198+0000","last_clean":"2023-09-23T22:50:34.659198+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:50:34.659198+0000","last_undegraded":"2023-09-23T22:50:34.659198+0000","last_fullsized":"2023-09-23T22:50:34.659198+0000","mapping_epoch":217,"log_start":"227'2700","ondisk_log_start":"227'2700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9695,"log_dups_size":0,"ondisk_log_size":9695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32112664,"num_objects":491,"num_object_clones":0,"num_object_copies":1473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":491,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12395,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":291,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.468331+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:50:19.468331+0000","last_peered":"2023-09-23T22:50:19.468331+0000","last_clean":"2023-09-23T22:50:19.468331+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:50:19.468331+0000","last_undegraded":"2023-09-23T22:50:19.468331+0000","last_fullsized":"2023-09-23T22:50:19.468331+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":155,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580764+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:50:29.580764+0000","last_peered":"2023-09-23T22:50:29.580764+0000","last_clean":"2023-09-23T22:50:29.580764+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:50:29.580764+0000","last_undegraded":"2023-09-23T22:50:29.580764+0000","last_fullsized":"2023-09-23T22:50:29.580764+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":209,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.455932+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:50:19.455932+0000","last_peered":"2023-09-23T22:50:19.455932+0000","last_clean":"2023-09-23T22:50:19.455932+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:50:19.455932+0000","last_undegraded":"2023-09-23T22:50:19.455932+0000","last_fullsized":"2023-09-23T22:50:19.455932+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":190,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.472501+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:50:19.472501+0000","last_peered":"2023-09-23T22:50:19.472501+0000","last_clean":"2023-09-23T22:50:19.472501+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:50:19.472501+0000","last_undegraded":"2023-09-23T22:50:19.472501+0000","last_fullsized":"2023-09-23T22:50:19.472501+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":32,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:24.743654+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:50:24.743654+0000","last_peered":"2023-09-23T22:50:24.743654+0000","last_clean":"2023-09-23T22:50:24.743654+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:50:24.743654+0000","last_undegraded":"2023-09-23T22:50:24.743654+0000","last_fullsized":"2023-09-23T22:50:24.743654+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":69,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456250+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:50:19.456250+0000","last_peered":"2023-09-23T22:50:19.456250+0000","last_clean":"2023-09-23T22:50:19.456250+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:50:19.456250+0000","last_undegraded":"2023-09-23T22:50:19.456250+0000","last_fullsized":"2023-09-23T22:50:19.456250+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":258,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456217+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:50:19.456217+0000","last_peered":"2023-09-23T22:50:19.456217+0000","last_clean":"2023-09-23T22:50:19.456217+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:50:19.456217+0000","last_undegraded":"2023-09-23T22:50:19.456217+0000","last_fullsized":"2023-09-23T22:50:19.456217+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":69,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456460+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:50:19.456460+0000","last_peered":"2023-09-23T22:50:19.456460+0000","last_clean":"2023-09-23T22:50:19.456460+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:50:19.456460+0000","last_undegraded":"2023-09-23T22:50:19.456460+0000","last_fullsized":"2023-09-23T22:50:19.456460+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":130,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.456061+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:50:19.456061+0000","last_peered":"2023-09-23T22:50:19.456061+0000","last_clean":"2023-09-23T22:50:19.456061+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:50:19.456061+0000","last_undegraded":"2023-09-23T22:50:19.456061+0000","last_fullsized":"2023-09-23T22:50:19.456061+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":253,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.452931+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:50:19.452931+0000","last_peered":"2023-09-23T22:50:19.452931+0000","last_clean":"2023-09-23T22:50:19.452931+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:50:19.452931+0000","last_undegraded":"2023-09-23T22:50:19.452931+0000","last_fullsized":"2023-09-23T22:50:19.452931+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":291,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.448743+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:50:19.448743+0000","last_peered":"2023-09-23T22:50:19.448743+0000","last_clean":"2023-09-23T22:50:19.448743+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:50:19.448743+0000","last_undegraded":"2023-09-23T22:50:19.448743+0000","last_fullsized":"2023-09-23T22:50:19.448743+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":204,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:22.894622+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:50:22.894622+0000","last_peered":"2023-09-23T22:50:22.894622+0000","last_clean":"2023-09-23T22:50:22.894622+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:50:22.894622+0000","last_undegraded":"2023-09-23T22:50:22.894622+0000","last_fullsized":"2023-09-23T22:50:22.894622+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":224,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:50:19.455886+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:50:19.455886+0000","last_peered":"2023-09-23T22:50:19.455886+0000","last_clean":"2023-09-23T22:50:19.455886+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:50:19.455886+0000","last_undegraded":"2023-09-23T22:50:19.455886+0000","last_fullsized":"2023-09-23T22:50:19.455886+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":498728984,"num_objects":7611,"num_object_clones":0,"num_object_copies":22833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2790,"num_objects_misplaced":454,"num_objects_unfound":0,"num_objects_dirty":7611,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":197287,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3377,"num_bytes_recovered":221315072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1937256448,"data_stored":1937244232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127787,"ondisk_log_size":127787,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430491,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626719,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484268,"kb_used_data":174756,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93260884,"statfs":{"total":95995035648,"available":95499145216,"internally_reserved":0,"allocated":178950144,"data_stored":178446387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":3.5,"15min":2.0259999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3890000000000002,"5min":117.45999999999999,"15min":117.45999999999999},"last":115.441},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7360000000000002,"15min":1.8959999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.9800000000000004,"5min":117.48699999999999,"15min":117.48699999999999},"last":115.501}]},{"osd":1,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":4.2599999999999998,"5min":2.577,"15min":2.4870000000000001},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":31.286000000000001,"5min":31.286000000000001,"15min":153.78399999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":4.5350000000000001,"5min":4.3220000000000001,"15min":3.0830000000000002},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26200000000000001},"max":{"1min":31.512,"5min":185.411,"15min":185.411},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:50:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.7229999999999999,"15min":3.1499999999999999},"min":{"1min":0.44900000000000001,"5min":0.309,"15min":0.28299999999999997},"max":{"1min":6.4100000000000001,"5min":33.174999999999997,"15min":100.785},"last":100.85299999999999},{"interface":"front","average":{"1min":1.825,"5min":4.2220000000000004,"15min":3.5459999999999998},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4829999999999997,"5min":185.21799999999999,"15min":185.21799999999999},"last":101.13}]},{"osd":3,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":5.9370000000000003,"15min":3.4489999999999998},"min":{"1min":0.32800000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.742999999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.5230000000000001,"5min":4.4880000000000004,"15min":3.0670000000000002},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":10.773999999999999,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.79800000000000004,"15min":0.67600000000000005},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6070000000000002,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.85399999999999998,"15min":0.68600000000000005},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":0.69199999999999995,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.73999999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.5620000000000003,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.93400000000000005,"15min":0.754},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.4429999999999996,"5min":9.7899999999999991,"15min":18.625},"last":0.318}]},{"osd":7,"last update":"Sat Sep 23 22:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.98799999999999999,"15min":0.748},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":1.9990000000000001,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.75600000000000001,"15min":0.73399999999999999},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":2.8650000000000002,"5min":5.1790000000000003,"15min":16.661999999999999},"last":0.39800000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253028,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":592452,"kb_used_data":280252,"kb_used_omap":9,"kb_used_meta":312182,"kb_avail":93152700,"statfs":{"total":95995035648,"available":95388364800,"internally_reserved":0,"allocated":286978048,"data_stored":286047285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":319675369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.367,"5min":8.0190000000000001,"15min":5.4509999999999996},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":26.042000000000002,"5min":101.605,"15min":101.605},"last":2.254},{"interface":"front","average":{"1min":4.7430000000000003,"5min":7.6180000000000003,"15min":5.5490000000000004},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":25.379999999999999,"5min":101.892,"15min":101.892},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":3.7850000000000001,"15min":2.6589999999999998},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":26.129000000000001,"5min":37.869999999999997,"15min":37.889000000000003},"last":2.008},{"interface":"front","average":{"1min":6.0439999999999996,"5min":4.0890000000000004,"15min":2.7549999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":26.001999999999999,"5min":37.957000000000001,"15min":37.957000000000001},"last":2.0390000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":3.46,"15min":2.1859999999999999},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":3.1179999999999999,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.63},{"interface":"front","average":{"1min":2.016,"5min":4.2999999999999998,"15min":2.6389999999999998},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.20999999999999999},"max":{"1min":6.7649999999999997,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.56299999999999994}]},{"osd":4,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":2.681,"15min":1.9990000000000001},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":5.6150000000000002,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.435},{"interface":"front","average":{"1min":3.4140000000000001,"5min":2.4729999999999999,"15min":1.9490000000000001},"min":{"1min":0.33500000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":14.058,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.871}]},{"osd":5,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.069,"15min":1.546},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":4.3230000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.343,"5min":2.7919999999999998,"15min":1.865},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":5.5650000000000004,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.495}]},{"osd":6,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":2.5550000000000002,"15min":1.7829999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":13.991,"5min":79.311000000000007,"15min":79.311000000000007},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.91400000000000003,"5min":2.0950000000000002,"15min":1.482},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":2.984,"5min":79.328999999999994,"15min":79.328999999999994},"last":3.5990000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1480000000000001,"15min":2.234},"min":{"1min":0.435,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":5.6539999999999999,"5min":11.481999999999999,"15min":159.018},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.548,"5min":2.0659999999999998,"15min":2.2890000000000001},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.37},"max":{"1min":14.032999999999999,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.91000000000000003}]}]},{"osd":0,"up_from":219,"seq":940597837877,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":471032,"kb_used_data":300848,"kb_used_omap":31,"kb_used_meta":170144,"kb_avail":93274120,"statfs":{"total":95995035648,"available":95512698880,"internally_reserved":0,"allocated":308068352,"data_stored":307089957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":174228402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":3.887,"15min":3.77},"min":{"1min":0.221,"5min":0.216,"15min":0.216},"max":{"1min":14.141999999999999,"5min":32.344999999999999,"15min":32.344999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":3.6520000000000001,"5min":3.9460000000000002,"15min":3.6880000000000002},"min":{"1min":0.26100000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.291,"5min":32.106000000000002,"15min":32.106000000000002},"last":5.8369999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":3.7509999999999999,"15min":2.649},"min":{"1min":0.26700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":16.643999999999998,"5min":24.253,"15min":24.253},"last":5.9160000000000004},{"interface":"front","average":{"1min":4.4619999999999997,"5min":2.9489999999999998,"15min":2.1539999999999999},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":21.151,"5min":21.151,"15min":21.151},"last":2.0979999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.4400000000000004,"5min":3.4319999999999999,"15min":2.4540000000000002},"min":{"1min":0.30199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.609999999999999,"5min":20.710000000000001,"15min":20.710000000000001},"last":6.5330000000000004},{"interface":"front","average":{"1min":4.3090000000000002,"5min":2.431,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.254},"max":{"1min":22.172000000000001,"5min":22.172000000000001,"15min":22.172000000000001},"last":2.1320000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.5529999999999999,"15min":2.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.2030000000000003,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":3.194,"5min":2.407,"15min":1.968},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":16.52,"5min":20.009,"15min":20.009},"last":5.8810000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":2.8650000000000002,"15min":1.845},"min":{"1min":0.38,"5min":0.318,"15min":0.318},"max":{"1min":61.756999999999998,"5min":61.756999999999998,"15min":61.756999999999998},"last":2.165},{"interface":"front","average":{"1min":5.8150000000000004,"5min":3.2080000000000002,"15min":2.1040000000000001},"min":{"1min":0.379,"5min":0.374,"15min":0.374},"max":{"1min":61.777000000000001,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":6.3579999999999997,"5min":2.6970000000000001,"15min":1.8129999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":59.146999999999998,"5min":59.146999999999998,"15min":59.146999999999998},"last":2.2160000000000002},{"interface":"front","average":{"1min":4.5300000000000002,"5min":3.1800000000000002,"15min":2.1840000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":59.164000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":2.4409999999999998,"15min":1.877},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.2350000000000003,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":3.4740000000000002,"5min":2,"15min":1.452},"min":{"1min":0.49399999999999999,"5min":0.432,"15min":0.432},"max":{"1min":21.954999999999998,"5min":21.954999999999998,"15min":21.954999999999998},"last":4.415}]}]},{"osd":2,"up_from":131,"seq":562640715890,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":463128,"kb_used_data":194960,"kb_used_omap":81,"kb_used_meta":267950,"kb_avail":93282024,"statfs":{"total":95995035648,"available":95520792576,"internally_reserved":0,"allocated":199639040,"data_stored":198691984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":274381018},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":10.039,"5min":5.8920000000000003,"15min":5.476},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":132.79499999999999,"5min":132.79499999999999,"15min":132.79499999999999},"last":3.4940000000000002},{"interface":"front","average":{"1min":10.228,"5min":6.9359999999999999,"15min":5.9050000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":132.58099999999999,"5min":132.58099999999999,"15min":132.58099999999999},"last":3.4049999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3079999999999998,"15min":3.1850000000000001},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.3650000000000002,"5min":11.759,"15min":88.316000000000003},"last":1.548},{"interface":"front","average":{"1min":3.8130000000000002,"5min":3.3490000000000002,"15min":2.9870000000000001},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.167,"5min":28.596,"15min":88.436999999999998},"last":3.8239999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":5.2830000000000004,"5min":3.992,"15min":3.3370000000000002},"min":{"1min":0.39800000000000002,"5min":0.223,"15min":0.223},"max":{"1min":20.41,"5min":21.210999999999999,"15min":88.653000000000006},"last":4.234},{"interface":"front","average":{"1min":3.7839999999999998,"5min":2.734,"15min":2.6040000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":13.836,"5min":13.836,"15min":88.474000000000004},"last":4.1909999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":4.6219999999999999,"15min":2.2389999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.965,"5min":108.878,"15min":108.878},"last":0.28899999999999998},{"interface":"front","average":{"1min":2.2690000000000001,"5min":4.8079999999999998,"15min":2.9969999999999999},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.2539999999999996,"5min":108.93000000000001,"15min":108.93000000000001},"last":1.615}]},{"osd":5,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":4.2210000000000001,"15min":2.8519999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.0869999999999997,"5min":97.322999999999993,"15min":97.322999999999993},"last":1.5880000000000001},{"interface":"front","average":{"1min":3.0720000000000001,"5min":4.4569999999999999,"15min":2.4889999999999999},"min":{"1min":0.34499999999999997,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":7.694,"5min":97.301000000000002,"15min":97.301000000000002},"last":3.8620000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":3.8780000000000001,"15min":2.4049999999999998},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6669999999999998,"5min":85.034999999999997,"15min":88.606999999999999},"last":3.8919999999999999},{"interface":"front","average":{"1min":2.2349999999999999,"5min":3.395,"15min":1.863},"min":{"1min":0.41099999999999998,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":8.1479999999999997,"5min":85.209000000000003,"15min":85.209000000000003},"last":0.313}]},{"osd":7,"last update":"Sat Sep 23 22:49:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.8,"15min":1.389},"min":{"1min":0.46200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.9159999999999999,"5min":2.3570000000000002,"15min":2.2759999999999998},"min":{"1min":0.434,"5min":0.37,"15min":0.37},"max":{"1min":9.0419999999999998,"5min":9.5549999999999997,"15min":88.584000000000003},"last":1.6439999999999999}]}]},{"osd":3,"up_from":94,"seq":403726925976,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":580096,"kb_used_data":174840,"kb_used_omap":7,"kb_used_meta":405176,"kb_avail":93165056,"statfs":{"total":95995035648,"available":95401017344,"internally_reserved":0,"allocated":179036160,"data_stored":178107154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":414900840},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,4,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":2.931,"5min":3.3849999999999998,"15min":2.3679999999999999},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.662000000000001,"5min":80.099000000000004,"15min":80.099000000000004},"last":0.246},{"interface":"front","average":{"1min":3.2810000000000001,"5min":3.3380000000000001,"15min":2.1779999999999999},"min":{"1min":0.374,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.128,"5min":80.150000000000006,"15min":80.150000000000006},"last":8.7769999999999992}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6949999999999998,"5min":2.6949999999999998,"15min":1.831},"min":{"1min":0.221,"5min":0.221,"15min":0.20000000000000001},"max":{"1min":11.086,"5min":17.231000000000002,"15min":22.818000000000001},"last":1.9410000000000001},{"interface":"front","average":{"1min":2.665,"5min":1.9570000000000001,"15min":1.4390000000000001},"min":{"1min":0.377,"5min":0.25600000000000001,"15min":0.222},"max":{"1min":11.757999999999999,"5min":15.098000000000001,"15min":23.917999999999999},"last":1.901}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":2.8610000000000002,"15min":2.6949999999999998},"min":{"1min":0.42899999999999999,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":17.170999999999999,"5min":20.431000000000001,"15min":125.446},"last":8.8230000000000004},{"interface":"front","average":{"1min":3.359,"5min":3.2599999999999998,"15min":3.2010000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":11.641,"5min":15.756,"15min":182.483},"last":2.8650000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5720000000000001,"5min":2.077,"15min":1.522},"min":{"1min":0.33500000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":8.1509999999999998,"5min":12.896000000000001,"15min":18.757000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.609,"5min":1.706,"15min":1.325},"min":{"1min":0.47799999999999998,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":11.834,"5min":12.131,"15min":12.131},"last":0.62}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.5449999999999999,"15min":1.2549999999999999},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":11.803000000000001,"5min":11.803000000000001,"15min":12.127000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":3.105,"5min":2.2930000000000001,"15min":1.4970000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.951000000000001,"5min":11.951000000000001,"15min":11.951000000000001},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.7390000000000001,"15min":1.3100000000000001},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.28999999999999998},"max":{"1min":11.867000000000001,"5min":11.867000000000001,"15min":12.16},"last":0.82999999999999996},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.2250000000000001,"15min":1.534},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.913,"5min":11.913,"15min":11.913},"last":0.42699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":3.1680000000000001,"15min":1.8320000000000001},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":11.694000000000001,"5min":91.757999999999996,"15min":91.757999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.8559999999999999,"5min":3.1019999999999999,"15min":1.9550000000000001},"min":{"1min":0.48699999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":8.2010000000000005,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.52300000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019175,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888168,"kb_used_data":578656,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92856984,"statfs":{"total":95995035648,"available":95085551616,"internally_reserved":0,"allocated":592543744,"data_stored":592043557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":4.1849999999999996,"15min":6.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.9770000000000003,"5min":110.08499999999999,"15min":110.08499999999999},"last":2.6629999999999998},{"interface":"front","average":{"1min":2.5230000000000001,"5min":5.077,"15min":7.3780000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.464,"5min":110.33799999999999,"15min":110.33799999999999},"last":0.27900000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1230000000000002,"15min":1.53},"min":{"1min":0.317,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.0269999999999992,"5min":41.643000000000001,"15min":45.320999999999998},"last":2.5430000000000001},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.9019999999999999,"15min":1.4279999999999999},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.4489999999999998,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.56299999999999994}]},{"osd":2,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.2749999999999999,"15min":3.2719999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":17.093,"5min":41.607999999999997,"15min":84.909000000000006},"last":0.36199999999999999},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.4249999999999998,"15min":3.206},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":21.704000000000001,"5min":41.670000000000002,"15min":83.840000000000003},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":7.3099999999999996,"15min":3.3730000000000002},"min":{"1min":0.442,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":100.51600000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":8.1150000000000002,"5min":7.3609999999999998,"15min":3.4169999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":100.637,"5min":129.197,"15min":129.197},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":0.81000000000000005,"15min":0.82399999999999995},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":12.238,"5min":12.238,"15min":43.189999999999998},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.83499999999999996,"15min":0.85199999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":7.5460000000000003,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.33400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:49:34 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.91400000000000003,"15min":0.78500000000000003},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.28199999999999997},"max":{"1min":10.952999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.72199999999999998,"15min":0.76700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":4.5359999999999996,"5min":5.8689999999999998,"15min":26.940999999999999},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.69799999999999995,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.181,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.75900000000000001,"15min":0.68400000000000005},"min":{"1min":0.38800000000000001,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":8.6739999999999995,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.54100000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822947,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":581652,"kb_used_data":276236,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93163500,"statfs":{"total":95995035648,"available":95399424000,"internally_reserved":0,"allocated":282865664,"data_stored":282381095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":10.109999999999999,"5min":7.133,"15min":3.5830000000000002},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":84.367999999999995,"5min":121.319,"15min":121.319},"last":4.1879999999999997},{"interface":"front","average":{"1min":10.358000000000001,"5min":7.2380000000000004,"15min":3.5099999999999998},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":84.311999999999998,"5min":121.075,"15min":121.075},"last":0.33000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.5269999999999999,"15min":1.8089999999999999},"min":{"1min":0.29299999999999998,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":6.9189999999999996,"5min":28.658999999999999,"15min":149.203},"last":3.9900000000000002},{"interface":"front","average":{"1min":1.581,"5min":1.9259999999999999,"15min":2.0030000000000001},"min":{"1min":0.34499999999999997,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.0119999999999996,"5min":28.664999999999999,"15min":149.554},"last":5.0359999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8179999999999996,"5min":4.048,"15min":6.4340000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":63.156999999999996,"5min":158.92599999999999,"15min":158.92599999999999},"last":3.7440000000000002},{"interface":"front","average":{"1min":4.4349999999999996,"5min":3.8959999999999999,"15min":6.4039999999999999},"min":{"1min":0.318,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":63.277999999999999,"5min":158.91399999999999,"15min":158.91399999999999},"last":4.0599999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":10.577999999999999,"5min":9.7460000000000004,"15min":4.899},"min":{"1min":0.31,"5min":0.30499999999999999,"15min":0.25700000000000001},"max":{"1min":98.641000000000005,"5min":257.05000000000001,"15min":257.05000000000001},"last":3.5859999999999999},{"interface":"front","average":{"1min":11.012,"5min":9.7850000000000001,"15min":4.7750000000000004},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":98.906000000000006,"5min":257.35000000000002,"15min":257.35000000000002},"last":3.7519999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66300000000000003,"15min":0.62},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":5.0309999999999997,"5min":5.0309999999999997,"15min":5.0309999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.75700000000000001,"15min":0.624},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":3.4409999999999998,"5min":10.535,"15min":10.535},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.89900000000000002,"15min":1.4099999999999999},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":4.367,"5min":12.615,"15min":140.07499999999999},"last":2.5640000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.0449999999999999,"15min":1.466},"min":{"1min":0.318,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":6.6219999999999999,"5min":12.74,"15min":139.739},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.74399999999999999,"15min":0.71999999999999997},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.738,"5min":4.2510000000000003,"15min":7.9809999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.73599999999999999,"15min":0.68000000000000005},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":3.5539999999999998,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.36199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":287309824,"data_stored":287309824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":264830976,"data_stored":264830976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":193531904,"data_stored":193527832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":172949504,"data_stored":172949504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":581173248,"data_stored":581173248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":265031680,"data_stored":265027608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":172429312,"data_stored":172425240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:50:39.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:39 smithi151 ceph-mon[159210]: from='client.55396 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:39.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:39 smithi151 ceph-mon[159210]: pgmap v722: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 430 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2790/20700 objects degraded (13.478%); 454/20700 objects misplaced (2.193%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:39.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2341533418' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:39.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:39 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:39.472 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:39.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:39.577 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:39.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[180979]: from='client.55396 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:39.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[180979]: pgmap v722: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 430 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2790/20700 objects degraded (13.478%); 454/20700 objects misplaced (2.193%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:39.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2341533418' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:39.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:39.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[187743]: from='client.55396 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:39.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[187743]: pgmap v722: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 430 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2790/20700 objects degraded (13.478%); 454/20700 objects misplaced (2.193%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:39.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2341533418' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:39.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:39 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:50:39.675 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:40.243 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10176 6.c e128/99 6:30687e13:::benchmark_data_smithi052_234456_object10175:head v 128'611, mlcod=128'609)", 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.04116574299997, 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26259265100000001, 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.244 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200006+0000", 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200005+0000", 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4515999999999999e-05 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200049+0000", 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0868e-05 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200060+0000", 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7479999999999999e-06 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200069+0000", 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8922999999999999e-05 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200118+0000", 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1420000000000002e-05 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200139+0000", 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26240534999999998 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462545+0000", 2023-09-23T22:50:40.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3596e-05 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462598+0000", 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26259265100000001 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10178 6.3 e128/104 6:cbb8686c:::benchmark_data_smithi052_234456_object10177:head v 128'684, mlcod=128'682)", 2023-09-23T22:50:40.249 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.04090383400001, 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26233074499999998, 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200268+0000", 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200267+0000", 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2943999999999999e-05 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200290+0000", 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0489000000000001e-05 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200300+0000", 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4090000000000002e-06 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200305+0000", 2023-09-23T22:50:40.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000490297 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200796+0000", 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2308999999999999e-05 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200818+0000", 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085734699999999995 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201675+0000", 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26092291299999998 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462598+0000", 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26233074499999998 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10170 6.1 e128/104 6:8aa77601:::benchmark_data_smithi052_234456_object10169:head v 128'666, mlcod=128'664)", 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.04038689599997, 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26275960799999998, 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200784+0000", 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1285999999999997e-05 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200825+0000", 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6779999999999995e-06 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200835+0000", 2023-09-23T22:50:40.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.976e-06 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200841+0000", 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0885999999999999e-05 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200902+0000", 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1832999999999999e-05 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200923+0000", 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26257463399999997 2023-09-23T22:50:40.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463498+0000", 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6022000000000001e-05 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463544+0000", 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26275960799999998 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.260 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10180 6.1 e128/104 6:8dfddb97:::benchmark_data_smithi052_234456_object10179:head v 128'667, mlcod=128'664)", 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.04029926199996, 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26309316700000002, 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200872+0000", 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9139e-05 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200901+0000", 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2439999999999996e-06 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200908+0000", 2023-09-23T22:50:40.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.200913+0000", 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045587999999999998 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201369+0000", 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8995e-05 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.201388+0000", 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26253594800000002 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463924+0000", 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1671000000000001e-05 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463965+0000", 2023-09-23T22:50:40.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26309316700000002 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10183 6.1 e128/104 6:8243606a:::benchmark_data_smithi052_234456_object10182:head v 128'668, mlcod=128'664)", 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.03906271599999, 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261887543, 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.266 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202109+0000", 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202108+0000", 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0642000000000002e-05 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202148+0000", 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26041415499999998 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462563+0000", 2023-09-23T22:50:40.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6589999999999996e-06 2023-09-23T22:50:40.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462571+0000", 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00052397000000000001 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463095+0000", 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2568000000000001e-05 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463118+0000", 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00085566799999999997 2023-09-23T22:50:40.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463973+0000", 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2731999999999999e-05 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463996+0000", 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261887543 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10181 6.3 e128/104 6:c0620c37:::benchmark_data_smithi052_234456_object10180:head v 128'685, mlcod=128'682)", 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.03858023099997, 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26238228600000002, 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202591+0000", 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202590+0000", 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5381999999999999e-05 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.202616+0000", 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25993996400000002 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462556+0000", 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0231000000000001e-05 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462566+0000", 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2098000000000001e-05 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462628+0000", 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0216999999999999e-05 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462658+0000", 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002274767 2023-09-23T22:50:40.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464933+0000", 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0459e-05 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.464973+0000", 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26238228600000002 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.276 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.34680.0:10192 6.a e128/126 6:5a145b56:::benchmark_data_smithi052_234456_object10191:head v 128'636, mlcod=128'634)", 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 586.03565086100002, 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.258119451, 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205521+0000", 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205519+0000", 2023-09-23T22:50:40.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8529000000000001e-05 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.205548+0000", 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25701668599999999 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462565+0000", 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9289999999999998e-06 2023-09-23T22:50:40.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462573+0000", 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4702000000000004e-05 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462647+0000", 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1945000000000001e-05 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.462669+0000", 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093275200000000002 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463602+0000", 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8049000000000001e-05 2023-09-23T22:50:40.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:40:54.463640+0000", 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.258119451 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.282 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61219 7.d 7:b3f702b2:::benchmark_data_smithi052_277165_object61218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 487.17206139799998, 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23283454100000001, 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61219 2023-09-23T22:50:40.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069110+0000", 2023-09-23T22:50:40.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069109+0000", 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8376000000000001e-05 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069137+0000", 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0046e-05 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069147+0000", 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3259999999999997e-06 2023-09-23T22:50:40.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069151+0000", 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9551999999999997e-05 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069191+0000", 2023-09-23T22:50:40.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112429 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069303+0000", 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114738 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069418+0000", 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003836182 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073254+0000", 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022611850000000002 2023-09-23T22:50:40.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075516+0000", 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226299794 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301815+0000", 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7392999999999995e-05 2023-09-23T22:50:40.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301903+0000", 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1786000000000003e-05 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301945+0000", 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23283454100000001 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.291 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61222 7.d 7:b9a46085:::benchmark_data_smithi052_277165_object61221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 487.17201240600002, 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23289720799999999, 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.292 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61222 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069159+0000", 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:50:40.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069158+0000", 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8464999999999998e-05 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069197+0000", 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3069999999999999e-06 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069203+0000", 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0140000000000001e-06 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.069210+0000", 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002195729 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071406+0000", 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010020499999999999 2023-09-23T22:50:40.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071506+0000", 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013088800000000001 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.071637+0000", 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018844230000000001 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.073521+0000", 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020741359999999999 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.075596+0000", 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226399296 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.301995+0000", 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6539999999999999e-05 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.302031+0000", 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4808e-05 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.302056+0000", 2023-09-23T22:50:40.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23289720799999999 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:61226 7.7 7:ebd42417:::benchmark_data_smithi052_277165_object61225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 487.16506053699999, 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.227997757, 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.299 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61226 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076111+0000", 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076110+0000", 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9333999999999999e-05 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076159+0000", 2023-09-23T22:50:40.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1613e-05 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076171+0000", 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129999999999996e-06 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076175+0000", 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6888000000000002e-05 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076222+0000", 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129424 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076352+0000", 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8393000000000005e-05 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.076450+0000", 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023315779999999999 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.078782+0000", 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001019966 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.079801+0000", 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.224238032 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304040+0000", 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3377000000000002e-05 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304083+0000", 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5698e-05 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:42:33.304109+0000", 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.227997757 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.306 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100776 7.c 7:350d58b3:::benchmark_data_smithi052_277165_object8284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 429.66446642699998, 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999, 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.307 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100776 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:40.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576703+0000", 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0799999999999995e-06 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576712+0000", 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2679e-05 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576725+0000", 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6329999999999999e-06 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576731+0000", 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18361677800000001 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760348+0000", 2023-09-23T22:50:40.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000140829 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760489+0000", 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016031800000000001 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760649+0000", 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010853742 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.771503+0000", 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023432639000000002 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.794936+0000", 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035400660000000001 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798476+0000", 2023-09-23T22:50:40.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5828999999999999e-05 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798511+0000", 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012960090000000001 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.799807+0000", 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.313 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100781 7.c 7:39965936:::benchmark_data_smithi052_277165_object8289:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 429.65718778299998, 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999, 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.314 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100781 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:50:40.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583981+0000", 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.59e-05 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583996+0000", 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.174193922 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758190+0000", 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2031000000000001e-05 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758202+0000", 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049640919999999998 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763167+0000", 2023-09-23T22:50:40.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143024 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763310+0000", 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016401100000000001 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763474+0000", 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012004406 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.775478+0000", 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019532911 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.795011+0000", 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072752770000000001 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802286+0000", 2023-09-23T22:50:40.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0192000000000002e-05 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802346+0000", 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018569999999999999 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.804203+0000", 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.320 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101958 7.7 7:e819df4a:::benchmark_data_smithi052_277165_object9466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 427.71443832400001, 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001, 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.321 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101958 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:40.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526731+0000", 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.396e-06 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526739+0000", 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3361e-05 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526752+0000", 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2920000000000003e-06 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526757+0000", 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1291000000000002e-05 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526799+0000", 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121802 2023-09-23T22:50:40.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526920+0000", 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014018900000000001 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527061+0000", 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033778649999999999 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530439+0000", 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000232978 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530671+0000", 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21847482400000001 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749146+0000", 2023-09-23T22:50:40.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749200+0000", 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5618000000000003e-05 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749245+0000", 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.327 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162201 7.c 7:3f19fea2:::benchmark_data_smithi052_277165_object69709:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 322.98591869099999, 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.27262330099999998, 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.328 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162201 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255252+0000", 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.712e-06 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255255+0000", 2023-09-23T22:50:40.330 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8510000000000001e-06 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255259+0000", 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5349999999999999e-06 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255261+0000", 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1329999999999999e-05 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255303+0000", 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132652 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255435+0000", 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012794600000000001 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.332 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255563+0000", 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040626919999999997 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.259626+0000", 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060452930000000002 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.265671+0000", 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261175247 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.526847+0000", 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4647000000000005e-05 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.526921+0000", 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095477599999999995 2023-09-23T22:50:40.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.527876+0000", 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.27262330099999998 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162214 7.c 7:33fa1cb0:::benchmark_data_smithi052_277165_object69722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:50:40.335 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 322.96989744699999, 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25716776299999999, 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162214 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.336 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271272+0000", 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9519999999999994e-06 2023-09-23T22:50:40.337 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271280+0000", 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2356e-05 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271292+0000", 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9319999999999999e-06 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.338 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271299+0000", 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8116999999999998e-05 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271377+0000", 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014364799999999999 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271521+0000", 2023-09-23T22:50:40.339 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014726999999999999 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271668+0000", 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089242190000000006 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280593+0000", 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6284000000000004e-05 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.340 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280659+0000", 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24732958699999999 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.527989+0000", 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4438999999999998e-05 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.341 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528043+0000", 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000398756 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528442+0000", 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25716776299999999 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.342 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162220 7.c 7:33dba7f1:::benchmark_data_smithi052_277165_object69728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 322.96514020199999, 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25290671399999998, 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162220 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.343 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.344 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276030+0000", 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2030000000000003e-06 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276037+0000", 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4914e-05 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276052+0000", 2023-09-23T22:50:40.345 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3910000000000003e-06 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276058+0000", 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010739300000000001 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276166+0000", 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018878300000000001 2023-09-23T22:50:40.346 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276355+0000", 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000217147 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276572+0000", 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041362409999999997 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.347 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280708+0000", 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2092e-05 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280720+0000", 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24777231399999999 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.348 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528492+0000", 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9494999999999999e-05 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528552+0000", 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00038610499999999998 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528938+0000", 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25290671399999998 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164712 7.7 7:e35a7399:::benchmark_data_smithi052_277165_object72220:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.84475388200002, 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29594756100000003, 2023-09-23T22:50:40.350 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164712 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.351 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396416+0000", 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6749999999999997e-06 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396425+0000", 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1613000000000001e-05 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396447+0000", 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6890000000000004e-06 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396455+0000", 2023-09-23T22:50:40.353 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021833180000000001 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398638+0000", 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139655 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398778+0000", 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015481100000000001 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398932+0000", 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045151879999999998 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.403448+0000", 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.288777124 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692225+0000", 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9673999999999999e-05 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692274+0000", 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0642000000000004e-05 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692365+0000", 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29594756100000003 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.356 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:168558 7.c 7:3f3c3116:::benchmark_data_smithi052_277165_object76066:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 312.90126378399998, 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.244409458, 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.357 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 168558 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:50:40.358 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339906+0000", 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0329999999999997e-06 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339915+0000", 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3118999999999999e-05 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.359 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339928+0000", 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8990000000000001e-06 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339936+0000", 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8579999999999998e-05 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339994+0000", 2023-09-23T22:50:40.360 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000148231 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.340142+0000", 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017634200000000001 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.340319+0000", 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037420520000000001 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344061+0000", 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056251469999999996 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.349686+0000", 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23382455599999999 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.583510+0000", 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4010999999999993e-05 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.583584+0000", 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073256099999999995 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584317+0000", 2023-09-23T22:50:40.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.244409458 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:168562 7.c 7:3c1c89f1:::benchmark_data_smithi052_277165_object76070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 312.89663196200001, 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.240143207, 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.364 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 168562 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.365 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4640000000000002e-06 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.366 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344542+0000", 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0480000000000004e-06 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344548+0000", 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1690000000000002e-06 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344553+0000", 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3362999999999996e-05 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344616+0000", 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014696300000000001 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344763+0000", 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018186 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:50:40.368 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344945+0000", 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039275380000000004 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.348872+0000", 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096992300000000003 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.349842+0000", 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.234521651 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584364+0000", 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3619999999999998e-05 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584418+0000", 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026506700000000002 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584683+0000", 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.240143207 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:48297 8.9 8:9a4ce8fa:::benchmark_data_smithi052_337357_object48296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 229.293246006, 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25518236, 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:50:40.371 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 48297 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.372 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947924+0000", 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9459000000000002e-05 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947974+0000", 2023-09-23T22:50:40.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9210000000000004e-06 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947984+0000", 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2860000000000001e-06 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947988+0000", 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4461000000000001e-05 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948022+0000", 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6387999999999999e-05 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948119+0000", 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9592999999999993e-05 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948218+0000", 2023-09-23T22:50:40.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001431501 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.949650+0000", 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00027939799999999998 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.949929+0000", 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25307412600000001 2023-09-23T22:50:40.376 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203003+0000", 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0455000000000003e-05 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203064+0000", 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3832999999999999e-05 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:50:40.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203108+0000", 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25518236 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:50:40.378 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:40.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:40.379 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:40.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:40.480 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:40.519 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:40 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:50:40.519 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:40 smithi151 ceph-mon[159210]: failed to encode map e260 with expected crc 2023-09-23T22:50:40.519 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:40 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:50:40.519 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:40 smithi151 ceph-mon[159210]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T22:50:40.519 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2790/20700 objects degraded (13.478%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:40.521 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:40.583 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:40.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:40.635 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:40.690 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:40.737 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:50:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[180979]: failed to encode map e260 with expected crc 2023-09-23T22:50:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:50:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[180979]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T22:50:40.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2790/20700 objects degraded (13.478%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:40.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:50:40.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[187743]: failed to encode map e260 with expected crc 2023-09-23T22:50:40.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:50:40.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[187743]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T22:50:40.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2790/20700 objects degraded (13.478%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:41.023 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:41.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:41.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:41.048 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:50:41.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:41.172 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:41.256 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:41.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:41.365 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:50:41.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:41.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:41 smithi052 ceph-mon[187743]: pgmap v724: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 407 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2790/19605 objects degraded (14.231%); 454/19605 objects misplaced (2.316%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:41.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:41 smithi052 ceph-mon[180979]: pgmap v724: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 407 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2790/19605 objects degraded (14.231%); 454/19605 objects misplaced (2.316%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:41.556 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:50:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:41.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:50:41.596 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:41.668 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:41.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:41 smithi151 ceph-mon[159210]: pgmap v724: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 407 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 2790/19605 objects degraded (14.231%); 454/19605 objects misplaced (2.316%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:41.702 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:41.797 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:41.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:41.901 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:41.982 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:42.200 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:42.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:42.327 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:42.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:50:42.420 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:50:42.451 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:42.706 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:42.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:42.828 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:42.870 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:42.976 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:42.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:42.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:43.080 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:43.096 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:50:43.136 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:43.137 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:50:43.168 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:50:43.215 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:43.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:43.219 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:50:43.247 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:50:43.247 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:50:43.248 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:50:43.248 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:50:43.248 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:50:43.302 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:50:43.319 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:43.356 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:50:43.370 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:43.399 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:50:43.417 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.417 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.417 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.417 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.417 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.418 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.418 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.418 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:50:43.606 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:50:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:50:43] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-09-23T22:50:43.607 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:43 smithi052 ceph-mon[180979]: pgmap v725: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 291 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 2836/14028 objects degraded (20.217%); 255/14028 objects misplaced (1.818%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:43.607 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/367502033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:43.607 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:43 smithi052 ceph-mon[187743]: pgmap v725: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 291 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 2836/14028 objects degraded (20.217%); 255/14028 objects misplaced (1.818%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:43.607 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/367502033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:43.608 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:43.610 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:43 smithi151 ceph-mon[159210]: pgmap v725: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 291 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 2836/14028 objects degraded (20.217%); 255/14028 objects misplaced (1.818%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:43.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/367502033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:44.026 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:50:44.509 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:44.941 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:44.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:45.048 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:45.076 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:45.118 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:45.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:45.222 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:45.257 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:45.316 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:45.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:45.420 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:45.450 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:45.656 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:45 smithi151 ceph-mon[159210]: pgmap v726: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 245 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 2836/11820 objects degraded (23.993%); 255/11820 objects misplaced (2.157%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:45.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2836/11820 objects degraded (23.993%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:45.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:45 smithi052 ceph-mon[187743]: pgmap v726: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 245 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 2836/11820 objects degraded (23.993%); 255/11820 objects misplaced (2.157%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:45.678 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2836/11820 objects degraded (23.993%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:45.681 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:45 smithi052 ceph-mon[180979]: pgmap v726: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 245 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 2836/11820 objects degraded (23.993%); 255/11820 objects misplaced (2.157%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:45.681 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2836/11820 objects degraded (23.993%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:45.693 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:45.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:45.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:45.839 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:45.927 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:45.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:46.030 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:46.203 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:50:46.203 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:50:46.416 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:46.574 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:46.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:46.678 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:46.771 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:46.957 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:50:47.289 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:47.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:47.393 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:47.466 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:50:47.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:47.699 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:50:47.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:47 smithi151 ceph-mon[159210]: pgmap v727: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 111 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2873/5370 objects degraded (53.501%); 70/5370 objects misplaced (1.304%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:47.715 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:47 smithi052 ceph-mon[187743]: pgmap v727: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 111 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2873/5370 objects degraded (53.501%); 70/5370 objects misplaced (1.304%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:47.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:47 smithi052 ceph-mon[180979]: pgmap v727: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 111 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2873/5370 objects degraded (53.501%); 70/5370 objects misplaced (1.304%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:47.779 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:47.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:47.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:47.943 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:48.031 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Removed 102448 objects 2023-09-23T22:50:48.031 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Clean up completed and total clean up time :174.289 2023-09-23T22:50:48.056 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:48.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:48.110 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:48.111 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:50:48.160 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:48.185 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:48.227 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:48.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:48.330 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:48.357 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:48.467 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:50:48.522 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2114769069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:48.522 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2114769069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:48.610 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:48.614 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2114769069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:50:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:48.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:48.716 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:48.744 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:48.838 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:48.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:48.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:48.943 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:48.971 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:48.996 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:50:48.996 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:50:49.014 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:49.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:49.119 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:49.147 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:49.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:49.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:49.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:49.344 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:49.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:49 smithi151 ceph-mon[159210]: from='client.55408 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:49.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:49 smithi151 ceph-mon[159210]: pgmap v728: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 65 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2873/3153 objects degraded (91.120%); 70/3153 objects misplaced (2.220%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:49.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:49 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:49.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:49 smithi151 ceph-mon[159210]: failed to encode map e261 with expected crc 2023-09-23T22:50:49.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2114769069' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:50:49.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:49 smithi151 ceph-mon[159210]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T22:50:49.575 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:50:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[187743]: from='client.55408 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[187743]: pgmap v728: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 65 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2873/3153 objects degraded (91.120%); 70/3153 objects misplaced (2.220%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[187743]: failed to encode map e261 with expected crc 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2114769069' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[187743]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[180979]: from='client.55408 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[180979]: pgmap v728: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 65 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 2873/3153 objects degraded (91.120%); 70/3153 objects misplaced (2.220%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[180979]: failed to encode map e261 with expected crc 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2114769069' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:50:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:49 smithi052 ceph-mon[180979]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T22:50:49.827 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:49.895 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:49.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:49.999 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:50.032 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:50.062 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T22:50:50.062 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T22:50:50.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[180979]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[180979]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[180979]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2873/3153 objects degraded (91.120%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[187743]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[187743]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[187743]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2873/3153 objects degraded (91.120%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:50.913 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:50.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:50:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:50 smithi151 ceph-mon[159210]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:50 smithi151 ceph-mon[159210]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:50 smithi151 ceph-mon[159210]: failed to encode map e261 with expected crc 2023-09-23T22:50:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2873/3153 objects degraded (91.120%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:51.017 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:51.050 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:51.101 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:51.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:51.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:51.207 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:51.235 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:51.379 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":727,"stamp":"2023-09-23T22:50:46.308149+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":116266936,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2873,"num_objects_misplaced":70,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":203392,"num_write_kb":6559118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3710,"num_bytes_recovered":241051722,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127826,"ondisk_log_size":127826,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2953784,"kb_used_data":877696,"kb_used_omap":129,"kb_used_meta":2075774,"kb_avail":653262280,"statfs":{"total":671965249536,"available":668940574720,"internally_reserved":0,"allocated":898760704,"data_stored":893481569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2125593309},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,6,8,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-383057920,"num_objects":-5845,"num_object_clones":0,"num_object_copies":-17535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":83,"num_objects_misplaced":-384,"num_objects_unfound":0,"num_objects_dirty":-5845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5845,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":301,"num_bytes_recovered":19726336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007596"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":257,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573764+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:50:29.573764+0000","last_peered":"2023-09-23T22:50:29.573764+0000","last_clean":"2023-09-23T22:50:29.573764+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:50:29.573764+0000","last_undegraded":"2023-09-23T22:50:29.573764+0000","last_fullsized":"2023-09-23T22:50:29.573764+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":233,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:36.088765+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:50:36.088765+0000","last_peered":"2023-09-23T22:50:36.088765+0000","last_clean":"2023-09-23T22:50:36.088765+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:50:36.088765+0000","last_undegraded":"2023-09-23T22:50:36.088765+0000","last_fullsized":"2023-09-23T22:50:36.088765+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579964+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:50:29.579964+0000","last_peered":"2023-09-23T22:50:29.579964+0000","last_clean":"2023-09-23T22:50:29.579964+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:50:29.579964+0000","last_undegraded":"2023-09-23T22:50:29.579964+0000","last_fullsized":"2023-09-23T22:50:29.579964+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":228,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580854+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:50:29.580854+0000","last_peered":"2023-09-23T22:50:29.580854+0000","last_clean":"2023-09-23T22:50:29.580854+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:50:29.580854+0000","last_undegraded":"2023-09-23T22:50:29.580854+0000","last_fullsized":"2023-09-23T22:50:29.580854+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":292,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579825+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:50:29.579825+0000","last_peered":"2023-09-23T22:50:29.579825+0000","last_clean":"2023-09-23T22:50:29.579825+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:50:29.579825+0000","last_undegraded":"2023-09-23T22:50:29.579825+0000","last_fullsized":"2023-09-23T22:50:29.579825+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.565340+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:50:29.565340+0000","last_peered":"2023-09-23T22:50:29.565340+0000","last_clean":"2023-09-23T22:50:29.565340+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:50:29.565340+0000","last_undegraded":"2023-09-23T22:50:29.565340+0000","last_fullsized":"2023-09-23T22:50:29.565340+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":159,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580836+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:50:29.580836+0000","last_peered":"2023-09-23T22:50:29.580836+0000","last_clean":"2023-09-23T22:50:29.580836+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:50:29.580836+0000","last_undegraded":"2023-09-23T22:50:29.580836+0000","last_fullsized":"2023-09-23T22:50:29.580836+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579895+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:50:29.579895+0000","last_peered":"2023-09-23T22:50:29.579895+0000","last_clean":"2023-09-23T22:50:29.579895+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:50:29.579895+0000","last_undegraded":"2023-09-23T22:50:29.579895+0000","last_fullsized":"2023-09-23T22:50:29.579895+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":229,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580364+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:50:29.580364+0000","last_peered":"2023-09-23T22:50:29.580364+0000","last_clean":"2023-09-23T22:50:29.580364+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:50:29.580364+0000","last_undegraded":"2023-09-23T22:50:29.580364+0000","last_fullsized":"2023-09-23T22:50:29.580364+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":218,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579875+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:50:29.579875+0000","last_peered":"2023-09-23T22:50:29.579875+0000","last_clean":"2023-09-23T22:50:29.579875+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:50:29.579875+0000","last_undegraded":"2023-09-23T22:50:29.579875+0000","last_fullsized":"2023-09-23T22:50:29.579875+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":135,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580797+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:50:29.580797+0000","last_peered":"2023-09-23T22:50:29.580797+0000","last_clean":"2023-09-23T22:50:29.580797+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:50:29.580797+0000","last_undegraded":"2023-09-23T22:50:29.580797+0000","last_fullsized":"2023-09-23T22:50:29.580797+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":134,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.564147+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:50:29.564147+0000","last_peered":"2023-09-23T22:50:29.564147+0000","last_clean":"2023-09-23T22:50:29.564147+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:50:29.564147+0000","last_undegraded":"2023-09-23T22:50:29.564147+0000","last_fullsized":"2023-09-23T22:50:29.564147+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":280,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574004+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:50:29.574004+0000","last_peered":"2023-09-23T22:50:29.574004+0000","last_clean":"2023-09-23T22:50:29.574004+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:50:29.574004+0000","last_undegraded":"2023-09-23T22:50:29.574004+0000","last_fullsized":"2023-09-23T22:50:29.574004+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573796+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:50:29.573796+0000","last_peered":"2023-09-23T22:50:29.573796+0000","last_clean":"2023-09-23T22:50:29.573796+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:50:29.573796+0000","last_undegraded":"2023-09-23T22:50:29.573796+0000","last_fullsized":"2023-09-23T22:50:29.573796+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":118,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573890+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:50:29.573890+0000","last_peered":"2023-09-23T22:50:29.573890+0000","last_clean":"2023-09-23T22:50:29.573890+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:50:29.573890+0000","last_undegraded":"2023-09-23T22:50:29.573890+0000","last_fullsized":"2023-09-23T22:50:29.573890+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":280,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581886+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:50:29.581886+0000","last_peered":"2023-09-23T22:50:29.581886+0000","last_clean":"2023-09-23T22:50:29.581886+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:50:29.581886+0000","last_undegraded":"2023-09-23T22:50:29.581886+0000","last_fullsized":"2023-09-23T22:50:29.581886+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":215,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581244+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:50:29.581244+0000","last_peered":"2023-09-23T22:50:29.581244+0000","last_clean":"2023-09-23T22:50:29.581244+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:50:29.581244+0000","last_undegraded":"2023-09-23T22:50:29.581244+0000","last_fullsized":"2023-09-23T22:50:29.581244+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":241,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:39.395665+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:50:39.395665+0000","last_peered":"2023-09-23T22:50:39.395665+0000","last_clean":"2023-09-23T22:50:39.395665+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:50:39.395665+0000","last_undegraded":"2023-09-23T22:50:39.395665+0000","last_fullsized":"2023-09-23T22:50:39.395665+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574619+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:50:29.574619+0000","last_peered":"2023-09-23T22:50:29.574619+0000","last_clean":"2023-09-23T22:50:29.574619+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:50:29.574619+0000","last_undegraded":"2023-09-23T22:50:29.574619+0000","last_fullsized":"2023-09-23T22:50:29.574619+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":216,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580929+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:50:29.580929+0000","last_peered":"2023-09-23T22:50:29.580929+0000","last_clean":"2023-09-23T22:50:29.580929+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:50:29.580929+0000","last_undegraded":"2023-09-23T22:50:29.580929+0000","last_fullsized":"2023-09-23T22:50:29.580929+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":304,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.564108+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:50:29.564108+0000","last_peered":"2023-09-23T22:50:29.564108+0000","last_clean":"2023-09-23T22:50:29.564108+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:50:29.564108+0000","last_undegraded":"2023-09-23T22:50:29.564108+0000","last_fullsized":"2023-09-23T22:50:29.564108+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":260,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.584956+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:50:29.584956+0000","last_peered":"2023-09-23T22:50:29.584956+0000","last_clean":"2023-09-23T22:50:29.584956+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:50:29.584956+0000","last_undegraded":"2023-09-23T22:50:29.584956+0000","last_fullsized":"2023-09-23T22:50:29.584956+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"260'12672","reported_seq":12723,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:45.588237+0000","last_change":"2023-09-23T22:46:12.238367+0000","last_active":"2023-09-23T22:50:45.588237+0000","last_peered":"2023-09-23T22:50:45.588237+0000","last_clean":"2023-09-23T22:50:45.588237+0000","last_became_active":"2023-09-23T22:46:12.238032+0000","last_became_peered":"2023-09-23T22:46:12.238032+0000","last_unstale":"2023-09-23T22:50:45.588237+0000","last_undegraded":"2023-09-23T22:50:45.588237+0000","last_fullsized":"2023-09-23T22:50:45.588237+0000","mapping_epoch":217,"log_start":"233'5300","ondisk_log_start":"233'5300","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7372,"log_dups_size":0,"ondisk_log_size":7372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5242880,"num_objects":80,"num_object_clones":0,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12672,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":129,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574011+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:50:29.574011+0000","last_peered":"2023-09-23T22:50:29.574011+0000","last_clean":"2023-09-23T22:50:29.574011+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:50:29.574011+0000","last_undegraded":"2023-09-23T22:50:29.574011+0000","last_fullsized":"2023-09-23T22:50:29.574011+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":224,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.564195+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:50:29.564195+0000","last_peered":"2023-09-23T22:50:29.564195+0000","last_clean":"2023-09-23T22:50:29.564195+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:50:29.564195+0000","last_undegraded":"2023-09-23T22:50:29.564195+0000","last_fullsized":"2023-09-23T22:50:29.564195+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"260'12473","reported_seq":12530,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:45.873202+0000","last_change":"2023-09-23T22:46:14.254861+0000","last_active":"2023-09-23T22:50:45.873202+0000","last_peered":"2023-09-23T22:50:45.873202+0000","last_clean":"2023-09-23T22:50:45.873202+0000","last_became_active":"2023-09-23T22:46:14.254426+0000","last_became_peered":"2023-09-23T22:46:14.254426+0000","last_unstale":"2023-09-23T22:50:45.873202+0000","last_undegraded":"2023-09-23T22:50:45.873202+0000","last_fullsized":"2023-09-23T22:50:45.873202+0000","mapping_epoch":219,"log_start":"230'3900","ondisk_log_start":"230'3900","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8573,"log_dups_size":0,"ondisk_log_size":8573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5308416,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12473,"num_write_kb":401728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":139,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574823+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:50:29.574823+0000","last_peered":"2023-09-23T22:50:29.574823+0000","last_clean":"2023-09-23T22:50:29.574823+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:50:29.574823+0000","last_undegraded":"2023-09-23T22:50:29.574823+0000","last_fullsized":"2023-09-23T22:50:29.574823+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574491+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:50:29.574491+0000","last_peered":"2023-09-23T22:50:29.574491+0000","last_clean":"2023-09-23T22:50:29.574491+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:50:29.574491+0000","last_undegraded":"2023-09-23T22:50:29.574491+0000","last_fullsized":"2023-09-23T22:50:29.574491+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"260'12406","reported_seq":12459,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:42.896688+0000","last_change":"2023-09-23T22:46:14.250207+0000","last_active":"2023-09-23T22:50:42.896688+0000","last_peered":"2023-09-23T22:50:42.896688+0000","last_clean":"2023-09-23T22:50:42.896688+0000","last_became_active":"2023-09-23T22:46:14.248801+0000","last_became_peered":"2023-09-23T22:46:14.248801+0000","last_unstale":"2023-09-23T22:50:42.896688+0000","last_undegraded":"2023-09-23T22:50:42.896688+0000","last_fullsized":"2023-09-23T22:50:42.896688+0000","mapping_epoch":219,"log_start":"232'4500","ondisk_log_start":"232'4500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7906,"log_dups_size":0,"ondisk_log_size":7906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:12.184234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12451840,"num_objects":190,"num_object_clones":0,"num_object_copies":570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12406,"num_write_kb":403072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":70,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.565636+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:50:29.565636+0000","last_peered":"2023-09-23T22:50:29.565636+0000","last_clean":"2023-09-23T22:50:29.565636+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:50:29.565636+0000","last_undegraded":"2023-09-23T22:50:29.565636+0000","last_fullsized":"2023-09-23T22:50:29.565636+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":252,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.564327+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:50:29.564327+0000","last_peered":"2023-09-23T22:50:29.564327+0000","last_clean":"2023-09-23T22:50:29.564327+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:50:29.564327+0000","last_undegraded":"2023-09-23T22:50:29.564327+0000","last_fullsized":"2023-09-23T22:50:29.564327+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"260'12970","reported_seq":13031,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:46.058194+0000","last_change":"2023-09-23T22:46:44.026957+0000","last_active":"2023-09-23T22:50:46.058194+0000","last_peered":"2023-09-23T22:50:46.058194+0000","last_clean":"2023-09-23T22:50:46.058194+0000","last_became_active":"2023-09-23T22:46:44.022288+0000","last_became_peered":"2023-09-23T22:46:44.022288+0000","last_unstale":"2023-09-23T22:50:46.058194+0000","last_undegraded":"2023-09-23T22:50:46.058194+0000","last_fullsized":"2023-09-23T22:50:46.058194+0000","mapping_epoch":226,"log_start":"234'5800","ondisk_log_start":"234'5800","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7170,"log_dups_size":0,"ondisk_log_size":7170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.384879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4718592,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12970,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.582114+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:50:29.582114+0000","last_peered":"2023-09-23T22:50:29.582114+0000","last_clean":"2023-09-23T22:50:29.582114+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:50:29.582114+0000","last_undegraded":"2023-09-23T22:50:29.582114+0000","last_fullsized":"2023-09-23T22:50:29.582114+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":153,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:39.409827+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:50:39.409827+0000","last_peered":"2023-09-23T22:50:39.409827+0000","last_clean":"2023-09-23T22:50:39.409827+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:50:39.409827+0000","last_undegraded":"2023-09-23T22:50:39.409827+0000","last_fullsized":"2023-09-23T22:50:39.409827+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"260'12565","reported_seq":12619,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:46.085670+0000","last_change":"2023-09-23T22:46:12.238263+0000","last_active":"2023-09-23T22:50:46.085670+0000","last_peered":"2023-09-23T22:50:46.085670+0000","last_clean":"2023-09-23T22:50:46.085670+0000","last_became_active":"2023-09-23T22:46:12.237976+0000","last_became_peered":"2023-09-23T22:46:12.237976+0000","last_unstale":"2023-09-23T22:50:46.085670+0000","last_undegraded":"2023-09-23T22:50:46.085670+0000","last_fullsized":"2023-09-23T22:50:46.085670+0000","mapping_epoch":217,"log_start":"233'5400","ondisk_log_start":"233'5400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7165,"log_dups_size":0,"ondisk_log_size":7165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:38.758255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":261,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581723+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:50:29.581723+0000","last_peered":"2023-09-23T22:50:29.581723+0000","last_clean":"2023-09-23T22:50:29.581723+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:50:29.581723+0000","last_undegraded":"2023-09-23T22:50:29.581723+0000","last_fullsized":"2023-09-23T22:50:29.581723+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573925+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:50:29.573925+0000","last_peered":"2023-09-23T22:50:29.573925+0000","last_clean":"2023-09-23T22:50:29.573925+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:50:29.573925+0000","last_undegraded":"2023-09-23T22:50:29.573925+0000","last_fullsized":"2023-09-23T22:50:29.573925+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"260'12588","reported_seq":12661,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:42.904964+0000","last_change":"2023-09-23T22:46:44.075632+0000","last_active":"2023-09-23T22:50:42.904964+0000","last_peered":"2023-09-23T22:50:42.904964+0000","last_clean":"2023-09-23T22:50:42.904964+0000","last_became_active":"2023-09-23T22:46:44.036750+0000","last_became_peered":"2023-09-23T22:46:44.036750+0000","last_unstale":"2023-09-23T22:50:42.904964+0000","last_undegraded":"2023-09-23T22:50:42.904964+0000","last_fullsized":"2023-09-23T22:50:42.904964+0000","mapping_epoch":226,"log_start":"232'4600","ondisk_log_start":"232'4600","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7988,"log_dups_size":0,"ondisk_log_size":7988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:02.225785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12588,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":244,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.564463+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:50:29.564463+0000","last_peered":"2023-09-23T22:50:29.564463+0000","last_clean":"2023-09-23T22:50:29.564463+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:50:29.564463+0000","last_undegraded":"2023-09-23T22:50:29.564463+0000","last_fullsized":"2023-09-23T22:50:29.564463+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":255,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573745+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:50:29.573745+0000","last_peered":"2023-09-23T22:50:29.573745+0000","last_clean":"2023-09-23T22:50:29.573745+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:50:29.573745+0000","last_undegraded":"2023-09-23T22:50:29.573745+0000","last_fullsized":"2023-09-23T22:50:29.573745+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"260'12904","reported_seq":12961,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:45.576873+0000","last_change":"2023-09-23T22:46:14.247927+0000","last_active":"2023-09-23T22:50:45.576873+0000","last_peered":"2023-09-23T22:50:45.576873+0000","last_clean":"2023-09-23T22:50:45.576873+0000","last_became_active":"2023-09-23T22:46:14.247415+0000","last_became_peered":"2023-09-23T22:46:14.247415+0000","last_unstale":"2023-09-23T22:50:45.576873+0000","last_undegraded":"2023-09-23T22:50:45.576873+0000","last_fullsized":"2023-09-23T22:50:45.576873+0000","mapping_epoch":219,"log_start":"233'5500","ondisk_log_start":"233'5500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12904,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":268,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574279+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:50:29.574279+0000","last_peered":"2023-09-23T22:50:29.574279+0000","last_clean":"2023-09-23T22:50:29.574279+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:50:29.574279+0000","last_undegraded":"2023-09-23T22:50:29.574279+0000","last_fullsized":"2023-09-23T22:50:29.574279+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574897+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:50:29.574897+0000","last_peered":"2023-09-23T22:50:29.574897+0000","last_clean":"2023-09-23T22:50:29.574897+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:50:29.574897+0000","last_undegraded":"2023-09-23T22:50:29.574897+0000","last_fullsized":"2023-09-23T22:50:29.574897+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"260'12361","reported_seq":12429,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:45.868642+0000","last_change":"2023-09-23T22:46:44.078273+0000","last_active":"2023-09-23T22:50:45.868642+0000","last_peered":"2023-09-23T22:50:45.868642+0000","last_clean":"2023-09-23T22:50:45.868642+0000","last_became_active":"2023-09-23T22:46:44.028766+0000","last_became_peered":"2023-09-23T22:46:44.028766+0000","last_unstale":"2023-09-23T22:50:45.868642+0000","last_undegraded":"2023-09-23T22:50:45.868642+0000","last_fullsized":"2023-09-23T22:50:45.868642+0000","mapping_epoch":226,"log_start":"230'3700","ondisk_log_start":"230'3700","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5570560,"num_objects":85,"num_object_clones":0,"num_object_copies":255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12361,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":261,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579680+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:50:29.579680+0000","last_peered":"2023-09-23T22:50:29.579680+0000","last_clean":"2023-09-23T22:50:29.579680+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:50:29.579680+0000","last_undegraded":"2023-09-23T22:50:29.579680+0000","last_fullsized":"2023-09-23T22:50:29.579680+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":187,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.569805+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:50:29.569805+0000","last_peered":"2023-09-23T22:50:29.569805+0000","last_clean":"2023-09-23T22:50:29.569805+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:50:29.569805+0000","last_undegraded":"2023-09-23T22:50:29.569805+0000","last_fullsized":"2023-09-23T22:50:29.569805+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"260'12628","reported_seq":12680,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:42.810464+0000","last_change":"2023-09-23T22:46:14.251012+0000","last_active":"2023-09-23T22:50:42.810464+0000","last_peered":"2023-09-23T22:50:42.810464+0000","last_clean":"2023-09-23T22:50:42.810464+0000","last_became_active":"2023-09-23T22:46:14.249264+0000","last_became_peered":"2023-09-23T22:46:14.249264+0000","last_unstale":"2023-09-23T22:50:42.810464+0000","last_undegraded":"2023-09-23T22:50:42.810464+0000","last_fullsized":"2023-09-23T22:50:42.810464+0000","mapping_epoch":219,"log_start":"232'4700","ondisk_log_start":"232'4700","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.309150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13238272,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12628,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":296,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580350+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:50:29.580350+0000","last_peered":"2023-09-23T22:50:29.580350+0000","last_clean":"2023-09-23T22:50:29.580350+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:50:29.580350+0000","last_undegraded":"2023-09-23T22:50:29.580350+0000","last_fullsized":"2023-09-23T22:50:29.580350+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":180,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581098+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:50:29.581098+0000","last_peered":"2023-09-23T22:50:29.581098+0000","last_clean":"2023-09-23T22:50:29.581098+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:50:29.581098+0000","last_undegraded":"2023-09-23T22:50:29.581098+0000","last_fullsized":"2023-09-23T22:50:29.581098+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":91,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573674+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:50:29.573674+0000","last_peered":"2023-09-23T22:50:29.573674+0000","last_clean":"2023-09-23T22:50:29.573674+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:50:29.573674+0000","last_undegraded":"2023-09-23T22:50:29.573674+0000","last_fullsized":"2023-09-23T22:50:29.573674+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"260'12652","reported_seq":12712,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:46.066511+0000","last_change":"2023-09-23T22:46:14.248433+0000","last_active":"2023-09-23T22:50:46.066511+0000","last_peered":"2023-09-23T22:50:46.066511+0000","last_clean":"2023-09-23T22:50:46.066511+0000","last_became_active":"2023-09-23T22:46:14.248009+0000","last_became_peered":"2023-09-23T22:46:14.248009+0000","last_unstale":"2023-09-23T22:50:46.066511+0000","last_undegraded":"2023-09-23T22:50:46.066511+0000","last_fullsized":"2023-09-23T22:50:46.066511+0000","mapping_epoch":219,"log_start":"233'5500","ondisk_log_start":"233'5500","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7152,"log_dups_size":0,"ondisk_log_size":7152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.888467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5111808,"num_objects":78,"num_object_clones":0,"num_object_copies":234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12652,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":94,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.564402+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:50:29.564402+0000","last_peered":"2023-09-23T22:50:29.564402+0000","last_clean":"2023-09-23T22:50:29.564402+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:50:29.564402+0000","last_undegraded":"2023-09-23T22:50:29.564402+0000","last_fullsized":"2023-09-23T22:50:29.564402+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":256,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573973+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:50:29.573973+0000","last_peered":"2023-09-23T22:50:29.573973+0000","last_clean":"2023-09-23T22:50:29.573973+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:50:29.573973+0000","last_undegraded":"2023-09-23T22:50:29.573973+0000","last_fullsized":"2023-09-23T22:50:29.573973+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":176,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.569896+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:50:29.569896+0000","last_peered":"2023-09-23T22:50:29.569896+0000","last_clean":"2023-09-23T22:50:29.569896+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:50:29.569896+0000","last_undegraded":"2023-09-23T22:50:29.569896+0000","last_fullsized":"2023-09-23T22:50:29.569896+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"260'12874","reported_seq":12927,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:46.085631+0000","last_change":"2023-09-23T22:46:12.238298+0000","last_active":"2023-09-23T22:50:46.085631+0000","last_peered":"2023-09-23T22:50:46.085631+0000","last_clean":"2023-09-23T22:50:46.085631+0000","last_became_active":"2023-09-23T22:46:12.238008+0000","last_became_peered":"2023-09-23T22:46:12.238008+0000","last_unstale":"2023-09-23T22:50:46.085631+0000","last_undegraded":"2023-09-23T22:50:46.085631+0000","last_fullsized":"2023-09-23T22:50:46.085631+0000","mapping_epoch":217,"log_start":"234'5700","ondisk_log_start":"234'5700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:29.068385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3538944,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12874,"num_write_kb":413696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":251,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.570213+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:50:29.570213+0000","last_peered":"2023-09-23T22:50:29.570213+0000","last_clean":"2023-09-23T22:50:29.570213+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:50:29.570213+0000","last_undegraded":"2023-09-23T22:50:29.570213+0000","last_fullsized":"2023-09-23T22:50:29.570213+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:39.385882+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:50:39.385882+0000","last_peered":"2023-09-23T22:50:39.385882+0000","last_clean":"2023-09-23T22:50:39.385882+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:50:39.385882+0000","last_undegraded":"2023-09-23T22:50:39.385882+0000","last_fullsized":"2023-09-23T22:50:39.385882+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":265,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580753+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:50:29.580753+0000","last_peered":"2023-09-23T22:50:29.580753+0000","last_clean":"2023-09-23T22:50:29.580753+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:50:29.580753+0000","last_undegraded":"2023-09-23T22:50:29.580753+0000","last_fullsized":"2023-09-23T22:50:29.580753+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"260'12536","reported_seq":12593,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:45.893448+0000","last_change":"2023-09-23T22:46:14.249217+0000","last_active":"2023-09-23T22:50:45.893448+0000","last_peered":"2023-09-23T22:50:45.893448+0000","last_clean":"2023-09-23T22:50:45.893448+0000","last_became_active":"2023-09-23T22:46:14.248894+0000","last_became_peered":"2023-09-23T22:46:14.248894+0000","last_unstale":"2023-09-23T22:50:45.893448+0000","last_undegraded":"2023-09-23T22:50:45.893448+0000","last_fullsized":"2023-09-23T22:50:45.893448+0000","mapping_epoch":219,"log_start":"230'3900","ondisk_log_start":"230'3900","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":8636,"log_dups_size":0,"ondisk_log_size":8636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5111808,"num_objects":78,"num_object_clones":0,"num_object_copies":234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12536,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":261,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.569568+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:50:29.569568+0000","last_peered":"2023-09-23T22:50:29.569568+0000","last_clean":"2023-09-23T22:50:29.569568+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:50:29.569568+0000","last_undegraded":"2023-09-23T22:50:29.569568+0000","last_fullsized":"2023-09-23T22:50:29.569568+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":347,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581484+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:50:29.581484+0000","last_peered":"2023-09-23T22:50:29.581484+0000","last_clean":"2023-09-23T22:50:29.581484+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:50:29.581484+0000","last_undegraded":"2023-09-23T22:50:29.581484+0000","last_fullsized":"2023-09-23T22:50:29.581484+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":192,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:36.088704+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:50:36.088704+0000","last_peered":"2023-09-23T22:50:36.088704+0000","last_clean":"2023-09-23T22:50:36.088704+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:50:36.088704+0000","last_undegraded":"2023-09-23T22:50:36.088704+0000","last_fullsized":"2023-09-23T22:50:36.088704+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":252,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.565715+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:50:29.565715+0000","last_peered":"2023-09-23T22:50:29.565715+0000","last_clean":"2023-09-23T22:50:29.565715+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:50:29.565715+0000","last_undegraded":"2023-09-23T22:50:29.565715+0000","last_fullsized":"2023-09-23T22:50:29.565715+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"260'12811","reported_seq":12871,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:44.776045+0000","last_change":"2023-09-23T22:46:44.028117+0000","last_active":"2023-09-23T22:50:44.776045+0000","last_peered":"2023-09-23T22:50:44.776045+0000","last_clean":"2023-09-23T22:50:44.776045+0000","last_became_active":"2023-09-23T22:46:44.025256+0000","last_became_peered":"2023-09-23T22:46:44.025256+0000","last_unstale":"2023-09-23T22:50:44.776045+0000","last_undegraded":"2023-09-23T22:50:44.776045+0000","last_fullsized":"2023-09-23T22:50:44.776045+0000","mapping_epoch":226,"log_start":"229'3100","ondisk_log_start":"229'3100","created":217,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9711,"log_dups_size":0,"ondisk_log_size":9711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:53.812817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8978432,"num_objects":137,"num_object_clones":0,"num_object_copies":411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12811,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581619+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:50:29.581619+0000","last_peered":"2023-09-23T22:50:29.581619+0000","last_clean":"2023-09-23T22:50:29.581619+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:50:29.581619+0000","last_undegraded":"2023-09-23T22:50:29.581619+0000","last_fullsized":"2023-09-23T22:50:29.581619+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":197,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.582802+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:50:29.582802+0000","last_peered":"2023-09-23T22:50:29.582802+0000","last_clean":"2023-09-23T22:50:29.582802+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:50:29.582802+0000","last_undegraded":"2023-09-23T22:50:29.582802+0000","last_fullsized":"2023-09-23T22:50:29.582802+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":184,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:32.899525+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:50:32.899525+0000","last_peered":"2023-09-23T22:50:32.899525+0000","last_clean":"2023-09-23T22:50:32.899525+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:50:32.899525+0000","last_undegraded":"2023-09-23T22:50:32.899525+0000","last_fullsized":"2023-09-23T22:50:32.899525+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"260'13182","reported_seq":20594,"reported_epoch":260,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:50:46.100913+0000","last_change":"2023-09-23T22:47:55.885382+0000","last_active":"2023-09-23T22:50:46.100913+0000","last_peered":"2023-09-23T22:50:46.100913+0000","last_clean":"2023-09-23T22:47:50.504847+0000","last_became_active":"2023-09-23T22:47:52.757427+0000","last_became_peered":"2023-09-23T22:47:52.757427+0000","last_unstale":"2023-09-23T22:50:46.100913+0000","last_undegraded":"2023-09-23T22:47:55.879652+0000","last_fullsized":"2023-09-23T22:47:52.505085+0000","mapping_epoch":239,"log_start":"235'6000","ondisk_log_start":"235'6000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.702788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5636096,"num_objects":86,"num_object_clones":0,"num_object_copies":258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2873,"num_objects_misplaced":70,"num_objects_unfound":0,"num_objects_dirty":86,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13182,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3675,"num_bytes_recovered":240844800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":-2857},{"shards":"1,3,4","objects":70}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":189,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.581835+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:50:29.581835+0000","last_peered":"2023-09-23T22:50:29.581835+0000","last_clean":"2023-09-23T22:50:29.581835+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:50:29.581835+0000","last_undegraded":"2023-09-23T22:50:29.581835+0000","last_fullsized":"2023-09-23T22:50:29.581835+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":138,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.570063+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:50:29.570063+0000","last_peered":"2023-09-23T22:50:29.570063+0000","last_clean":"2023-09-23T22:50:29.570063+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:50:29.570063+0000","last_undegraded":"2023-09-23T22:50:29.570063+0000","last_fullsized":"2023-09-23T22:50:29.570063+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":189,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580658+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:50:29.580658+0000","last_peered":"2023-09-23T22:50:29.580658+0000","last_clean":"2023-09-23T22:50:29.580658+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:50:29.580658+0000","last_undegraded":"2023-09-23T22:50:29.580658+0000","last_fullsized":"2023-09-23T22:50:29.580658+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"260'12729","reported_seq":12781,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:42.767377+0000","last_change":"2023-09-23T22:46:14.246487+0000","last_active":"2023-09-23T22:50:42.767377+0000","last_peered":"2023-09-23T22:50:42.767377+0000","last_clean":"2023-09-23T22:50:42.767377+0000","last_became_active":"2023-09-23T22:46:14.246173+0000","last_became_peered":"2023-09-23T22:46:14.246173+0000","last_unstale":"2023-09-23T22:50:42.767377+0000","last_undegraded":"2023-09-23T22:50:42.767377+0000","last_fullsized":"2023-09-23T22:50:42.767377+0000","mapping_epoch":219,"log_start":"232'4800","ondisk_log_start":"232'4800","created":217,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":7929,"log_dups_size":0,"ondisk_log_size":7929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:10.271838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11730944,"num_objects":179,"num_object_clones":0,"num_object_copies":537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12729,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":300,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.584877+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:50:29.584877+0000","last_peered":"2023-09-23T22:50:29.584877+0000","last_clean":"2023-09-23T22:50:29.584877+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:50:29.584877+0000","last_undegraded":"2023-09-23T22:50:29.584877+0000","last_fullsized":"2023-09-23T22:50:29.584877+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":58,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.565453+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:50:29.565453+0000","last_peered":"2023-09-23T22:50:29.565453+0000","last_clean":"2023-09-23T22:50:29.565453+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:50:29.565453+0000","last_undegraded":"2023-09-23T22:50:29.565453+0000","last_fullsized":"2023-09-23T22:50:29.565453+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":286,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579754+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:50:29.579754+0000","last_peered":"2023-09-23T22:50:29.579754+0000","last_clean":"2023-09-23T22:50:29.579754+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:50:29.579754+0000","last_undegraded":"2023-09-23T22:50:29.579754+0000","last_fullsized":"2023-09-23T22:50:29.579754+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"260'12781","reported_seq":12836,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T22:50:44.791250+0000","last_change":"2023-09-23T22:46:12.238725+0000","last_active":"2023-09-23T22:50:44.791250+0000","last_peered":"2023-09-23T22:50:44.791250+0000","last_clean":"2023-09-23T22:50:44.791250+0000","last_became_active":"2023-09-23T22:46:12.238443+0000","last_became_peered":"2023-09-23T22:46:12.238443+0000","last_unstale":"2023-09-23T22:50:44.791250+0000","last_undegraded":"2023-09-23T22:50:44.791250+0000","last_fullsized":"2023-09-23T22:50:44.791250+0000","mapping_epoch":217,"log_start":"229'3100","ondisk_log_start":"229'3100","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:46:11.224396+0000","last_clean_scrub_stamp":"2023-09-23T22:46:11.224396+0000","objects_scrubbed":0,"log_size":9681,"log_dups_size":0,"ondisk_log_size":9681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:00.930429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6815768,"num_objects":105,"num_object_clones":0,"num_object_copies":315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12781,"num_write_kb":412289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":292,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.565267+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:50:29.565267+0000","last_peered":"2023-09-23T22:50:29.565267+0000","last_clean":"2023-09-23T22:50:29.565267+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:50:29.565267+0000","last_undegraded":"2023-09-23T22:50:29.565267+0000","last_fullsized":"2023-09-23T22:50:29.565267+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":156,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:32.899546+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:50:32.899546+0000","last_peered":"2023-09-23T22:50:32.899546+0000","last_clean":"2023-09-23T22:50:32.899546+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:50:32.899546+0000","last_undegraded":"2023-09-23T22:50:32.899546+0000","last_fullsized":"2023-09-23T22:50:32.899546+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574358+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:50:29.574358+0000","last_peered":"2023-09-23T22:50:29.574358+0000","last_clean":"2023-09-23T22:50:29.574358+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:50:29.574358+0000","last_undegraded":"2023-09-23T22:50:29.574358+0000","last_fullsized":"2023-09-23T22:50:29.574358+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":191,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580519+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:50:29.580519+0000","last_peered":"2023-09-23T22:50:29.580519+0000","last_clean":"2023-09-23T22:50:29.580519+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:50:29.580519+0000","last_undegraded":"2023-09-23T22:50:29.580519+0000","last_fullsized":"2023-09-23T22:50:29.580519+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.569977+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:50:29.569977+0000","last_peered":"2023-09-23T22:50:29.569977+0000","last_clean":"2023-09-23T22:50:29.569977+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:50:29.569977+0000","last_undegraded":"2023-09-23T22:50:29.569977+0000","last_fullsized":"2023-09-23T22:50:29.569977+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":70,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573845+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:50:29.573845+0000","last_peered":"2023-09-23T22:50:29.573845+0000","last_clean":"2023-09-23T22:50:29.573845+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:50:29.573845+0000","last_undegraded":"2023-09-23T22:50:29.573845+0000","last_fullsized":"2023-09-23T22:50:29.573845+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":259,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574032+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:50:29.574032+0000","last_peered":"2023-09-23T22:50:29.574032+0000","last_clean":"2023-09-23T22:50:29.574032+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:50:29.574032+0000","last_undegraded":"2023-09-23T22:50:29.574032+0000","last_fullsized":"2023-09-23T22:50:29.574032+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":70,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.574207+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:50:29.574207+0000","last_peered":"2023-09-23T22:50:29.574207+0000","last_clean":"2023-09-23T22:50:29.574207+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:50:29.574207+0000","last_undegraded":"2023-09-23T22:50:29.574207+0000","last_fullsized":"2023-09-23T22:50:29.574207+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573846+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:50:29.573846+0000","last_peered":"2023-09-23T22:50:29.573846+0000","last_clean":"2023-09-23T22:50:29.573846+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:50:29.573846+0000","last_undegraded":"2023-09-23T22:50:29.573846+0000","last_fullsized":"2023-09-23T22:50:29.573846+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":254,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580454+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:50:29.580454+0000","last_peered":"2023-09-23T22:50:29.580454+0000","last_clean":"2023-09-23T22:50:29.580454+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:50:29.580454+0000","last_undegraded":"2023-09-23T22:50:29.580454+0000","last_fullsized":"2023-09-23T22:50:29.580454+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":292,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.579793+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:50:29.579793+0000","last_peered":"2023-09-23T22:50:29.579793+0000","last_clean":"2023-09-23T22:50:29.579793+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:50:29.579793+0000","last_undegraded":"2023-09-23T22:50:29.579793+0000","last_fullsized":"2023-09-23T22:50:29.579793+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":205,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.580935+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:50:29.580935+0000","last_peered":"2023-09-23T22:50:29.580935+0000","last_clean":"2023-09-23T22:50:29.580935+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:50:29.580935+0000","last_undegraded":"2023-09-23T22:50:29.580935+0000","last_fullsized":"2023-09-23T22:50:29.580935+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:50:29.573657+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:50:29.573657+0000","last_peered":"2023-09-23T22:50:29.573657+0000","last_clean":"2023-09-23T22:50:29.573657+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:50:29.573657+0000","last_undegraded":"2023-09-23T22:50:29.573657+0000","last_fullsized":"2023-09-23T22:50:29.573657+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":115671064,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2873,"num_objects_misplaced":70,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":203132,"num_write_kb":6556673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3678,"num_bytes_recovered":241041408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":814493696,"data_stored":814481480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127632,"ondisk_log_size":127632,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430493,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626721,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339572,"kb_used_data":38252,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93405580,"statfs":{"total":95995035648,"available":95647313920,"internally_reserved":0,"allocated":39170048,"data_stored":38666053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":3.5,"15min":2.0259999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3890000000000002,"5min":117.45999999999999,"15min":117.45999999999999},"last":1.3540000000000001},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7360000000000002,"15min":1.8959999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.9800000000000004,"5min":117.48699999999999,"15min":117.48699999999999},"last":0.70099999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.1309999999999998,"15min":2.532},"min":{"1min":0.28599999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.4039999999999999,"5min":31.286000000000001,"15min":153.78399999999999},"last":5.3760000000000003},{"interface":"front","average":{"1min":1.7150000000000001,"5min":4.1219999999999999,"15min":3.1600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6799999999999997,"5min":185.411,"15min":185.411},"last":0.40400000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:50:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.7229999999999999,"15min":3.1499999999999999},"min":{"1min":0.44900000000000001,"5min":0.309,"15min":0.28299999999999997},"max":{"1min":6.4100000000000001,"5min":33.174999999999997,"15min":100.785},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.825,"5min":4.2220000000000004,"15min":3.5459999999999998},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4829999999999997,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.34300000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":5.5170000000000003,"15min":3.5099999999999998},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.6020000000000003,"5min":185.19399999999999,"15min":185.19399999999999},"last":5.3520000000000003},{"interface":"front","average":{"1min":2.605,"5min":4.3419999999999996,"15min":3.1989999999999998},"min":{"1min":0.36799999999999999,"5min":0.35599999999999998,"15min":0.28899999999999998},"max":{"1min":9.4410000000000007,"5min":128.19800000000001,"15min":128.19800000000001},"last":2.4300000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.79200000000000004,"15min":0.68799999999999994},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8239999999999998,"5min":6.5309999999999997,"15min":18.085999999999999},"last":3.198},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.86599999999999999,"15min":0.70699999999999996},"min":{"1min":0.318,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":4.2530000000000001,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.0029999999999999,"15min":0.79600000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":7.1130000000000004,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.93700000000000006,"15min":0.76900000000000002},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.9089999999999998,"5min":9.7899999999999991,"15min":18.625},"last":3.1629999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.98799999999999999,"15min":0.748},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":1.9990000000000001,"5min":9.8800000000000008,"15min":12.481999999999999},"last":2.0569999999999999},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.75600000000000001,"15min":0.73399999999999999},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":2.8650000000000002,"5min":5.1790000000000003,"15min":16.661999999999999},"last":5.3150000000000004}]}]},{"osd":1,"up_from":84,"seq":360777253030,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":419340,"kb_used_data":98948,"kb_used_omap":9,"kb_used_meta":320374,"kb_avail":93325812,"statfs":{"total":95995035648,"available":95565631488,"internally_reserved":0,"allocated":101322752,"data_stored":100391751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":328063977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.367,"5min":8.0190000000000001,"15min":5.4509999999999996},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":26.042000000000002,"5min":101.605,"15min":101.605},"last":0.56699999999999995},{"interface":"front","average":{"1min":4.7430000000000003,"5min":7.6180000000000003,"15min":5.5490000000000004},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":25.379999999999999,"5min":101.892,"15min":101.892},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":3.7850000000000001,"15min":2.6589999999999998},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":26.129000000000001,"5min":37.869999999999997,"15min":37.889000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":6.0439999999999996,"5min":4.0890000000000004,"15min":2.7549999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":26.001999999999999,"5min":37.957000000000001,"15min":37.957000000000001},"last":0.53100000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":3.46,"15min":2.1859999999999999},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":3.1179999999999999,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.221},{"interface":"front","average":{"1min":2.016,"5min":4.2999999999999998,"15min":2.6389999999999998},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.20999999999999999},"max":{"1min":6.7649999999999997,"5min":89.861999999999995,"15min":89.861999999999995},"last":9.8699999999999992}]},{"osd":4,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":2.681,"15min":1.9990000000000001},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":5.6150000000000002,"5min":78.948999999999998,"15min":78.948999999999998},"last":9.9039999999999999},{"interface":"front","average":{"1min":3.4140000000000001,"5min":2.4729999999999999,"15min":1.9490000000000001},"min":{"1min":0.33500000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":14.058,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.313}]},{"osd":5,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.069,"15min":1.546},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":4.3230000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.343,"5min":2.7919999999999998,"15min":1.865},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":5.5650000000000004,"5min":87.515000000000001,"15min":87.515000000000001},"last":9.8170000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":2.5550000000000002,"15min":1.7829999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":13.991,"5min":79.311000000000007,"15min":79.311000000000007},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":2.0950000000000002,"15min":1.482},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":2.984,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1480000000000001,"15min":2.234},"min":{"1min":0.435,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":5.6539999999999999,"5min":11.481999999999999,"15min":159.018},"last":9.9329999999999998},{"interface":"front","average":{"1min":3.548,"5min":2.0659999999999998,"15min":2.2890000000000001},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.37},"max":{"1min":14.032999999999999,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.499}]}]},{"osd":0,"up_from":219,"seq":940597837879,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":293120,"kb_used_data":114616,"kb_used_omap":31,"kb_used_meta":178336,"kb_avail":93452032,"statfs":{"total":95995035648,"available":95694880768,"internally_reserved":0,"allocated":117366784,"data_stored":116388151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":182617010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,0,8],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":3.887,"15min":3.77},"min":{"1min":0.221,"5min":0.216,"15min":0.216},"max":{"1min":14.141999999999999,"5min":32.344999999999999,"15min":32.344999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":3.6520000000000001,"5min":3.9460000000000002,"15min":3.6880000000000002},"min":{"1min":0.26100000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.291,"5min":32.106000000000002,"15min":32.106000000000002},"last":0.24099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":3.7509999999999999,"15min":2.649},"min":{"1min":0.26700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":16.643999999999998,"5min":24.253,"15min":24.253},"last":2.9980000000000002},{"interface":"front","average":{"1min":4.4619999999999997,"5min":2.9489999999999998,"15min":2.1539999999999999},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":21.151,"5min":21.151,"15min":21.151},"last":2.9820000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.4400000000000004,"5min":3.4319999999999999,"15min":2.4540000000000002},"min":{"1min":0.30199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.609999999999999,"5min":20.710000000000001,"15min":20.710000000000001},"last":11.068},{"interface":"front","average":{"1min":4.3090000000000002,"5min":2.431,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.254},"max":{"1min":22.172000000000001,"5min":22.172000000000001,"15min":22.172000000000001},"last":10.779999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.5529999999999999,"15min":2.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.2030000000000003,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":3.194,"5min":2.407,"15min":1.968},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":16.52,"5min":20.009,"15min":20.009},"last":2.411}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":2.8650000000000002,"15min":1.845},"min":{"1min":0.38,"5min":0.318,"15min":0.318},"max":{"1min":61.756999999999998,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":5.8150000000000004,"5min":3.2080000000000002,"15min":2.1040000000000001},"min":{"1min":0.379,"5min":0.374,"15min":0.374},"max":{"1min":61.777000000000001,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.71999999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":6.3579999999999997,"5min":2.6970000000000001,"15min":1.8129999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":59.146999999999998,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":4.5300000000000002,"5min":3.1800000000000002,"15min":2.1840000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":59.164000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.65700000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":2.4409999999999998,"15min":1.877},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.2350000000000003,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.749},{"interface":"front","average":{"1min":3.4740000000000002,"5min":2,"15min":1.452},"min":{"1min":0.49399999999999999,"5min":0.432,"15min":0.432},"max":{"1min":21.954999999999998,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.46000000000000002}]}]},{"osd":2,"up_from":131,"seq":562640715892,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":335392,"kb_used_data":55064,"kb_used_omap":81,"kb_used_meta":280238,"kb_avail":93409760,"statfs":{"total":95995035648,"available":95651594240,"internally_reserved":0,"allocated":56385536,"data_stored":55438242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":286963930},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":10.039,"5min":5.8920000000000003,"15min":5.476},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":132.79499999999999,"5min":132.79499999999999,"15min":132.79499999999999},"last":4.5510000000000002},{"interface":"front","average":{"1min":10.228,"5min":6.9359999999999999,"15min":5.9050000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":132.58099999999999,"5min":132.58099999999999,"15min":132.58099999999999},"last":4.4930000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":4.0119999999999996,"5min":2.5299999999999998,"15min":3.2469999999999999},"min":{"1min":0.253,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.622999999999999,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":3.581,"5min":3.298,"15min":3.0859999999999999},"min":{"1min":0.27200000000000002,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.143000000000001,"5min":28.596,"15min":88.436999999999998},"last":5.4199999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9020000000000001,"5min":4.0270000000000001,"15min":3.6579999999999999},"min":{"1min":0.34200000000000003,"5min":0.223,"15min":0.223},"max":{"1min":35.198999999999998,"5min":35.198999999999998,"15min":88.653000000000006},"last":5.5490000000000004},{"interface":"front","average":{"1min":5.9589999999999996,"5min":3.3559999999999999,"15min":2.931},"min":{"1min":0.32900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":35.136000000000003,"5min":35.136000000000003,"15min":88.474000000000004},"last":4.4029999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.8230000000000004,"15min":2.3530000000000002},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":10.433,"5min":108.878,"15min":108.878},"last":4.46},{"interface":"front","average":{"1min":2.3839999999999999,"5min":4.9279999999999999,"15min":3.0680000000000001},"min":{"1min":0.39300000000000002,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":11.273,"5min":108.93000000000001,"15min":108.93000000000001},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":4.2460000000000004,"15min":2.9209999999999998},"min":{"1min":0.36299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.239000000000001,"5min":97.322999999999993,"15min":97.322999999999993},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.2549999999999999,"5min":4.3730000000000002,"15min":2.589},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":10.327,"5min":97.301000000000002,"15min":97.301000000000002},"last":5.3849999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":6.827,"5min":4.7030000000000003,"15min":2.8069999999999999},"min":{"1min":0.36899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":98.597999999999999,"5min":98.597999999999999,"15min":98.597999999999999},"last":10.249000000000001},{"interface":"front","average":{"1min":6.8230000000000004,"5min":4.4189999999999996,"15min":2.258},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":98.509,"5min":98.509,"15min":98.509},"last":4.5279999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9990000000000001,"15min":1.506},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.477,"5min":10.92,"15min":10.92},"last":4.5030000000000001},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.339,"15min":2.3530000000000002},"min":{"1min":0.49399999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.311,"5min":11.311,"15min":88.584000000000003},"last":0.52000000000000002}]}]},{"osd":3,"up_from":94,"seq":403726925978,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":441164,"kb_used_data":44164,"kb_used_omap":7,"kb_used_meta":396984,"kb_avail":93303988,"statfs":{"total":95995035648,"available":95543283712,"internally_reserved":0,"allocated":45223936,"data_stored":44290956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":406512232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":2.931,"5min":3.3849999999999998,"15min":2.3679999999999999},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.662000000000001,"5min":80.099000000000004,"15min":80.099000000000004},"last":2.911},{"interface":"front","average":{"1min":3.2810000000000001,"5min":3.3380000000000001,"15min":2.1779999999999999},"min":{"1min":0.374,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.128,"5min":80.150000000000006,"15min":80.150000000000006},"last":0.438}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6949999999999998,"5min":2.6949999999999998,"15min":1.831},"min":{"1min":0.221,"5min":0.221,"15min":0.20000000000000001},"max":{"1min":11.086,"5min":17.231000000000002,"15min":22.818000000000001},"last":2.0739999999999998},{"interface":"front","average":{"1min":2.665,"5min":1.9570000000000001,"15min":1.4390000000000001},"min":{"1min":0.377,"5min":0.25600000000000001,"15min":0.222},"max":{"1min":11.757999999999999,"5min":15.098000000000001,"15min":23.917999999999999},"last":2.0390000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":2.8610000000000002,"15min":2.6949999999999998},"min":{"1min":0.42899999999999999,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":17.170999999999999,"5min":20.431000000000001,"15min":125.446},"last":1.27},{"interface":"front","average":{"1min":3.359,"5min":3.2599999999999998,"15min":3.2010000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":11.641,"5min":15.756,"15min":182.483},"last":5.6479999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5720000000000001,"5min":2.077,"15min":1.522},"min":{"1min":0.33500000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":8.1509999999999998,"5min":12.896000000000001,"15min":18.757000000000001},"last":1.9039999999999999},{"interface":"front","average":{"1min":2.609,"5min":1.706,"15min":1.325},"min":{"1min":0.47799999999999998,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":11.834,"5min":12.131,"15min":12.131},"last":0.498}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.5449999999999999,"15min":1.2549999999999999},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":11.803000000000001,"5min":11.803000000000001,"15min":12.127000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.105,"5min":2.2930000000000001,"15min":1.4970000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.951000000000001,"5min":11.951000000000001,"15min":11.951000000000001},"last":5.1059999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.7390000000000001,"15min":1.3100000000000001},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.28999999999999998},"max":{"1min":11.867000000000001,"5min":11.867000000000001,"15min":12.16},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.2250000000000001,"15min":1.534},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.913,"5min":11.913,"15min":11.913},"last":5.0679999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":3.1680000000000001,"15min":1.8320000000000001},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":11.694000000000001,"5min":91.757999999999996,"15min":91.757999999999996},"last":5.1349999999999998},{"interface":"front","average":{"1min":2.8559999999999999,"5min":3.1019999999999999,"15min":1.9550000000000001},"min":{"1min":0.48699999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":8.2010000000000005,"5min":91.709999999999994,"15min":91.709999999999994},"last":1.952}]}]},{"osd":4,"up_from":30,"seq":128849019177,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":759984,"kb_used_data":462760,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92985168,"statfs":{"total":95995035648,"available":95216812032,"internally_reserved":0,"allocated":473866240,"data_stored":473365815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":4.1849999999999996,"15min":6.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.9770000000000003,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.247},{"interface":"front","average":{"1min":2.5230000000000001,"5min":5.077,"15min":7.3780000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.464,"5min":110.33799999999999,"15min":110.33799999999999},"last":1.9810000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":2.4319999999999999,"15min":1.6850000000000001},"min":{"1min":0.29099999999999998,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":12.192,"5min":41.643000000000001,"15min":45.320999999999998},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.514,"5min":1.9990000000000001,"15min":1.492},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.4130000000000003,"5min":41.712000000000003,"15min":45.372999999999998},"last":1.704}]},{"osd":2,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.2749999999999999,"15min":3.2719999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":17.093,"5min":41.607999999999997,"15min":84.909000000000006},"last":2.0310000000000001},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.4249999999999998,"15min":3.206},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":21.704000000000001,"5min":41.670000000000002,"15min":83.840000000000003},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":6.4089999999999998,"15min":3.5209999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":10.329000000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.794,"5min":6.327,"15min":3.5640000000000001},"min":{"1min":0.38300000000000001,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":12.345000000000001,"5min":129.197,"15min":129.197},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":0.81000000000000005,"15min":0.82399999999999995},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":12.238,"5min":12.238,"15min":43.189999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.83499999999999996,"15min":0.85199999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":7.5460000000000003,"5min":7.5460000000000003,"15min":42.450000000000003},"last":1.9410000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85699999999999998,"15min":0.80900000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":7.4560000000000004,"5min":10.952999999999999,"15min":27.056000000000001},"last":3.3460000000000001},{"interface":"front","average":{"1min":1.47,"5min":0.85199999999999998,"15min":0.82699999999999996},"min":{"1min":0.42999999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":12.269,"5min":12.269,"15min":26.940999999999999},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.69799999999999995,"15min":0.68799999999999994},"min":{"1min":0.40100000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.181,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.75900000000000001,"15min":0.68400000000000005},"min":{"1min":0.38800000000000001,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":8.6739999999999995,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.442}]}]},{"osd":5,"up_from":36,"seq":154618822949,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365212,"kb_used_data":63892,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93379940,"statfs":{"total":95995035648,"available":95621058560,"internally_reserved":0,"allocated":65425408,"data_stored":64940601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":10.109999999999999,"5min":7.133,"15min":3.5830000000000002},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":84.367999999999995,"5min":121.319,"15min":121.319},"last":0.26700000000000002},{"interface":"front","average":{"1min":10.358000000000001,"5min":7.2380000000000004,"15min":3.5099999999999998},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":84.311999999999998,"5min":121.075,"15min":121.075},"last":2.6280000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.5269999999999999,"15min":1.8089999999999999},"min":{"1min":0.29299999999999998,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":6.9189999999999996,"5min":28.658999999999999,"15min":149.203},"last":2.3460000000000001},{"interface":"front","average":{"1min":1.581,"5min":1.9259999999999999,"15min":2.0030000000000001},"min":{"1min":0.34499999999999997,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.0119999999999996,"5min":28.664999999999999,"15min":149.554},"last":2.492}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8179999999999996,"5min":4.048,"15min":6.4340000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":63.156999999999996,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":4.4349999999999996,"5min":3.8959999999999999,"15min":6.4039999999999999},"min":{"1min":0.318,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":63.277999999999999,"5min":158.91399999999999,"15min":158.91399999999999},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":10.577999999999999,"5min":9.7460000000000004,"15min":4.899},"min":{"1min":0.31,"5min":0.30499999999999999,"15min":0.25700000000000001},"max":{"1min":98.641000000000005,"5min":257.05000000000001,"15min":257.05000000000001},"last":1.335},{"interface":"front","average":{"1min":11.012,"5min":9.7850000000000001,"15min":4.7750000000000004},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":98.906000000000006,"5min":257.35000000000002,"15min":257.35000000000002},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66300000000000003,"15min":0.62},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":5.0309999999999997,"5min":5.0309999999999997,"15min":5.0309999999999997},"last":0.309},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.75700000000000001,"15min":0.624},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":3.4409999999999998,"5min":10.535,"15min":10.535},"last":0.28499999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.89900000000000002,"15min":1.4099999999999999},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":4.367,"5min":12.615,"15min":140.07499999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.0449999999999999,"15min":1.466},"min":{"1min":0.318,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":6.6219999999999999,"5min":12.74,"15min":139.739},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.74399999999999999,"15min":0.71999999999999997},"min":{"1min":0.34699999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.738,"5min":4.2510000000000003,"15min":7.9809999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.73599999999999999,"15min":0.68000000000000005},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":3.5539999999999998,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.47899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":97714176,"data_stored":97714176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":97648640,"data_stored":97648640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":48828416,"data_stored":48824344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":27918336,"data_stored":27918336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":461766656,"data_stored":461766656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":48959488,"data_stored":48955416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":31657984,"data_stored":31653912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:50:51.412 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:50:51.418 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:51.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:51.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:51.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:51 smithi052 ceph-mon[187743]: pgmap v730: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 50 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 2873/2427 objects degraded (118.377%); 70/2427 objects misplaced (2.884%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:51.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:51 smithi052 ceph-mon[180979]: pgmap v730: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 50 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 2873/2427 objects degraded (118.377%); 70/2427 objects misplaced (2.884%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:51.524 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:51.556 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:51.605 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:51.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:51.709 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:51.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:51 smithi151 ceph-mon[159210]: pgmap v730: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 50 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 2873/2427 objects degraded (118.377%); 70/2427 objects misplaced (2.884%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T22:50:51.737 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:51.781 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:51.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:51.794 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:51.885 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:51.912 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:51.954 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:51.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:52.060 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:50:52.086 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:52.127 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:50:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:52.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:50:52.232 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:50:52.258 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:52.300 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:50:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:52.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:50:52.404 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:52.430 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:52.471 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:52.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:52.510 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:52.577 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:52.603 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:52.644 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:52.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:52.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:52.712 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:52.752 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:52.763 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:52.904 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:53.355 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:50:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:50:53] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:50:53.368 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:53.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:53.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:53.472 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:53.502 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:53.622 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:50:53.649 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:53 smithi052 ceph-mon[180979]: pgmap v731: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 2343/114 objects degraded (2055.263%); 1.3 MiB/s, 74 objects/s recovering 2023-09-23T22:50:53.650 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:53 smithi052 ceph-mon[187743]: pgmap v731: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 2343/114 objects degraded (2055.263%); 1.3 MiB/s, 74 objects/s recovering 2023-09-23T22:50:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:53 smithi151 ceph-mon[159210]: pgmap v731: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 463 op/s; 2343/114 objects degraded (2055.263%); 1.3 MiB/s, 74 objects/s recovering 2023-09-23T22:50:54.469 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:50:55.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:55 smithi052 ceph-mon[187743]: pgmap v732: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 2343/72 objects degraded (3254.167%); 1.3 MiB/s, 74 objects/s recovering 2023-09-23T22:50:55.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 2343/72 objects degraded (3254.167%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:55.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:55 smithi052 ceph-mon[180979]: pgmap v732: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 2343/72 objects degraded (3254.167%); 1.3 MiB/s, 74 objects/s recovering 2023-09-23T22:50:55.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 2343/72 objects degraded (3254.167%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:55.835 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:55.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:55.939 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:50:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:55 smithi151 ceph-mon[159210]: pgmap v732: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 2343/72 objects degraded (3254.167%); 1.3 MiB/s, 74 objects/s recovering 2023-09-23T22:50:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 2343/72 objects degraded (3254.167%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:50:55.975 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:56.284 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:50:56.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:56.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:50:56.388 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:56.427 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:56.473 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4130598745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:56.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4130598745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:56.486 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:50:56.509 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:50:56.533 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:50:56.563 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:50:56.585 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:50:56.604 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:50:56.624 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:50:56.644 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:50:56.646 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.646 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.646 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.647 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.647 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.647 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.647 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.647 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:50:56.740 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:56.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:56.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:56.844 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:50:56.863 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:50:56.864 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":989,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":261,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":7,"osd_in_since":1695509092,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":24,"data_bytes":595872,"bytes_used":2567098368,"bytes_avail":669398151168,"bytes_total":671965249536,"degraded_objects":1558,"degraded_total":72,"degraded_ratio":21.638888888888893,"recovering_objects_per_sec":138,"recovering_bytes_per_sec":373451,"recovering_keys_per_sec":0,"num_objects_recovered":1385,"num_bytes_recovered":3735552,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":176},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":27,"modified":"2023-09-23T22:50:10.280818+0000","services":{}},"progress_events":{"3d9c7ac0-c827-48ce-9357-8c3e026dd16f":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:50:56.876 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4130598745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:50:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[187743]: pgmap v733: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 1558/72 objects degraded (2163.889%); 365 KiB/s, 138 objects/s recovering 2023-09-23T22:50:57.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3792636840' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:57.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[180979]: pgmap v733: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 1558/72 objects degraded (2163.889%); 365 KiB/s, 138 objects/s recovering 2023-09-23T22:50:57.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3792636840' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:57.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:57.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:57 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:57 smithi151 ceph-mon[159210]: pgmap v733: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 1558/72 objects degraded (2163.889%); 365 KiB/s, 138 objects/s recovering 2023-09-23T22:50:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3792636840' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:50:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:57 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:57 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:50:58.431 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:50:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:58.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:50:58.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:58.571 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:58.623 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:58.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:58.727 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:50:58.763 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:58.813 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:50:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:58.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:50:58.919 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:50:58.947 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:59.676 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:50:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:59.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:50:59.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:50:59 smithi052 ceph-mon[187743]: pgmap v734: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 1558/72 objects degraded (2163.889%); 365 KiB/s, 138 objects/s recovering 2023-09-23T22:50:59.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:50:59 smithi052 ceph-mon[180979]: pgmap v734: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 1558/72 objects degraded (2163.889%); 365 KiB/s, 138 objects/s recovering 2023-09-23T22:50:59.771 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:50:59.780 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:59.794 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:50:59.794 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:50:59.810 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:50:59 smithi151 ceph-mon[159210]: pgmap v734: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 1558/72 objects degraded (2163.889%); 365 KiB/s, 138 objects/s recovering 2023-09-23T22:50:59.813 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:50:59.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:50:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:50:59.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:50:59.967 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:50:59.995 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:00.040 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:00.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:00.145 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:00.181 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:00.249 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:00.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:00.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:00.395 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:00.478 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:51:00.479 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:51:00.505 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:51:00.505 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:51:00.505 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T22:51:00.505 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:51:00.709 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:00.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:00.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:00.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:51:00.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:00.888 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:00.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:00.993 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:01.026 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:01.503 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_2' removed 2023-09-23T22:51:01.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:01 smithi052 ceph-mon[180979]: pgmap v735: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 1558/72 objects degraded (2163.889%); 308 KiB/s, 116 objects/s recovering 2023-09-23T22:51:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3446971922' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:51:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:51:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:01 smithi052 ceph-mon[187743]: pgmap v735: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 1558/72 objects degraded (2163.889%); 308 KiB/s, 116 objects/s recovering 2023-09-23T22:51:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3446971922' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:51:01.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:51:01.748 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:51:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:01 smithi151 ceph-mon[159210]: pgmap v735: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 1558/72 objects degraded (2163.889%); 308 KiB/s, 116 objects/s recovering 2023-09-23T22:51:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3446971922' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:51:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:01 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:51:02.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:51:02.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[187743]: failed to encode map e262 with expected crc 2023-09-23T22:51:02.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:51:02.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[187743]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T22:51:02.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[187743]: failed to encode map e262 with expected crc 2023-09-23T22:51:02.697 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[180979]: failed to encode map e262 with expected crc 2023-09-23T22:51:02.697 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:51:02.698 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[180979]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T22:51:02.698 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:02 smithi052 ceph-mon[180979]: failed to encode map e262 with expected crc 2023-09-23T22:51:02.699 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:51:02.699 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:51:02.699 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T22:51:02.699 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T22:51:02.837 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:02.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:02.940 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:02.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:02 smithi151 ceph-mon[159210]: failed to encode map e262 with expected crc 2023-09-23T22:51:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:02 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:51:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:02 smithi151 ceph-mon[159210]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T22:51:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:02 smithi151 ceph-mon[159210]: failed to encode map e262 with expected crc 2023-09-23T22:51:02.971 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:02.972 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:03.111 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:03.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:03.214 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:03.246 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:03.291 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:03.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:03.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:03.400 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:03.437 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:03.531 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:03.623 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:03.649 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:03.656 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:51:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:51:03] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:51:03.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:03 smithi052 ceph-mon[187743]: failed to encode map e262 with expected crc 2023-09-23T22:51:03.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:03 smithi052 ceph-mon[187743]: pgmap v737: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:03.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:03 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized) 2023-09-23T22:51:03.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:03 smithi052 ceph-mon[180979]: failed to encode map e262 with expected crc 2023-09-23T22:51:03.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:03 smithi052 ceph-mon[180979]: pgmap v737: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:03.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:03 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized) 2023-09-23T22:51:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:03.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:03 smithi151 ceph-mon[159210]: failed to encode map e262 with expected crc 2023-09-23T22:51:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:03 smithi151 ceph-mon[159210]: pgmap v737: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:03 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1558/72 objects degraded (2163.889%), 1 pg degraded, 1 pg undersized) 2023-09-23T22:51:03.759 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:03.789 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:03.916 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:03.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:04.020 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:04.052 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:04.091 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:04.101 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:04.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:04.208 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:04.236 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:04.280 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:04.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:04.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:04.412 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:04.454 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:04.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:04.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:04.503 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:51:04.560 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:04.593 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3284602294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:04.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3284602294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:04.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3284602294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: pgmap v738: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3284602294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[187743]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: pgmap v738: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3284602294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:05 smithi052 ceph-mon[180979]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: pgmap v738: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3284602294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T22:51:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: failed to encode map e263 with expected crc 2023-09-23T22:51:05.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:05 smithi151 ceph-mon[159210]: failed to encode map e263 with expected crc 2023-09-23T22:51:06.867 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:06.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:06.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:07.004 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:07.234 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:51:07.253 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:51:07.272 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:51:07.290 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:51:07.309 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:51:07.328 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:51:07.351 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:51:07.370 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.372 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:51:07.505 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:07.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:07.590 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:51:07.590 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:51:07.610 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:07.611 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:07 smithi052 ceph-mon[187743]: pgmap v740: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:07.611 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4134524416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:07.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:07 smithi052 ceph-mon[180979]: pgmap v740: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:07.611 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4134524416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:07.643 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:07.692 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:07.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:07.798 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:07.829 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:07 smithi151 ceph-mon[159210]: pgmap v740: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4134524416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:08.964 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:51:09.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:51:09.307 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:09.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:09.416 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:09.449 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:09.496 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:09.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:09.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:09.515 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:09.521 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_3' created 2023-09-23T22:51:09.601 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:09.628 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:09 smithi052 ceph-mon[187743]: pgmap v741: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:09.628 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1279928604' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:51:09.628 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:51:09.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:09 smithi052 ceph-mon[180979]: pgmap v741: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:09.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1279928604' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:51:09.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:51:09.631 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:09 smithi151 ceph-mon[159210]: pgmap v741: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1279928604' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:51:09.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:51:09.750 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:51:09.751 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:51:10.242 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:10.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:10.346 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:10.374 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:10.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T22:51:10.830 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T22:51:10.830 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.830 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.830 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[187743]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.831 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:10.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:10 smithi052 ceph-mon[180979]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T22:51:10.833 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":742,"stamp":"2023-09-23T22:51:09.498915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":220,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2491284,"kb_used_data":435804,"kb_used_omap":129,"kb_used_meta":2055294,"kb_avail":653724780,"statfs":{"total":671965249536,"available":669414174720,"internally_reserved":0,"allocated":446263296,"data_stored":440919398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2104621789},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":52,"apply_latency_ms":52,"commit_latency_ns":52000000,"apply_latency_ns":52000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.015329"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":260,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972627+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:51:01.972627+0000","last_peered":"2023-09-23T22:51:01.972627+0000","last_clean":"2023-09-23T22:51:01.972627+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:51:01.972627+0000","last_undegraded":"2023-09-23T22:51:01.972627+0000","last_fullsized":"2023-09-23T22:51:01.972627+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.509433+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:51:01.509433+0000","last_peered":"2023-09-23T22:51:01.509433+0000","last_clean":"2023-09-23T22:51:01.509433+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:51:01.509433+0000","last_undegraded":"2023-09-23T22:51:01.509433+0000","last_fullsized":"2023-09-23T22:51:01.509433+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":88,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.508012+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:51:01.508012+0000","last_peered":"2023-09-23T22:51:01.508012+0000","last_clean":"2023-09-23T22:51:01.508012+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:51:01.508012+0000","last_undegraded":"2023-09-23T22:51:01.508012+0000","last_fullsized":"2023-09-23T22:51:01.508012+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":231,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.745338+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:51:01.745338+0000","last_peered":"2023-09-23T22:51:01.745338+0000","last_clean":"2023-09-23T22:51:01.745338+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:51:01.745338+0000","last_undegraded":"2023-09-23T22:51:01.745338+0000","last_fullsized":"2023-09-23T22:51:01.745338+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":295,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.512280+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:51:01.512280+0000","last_peered":"2023-09-23T22:51:01.512280+0000","last_clean":"2023-09-23T22:51:01.512280+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:51:01.512280+0000","last_undegraded":"2023-09-23T22:51:01.512280+0000","last_fullsized":"2023-09-23T22:51:01.512280+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":228,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489416+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:51:01.489416+0000","last_peered":"2023-09-23T22:51:01.489416+0000","last_clean":"2023-09-23T22:51:01.489416+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:51:01.489416+0000","last_undegraded":"2023-09-23T22:51:01.489416+0000","last_fullsized":"2023-09-23T22:51:01.489416+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":163,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:02.923107+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:51:02.923107+0000","last_peered":"2023-09-23T22:51:02.923107+0000","last_clean":"2023-09-23T22:51:02.923107+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:51:02.923107+0000","last_undegraded":"2023-09-23T22:51:02.923107+0000","last_fullsized":"2023-09-23T22:51:02.923107+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":263,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.512350+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:51:01.512350+0000","last_peered":"2023-09-23T22:51:01.512350+0000","last_clean":"2023-09-23T22:51:01.512350+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:51:01.512350+0000","last_undegraded":"2023-09-23T22:51:01.512350+0000","last_fullsized":"2023-09-23T22:51:01.512350+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":232,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.510450+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:51:01.510450+0000","last_peered":"2023-09-23T22:51:01.510450+0000","last_clean":"2023-09-23T22:51:01.510450+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:51:01.510450+0000","last_undegraded":"2023-09-23T22:51:01.510450+0000","last_fullsized":"2023-09-23T22:51:01.510450+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":221,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.507937+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:51:01.507937+0000","last_peered":"2023-09-23T22:51:01.507937+0000","last_clean":"2023-09-23T22:51:01.507937+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:51:01.507937+0000","last_undegraded":"2023-09-23T22:51:01.507937+0000","last_fullsized":"2023-09-23T22:51:01.507937+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":138,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.509360+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:51:01.509360+0000","last_peered":"2023-09-23T22:51:01.509360+0000","last_clean":"2023-09-23T22:51:01.509360+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:51:01.509360+0000","last_undegraded":"2023-09-23T22:51:01.509360+0000","last_fullsized":"2023-09-23T22:51:01.509360+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489091+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:51:01.489091+0000","last_peered":"2023-09-23T22:51:01.489091+0000","last_clean":"2023-09-23T22:51:01.489091+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:51:01.489091+0000","last_undegraded":"2023-09-23T22:51:01.489091+0000","last_fullsized":"2023-09-23T22:51:01.489091+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":283,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972266+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:51:01.972266+0000","last_peered":"2023-09-23T22:51:01.972266+0000","last_clean":"2023-09-23T22:51:01.972266+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:51:01.972266+0000","last_undegraded":"2023-09-23T22:51:01.972266+0000","last_fullsized":"2023-09-23T22:51:01.972266+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":228,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972735+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:51:01.972735+0000","last_peered":"2023-09-23T22:51:01.972735+0000","last_clean":"2023-09-23T22:51:01.972735+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:51:01.972735+0000","last_undegraded":"2023-09-23T22:51:01.972735+0000","last_fullsized":"2023-09-23T22:51:01.972735+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":121,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972338+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:51:01.972338+0000","last_peered":"2023-09-23T22:51:01.972338+0000","last_clean":"2023-09-23T22:51:01.972338+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:51:01.972338+0000","last_undegraded":"2023-09-23T22:51:01.972338+0000","last_fullsized":"2023-09-23T22:51:01.972338+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":283,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.509733+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:51:01.509733+0000","last_peered":"2023-09-23T22:51:01.509733+0000","last_clean":"2023-09-23T22:51:01.509733+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:51:01.509733+0000","last_undegraded":"2023-09-23T22:51:01.509733+0000","last_fullsized":"2023-09-23T22:51:01.509733+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":219,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738281+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:51:01.738281+0000","last_peered":"2023-09-23T22:51:01.738281+0000","last_clean":"2023-09-23T22:51:01.738281+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:51:01.738281+0000","last_undegraded":"2023-09-23T22:51:01.738281+0000","last_fullsized":"2023-09-23T22:51:01.738281+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":244,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.509499+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:51:01.509499+0000","last_peered":"2023-09-23T22:51:01.509499+0000","last_clean":"2023-09-23T22:51:01.509499+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:51:01.509499+0000","last_undegraded":"2023-09-23T22:51:01.509499+0000","last_fullsized":"2023-09-23T22:51:01.509499+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.973107+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:51:01.973107+0000","last_peered":"2023-09-23T22:51:01.973107+0000","last_clean":"2023-09-23T22:51:01.973107+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:51:01.973107+0000","last_undegraded":"2023-09-23T22:51:01.973107+0000","last_fullsized":"2023-09-23T22:51:01.973107+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":220,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:02.923062+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:51:02.923062+0000","last_peered":"2023-09-23T22:51:02.923062+0000","last_clean":"2023-09-23T22:51:02.923062+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:51:02.923062+0000","last_undegraded":"2023-09-23T22:51:02.923062+0000","last_fullsized":"2023-09-23T22:51:02.923062+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":307,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.488813+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:51:01.488813+0000","last_peered":"2023-09-23T22:51:01.488813+0000","last_clean":"2023-09-23T22:51:01.488813+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:51:01.488813+0000","last_undegraded":"2023-09-23T22:51:01.488813+0000","last_fullsized":"2023-09-23T22:51:01.488813+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":263,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.515769+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:51:01.515769+0000","last_peered":"2023-09-23T22:51:01.515769+0000","last_clean":"2023-09-23T22:51:01.515769+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:51:01.515769+0000","last_undegraded":"2023-09-23T22:51:01.515769+0000","last_fullsized":"2023-09-23T22:51:01.515769+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":132,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.973592+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:51:01.973592+0000","last_peered":"2023-09-23T22:51:01.973592+0000","last_clean":"2023-09-23T22:51:01.973592+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:51:01.973592+0000","last_undegraded":"2023-09-23T22:51:01.973592+0000","last_fullsized":"2023-09-23T22:51:01.973592+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":227,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.488796+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:51:01.488796+0000","last_peered":"2023-09-23T22:51:01.488796+0000","last_clean":"2023-09-23T22:51:01.488796+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:51:01.488796+0000","last_undegraded":"2023-09-23T22:51:01.488796+0000","last_fullsized":"2023-09-23T22:51:01.488796+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":142,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.973999+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:51:01.973999+0000","last_peered":"2023-09-23T22:51:01.973999+0000","last_clean":"2023-09-23T22:51:01.973999+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:51:01.973999+0000","last_undegraded":"2023-09-23T22:51:01.973999+0000","last_fullsized":"2023-09-23T22:51:01.973999+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":243,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972502+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:51:01.972502+0000","last_peered":"2023-09-23T22:51:01.972502+0000","last_clean":"2023-09-23T22:51:01.972502+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:51:01.972502+0000","last_undegraded":"2023-09-23T22:51:01.972502+0000","last_fullsized":"2023-09-23T22:51:01.972502+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":73,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489842+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:51:01.489842+0000","last_peered":"2023-09-23T22:51:01.489842+0000","last_clean":"2023-09-23T22:51:01.489842+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:51:01.489842+0000","last_undegraded":"2023-09-23T22:51:01.489842+0000","last_fullsized":"2023-09-23T22:51:01.489842+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":255,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489014+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:51:01.489014+0000","last_peered":"2023-09-23T22:51:01.489014+0000","last_clean":"2023-09-23T22:51:01.489014+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:51:01.489014+0000","last_undegraded":"2023-09-23T22:51:01.489014+0000","last_fullsized":"2023-09-23T22:51:01.489014+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":116,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.509922+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:51:01.509922+0000","last_peered":"2023-09-23T22:51:01.509922+0000","last_clean":"2023-09-23T22:51:01.509922+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:51:01.509922+0000","last_undegraded":"2023-09-23T22:51:01.509922+0000","last_fullsized":"2023-09-23T22:51:01.509922+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":156,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738329+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:51:01.738329+0000","last_peered":"2023-09-23T22:51:01.738329+0000","last_clean":"2023-09-23T22:51:01.738329+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:51:01.738329+0000","last_undegraded":"2023-09-23T22:51:01.738329+0000","last_fullsized":"2023-09-23T22:51:01.738329+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":264,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.508581+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:51:01.508581+0000","last_peered":"2023-09-23T22:51:01.508581+0000","last_clean":"2023-09-23T22:51:01.508581+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:51:01.508581+0000","last_undegraded":"2023-09-23T22:51:01.508581+0000","last_fullsized":"2023-09-23T22:51:01.508581+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":228,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972780+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:51:01.972780+0000","last_peered":"2023-09-23T22:51:01.972780+0000","last_clean":"2023-09-23T22:51:01.972780+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:51:01.972780+0000","last_undegraded":"2023-09-23T22:51:01.972780+0000","last_fullsized":"2023-09-23T22:51:01.972780+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":247,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489174+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:51:01.489174+0000","last_peered":"2023-09-23T22:51:01.489174+0000","last_clean":"2023-09-23T22:51:01.489174+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:51:01.489174+0000","last_undegraded":"2023-09-23T22:51:01.489174+0000","last_fullsized":"2023-09-23T22:51:01.489174+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":258,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972742+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:51:01.972742+0000","last_peered":"2023-09-23T22:51:01.972742+0000","last_clean":"2023-09-23T22:51:01.972742+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:51:01.972742+0000","last_undegraded":"2023-09-23T22:51:01.972742+0000","last_fullsized":"2023-09-23T22:51:01.972742+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":271,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972205+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:51:01.972205+0000","last_peered":"2023-09-23T22:51:01.972205+0000","last_clean":"2023-09-23T22:51:01.972205+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:51:01.972205+0000","last_undegraded":"2023-09-23T22:51:01.972205+0000","last_fullsized":"2023-09-23T22:51:01.972205+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972247+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:51:01.972247+0000","last_peered":"2023-09-23T22:51:01.972247+0000","last_clean":"2023-09-23T22:51:01.972247+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:51:01.972247+0000","last_undegraded":"2023-09-23T22:51:01.972247+0000","last_fullsized":"2023-09-23T22:51:01.972247+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":265,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:51:04.498137+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:51:04.498137+0000","last_peered":"2023-09-23T22:51:04.498137+0000","last_clean":"2023-09-23T22:51:04.498137+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:51:04.498137+0000","last_undegraded":"2023-09-23T22:51:04.498137+0000","last_fullsized":"2023-09-23T22:51:04.498137+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.495486+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:51:01.495486+0000","last_peered":"2023-09-23T22:51:01.495486+0000","last_clean":"2023-09-23T22:51:01.495486+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:51:01.495486+0000","last_undegraded":"2023-09-23T22:51:01.495486+0000","last_fullsized":"2023-09-23T22:51:01.495486+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":300,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738783+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:51:01.738783+0000","last_peered":"2023-09-23T22:51:01.738783+0000","last_clean":"2023-09-23T22:51:01.738783+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:51:01.738783+0000","last_undegraded":"2023-09-23T22:51:01.738783+0000","last_fullsized":"2023-09-23T22:51:01.738783+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.509274+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:51:01.509274+0000","last_peered":"2023-09-23T22:51:01.509274+0000","last_clean":"2023-09-23T22:51:01.509274+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:51:01.509274+0000","last_undegraded":"2023-09-23T22:51:01.509274+0000","last_fullsized":"2023-09-23T22:51:01.509274+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":94,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972524+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:51:01.972524+0000","last_peered":"2023-09-23T22:51:01.972524+0000","last_clean":"2023-09-23T22:51:01.972524+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:51:01.972524+0000","last_undegraded":"2023-09-23T22:51:01.972524+0000","last_fullsized":"2023-09-23T22:51:01.972524+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":97,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489092+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:51:01.489092+0000","last_peered":"2023-09-23T22:51:01.489092+0000","last_clean":"2023-09-23T22:51:01.489092+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:51:01.489092+0000","last_undegraded":"2023-09-23T22:51:01.489092+0000","last_fullsized":"2023-09-23T22:51:01.489092+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":259,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972847+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:51:01.972847+0000","last_peered":"2023-09-23T22:51:01.972847+0000","last_clean":"2023-09-23T22:51:01.972847+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:51:01.972847+0000","last_undegraded":"2023-09-23T22:51:01.972847+0000","last_fullsized":"2023-09-23T22:51:01.972847+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":179,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.488414+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:51:01.488414+0000","last_peered":"2023-09-23T22:51:01.488414+0000","last_clean":"2023-09-23T22:51:01.488414+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:51:01.488414+0000","last_undegraded":"2023-09-23T22:51:01.488414+0000","last_fullsized":"2023-09-23T22:51:01.488414+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":254,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.485197+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:51:01.485197+0000","last_peered":"2023-09-23T22:51:01.485197+0000","last_clean":"2023-09-23T22:51:01.485197+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:51:01.485197+0000","last_undegraded":"2023-09-23T22:51:01.485197+0000","last_fullsized":"2023-09-23T22:51:01.485197+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":143,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.485047+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:51:01.485047+0000","last_peered":"2023-09-23T22:51:01.485047+0000","last_clean":"2023-09-23T22:51:01.485047+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:51:01.485047+0000","last_undegraded":"2023-09-23T22:51:01.485047+0000","last_fullsized":"2023-09-23T22:51:01.485047+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":269,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:51:04.771371+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:51:04.771371+0000","last_peered":"2023-09-23T22:51:04.771371+0000","last_clean":"2023-09-23T22:51:04.771371+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:51:04.771371+0000","last_undegraded":"2023-09-23T22:51:04.771371+0000","last_fullsized":"2023-09-23T22:51:04.771371+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":264,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.488201+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:51:01.488201+0000","last_peered":"2023-09-23T22:51:01.488201+0000","last_clean":"2023-09-23T22:51:01.488201+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:51:01.488201+0000","last_undegraded":"2023-09-23T22:51:01.488201+0000","last_fullsized":"2023-09-23T22:51:01.488201+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":350,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738254+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:51:01.738254+0000","last_peered":"2023-09-23T22:51:01.738254+0000","last_clean":"2023-09-23T22:51:01.738254+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:51:01.738254+0000","last_undegraded":"2023-09-23T22:51:01.738254+0000","last_fullsized":"2023-09-23T22:51:01.738254+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":195,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.508215+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:51:01.508215+0000","last_peered":"2023-09-23T22:51:01.508215+0000","last_clean":"2023-09-23T22:51:01.508215+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:51:01.508215+0000","last_undegraded":"2023-09-23T22:51:01.508215+0000","last_fullsized":"2023-09-23T22:51:01.508215+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":255,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489990+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:51:01.489990+0000","last_peered":"2023-09-23T22:51:01.489990+0000","last_clean":"2023-09-23T22:51:01.489990+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:51:01.489990+0000","last_undegraded":"2023-09-23T22:51:01.489990+0000","last_fullsized":"2023-09-23T22:51:01.489990+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":229,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738461+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:51:01.738461+0000","last_peered":"2023-09-23T22:51:01.738461+0000","last_clean":"2023-09-23T22:51:01.738461+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:51:01.738461+0000","last_undegraded":"2023-09-23T22:51:01.738461+0000","last_fullsized":"2023-09-23T22:51:01.738461+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":200,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738183+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:51:01.738183+0000","last_peered":"2023-09-23T22:51:01.738183+0000","last_clean":"2023-09-23T22:51:01.738183+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:51:01.738183+0000","last_undegraded":"2023-09-23T22:51:01.738183+0000","last_fullsized":"2023-09-23T22:51:01.738183+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":187,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.745170+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:51:01.745170+0000","last_peered":"2023-09-23T22:51:01.745170+0000","last_clean":"2023-09-23T22:51:01.745170+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:51:01.745170+0000","last_undegraded":"2023-09-23T22:51:01.745170+0000","last_fullsized":"2023-09-23T22:51:01.745170+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":192,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.739461+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:51:01.739461+0000","last_peered":"2023-09-23T22:51:01.739461+0000","last_clean":"2023-09-23T22:51:01.739461+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:51:01.739461+0000","last_undegraded":"2023-09-23T22:51:01.739461+0000","last_fullsized":"2023-09-23T22:51:01.739461+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.488796+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:51:01.488796+0000","last_peered":"2023-09-23T22:51:01.488796+0000","last_clean":"2023-09-23T22:51:01.488796+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:51:01.488796+0000","last_undegraded":"2023-09-23T22:51:01.488796+0000","last_fullsized":"2023-09-23T22:51:01.488796+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":192,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738603+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:51:01.738603+0000","last_peered":"2023-09-23T22:51:01.738603+0000","last_clean":"2023-09-23T22:51:01.738603+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:51:01.738603+0000","last_undegraded":"2023-09-23T22:51:01.738603+0000","last_fullsized":"2023-09-23T22:51:01.738603+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":304,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:51:04.501595+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:51:04.501595+0000","last_peered":"2023-09-23T22:51:04.501595+0000","last_clean":"2023-09-23T22:51:04.501595+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:51:04.501595+0000","last_undegraded":"2023-09-23T22:51:04.501595+0000","last_fullsized":"2023-09-23T22:51:04.501595+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":61,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489555+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:51:01.489555+0000","last_peered":"2023-09-23T22:51:01.489555+0000","last_clean":"2023-09-23T22:51:01.489555+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:51:01.489555+0000","last_undegraded":"2023-09-23T22:51:01.489555+0000","last_fullsized":"2023-09-23T22:51:01.489555+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":289,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.512204+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:51:01.512204+0000","last_peered":"2023-09-23T22:51:01.512204+0000","last_clean":"2023-09-23T22:51:01.512204+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:51:01.512204+0000","last_undegraded":"2023-09-23T22:51:01.512204+0000","last_fullsized":"2023-09-23T22:51:01.512204+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:51:09.495811+0000","last_change":"2023-09-23T22:51:09.495811+0000","last_active":"2023-09-23T22:51:09.495811+0000","last_peered":"2023-09-23T22:51:09.495811+0000","last_clean":"2023-09-23T22:51:09.495811+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:51:09.495811+0000","last_undegraded":"2023-09-23T22:51:09.495811+0000","last_fullsized":"2023-09-23T22:51:09.495811+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":295,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.489327+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:51:01.489327+0000","last_peered":"2023-09-23T22:51:01.489327+0000","last_clean":"2023-09-23T22:51:01.489327+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:51:01.489327+0000","last_undegraded":"2023-09-23T22:51:01.489327+0000","last_fullsized":"2023-09-23T22:51:01.489327+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":159,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.745265+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:51:01.745265+0000","last_peered":"2023-09-23T22:51:01.745265+0000","last_clean":"2023-09-23T22:51:01.745265+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:51:01.745265+0000","last_undegraded":"2023-09-23T22:51:01.745265+0000","last_fullsized":"2023-09-23T22:51:01.745265+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":213,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972401+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:51:01.972401+0000","last_peered":"2023-09-23T22:51:01.972401+0000","last_clean":"2023-09-23T22:51:01.972401+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:51:01.972401+0000","last_undegraded":"2023-09-23T22:51:01.972401+0000","last_fullsized":"2023-09-23T22:51:01.972401+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":194,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.738626+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:51:01.738626+0000","last_peered":"2023-09-23T22:51:01.738626+0000","last_clean":"2023-09-23T22:51:01.738626+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:51:01.738626+0000","last_undegraded":"2023-09-23T22:51:01.738626+0000","last_fullsized":"2023-09-23T22:51:01.738626+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":36,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.488492+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:51:01.488492+0000","last_peered":"2023-09-23T22:51:01.488492+0000","last_clean":"2023-09-23T22:51:01.488492+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:51:01.488492+0000","last_undegraded":"2023-09-23T22:51:01.488492+0000","last_fullsized":"2023-09-23T22:51:01.488492+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972681+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:51:01.972681+0000","last_peered":"2023-09-23T22:51:01.972681+0000","last_clean":"2023-09-23T22:51:01.972681+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:51:01.972681+0000","last_undegraded":"2023-09-23T22:51:01.972681+0000","last_fullsized":"2023-09-23T22:51:01.972681+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":262,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972202+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:51:01.972202+0000","last_peered":"2023-09-23T22:51:01.972202+0000","last_clean":"2023-09-23T22:51:01.972202+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:51:01.972202+0000","last_undegraded":"2023-09-23T22:51:01.972202+0000","last_fullsized":"2023-09-23T22:51:01.972202+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":73,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972880+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:51:01.972880+0000","last_peered":"2023-09-23T22:51:01.972880+0000","last_clean":"2023-09-23T22:51:01.972880+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:51:01.972880+0000","last_undegraded":"2023-09-23T22:51:01.972880+0000","last_fullsized":"2023-09-23T22:51:01.972880+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972893+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:51:01.972893+0000","last_peered":"2023-09-23T22:51:01.972893+0000","last_clean":"2023-09-23T22:51:01.972893+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:51:01.972893+0000","last_undegraded":"2023-09-23T22:51:01.972893+0000","last_fullsized":"2023-09-23T22:51:01.972893+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":257,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.510530+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:51:01.510530+0000","last_peered":"2023-09-23T22:51:01.510530+0000","last_clean":"2023-09-23T22:51:01.510530+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:51:01.510530+0000","last_undegraded":"2023-09-23T22:51:01.510530+0000","last_fullsized":"2023-09-23T22:51:01.510530+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":295,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.507868+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:51:01.507868+0000","last_peered":"2023-09-23T22:51:01.507868+0000","last_clean":"2023-09-23T22:51:01.507868+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:51:01.507868+0000","last_undegraded":"2023-09-23T22:51:01.507868+0000","last_fullsized":"2023-09-23T22:51:01.507868+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":208,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.739050+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:51:01.739050+0000","last_peered":"2023-09-23T22:51:01.739050+0000","last_clean":"2023-09-23T22:51:01.739050+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:51:01.739050+0000","last_undegraded":"2023-09-23T22:51:01.739050+0000","last_fullsized":"2023-09-23T22:51:01.739050+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":228,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:51:01.972390+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:51:01.972390+0000","last_peered":"2023-09-23T22:51:01.972390+0000","last_clean":"2023-09-23T22:51:01.972390+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:51:01.972390+0000","last_undegraded":"2023-09-23T22:51:01.972390+0000","last_fullsized":"2023-09-23T22:51:01.972390+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430497,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626725,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304204,"kb_used_data":2884,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93440948,"statfs":{"total":95995035648,"available":95683530752,"internally_reserved":0,"allocated":2953216,"data_stored":2448060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":3.5,"15min":2.0259999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3890000000000002,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7360000000000002,"15min":1.8959999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.9800000000000004,"5min":117.48699999999999,"15min":117.48699999999999},"last":0.53400000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.1309999999999998,"15min":2.532},"min":{"1min":0.28599999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.4039999999999999,"5min":31.286000000000001,"15min":153.78399999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.7150000000000001,"5min":4.1219999999999999,"15min":3.1600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6799999999999997,"5min":185.411,"15min":185.411},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:50:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.7229999999999999,"15min":3.1499999999999999},"min":{"1min":0.44900000000000001,"5min":0.309,"15min":0.28299999999999997},"max":{"1min":6.4100000000000001,"5min":33.174999999999997,"15min":100.785},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.825,"5min":4.2220000000000004,"15min":3.5459999999999998},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4829999999999997,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":5.5170000000000003,"15min":3.5099999999999998},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.6020000000000003,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.605,"5min":4.3419999999999996,"15min":3.1989999999999998},"min":{"1min":0.36799999999999999,"5min":0.35599999999999998,"15min":0.28899999999999998},"max":{"1min":9.4410000000000007,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.76800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.79200000000000004,"15min":0.68799999999999994},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8239999999999998,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.86599999999999999,"15min":0.70699999999999996},"min":{"1min":0.318,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":4.2530000000000001,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.36599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.0029999999999999,"15min":0.79600000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":7.1130000000000004,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.93700000000000006,"15min":0.76900000000000002},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.9089999999999998,"5min":9.7899999999999991,"15min":18.625},"last":0.435}]},{"osd":7,"last update":"Sat Sep 23 22:50:54 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.93200000000000005,"15min":0.77900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34399999999999997,"15min":0.316},"max":{"1min":4.3049999999999997,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.80400000000000005,"15min":0.75600000000000001},"min":{"1min":0.33800000000000002,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":5.3150000000000004,"5min":5.3150000000000004,"15min":16.661999999999999},"last":0.377}]}]},{"osd":1,"up_from":84,"seq":360777253035,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315572,"kb_used_data":3372,"kb_used_omap":9,"kb_used_meta":312182,"kb_avail":93429580,"statfs":{"total":95995035648,"available":95671889920,"internally_reserved":0,"allocated":3452928,"data_stored":2505515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":319675369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.367,"5min":8.0190000000000001,"15min":5.4509999999999996},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":26.042000000000002,"5min":101.605,"15min":101.605},"last":0.32400000000000001},{"interface":"front","average":{"1min":4.7430000000000003,"5min":7.6180000000000003,"15min":5.5490000000000004},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":25.379999999999999,"5min":101.892,"15min":101.892},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":3.7850000000000001,"15min":2.6589999999999998},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":26.129000000000001,"5min":37.869999999999997,"15min":37.889000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":6.0439999999999996,"5min":4.0890000000000004,"15min":2.7549999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":26.001999999999999,"5min":37.957000000000001,"15min":37.957000000000001},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":3.46,"15min":2.1859999999999999},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":3.1179999999999999,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.375},{"interface":"front","average":{"1min":2.016,"5min":4.2999999999999998,"15min":2.6389999999999998},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.20999999999999999},"max":{"1min":6.7649999999999997,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":2.681,"15min":1.9990000000000001},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":5.6150000000000002,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":3.4140000000000001,"5min":2.4729999999999999,"15min":1.9490000000000001},"min":{"1min":0.33500000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":14.058,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.069,"15min":1.546},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":4.3230000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.343,"5min":2.7919999999999998,"15min":1.865},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":5.5650000000000004,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":2.5550000000000002,"15min":1.7829999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":13.991,"5min":79.311000000000007,"15min":79.311000000000007},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.91400000000000003,"5min":2.0950000000000002,"15min":1.482},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":2.984,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1480000000000001,"15min":2.234},"min":{"1min":0.435,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":5.6539999999999999,"5min":11.481999999999999,"15min":159.018},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.548,"5min":2.0659999999999998,"15min":2.2890000000000001},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.37},"max":{"1min":14.032999999999999,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.55400000000000005}]}]},{"osd":0,"up_from":219,"seq":940597837884,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":173608,"kb_used_data":3424,"kb_used_omap":31,"kb_used_meta":170144,"kb_avail":93571544,"statfs":{"total":95995035648,"available":95817261056,"internally_reserved":0,"allocated":3506176,"data_stored":2511131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":174228402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":3.887,"15min":3.77},"min":{"1min":0.221,"5min":0.216,"15min":0.216},"max":{"1min":14.141999999999999,"5min":32.344999999999999,"15min":32.344999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":3.6520000000000001,"5min":3.9460000000000002,"15min":3.6880000000000002},"min":{"1min":0.26100000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.291,"5min":32.106000000000002,"15min":32.106000000000002},"last":0.503}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":3.7509999999999999,"15min":2.649},"min":{"1min":0.26700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":16.643999999999998,"5min":24.253,"15min":24.253},"last":0.53200000000000003},{"interface":"front","average":{"1min":4.4619999999999997,"5min":2.9489999999999998,"15min":2.1539999999999999},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":21.151,"5min":21.151,"15min":21.151},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.4400000000000004,"5min":3.4319999999999999,"15min":2.4540000000000002},"min":{"1min":0.30199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.609999999999999,"5min":20.710000000000001,"15min":20.710000000000001},"last":0.439},{"interface":"front","average":{"1min":4.3090000000000002,"5min":2.431,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.254},"max":{"1min":22.172000000000001,"5min":22.172000000000001,"15min":22.172000000000001},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.5529999999999999,"15min":2.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.2030000000000003,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":3.194,"5min":2.407,"15min":1.968},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":16.52,"5min":20.009,"15min":20.009},"last":0.93300000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":2.8650000000000002,"15min":1.845},"min":{"1min":0.38,"5min":0.318,"15min":0.318},"max":{"1min":61.756999999999998,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.995},{"interface":"front","average":{"1min":5.8150000000000004,"5min":3.2080000000000002,"15min":2.1040000000000001},"min":{"1min":0.379,"5min":0.374,"15min":0.374},"max":{"1min":61.777000000000001,"5min":61.777000000000001,"15min":61.777000000000001},"last":1.0089999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":6.3579999999999997,"5min":2.6970000000000001,"15min":1.8129999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":59.146999999999998,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.88300000000000001},{"interface":"front","average":{"1min":4.5300000000000002,"5min":3.1800000000000002,"15min":2.1840000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":59.164000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.95899999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":2.4409999999999998,"15min":1.877},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.2350000000000003,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":3.4740000000000002,"5min":2,"15min":1.452},"min":{"1min":0.49399999999999999,"5min":0.432,"15min":0.432},"max":{"1min":21.954999999999998,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.77600000000000002}]}]},{"osd":2,"up_from":131,"seq":562640715896,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270840,"kb_used_data":2800,"kb_used_omap":81,"kb_used_meta":267950,"kb_avail":93474312,"statfs":{"total":95995035648,"available":95717695488,"internally_reserved":0,"allocated":2867200,"data_stored":1911934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":274381018},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":10.039,"5min":5.8920000000000003,"15min":5.476},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":132.79499999999999,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.251},{"interface":"front","average":{"1min":10.228,"5min":6.9359999999999999,"15min":5.9050000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":132.58099999999999,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.314}]},{"osd":1,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":4.0119999999999996,"5min":2.5299999999999998,"15min":3.2469999999999999},"min":{"1min":0.253,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.622999999999999,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.26900000000000002},{"interface":"front","average":{"1min":3.581,"5min":3.298,"15min":3.0859999999999999},"min":{"1min":0.27200000000000002,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.143000000000001,"5min":28.596,"15min":88.436999999999998},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9020000000000001,"5min":4.0270000000000001,"15min":3.6579999999999999},"min":{"1min":0.34200000000000003,"5min":0.223,"15min":0.223},"max":{"1min":35.198999999999998,"5min":35.198999999999998,"15min":88.653000000000006},"last":0.33700000000000002},{"interface":"front","average":{"1min":5.9589999999999996,"5min":3.3559999999999999,"15min":2.931},"min":{"1min":0.32900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":35.136000000000003,"5min":35.136000000000003,"15min":88.474000000000004},"last":0.32400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.8230000000000004,"15min":2.3530000000000002},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":10.433,"5min":108.878,"15min":108.878},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.3839999999999999,"5min":4.9279999999999999,"15min":3.0680000000000001},"min":{"1min":0.39300000000000002,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":11.273,"5min":108.93000000000001,"15min":108.93000000000001},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":4.2460000000000004,"15min":2.9209999999999998},"min":{"1min":0.36299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.239000000000001,"5min":97.322999999999993,"15min":97.322999999999993},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.2549999999999999,"5min":4.3730000000000002,"15min":2.589},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":10.327,"5min":97.301000000000002,"15min":97.301000000000002},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":6.827,"5min":4.7030000000000003,"15min":2.8069999999999999},"min":{"1min":0.36899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":98.597999999999999,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.434},{"interface":"front","average":{"1min":6.8230000000000004,"5min":4.4189999999999996,"15min":2.258},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":98.509,"5min":98.509,"15min":98.509},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9990000000000001,"15min":1.506},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.477,"5min":10.92,"15min":10.92},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.339,"15min":2.3530000000000002},"min":{"1min":0.49399999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.311,"5min":11.311,"15min":88.584000000000003},"last":0.44}]}]},{"osd":3,"up_from":94,"seq":403726925982,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":399796,"kb_used_data":2796,"kb_used_omap":7,"kb_used_meta":396984,"kb_avail":93345356,"statfs":{"total":95995035648,"available":95585644544,"internally_reserved":0,"allocated":2863104,"data_stored":1913712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":406512232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":2.931,"5min":3.3849999999999998,"15min":2.3679999999999999},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.662000000000001,"5min":80.099000000000004,"15min":80.099000000000004},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.2810000000000001,"5min":3.3380000000000001,"15min":2.1779999999999999},"min":{"1min":0.374,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.128,"5min":80.150000000000006,"15min":80.150000000000006},"last":0.26300000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6949999999999998,"5min":2.6949999999999998,"15min":1.831},"min":{"1min":0.221,"5min":0.221,"15min":0.20000000000000001},"max":{"1min":11.086,"5min":17.231000000000002,"15min":22.818000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.665,"5min":1.9570000000000001,"15min":1.4390000000000001},"min":{"1min":0.377,"5min":0.25600000000000001,"15min":0.222},"max":{"1min":11.757999999999999,"5min":15.098000000000001,"15min":23.917999999999999},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":2.8610000000000002,"15min":2.6949999999999998},"min":{"1min":0.42899999999999999,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":17.170999999999999,"5min":20.431000000000001,"15min":125.446},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.359,"5min":3.2599999999999998,"15min":3.2010000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":11.641,"5min":15.756,"15min":182.483},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5720000000000001,"5min":2.077,"15min":1.522},"min":{"1min":0.33500000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":8.1509999999999998,"5min":12.896000000000001,"15min":18.757000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.609,"5min":1.706,"15min":1.325},"min":{"1min":0.47799999999999998,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":11.834,"5min":12.131,"15min":12.131},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.5449999999999999,"15min":1.2549999999999999},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":11.803000000000001,"5min":11.803000000000001,"15min":12.127000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":3.105,"5min":2.2930000000000001,"15min":1.4970000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.951000000000001,"5min":11.951000000000001,"15min":11.951000000000001},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.7390000000000001,"15min":1.3100000000000001},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.28999999999999998},"max":{"1min":11.867000000000001,"5min":11.867000000000001,"15min":12.16},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.2250000000000001,"15min":1.534},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.913,"5min":11.913,"15min":11.913},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":3.1680000000000001,"15min":1.8320000000000001},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":11.694000000000001,"5min":91.757999999999996,"15min":91.757999999999996},"last":0.63},{"interface":"front","average":{"1min":2.8559999999999999,"5min":3.1019999999999999,"15min":1.9550000000000001},"min":{"1min":0.48699999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":8.2010000000000005,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.53100000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019181,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":719564,"kb_used_data":418244,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93025588,"statfs":{"total":95995035648,"available":95258202112,"internally_reserved":0,"allocated":428281856,"data_stored":427776198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":4.1849999999999996,"15min":6.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.9770000000000003,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.5230000000000001,"5min":5.077,"15min":7.3780000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.464,"5min":110.33799999999999,"15min":110.33799999999999},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":2.4319999999999999,"15min":1.6850000000000001},"min":{"1min":0.29099999999999998,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":12.192,"5min":41.643000000000001,"15min":45.320999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.514,"5min":1.9990000000000001,"15min":1.492},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.4130000000000003,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.70499999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.2749999999999999,"15min":3.2719999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":17.093,"5min":41.607999999999997,"15min":84.909000000000006},"last":0.58599999999999997},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.4249999999999998,"15min":3.206},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":21.704000000000001,"5min":41.670000000000002,"15min":83.840000000000003},"last":0.621}]},{"osd":3,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":6.4089999999999998,"15min":3.5209999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":10.329000000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.794,"5min":6.327,"15min":3.5640000000000001},"min":{"1min":0.38300000000000001,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":12.345000000000001,"5min":129.197,"15min":129.197},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":0.81000000000000005,"15min":0.82399999999999995},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":12.238,"5min":12.238,"15min":43.189999999999998},"last":0.504},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.83499999999999996,"15min":0.85199999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":7.5460000000000003,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.68000000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85699999999999998,"15min":0.80900000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":7.4560000000000004,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.47,"5min":0.85199999999999998,"15min":0.82699999999999996},"min":{"1min":0.42999999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":12.269,"5min":12.269,"15min":26.940999999999999},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:50:55 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":0.88900000000000001,"15min":0.753},"min":{"1min":0.39200000000000002,"5min":0.33600000000000002,"15min":0.29899999999999999},"max":{"1min":12.311,"5min":12.311,"15min":12.311},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.1659999999999999,"5min":0.88200000000000001,"15min":0.72199999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29399999999999998},"max":{"1min":6.7919999999999998,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.75900000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822953,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307700,"kb_used_data":2284,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437452,"statfs":{"total":95995035648,"available":95679950848,"internally_reserved":0,"allocated":2338816,"data_stored":1852848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":10.109999999999999,"5min":7.133,"15min":3.5830000000000002},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":84.367999999999995,"5min":121.319,"15min":121.319},"last":0.40699999999999997},{"interface":"front","average":{"1min":10.358000000000001,"5min":7.2380000000000004,"15min":3.5099999999999998},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":84.311999999999998,"5min":121.075,"15min":121.075},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.5269999999999999,"15min":1.8089999999999999},"min":{"1min":0.29299999999999998,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":6.9189999999999996,"5min":28.658999999999999,"15min":149.203},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.581,"5min":1.9259999999999999,"15min":2.0030000000000001},"min":{"1min":0.34499999999999997,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.0119999999999996,"5min":28.664999999999999,"15min":149.554},"last":0.71999999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8179999999999996,"5min":4.048,"15min":6.4340000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":63.156999999999996,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":4.4349999999999996,"5min":3.8959999999999999,"15min":6.4039999999999999},"min":{"1min":0.318,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":63.277999999999999,"5min":158.91399999999999,"15min":158.91399999999999},"last":0.60099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":10.577999999999999,"5min":9.7460000000000004,"15min":4.899},"min":{"1min":0.31,"5min":0.30499999999999999,"15min":0.25700000000000001},"max":{"1min":98.641000000000005,"5min":257.05000000000001,"15min":257.05000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":11.012,"5min":9.7850000000000001,"15min":4.7750000000000004},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":98.906000000000006,"5min":257.35000000000002,"15min":257.35000000000002},"last":0.64100000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66300000000000003,"15min":0.62},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":5.0309999999999997,"5min":5.0309999999999997,"15min":5.0309999999999997},"last":0.625},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.75700000000000001,"15min":0.624},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":3.4409999999999998,"5min":10.535,"15min":10.535},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.89900000000000002,"15min":1.4099999999999999},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":4.367,"5min":12.615,"15min":140.07499999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.0449999999999999,"15min":1.466},"min":{"1min":0.318,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":6.6219999999999999,"5min":12.74,"15min":139.739},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:51:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.67000000000000004,"15min":0.72199999999999998},"min":{"1min":0.36599999999999999,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.3359999999999999,"5min":2.738,"15min":7.9809999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.77400000000000002,"15min":0.70299999999999996},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.0459999999999994,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.754}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:51:10.904 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:10.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:10.923 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: failed to encode map e264 with expected crc 2023-09-23T22:51:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:10.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:10 smithi151 ceph-mon[159210]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T22:51:11.008 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:11.040 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:11.070 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:51:11.087 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:11.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:11.196 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:11.241 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:11.404 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:11.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:11.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:11.508 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:11.538 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:11.594 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:11.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:11.650 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:11.697 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:11.729 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: from='client.45560 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:51:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: pgmap v743: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[180979]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: from='client.45560 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: pgmap v743: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:11 smithi052 ceph-mon[187743]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T22:51:11.777 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:11.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: from='client.45560 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:51:11.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: pgmap v743: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:11.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: failed to encode map e265 with expected crc 2023-09-23T22:51:11.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:11 smithi151 ceph-mon[159210]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T22:51:11.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:11.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:11.884 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:11.911 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:11.954 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:11.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:12.061 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:12.087 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:12.128 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:12.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:12.235 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:12.266 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:12.476 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:12.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:12.585 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:12.612 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:12.653 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:12.653 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:12.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:12.760 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:12.786 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:12.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: failed to encode map e266 with expected crc 2023-09-23T22:51:12.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: failed to encode map e266 with expected crc 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: failed to encode map e266 with expected crc 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: failed to encode map e266 with expected crc 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:12.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:12.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.828 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:12.832 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: failed to encode map e266 with expected crc 2023-09-23T22:51:12.833 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.833 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.833 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: failed to encode map e266 with expected crc 2023-09-23T22:51:12.833 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:51:12.833 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:51:12.833 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:12.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:12.935 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:12.965 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:13.013 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:13.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:13.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:13.117 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:13.145 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:13.460 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:51:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:51:13] "GET /metrics HTTP/1.1" 200 34176 "" "Prometheus/2.33.4" 2023-09-23T22:51:13.562 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:13 smithi052 ceph-mon[180979]: failed to encode map e266 with expected crc 2023-09-23T22:51:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:13 smithi052 ceph-mon[180979]: pgmap v746: 89 pgs: 1 creating+peering, 3 unknown, 85 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:13 smithi052 ceph-mon[187743]: failed to encode map e266 with expected crc 2023-09-23T22:51:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:13 smithi052 ceph-mon[187743]: pgmap v746: 89 pgs: 1 creating+peering, 3 unknown, 85 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:13.794 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:13.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:13.840 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:51:13.899 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:13.928 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:13 smithi151 ceph-mon[159210]: failed to encode map e266 with expected crc 2023-09-23T22:51:13.928 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:13 smithi151 ceph-mon[159210]: pgmap v746: 89 pgs: 1 creating+peering, 3 unknown, 85 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:13.931 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:13.981 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:13.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:13.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:14.086 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:14.114 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:14.158 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:14.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:14.263 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:14.290 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:14.331 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:14.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:14.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:14.464 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:14.505 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:14.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:14.564 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:51:14.609 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:14.618 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:14.635 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:14.677 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:14.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:14.727 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:14 smithi052 ceph-mon[187743]: failed to encode map e266 with expected crc 2023-09-23T22:51:14.800 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/373069371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:51:14.800 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:14 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:51:14.800 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:14 smithi052 ceph-mon[180979]: failed to encode map e266 with expected crc 2023-09-23T22:51:14.800 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/373069371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:51:14.800 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:14 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:51:14.801 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:14.834 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:14 smithi151 ceph-mon[159210]: failed to encode map e266 with expected crc 2023-09-23T22:51:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/373069371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:51:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:14 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:51:15.574 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:51:15.574 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:51:15.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[187743]: pgmap v747: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:15.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:15.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[187743]: failed to encode map e267 with expected crc 2023-09-23T22:51:15.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[187743]: failed to encode map e267 with expected crc 2023-09-23T22:51:15.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:51:15.654 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[187743]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T22:51:15.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[180979]: pgmap v747: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:15.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:15.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[180979]: failed to encode map e267 with expected crc 2023-09-23T22:51:15.655 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[180979]: failed to encode map e267 with expected crc 2023-09-23T22:51:15.655 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:51:15.655 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:15 smithi052 ceph-mon[180979]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T22:51:15.695 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:15.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:15.800 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:15.829 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:15 smithi151 ceph-mon[159210]: pgmap v747: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:15 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:15 smithi151 ceph-mon[159210]: failed to encode map e267 with expected crc 2023-09-23T22:51:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:15 smithi151 ceph-mon[159210]: failed to encode map e267 with expected crc 2023-09-23T22:51:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:51:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:15 smithi151 ceph-mon[159210]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T22:51:16.355 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:16.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:16.459 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:16.494 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:16.548 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:16.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:16.555 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:51:16.568 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T22:51:16.652 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:16.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3344448692' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:51:16.665 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:51:16.665 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3344448692' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:51:16.665 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:51:16.690 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3344448692' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:51:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:51:16.976 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:51:16.994 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:51:17.012 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:51:17.031 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:51:17.049 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:51:17.066 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:51:17.085 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:51:17.103 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:51:17.104 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.104 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.105 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:51:17.246 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:17.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:17.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:17.351 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:17.383 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:17.431 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:17.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:17.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:17.538 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:17.554 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:51:17.567 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:17 smithi151 ceph-mon[159210]: pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:17 smithi151 ceph-mon[159210]: failed to encode map e268 with expected crc 2023-09-23T22:51:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:51:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:17 smithi151 ceph-mon[159210]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T22:51:17.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1026655246' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[187743]: pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[187743]: failed to encode map e268 with expected crc 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[187743]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1026655246' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[180979]: pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[180979]: failed to encode map e268 with expected crc 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[180979]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T22:51:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1026655246' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:18.320 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:18.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:18.423 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:18.455 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:18.504 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:18.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:18.534 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:18.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:18 smithi052 ceph-mon[187743]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:18 smithi052 ceph-mon[187743]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.568 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:18 smithi052 ceph-mon[187743]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.570 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:18 smithi052 ceph-mon[180979]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.570 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:18 smithi052 ceph-mon[180979]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.571 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:18 smithi052 ceph-mon[180979]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:18.646 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:18.776 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:18.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:18.795 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:18.880 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:18.914 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:18 smithi151 ceph-mon[159210]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:18 smithi151 ceph-mon[159210]: failed to encode map e268 with expected crc 2023-09-23T22:51:18.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:18 smithi151 ceph-mon[159210]: failed to encode map e268 with expected crc 2023-09-23T22:51:19.092 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:51:19.094 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:51:19.320 DEBUG:teuthology.orchestra.run.smithi052:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T22:51:19.323 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:51:19.323 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:51:19.523 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:hints = 1 2023-09-23T22:51:19.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:51:19.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object prefix: benchmark_data_smithi052_402718 2023-09-23T22:51:19.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:51:19.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:51:19.734 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:19.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:19.840 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:19.844 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:19 smithi052 ceph-mon[180979]: pgmap v751: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:19.845 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:19.845 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:19 smithi052 ceph-mon[187743]: pgmap v751: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:19.845 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:19.900 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:19 smithi151 ceph-mon[159210]: pgmap v751: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T22:51:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:20.053 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:51:20.112 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:20.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:20.221 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:20.338 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:20.524 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 1 16 2926 2910 181.732 181.875 0.00487556 0.00548744 2023-09-23T22:51:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:20 smithi151 ceph-mon[159210]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:51:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:20 smithi052 ceph-mon[187743]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:51:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:20 smithi052 ceph-mon[180979]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:51:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:21.525 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 2 16 5680 5664 176.923 172.125 0.00504964 0.00564118 2023-09-23T22:51:21.635 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:21.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:21.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:21.769 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:21 smithi151 ceph-mon[159210]: pgmap v752: 89 pgs: 89 active+clean; 3.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 360 KiB/s wr, 5 op/s 2023-09-23T22:51:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:21 smithi052 ceph-mon[187743]: pgmap v752: 89 pgs: 89 active+clean; 3.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 360 KiB/s wr, 5 op/s 2023-09-23T22:51:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:21 smithi052 ceph-mon[180979]: pgmap v752: 89 pgs: 89 active+clean; 3.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 360 KiB/s wr, 5 op/s 2023-09-23T22:51:22.238 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:22.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:22.342 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:22.344 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":751,"stamp":"2023-09-23T22:51:18.316246+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2472356,"kb_used_data":416876,"kb_used_omap":129,"kb_used_meta":2055294,"kb_avail":653743708,"statfs":{"total":671965249536,"available":669433556992,"internally_reserved":0,"allocated":426881024,"data_stored":421480279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":132387,"internal_metadata":2104621789},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.814875"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":264,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059512+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:51:12.059512+0000","last_peered":"2023-09-23T22:51:12.059512+0000","last_clean":"2023-09-23T22:51:12.059512+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:51:12.059512+0000","last_undegraded":"2023-09-23T22:51:12.059512+0000","last_fullsized":"2023-09-23T22:51:12.059512+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":240,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502895+0000","last_change":"2023-09-23T22:38:06.150667+0000","last_active":"2023-09-23T22:51:11.502895+0000","last_peered":"2023-09-23T22:51:11.502895+0000","last_clean":"2023-09-23T22:51:11.502895+0000","last_became_active":"2023-09-23T22:38:06.150363+0000","last_became_peered":"2023-09-23T22:38:06.150363+0000","last_unstale":"2023-09-23T22:51:11.502895+0000","last_undegraded":"2023-09-23T22:51:11.502895+0000","last_fullsized":"2023-09-23T22:51:11.502895+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:37.872309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":92,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502953+0000","last_change":"2023-09-23T22:46:44.022589+0000","last_active":"2023-09-23T22:51:11.502953+0000","last_peered":"2023-09-23T22:51:11.502953+0000","last_clean":"2023-09-23T22:51:11.502953+0000","last_became_active":"2023-09-23T22:46:44.022292+0000","last_became_peered":"2023-09-23T22:46:44.022292+0000","last_unstale":"2023-09-23T22:51:11.502953+0000","last_undegraded":"2023-09-23T22:51:11.502953+0000","last_fullsized":"2023-09-23T22:51:11.502953+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:09.964847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":236,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640790+0000","last_change":"2023-09-23T22:46:51.235547+0000","last_active":"2023-09-23T22:51:14.640790+0000","last_peered":"2023-09-23T22:51:14.640790+0000","last_clean":"2023-09-23T22:51:14.640790+0000","last_became_active":"2023-09-23T22:46:51.235195+0000","last_became_peered":"2023-09-23T22:46:51.235195+0000","last_unstale":"2023-09-23T22:51:14.640790+0000","last_undegraded":"2023-09-23T22:51:14.640790+0000","last_fullsized":"2023-09-23T22:51:14.640790+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:49.325093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502534+0000","last_change":"2023-09-23T22:46:14.248315+0000","last_active":"2023-09-23T22:51:11.502534+0000","last_peered":"2023-09-23T22:51:11.502534+0000","last_clean":"2023-09-23T22:51:11.502534+0000","last_became_active":"2023-09-23T22:46:14.247904+0000","last_became_peered":"2023-09-23T22:46:14.247904+0000","last_unstale":"2023-09-23T22:51:11.502534+0000","last_undegraded":"2023-09-23T22:51:11.502534+0000","last_fullsized":"2023-09-23T22:51:11.502534+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:30.038091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":231,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.507778+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:51:10.507778+0000","last_peered":"2023-09-23T22:51:10.507778+0000","last_clean":"2023-09-23T22:51:10.507778+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:51:10.507778+0000","last_undegraded":"2023-09-23T22:51:10.507778+0000","last_fullsized":"2023-09-23T22:51:10.507778+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":168,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640715+0000","last_change":"2023-09-23T22:46:14.253380+0000","last_active":"2023-09-23T22:51:14.640715+0000","last_peered":"2023-09-23T22:51:14.640715+0000","last_clean":"2023-09-23T22:51:14.640715+0000","last_became_active":"2023-09-23T22:46:14.253126+0000","last_became_peered":"2023-09-23T22:46:14.253126+0000","last_unstale":"2023-09-23T22:51:14.640715+0000","last_undegraded":"2023-09-23T22:51:14.640715+0000","last_fullsized":"2023-09-23T22:51:14.640715+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.251875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":267,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502561+0000","last_change":"2023-09-23T22:41:10.930659+0000","last_active":"2023-09-23T22:51:11.502561+0000","last_peered":"2023-09-23T22:51:11.502561+0000","last_clean":"2023-09-23T22:51:11.502561+0000","last_became_active":"2023-09-23T22:41:10.930348+0000","last_became_peered":"2023-09-23T22:41:10.930348+0000","last_unstale":"2023-09-23T22:51:11.502561+0000","last_undegraded":"2023-09-23T22:51:11.502561+0000","last_fullsized":"2023-09-23T22:51:11.502561+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:17.048455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":236,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503093+0000","last_change":"2023-09-23T22:46:44.018308+0000","last_active":"2023-09-23T22:51:11.503093+0000","last_peered":"2023-09-23T22:51:11.503093+0000","last_clean":"2023-09-23T22:51:11.503093+0000","last_became_active":"2023-09-23T22:46:44.017981+0000","last_became_peered":"2023-09-23T22:46:44.017981+0000","last_unstale":"2023-09-23T22:51:11.503093+0000","last_undegraded":"2023-09-23T22:51:11.503093+0000","last_fullsized":"2023-09-23T22:51:11.503093+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:04.336185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":225,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502856+0000","last_change":"2023-09-23T22:44:53.861278+0000","last_active":"2023-09-23T22:51:11.502856+0000","last_peered":"2023-09-23T22:51:11.502856+0000","last_clean":"2023-09-23T22:51:11.502856+0000","last_became_active":"2023-09-23T22:44:53.860956+0000","last_became_peered":"2023-09-23T22:44:53.860956+0000","last_unstale":"2023-09-23T22:51:11.502856+0000","last_undegraded":"2023-09-23T22:51:11.502856+0000","last_fullsized":"2023-09-23T22:51:11.502856+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:57.774528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":142,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502788+0000","last_change":"2023-09-23T22:46:14.247747+0000","last_active":"2023-09-23T22:51:11.502788+0000","last_peered":"2023-09-23T22:51:11.502788+0000","last_clean":"2023-09-23T22:51:11.502788+0000","last_became_active":"2023-09-23T22:46:14.247449+0000","last_became_peered":"2023-09-23T22:46:14.247449+0000","last_unstale":"2023-09-23T22:51:11.502788+0000","last_undegraded":"2023-09-23T22:51:11.502788+0000","last_fullsized":"2023-09-23T22:51:11.502788+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:32:46.237735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":140,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506651+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:51:10.506651+0000","last_peered":"2023-09-23T22:51:10.506651+0000","last_clean":"2023-09-23T22:51:10.506651+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:51:10.506651+0000","last_undegraded":"2023-09-23T22:51:10.506651+0000","last_fullsized":"2023-09-23T22:51:10.506651+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":287,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.060005+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:51:12.060005+0000","last_peered":"2023-09-23T22:51:12.060005+0000","last_clean":"2023-09-23T22:51:12.060005+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:51:12.060005+0000","last_undegraded":"2023-09-23T22:51:12.060005+0000","last_fullsized":"2023-09-23T22:51:12.060005+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059338+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:51:12.059338+0000","last_peered":"2023-09-23T22:51:12.059338+0000","last_clean":"2023-09-23T22:51:12.059338+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:51:12.059338+0000","last_undegraded":"2023-09-23T22:51:12.059338+0000","last_fullsized":"2023-09-23T22:51:12.059338+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":125,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059657+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:51:12.059657+0000","last_peered":"2023-09-23T22:51:12.059657+0000","last_clean":"2023-09-23T22:51:12.059657+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:51:12.059657+0000","last_undegraded":"2023-09-23T22:51:12.059657+0000","last_fullsized":"2023-09-23T22:51:12.059657+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":287,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502793+0000","last_change":"2023-09-23T22:44:53.871800+0000","last_active":"2023-09-23T22:51:11.502793+0000","last_peered":"2023-09-23T22:51:11.502793+0000","last_clean":"2023-09-23T22:51:11.502793+0000","last_became_active":"2023-09-23T22:44:53.871467+0000","last_became_peered":"2023-09-23T22:44:53.871467+0000","last_unstale":"2023-09-23T22:51:11.502793+0000","last_undegraded":"2023-09-23T22:51:11.502793+0000","last_fullsized":"2023-09-23T22:51:11.502793+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:52.749249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":224,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640551+0000","last_change":"2023-09-23T22:46:14.246527+0000","last_active":"2023-09-23T22:51:14.640551+0000","last_peered":"2023-09-23T22:51:14.640551+0000","last_clean":"2023-09-23T22:51:14.640551+0000","last_became_active":"2023-09-23T22:46:14.246210+0000","last_became_peered":"2023-09-23T22:46:14.246210+0000","last_unstale":"2023-09-23T22:51:14.640551+0000","last_undegraded":"2023-09-23T22:51:14.640551+0000","last_fullsized":"2023-09-23T22:51:14.640551+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.208441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502982+0000","last_change":"2023-09-23T22:45:51.508229+0000","last_active":"2023-09-23T22:51:11.502982+0000","last_peered":"2023-09-23T22:51:11.502982+0000","last_clean":"2023-09-23T22:51:11.502982+0000","last_became_active":"2023-09-23T22:45:51.507869+0000","last_became_peered":"2023-09-23T22:45:51.507869+0000","last_unstale":"2023-09-23T22:51:11.502982+0000","last_undegraded":"2023-09-23T22:51:11.502982+0000","last_fullsized":"2023-09-23T22:51:11.502982+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:01.458285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":117,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059681+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:51:12.059681+0000","last_peered":"2023-09-23T22:51:12.059681+0000","last_clean":"2023-09-23T22:51:12.059681+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:51:12.059681+0000","last_undegraded":"2023-09-23T22:51:12.059681+0000","last_fullsized":"2023-09-23T22:51:12.059681+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":225,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640882+0000","last_change":"2023-09-23T22:46:14.253721+0000","last_active":"2023-09-23T22:51:14.640882+0000","last_peered":"2023-09-23T22:51:14.640882+0000","last_clean":"2023-09-23T22:51:14.640882+0000","last_became_active":"2023-09-23T22:46:14.253462+0000","last_became_peered":"2023-09-23T22:46:14.253462+0000","last_unstale":"2023-09-23T22:51:14.640882+0000","last_undegraded":"2023-09-23T22:51:14.640882+0000","last_fullsized":"2023-09-23T22:51:14.640882+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:34.957074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":310,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.507066+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:51:10.507066+0000","last_peered":"2023-09-23T22:51:10.507066+0000","last_clean":"2023-09-23T22:51:10.507066+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:51:10.507066+0000","last_undegraded":"2023-09-23T22:51:10.507066+0000","last_fullsized":"2023-09-23T22:51:10.507066+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":267,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503488+0000","last_change":"2023-09-23T22:44:53.876333+0000","last_active":"2023-09-23T22:51:11.503488+0000","last_peered":"2023-09-23T22:51:11.503488+0000","last_clean":"2023-09-23T22:51:11.503488+0000","last_became_active":"2023-09-23T22:44:53.871473+0000","last_became_peered":"2023-09-23T22:44:53.871473+0000","last_unstale":"2023-09-23T22:51:11.503488+0000","last_undegraded":"2023-09-23T22:51:11.503488+0000","last_fullsized":"2023-09-23T22:51:11.503488+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:54.852070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":13,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640631+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:51:14.640631+0000","last_peered":"2023-09-23T22:51:14.640631+0000","last_clean":"2023-09-23T22:51:14.640631+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:51:14.640631+0000","last_undegraded":"2023-09-23T22:51:14.640631+0000","last_fullsized":"2023-09-23T22:51:14.640631+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":136,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059969+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:51:12.059969+0000","last_peered":"2023-09-23T22:51:12.059969+0000","last_clean":"2023-09-23T22:51:12.059969+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:51:12.059969+0000","last_undegraded":"2023-09-23T22:51:12.059969+0000","last_fullsized":"2023-09-23T22:51:12.059969+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.507167+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:51:10.507167+0000","last_peered":"2023-09-23T22:51:10.507167+0000","last_clean":"2023-09-23T22:51:10.507167+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:51:10.507167+0000","last_undegraded":"2023-09-23T22:51:10.507167+0000","last_fullsized":"2023-09-23T22:51:10.507167+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059733+0000","last_change":"2023-09-23T22:51:10.575708+0000","last_active":"2023-09-23T22:51:12.059733+0000","last_peered":"2023-09-23T22:51:12.059733+0000","last_clean":"2023-09-23T22:51:12.059733+0000","last_became_active":"2023-09-23T22:51:10.575362+0000","last_became_peered":"2023-09-23T22:51:10.575362+0000","last_unstale":"2023-09-23T22:51:12.059733+0000","last_undegraded":"2023-09-23T22:51:12.059733+0000","last_fullsized":"2023-09-23T22:51:12.059733+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":146,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.060004+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:51:12.060004+0000","last_peered":"2023-09-23T22:51:12.060004+0000","last_clean":"2023-09-23T22:51:12.060004+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:51:12.060004+0000","last_undegraded":"2023-09-23T22:51:12.060004+0000","last_fullsized":"2023-09-23T22:51:12.060004+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059666+0000","last_change":"2023-09-23T22:38:06.154190+0000","last_active":"2023-09-23T22:51:12.059666+0000","last_peered":"2023-09-23T22:51:12.059666+0000","last_clean":"2023-09-23T22:51:12.059666+0000","last_became_active":"2023-09-23T22:38:06.153770+0000","last_became_peered":"2023-09-23T22:38:06.153770+0000","last_unstale":"2023-09-23T22:51:12.059666+0000","last_undegraded":"2023-09-23T22:51:12.059666+0000","last_fullsized":"2023-09-23T22:51:12.059666+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503354+0000","last_change":"2023-09-23T22:51:10.508685+0000","last_active":"2023-09-23T22:51:11.503354+0000","last_peered":"2023-09-23T22:51:11.503354+0000","last_clean":"2023-09-23T22:51:11.503354+0000","last_became_active":"2023-09-23T22:51:10.508401+0000","last_became_peered":"2023-09-23T22:51:10.508401+0000","last_unstale":"2023-09-23T22:51:11.503354+0000","last_undegraded":"2023-09-23T22:51:11.503354+0000","last_fullsized":"2023-09-23T22:51:11.503354+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:30.071055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":76,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506378+0000","last_change":"2023-09-23T22:45:51.511852+0000","last_active":"2023-09-23T22:51:10.506378+0000","last_peered":"2023-09-23T22:51:10.506378+0000","last_clean":"2023-09-23T22:51:10.506378+0000","last_became_active":"2023-09-23T22:45:51.511457+0000","last_became_peered":"2023-09-23T22:45:51.511457+0000","last_unstale":"2023-09-23T22:51:10.506378+0000","last_undegraded":"2023-09-23T22:51:10.506378+0000","last_fullsized":"2023-09-23T22:51:10.506378+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":258,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506794+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:51:10.506794+0000","last_peered":"2023-09-23T22:51:10.506794+0000","last_clean":"2023-09-23T22:51:10.506794+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:51:10.506794+0000","last_undegraded":"2023-09-23T22:51:10.506794+0000","last_fullsized":"2023-09-23T22:51:10.506794+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":11,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.574420+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:51:10.574420+0000","last_peered":"2023-09-23T22:51:10.574420+0000","last_clean":"2023-09-23T22:51:10.574420+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:51:10.574420+0000","last_undegraded":"2023-09-23T22:51:10.574420+0000","last_fullsized":"2023-09-23T22:51:10.574420+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":120,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502985+0000","last_change":"2023-09-23T22:45:51.508233+0000","last_active":"2023-09-23T22:51:11.502985+0000","last_peered":"2023-09-23T22:51:11.502985+0000","last_clean":"2023-09-23T22:51:11.502985+0000","last_became_active":"2023-09-23T22:45:51.507905+0000","last_became_peered":"2023-09-23T22:45:51.507905+0000","last_unstale":"2023-09-23T22:51:11.502985+0000","last_undegraded":"2023-09-23T22:51:11.502985+0000","last_fullsized":"2023-09-23T22:51:11.502985+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:12.808604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":161,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.641020+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:51:14.641020+0000","last_peered":"2023-09-23T22:51:14.641020+0000","last_clean":"2023-09-23T22:51:14.641020+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:51:14.641020+0000","last_undegraded":"2023-09-23T22:51:14.641020+0000","last_fullsized":"2023-09-23T22:51:14.641020+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502591+0000","last_change":"2023-09-23T22:51:10.509910+0000","last_active":"2023-09-23T22:51:11.502591+0000","last_peered":"2023-09-23T22:51:11.502591+0000","last_clean":"2023-09-23T22:51:11.502591+0000","last_became_active":"2023-09-23T22:51:10.509608+0000","last_became_peered":"2023-09-23T22:51:10.509608+0000","last_unstale":"2023-09-23T22:51:11.502591+0000","last_undegraded":"2023-09-23T22:51:11.502591+0000","last_fullsized":"2023-09-23T22:51:11.502591+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:03.178571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":268,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502808+0000","last_change":"2023-09-23T22:46:14.247740+0000","last_active":"2023-09-23T22:51:11.502808+0000","last_peered":"2023-09-23T22:51:11.502808+0000","last_clean":"2023-09-23T22:51:11.502808+0000","last_became_active":"2023-09-23T22:46:14.247325+0000","last_became_peered":"2023-09-23T22:46:14.247325+0000","last_unstale":"2023-09-23T22:51:11.502808+0000","last_undegraded":"2023-09-23T22:51:11.502808+0000","last_fullsized":"2023-09-23T22:51:11.502808+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:16.926705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059935+0000","last_change":"2023-09-23T22:38:06.152590+0000","last_active":"2023-09-23T22:51:12.059935+0000","last_peered":"2023-09-23T22:51:12.059935+0000","last_clean":"2023-09-23T22:51:12.059935+0000","last_became_active":"2023-09-23T22:38:06.152258+0000","last_became_peered":"2023-09-23T22:38:06.152258+0000","last_unstale":"2023-09-23T22:51:12.059935+0000","last_undegraded":"2023-09-23T22:51:12.059935+0000","last_fullsized":"2023-09-23T22:51:12.059935+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":10,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.575428+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:51:10.575428+0000","last_peered":"2023-09-23T22:51:10.575428+0000","last_clean":"2023-09-23T22:51:10.575428+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:51:10.575428+0000","last_undegraded":"2023-09-23T22:51:10.575428+0000","last_fullsized":"2023-09-23T22:51:10.575428+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":250,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506934+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:51:10.506934+0000","last_peered":"2023-09-23T22:51:10.506934+0000","last_clean":"2023-09-23T22:51:10.506934+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:51:10.506934+0000","last_undegraded":"2023-09-23T22:51:10.506934+0000","last_fullsized":"2023-09-23T22:51:10.506934+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":262,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059848+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:51:12.059848+0000","last_peered":"2023-09-23T22:51:12.059848+0000","last_clean":"2023-09-23T22:51:12.059848+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:51:12.059848+0000","last_undegraded":"2023-09-23T22:51:12.059848+0000","last_fullsized":"2023-09-23T22:51:12.059848+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502598+0000","last_change":"2023-09-23T22:51:10.573950+0000","last_active":"2023-09-23T22:51:11.502598+0000","last_peered":"2023-09-23T22:51:11.502598+0000","last_clean":"2023-09-23T22:51:11.502598+0000","last_became_active":"2023-09-23T22:51:10.573654+0000","last_became_peered":"2023-09-23T22:51:10.573654+0000","last_unstale":"2023-09-23T22:51:11.502598+0000","last_undegraded":"2023-09-23T22:51:11.502598+0000","last_fullsized":"2023-09-23T22:51:11.502598+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:45.751769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":275,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059492+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:51:12.059492+0000","last_peered":"2023-09-23T22:51:12.059492+0000","last_clean":"2023-09-23T22:51:12.059492+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:51:12.059492+0000","last_undegraded":"2023-09-23T22:51:12.059492+0000","last_fullsized":"2023-09-23T22:51:12.059492+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.060018+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:51:12.060018+0000","last_peered":"2023-09-23T22:51:12.060018+0000","last_clean":"2023-09-23T22:51:12.060018+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:51:12.060018+0000","last_undegraded":"2023-09-23T22:51:12.060018+0000","last_fullsized":"2023-09-23T22:51:12.060018+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502922+0000","last_change":"2023-09-23T22:51:10.575876+0000","last_active":"2023-09-23T22:51:11.502922+0000","last_peered":"2023-09-23T22:51:11.502922+0000","last_clean":"2023-09-23T22:51:11.502922+0000","last_became_active":"2023-09-23T22:51:10.575516+0000","last_became_peered":"2023-09-23T22:51:10.575516+0000","last_unstale":"2023-09-23T22:51:11.502922+0000","last_undegraded":"2023-09-23T22:51:11.502922+0000","last_fullsized":"2023-09-23T22:51:11.502922+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:39.330732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":269,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502440+0000","last_change":"2023-09-23T22:44:53.874557+0000","last_active":"2023-09-23T22:51:11.502440+0000","last_peered":"2023-09-23T22:51:11.502440+0000","last_clean":"2023-09-23T22:51:11.502440+0000","last_became_active":"2023-09-23T22:44:53.866574+0000","last_became_peered":"2023-09-23T22:44:53.866574+0000","last_unstale":"2023-09-23T22:51:11.502440+0000","last_undegraded":"2023-09-23T22:51:11.502440+0000","last_fullsized":"2023-09-23T22:51:11.502440+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:46.668189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":193,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.505575+0000","last_change":"2023-09-23T22:41:10.931298+0000","last_active":"2023-09-23T22:51:10.505575+0000","last_peered":"2023-09-23T22:51:10.505575+0000","last_clean":"2023-09-23T22:51:10.505575+0000","last_became_active":"2023-09-23T22:41:10.930875+0000","last_became_peered":"2023-09-23T22:41:10.930875+0000","last_unstale":"2023-09-23T22:51:10.505575+0000","last_undegraded":"2023-09-23T22:51:10.505575+0000","last_fullsized":"2023-09-23T22:51:10.505575+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:21:58.494750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502673+0000","last_change":"2023-09-23T22:51:10.576870+0000","last_active":"2023-09-23T22:51:11.502673+0000","last_peered":"2023-09-23T22:51:11.502673+0000","last_clean":"2023-09-23T22:51:11.502673+0000","last_became_active":"2023-09-23T22:51:10.576598+0000","last_became_peered":"2023-09-23T22:51:10.576598+0000","last_unstale":"2023-09-23T22:51:11.502673+0000","last_undegraded":"2023-09-23T22:51:11.502673+0000","last_fullsized":"2023-09-23T22:51:11.502673+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:18.347280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":305,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640574+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:51:14.640574+0000","last_peered":"2023-09-23T22:51:14.640574+0000","last_clean":"2023-09-23T22:51:14.640574+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:51:14.640574+0000","last_undegraded":"2023-09-23T22:51:14.640574+0000","last_fullsized":"2023-09-23T22:51:14.640574+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":187,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502727+0000","last_change":"2023-09-23T22:45:51.502149+0000","last_active":"2023-09-23T22:51:11.502727+0000","last_peered":"2023-09-23T22:51:11.502727+0000","last_clean":"2023-09-23T22:51:11.502727+0000","last_became_active":"2023-09-23T22:45:51.501762+0000","last_became_peered":"2023-09-23T22:45:51.501762+0000","last_unstale":"2023-09-23T22:51:11.502727+0000","last_undegraded":"2023-09-23T22:51:11.502727+0000","last_fullsized":"2023-09-23T22:51:11.502727+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:20.644414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059511+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:51:12.059511+0000","last_peered":"2023-09-23T22:51:12.059511+0000","last_clean":"2023-09-23T22:51:12.059511+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:51:12.059511+0000","last_undegraded":"2023-09-23T22:51:12.059511+0000","last_fullsized":"2023-09-23T22:51:12.059511+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":13,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640306+0000","last_change":"2023-09-23T22:51:10.510088+0000","last_active":"2023-09-23T22:51:14.640306+0000","last_peered":"2023-09-23T22:51:14.640306+0000","last_clean":"2023-09-23T22:51:14.640306+0000","last_became_active":"2023-09-23T22:51:10.509653+0000","last_became_peered":"2023-09-23T22:51:10.509653+0000","last_unstale":"2023-09-23T22:51:14.640306+0000","last_undegraded":"2023-09-23T22:51:14.640306+0000","last_fullsized":"2023-09-23T22:51:14.640306+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:28.032722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":100,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506853+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:51:10.506853+0000","last_peered":"2023-09-23T22:51:10.506853+0000","last_clean":"2023-09-23T22:51:10.506853+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:51:10.506853+0000","last_undegraded":"2023-09-23T22:51:10.506853+0000","last_fullsized":"2023-09-23T22:51:10.506853+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":263,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059669+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:51:12.059669+0000","last_peered":"2023-09-23T22:51:12.059669+0000","last_clean":"2023-09-23T22:51:12.059669+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:51:12.059669+0000","last_undegraded":"2023-09-23T22:51:12.059669+0000","last_fullsized":"2023-09-23T22:51:12.059669+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":182,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.505720+0000","last_change":"2023-09-23T22:45:51.501902+0000","last_active":"2023-09-23T22:51:10.505720+0000","last_peered":"2023-09-23T22:51:10.505720+0000","last_clean":"2023-09-23T22:51:10.505720+0000","last_became_active":"2023-09-23T22:45:51.501597+0000","last_became_peered":"2023-09-23T22:45:51.501597+0000","last_unstale":"2023-09-23T22:51:10.505720+0000","last_undegraded":"2023-09-23T22:51:10.505720+0000","last_fullsized":"2023-09-23T22:51:10.505720+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:56.703152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503149+0000","last_change":"2023-09-23T22:51:10.576240+0000","last_active":"2023-09-23T22:51:11.503149+0000","last_peered":"2023-09-23T22:51:11.503149+0000","last_clean":"2023-09-23T22:51:11.503149+0000","last_became_active":"2023-09-23T22:51:10.575936+0000","last_became_peered":"2023-09-23T22:51:10.575936+0000","last_unstale":"2023-09-23T22:51:11.503149+0000","last_undegraded":"2023-09-23T22:51:11.503149+0000","last_fullsized":"2023-09-23T22:51:11.503149+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:33.586721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":257,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.504908+0000","last_change":"2023-09-23T22:46:14.244543+0000","last_active":"2023-09-23T22:51:10.504908+0000","last_peered":"2023-09-23T22:51:10.504908+0000","last_clean":"2023-09-23T22:51:10.504908+0000","last_became_active":"2023-09-23T22:46:14.244214+0000","last_became_peered":"2023-09-23T22:46:14.244214+0000","last_unstale":"2023-09-23T22:51:10.504908+0000","last_undegraded":"2023-09-23T22:51:10.504908+0000","last_fullsized":"2023-09-23T22:51:10.504908+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:22.584673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":146,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.504523+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:51:10.504523+0000","last_peered":"2023-09-23T22:51:10.504523+0000","last_clean":"2023-09-23T22:51:10.504523+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:51:10.504523+0000","last_undegraded":"2023-09-23T22:51:10.504523+0000","last_fullsized":"2023-09-23T22:51:10.504523+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":274,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640475+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:51:14.640475+0000","last_peered":"2023-09-23T22:51:14.640475+0000","last_clean":"2023-09-23T22:51:14.640475+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:51:14.640475+0000","last_undegraded":"2023-09-23T22:51:14.640475+0000","last_fullsized":"2023-09-23T22:51:14.640475+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502372+0000","last_change":"2023-09-23T22:51:10.508706+0000","last_active":"2023-09-23T22:51:11.502372+0000","last_peered":"2023-09-23T22:51:11.502372+0000","last_clean":"2023-09-23T22:51:11.502372+0000","last_became_active":"2023-09-23T22:51:10.508417+0000","last_became_peered":"2023-09-23T22:51:10.508417+0000","last_unstale":"2023-09-23T22:51:11.502372+0000","last_undegraded":"2023-09-23T22:51:11.502372+0000","last_fullsized":"2023-09-23T22:51:11.502372+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.714716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":267,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.504636+0000","last_change":"2023-09-23T22:45:51.502544+0000","last_active":"2023-09-23T22:51:10.504636+0000","last_peered":"2023-09-23T22:51:10.504636+0000","last_clean":"2023-09-23T22:51:10.504636+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:51:10.504636+0000","last_undegraded":"2023-09-23T22:51:10.504636+0000","last_fullsized":"2023-09-23T22:51:10.504636+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:43.188394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":355,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640326+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:51:14.640326+0000","last_peered":"2023-09-23T22:51:14.640326+0000","last_clean":"2023-09-23T22:51:14.640326+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:51:14.640326+0000","last_undegraded":"2023-09-23T22:51:14.640326+0000","last_fullsized":"2023-09-23T22:51:14.640326+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":199,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503004+0000","last_change":"2023-09-23T22:38:06.154384+0000","last_active":"2023-09-23T22:51:11.503004+0000","last_peered":"2023-09-23T22:51:11.503004+0000","last_clean":"2023-09-23T22:51:11.503004+0000","last_became_active":"2023-09-23T22:38:06.153992+0000","last_became_peered":"2023-09-23T22:38:06.153992+0000","last_unstale":"2023-09-23T22:51:11.503004+0000","last_undegraded":"2023-09-23T22:51:11.503004+0000","last_fullsized":"2023-09-23T22:51:11.503004+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:51.433359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":258,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506548+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:51:10.506548+0000","last_peered":"2023-09-23T22:51:10.506548+0000","last_clean":"2023-09-23T22:51:10.506548+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:51:10.506548+0000","last_undegraded":"2023-09-23T22:51:10.506548+0000","last_fullsized":"2023-09-23T22:51:10.506548+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":13,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640946+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:51:14.640946+0000","last_peered":"2023-09-23T22:51:14.640946+0000","last_clean":"2023-09-23T22:51:14.640946+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:51:14.640946+0000","last_undegraded":"2023-09-23T22:51:14.640946+0000","last_fullsized":"2023-09-23T22:51:14.640946+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":234,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640413+0000","last_change":"2023-09-23T22:46:44.021989+0000","last_active":"2023-09-23T22:51:14.640413+0000","last_peered":"2023-09-23T22:51:14.640413+0000","last_clean":"2023-09-23T22:51:14.640413+0000","last_became_active":"2023-09-23T22:46:44.021187+0000","last_became_peered":"2023-09-23T22:46:44.021187+0000","last_unstale":"2023-09-23T22:51:14.640413+0000","last_undegraded":"2023-09-23T22:51:14.640413+0000","last_fullsized":"2023-09-23T22:51:14.640413+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.101289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":205,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640348+0000","last_change":"2023-09-23T22:46:14.246997+0000","last_active":"2023-09-23T22:51:14.640348+0000","last_peered":"2023-09-23T22:51:14.640348+0000","last_clean":"2023-09-23T22:51:14.640348+0000","last_became_active":"2023-09-23T22:46:14.246698+0000","last_became_peered":"2023-09-23T22:46:14.246698+0000","last_unstale":"2023-09-23T22:51:14.640348+0000","last_undegraded":"2023-09-23T22:51:14.640348+0000","last_fullsized":"2023-09-23T22:51:14.640348+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.622015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":192,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640571+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:51:14.640571+0000","last_peered":"2023-09-23T22:51:14.640571+0000","last_clean":"2023-09-23T22:51:14.640571+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:51:14.640571+0000","last_undegraded":"2023-09-23T22:51:14.640571+0000","last_fullsized":"2023-09-23T22:51:14.640571+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059867+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:51:12.059867+0000","last_peered":"2023-09-23T22:51:12.059867+0000","last_clean":"2023-09-23T22:51:12.059867+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:51:12.059867+0000","last_undegraded":"2023-09-23T22:51:12.059867+0000","last_fullsized":"2023-09-23T22:51:12.059867+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":197,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640796+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:51:14.640796+0000","last_peered":"2023-09-23T22:51:14.640796+0000","last_clean":"2023-09-23T22:51:14.640796+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:51:14.640796+0000","last_undegraded":"2023-09-23T22:51:14.640796+0000","last_fullsized":"2023-09-23T22:51:14.640796+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.505889+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:51:10.505889+0000","last_peered":"2023-09-23T22:51:10.505889+0000","last_clean":"2023-09-23T22:51:10.505889+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:51:10.505889+0000","last_undegraded":"2023-09-23T22:51:10.505889+0000","last_fullsized":"2023-09-23T22:51:10.505889+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":197,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.641319+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:51:14.641319+0000","last_peered":"2023-09-23T22:51:14.641319+0000","last_clean":"2023-09-23T22:51:14.641319+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:51:14.641319+0000","last_undegraded":"2023-09-23T22:51:14.641319+0000","last_fullsized":"2023-09-23T22:51:14.641319+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":11,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059773+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:51:12.059773+0000","last_peered":"2023-09-23T22:51:12.059773+0000","last_clean":"2023-09-23T22:51:12.059773+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:51:12.059773+0000","last_undegraded":"2023-09-23T22:51:12.059773+0000","last_fullsized":"2023-09-23T22:51:12.059773+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":308,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503480+0000","last_change":"2023-09-23T22:46:14.247751+0000","last_active":"2023-09-23T22:51:11.503480+0000","last_peered":"2023-09-23T22:51:11.503480+0000","last_clean":"2023-09-23T22:51:11.503480+0000","last_became_active":"2023-09-23T22:46:14.247347+0000","last_became_peered":"2023-09-23T22:46:14.247347+0000","last_unstale":"2023-09-23T22:51:11.503480+0000","last_undegraded":"2023-09-23T22:51:11.503480+0000","last_fullsized":"2023-09-23T22:51:11.503480+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:42.718925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":64,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.507232+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:51:10.507232+0000","last_peered":"2023-09-23T22:51:10.507232+0000","last_clean":"2023-09-23T22:51:10.507232+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:51:10.507232+0000","last_undegraded":"2023-09-23T22:51:10.507232+0000","last_fullsized":"2023-09-23T22:51:10.507232+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":293,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502469+0000","last_change":"2023-09-23T22:46:14.248227+0000","last_active":"2023-09-23T22:51:11.502469+0000","last_peered":"2023-09-23T22:51:11.502469+0000","last_clean":"2023-09-23T22:51:11.502469+0000","last_became_active":"2023-09-23T22:46:14.247282+0000","last_became_peered":"2023-09-23T22:46:14.247282+0000","last_unstale":"2023-09-23T22:51:11.502469+0000","last_undegraded":"2023-09-23T22:51:11.502469+0000","last_fullsized":"2023-09-23T22:51:11.502469+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:36.543705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":12,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502704+0000","last_change":"2023-09-23T22:51:10.509990+0000","last_active":"2023-09-23T22:51:11.502704+0000","last_peered":"2023-09-23T22:51:11.502704+0000","last_clean":"2023-09-23T22:51:11.502704+0000","last_became_active":"2023-09-23T22:51:10.509687+0000","last_became_peered":"2023-09-23T22:51:10.509687+0000","last_unstale":"2023-09-23T22:51:11.502704+0000","last_undegraded":"2023-09-23T22:51:11.502704+0000","last_fullsized":"2023-09-23T22:51:11.502704+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:40.576810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":298,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.506188+0000","last_change":"2023-09-23T22:44:53.870416+0000","last_active":"2023-09-23T22:51:10.506188+0000","last_peered":"2023-09-23T22:51:10.506188+0000","last_clean":"2023-09-23T22:51:10.506188+0000","last_became_active":"2023-09-23T22:44:53.870079+0000","last_became_peered":"2023-09-23T22:44:53.870079+0000","last_unstale":"2023-09-23T22:51:10.506188+0000","last_undegraded":"2023-09-23T22:51:10.506188+0000","last_fullsized":"2023-09-23T22:51:10.506188+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":164,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640685+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:51:14.640685+0000","last_peered":"2023-09-23T22:51:14.640685+0000","last_clean":"2023-09-23T22:51:14.640685+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:51:14.640685+0000","last_undegraded":"2023-09-23T22:51:14.640685+0000","last_fullsized":"2023-09-23T22:51:14.640685+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":217,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059581+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:51:12.059581+0000","last_peered":"2023-09-23T22:51:12.059581+0000","last_clean":"2023-09-23T22:51:12.059581+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:51:12.059581+0000","last_undegraded":"2023-09-23T22:51:12.059581+0000","last_fullsized":"2023-09-23T22:51:12.059581+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":200,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.641185+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:51:14.641185+0000","last_peered":"2023-09-23T22:51:14.641185+0000","last_clean":"2023-09-23T22:51:14.641185+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:51:14.641185+0000","last_undegraded":"2023-09-23T22:51:14.641185+0000","last_fullsized":"2023-09-23T22:51:14.641185+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":39,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:51:10.505798+0000","last_change":"2023-09-23T22:47:51.503374+0000","last_active":"2023-09-23T22:51:10.505798+0000","last_peered":"2023-09-23T22:51:10.505798+0000","last_clean":"2023-09-23T22:51:10.505798+0000","last_became_active":"2023-09-23T22:47:51.502974+0000","last_became_peered":"2023-09-23T22:47:51.502974+0000","last_unstale":"2023-09-23T22:51:10.505798+0000","last_undegraded":"2023-09-23T22:51:10.505798+0000","last_fullsized":"2023-09-23T22:51:10.505798+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.883637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":77,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059339+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:51:12.059339+0000","last_peered":"2023-09-23T22:51:12.059339+0000","last_clean":"2023-09-23T22:51:12.059339+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:51:12.059339+0000","last_undegraded":"2023-09-23T22:51:12.059339+0000","last_fullsized":"2023-09-23T22:51:12.059339+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059767+0000","last_change":"2023-09-23T22:40:13.761638+0000","last_active":"2023-09-23T22:51:12.059767+0000","last_peered":"2023-09-23T22:51:12.059767+0000","last_clean":"2023-09-23T22:51:12.059767+0000","last_became_active":"2023-09-23T22:40:13.761298+0000","last_became_peered":"2023-09-23T22:40:13.761298+0000","last_unstale":"2023-09-23T22:51:12.059767+0000","last_undegraded":"2023-09-23T22:51:12.059767+0000","last_fullsized":"2023-09-23T22:51:12.059767+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":77,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059359+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:51:12.059359+0000","last_peered":"2023-09-23T22:51:12.059359+0000","last_clean":"2023-09-23T22:51:12.059359+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:51:12.059359+0000","last_undegraded":"2023-09-23T22:51:12.059359+0000","last_fullsized":"2023-09-23T22:51:12.059359+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":138,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059621+0000","last_change":"2023-09-23T22:46:14.247739+0000","last_active":"2023-09-23T22:51:12.059621+0000","last_peered":"2023-09-23T22:51:12.059621+0000","last_clean":"2023-09-23T22:51:12.059621+0000","last_became_active":"2023-09-23T22:46:14.246992+0000","last_became_peered":"2023-09-23T22:46:14.246992+0000","last_unstale":"2023-09-23T22:51:12.059621+0000","last_undegraded":"2023-09-23T22:51:12.059621+0000","last_fullsized":"2023-09-23T22:51:12.059621+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":261,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.503151+0000","last_change":"2023-09-23T22:41:10.931140+0000","last_active":"2023-09-23T22:51:11.503151+0000","last_peered":"2023-09-23T22:51:11.503151+0000","last_clean":"2023-09-23T22:51:11.503151+0000","last_became_active":"2023-09-23T22:41:10.930458+0000","last_became_peered":"2023-09-23T22:41:10.930458+0000","last_unstale":"2023-09-23T22:51:11.503151+0000","last_undegraded":"2023-09-23T22:51:11.503151+0000","last_fullsized":"2023-09-23T22:51:11.503151+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:39.543001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":299,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:11.502733+0000","last_change":"2023-09-23T22:46:14.248216+0000","last_active":"2023-09-23T22:51:11.502733+0000","last_peered":"2023-09-23T22:51:11.502733+0000","last_clean":"2023-09-23T22:51:11.502733+0000","last_became_active":"2023-09-23T22:46:14.247794+0000","last_became_peered":"2023-09-23T22:46:14.247794+0000","last_unstale":"2023-09-23T22:51:11.502733+0000","last_undegraded":"2023-09-23T22:51:11.502733+0000","last_fullsized":"2023-09-23T22:51:11.502733+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:27.604528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":213,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T22:51:14.640763+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:51:14.640763+0000","last_peered":"2023-09-23T22:51:14.640763+0000","last_clean":"2023-09-23T22:51:14.640763+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:51:14.640763+0000","last_undegraded":"2023-09-23T22:51:14.640763+0000","last_fullsized":"2023-09-23T22:51:14.640763+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:51:12.059316+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:51:12.059316+0000","last_peered":"2023-09-23T22:51:12.059316+0000","last_clean":"2023-09-23T22:51:12.059316+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:51:12.059316+0000","last_undegraded":"2023-09-23T22:51:12.059316+0000","last_fullsized":"2023-09-23T22:51:12.059316+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":10314,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430499,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626727,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304228,"kb_used_data":2908,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93440924,"statfs":{"total":95995035648,"available":95683506176,"internally_reserved":0,"allocated":2977792,"data_stored":2471816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":3.5,"15min":2.0259999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3890000000000002,"5min":117.45999999999999,"15min":117.45999999999999},"last":0.434},{"interface":"front","average":{"1min":2.5459999999999998,"5min":3.7360000000000002,"15min":1.8959999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.9800000000000004,"5min":117.48699999999999,"15min":117.48699999999999},"last":0.48599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.1309999999999998,"15min":2.532},"min":{"1min":0.28599999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.4039999999999999,"5min":31.286000000000001,"15min":153.78399999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.7150000000000001,"5min":4.1219999999999999,"15min":3.1600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6799999999999997,"5min":185.411,"15min":185.411},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:50:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.7229999999999999,"15min":3.1499999999999999},"min":{"1min":0.44900000000000001,"5min":0.309,"15min":0.28299999999999997},"max":{"1min":6.4100000000000001,"5min":33.174999999999997,"15min":100.785},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.825,"5min":4.2220000000000004,"15min":3.5459999999999998},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4829999999999997,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":5.5170000000000003,"15min":3.5099999999999998},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.6020000000000003,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.605,"5min":4.3419999999999996,"15min":3.1989999999999998},"min":{"1min":0.36799999999999999,"5min":0.35599999999999998,"15min":0.28899999999999998},"max":{"1min":9.4410000000000007,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.79200000000000004,"15min":0.68799999999999994},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8239999999999998,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.86599999999999999,"15min":0.70699999999999996},"min":{"1min":0.318,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":4.2530000000000001,"5min":9.9740000000000002,"15min":18.597000000000001},"last":0.309}]},{"osd":5,"last update":"Sat Sep 23 22:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.0029999999999999,"15min":0.79600000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":7.1130000000000004,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.93700000000000006,"15min":0.76900000000000002},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":4.9089999999999998,"5min":9.7899999999999991,"15min":18.625},"last":0.374}]},{"osd":7,"last update":"Sat Sep 23 22:50:54 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.93200000000000005,"15min":0.77900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34399999999999997,"15min":0.316},"max":{"1min":4.3049999999999997,"5min":9.8800000000000008,"15min":12.481999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.80400000000000005,"15min":0.75600000000000001},"min":{"1min":0.33800000000000002,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":5.3150000000000004,"5min":5.3150000000000004,"15min":16.661999999999999},"last":0.38}]}]},{"osd":1,"up_from":84,"seq":360777253037,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315616,"kb_used_data":3416,"kb_used_omap":9,"kb_used_meta":312182,"kb_avail":93429536,"statfs":{"total":95995035648,"available":95671844864,"internally_reserved":0,"allocated":3497984,"data_stored":2538982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":319675369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.367,"5min":8.0190000000000001,"15min":5.4509999999999996},"min":{"1min":0.22700000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":26.042000000000002,"5min":101.605,"15min":101.605},"last":0.47699999999999998},{"interface":"front","average":{"1min":4.7430000000000003,"5min":7.6180000000000003,"15min":5.5490000000000004},"min":{"1min":0.22600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":25.379999999999999,"5min":101.892,"15min":101.892},"last":0.36199999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":6.3659999999999997,"5min":3.7850000000000001,"15min":2.6589999999999998},"min":{"1min":0.26600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":26.129000000000001,"5min":37.869999999999997,"15min":37.889000000000003},"last":2.0089999999999999},{"interface":"front","average":{"1min":6.0439999999999996,"5min":4.0890000000000004,"15min":2.7549999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":26.001999999999999,"5min":37.957000000000001,"15min":37.957000000000001},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":3.46,"15min":2.1859999999999999},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":3.1179999999999999,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.016,"5min":4.2999999999999998,"15min":2.6389999999999998},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.20999999999999999},"max":{"1min":6.7649999999999997,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":2.681,"15min":1.9990000000000001},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":5.6150000000000002,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":3.4140000000000001,"5min":2.4729999999999999,"15min":1.9490000000000001},"min":{"1min":0.33500000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":14.058,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.069,"15min":1.546},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":4.3230000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.343,"5min":2.7919999999999998,"15min":1.865},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":5.5650000000000004,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.80000000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":2.5550000000000002,"15min":1.7829999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":13.991,"5min":79.311000000000007,"15min":79.311000000000007},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.91400000000000003,"5min":2.0950000000000002,"15min":1.482},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":2.984,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1480000000000001,"15min":2.234},"min":{"1min":0.435,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":5.6539999999999999,"5min":11.481999999999999,"15min":159.018},"last":0.73699999999999999},{"interface":"front","average":{"1min":3.548,"5min":2.0659999999999998,"15min":2.2890000000000001},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.37},"max":{"1min":14.032999999999999,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.78000000000000003}]}]},{"osd":0,"up_from":219,"seq":940597837886,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":173664,"kb_used_data":3480,"kb_used_omap":31,"kb_used_meta":170144,"kb_avail":93571488,"statfs":{"total":95995035648,"available":95817203712,"internally_reserved":0,"allocated":3563520,"data_stored":2553223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":174228402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":3.887,"15min":3.77},"min":{"1min":0.221,"5min":0.216,"15min":0.216},"max":{"1min":14.141999999999999,"5min":32.344999999999999,"15min":32.344999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.6520000000000001,"5min":3.9460000000000002,"15min":3.6880000000000002},"min":{"1min":0.26100000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.291,"5min":32.106000000000002,"15min":32.106000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5499999999999998,"5min":3.7509999999999999,"15min":2.649},"min":{"1min":0.26700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":16.643999999999998,"5min":24.253,"15min":24.253},"last":0.32700000000000001},{"interface":"front","average":{"1min":4.4619999999999997,"5min":2.9489999999999998,"15min":2.1539999999999999},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":21.151,"5min":21.151,"15min":21.151},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.4400000000000004,"5min":3.4319999999999999,"15min":2.4540000000000002},"min":{"1min":0.30199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.609999999999999,"5min":20.710000000000001,"15min":20.710000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":4.3090000000000002,"5min":2.431,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.254,"15min":0.254},"max":{"1min":22.172000000000001,"5min":22.172000000000001,"15min":22.172000000000001},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.5529999999999999,"15min":2.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.2030000000000003,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.194,"5min":2.407,"15min":1.968},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":16.52,"5min":20.009,"15min":20.009},"last":0.70399999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":2.8650000000000002,"15min":1.845},"min":{"1min":0.38,"5min":0.318,"15min":0.318},"max":{"1min":61.756999999999998,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.73099999999999998},{"interface":"front","average":{"1min":5.8150000000000004,"5min":3.2080000000000002,"15min":2.1040000000000001},"min":{"1min":0.379,"5min":0.374,"15min":0.374},"max":{"1min":61.777000000000001,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.67700000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":6.3579999999999997,"5min":2.6970000000000001,"15min":1.8129999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":59.146999999999998,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":4.5300000000000002,"5min":3.1800000000000002,"15min":2.1840000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":59.164000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":2.4409999999999998,"15min":1.877},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.2350000000000003,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":3.4740000000000002,"5min":2,"15min":1.452},"min":{"1min":0.49399999999999999,"5min":0.432,"15min":0.432},"max":{"1min":21.954999999999998,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.78700000000000003}]}]},{"osd":2,"up_from":131,"seq":562640715898,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270888,"kb_used_data":2848,"kb_used_omap":81,"kb_used_meta":267950,"kb_avail":93474264,"statfs":{"total":95995035648,"available":95717646336,"internally_reserved":0,"allocated":2916352,"data_stored":1945409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":274381018},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":10.039,"5min":5.8920000000000003,"15min":5.476},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":132.79499999999999,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":10.228,"5min":6.9359999999999999,"15min":5.9050000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":132.58099999999999,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":4.0119999999999996,"5min":2.5299999999999998,"15min":3.2469999999999999},"min":{"1min":0.253,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.622999999999999,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.29199999999999998},{"interface":"front","average":{"1min":3.581,"5min":3.298,"15min":3.0859999999999999},"min":{"1min":0.27200000000000002,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":11.143000000000001,"5min":28.596,"15min":88.436999999999998},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9020000000000001,"5min":4.0270000000000001,"15min":3.6579999999999999},"min":{"1min":0.34200000000000003,"5min":0.223,"15min":0.223},"max":{"1min":35.198999999999998,"5min":35.198999999999998,"15min":88.653000000000006},"last":0.40899999999999997},{"interface":"front","average":{"1min":5.9589999999999996,"5min":3.3559999999999999,"15min":2.931},"min":{"1min":0.32900000000000001,"5min":0.251,"15min":0.251},"max":{"1min":35.136000000000003,"5min":35.136000000000003,"15min":88.474000000000004},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.8230000000000004,"15min":2.3530000000000002},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":10.433,"5min":108.878,"15min":108.878},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.3839999999999999,"5min":4.9279999999999999,"15min":3.0680000000000001},"min":{"1min":0.39300000000000002,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":11.273,"5min":108.93000000000001,"15min":108.93000000000001},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":4.2460000000000004,"15min":2.9209999999999998},"min":{"1min":0.36299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":11.239000000000001,"5min":97.322999999999993,"15min":97.322999999999993},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.2549999999999999,"5min":4.3730000000000002,"15min":2.589},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":10.327,"5min":97.301000000000002,"15min":97.301000000000002},"last":0.61099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":6.827,"5min":4.7030000000000003,"15min":2.8069999999999999},"min":{"1min":0.36899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":98.597999999999999,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":6.8230000000000004,"5min":4.4189999999999996,"15min":2.258},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":98.509,"5min":98.509,"15min":98.509},"last":0.66400000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9990000000000001,"15min":1.506},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.477,"5min":10.92,"15min":10.92},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.339,"15min":2.3530000000000002},"min":{"1min":0.49399999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.311,"5min":11.311,"15min":88.584000000000003},"last":0.64600000000000002}]}]},{"osd":3,"up_from":94,"seq":403726925984,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":399832,"kb_used_data":2832,"kb_used_omap":7,"kb_used_meta":396984,"kb_avail":93345320,"statfs":{"total":95995035648,"available":95585607680,"internally_reserved":0,"allocated":2899968,"data_stored":1939267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":406512232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":2.931,"5min":3.3849999999999998,"15min":2.3679999999999999},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.662000000000001,"5min":80.099000000000004,"15min":80.099000000000004},"last":0.311},{"interface":"front","average":{"1min":3.2810000000000001,"5min":3.3380000000000001,"15min":2.1779999999999999},"min":{"1min":0.374,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.128,"5min":80.150000000000006,"15min":80.150000000000006},"last":0.27200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6949999999999998,"5min":2.6949999999999998,"15min":1.831},"min":{"1min":0.221,"5min":0.221,"15min":0.20000000000000001},"max":{"1min":11.086,"5min":17.231000000000002,"15min":22.818000000000001},"last":0.24299999999999999},{"interface":"front","average":{"1min":2.665,"5min":1.9570000000000001,"15min":1.4390000000000001},"min":{"1min":0.377,"5min":0.25600000000000001,"15min":0.222},"max":{"1min":11.757999999999999,"5min":15.098000000000001,"15min":23.917999999999999},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":2.8610000000000002,"15min":2.6949999999999998},"min":{"1min":0.42899999999999999,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":17.170999999999999,"5min":20.431000000000001,"15min":125.446},"last":0.34799999999999998},{"interface":"front","average":{"1min":3.359,"5min":3.2599999999999998,"15min":3.2010000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":11.641,"5min":15.756,"15min":182.483},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5720000000000001,"5min":2.077,"15min":1.522},"min":{"1min":0.33500000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":8.1509999999999998,"5min":12.896000000000001,"15min":18.757000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.609,"5min":1.706,"15min":1.325},"min":{"1min":0.47799999999999998,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":11.834,"5min":12.131,"15min":12.131},"last":0.48199999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.5449999999999999,"15min":1.2549999999999999},"min":{"1min":0.34699999999999998,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":11.803000000000001,"5min":11.803000000000001,"15min":12.127000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":3.105,"5min":2.2930000000000001,"15min":1.4970000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.951000000000001,"5min":11.951000000000001,"15min":11.951000000000001},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.7390000000000001,"15min":1.3100000000000001},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.28999999999999998},"max":{"1min":11.867000000000001,"5min":11.867000000000001,"15min":12.16},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.2250000000000001,"15min":1.534},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.913,"5min":11.913,"15min":11.913},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":3.1680000000000001,"15min":1.8320000000000001},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":11.694000000000001,"5min":91.757999999999996,"15min":91.757999999999996},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.8559999999999999,"5min":3.1019999999999999,"15min":1.9550000000000001},"min":{"1min":0.48699999999999999,"5min":0.36799999999999999,"15min":0.33800000000000002},"max":{"1min":8.2010000000000005,"5min":91.709999999999994,"15min":91.709999999999994},"last":0.56899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019183,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700396,"kb_used_data":399076,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93044756,"statfs":{"total":95995035648,"available":95277830144,"internally_reserved":0,"allocated":408653824,"data_stored":408147066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":4.1849999999999996,"15min":6.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.9770000000000003,"5min":110.08499999999999,"15min":110.08499999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.5230000000000001,"5min":5.077,"15min":7.3780000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.464,"5min":110.33799999999999,"15min":110.33799999999999},"last":0.89200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":2.4319999999999999,"15min":1.6850000000000001},"min":{"1min":0.29099999999999998,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":12.192,"5min":41.643000000000001,"15min":45.320999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.514,"5min":1.9990000000000001,"15min":1.492},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.4130000000000003,"5min":41.712000000000003,"15min":45.372999999999998},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:50:25 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.2749999999999999,"15min":3.2719999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":17.093,"5min":41.607999999999997,"15min":84.909000000000006},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.4249999999999998,"15min":3.206},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":21.704000000000001,"5min":41.670000000000002,"15min":83.840000000000003},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":6.4089999999999998,"15min":3.5209999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":10.329000000000001,"5min":128.68299999999999,"15min":128.68299999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.794,"5min":6.327,"15min":3.5640000000000001},"min":{"1min":0.38300000000000001,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":12.345000000000001,"5min":129.197,"15min":129.197},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":0.81000000000000005,"15min":0.82399999999999995},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":12.238,"5min":12.238,"15min":43.189999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.83499999999999996,"15min":0.85199999999999998},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":7.5460000000000003,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.85699999999999998,"15min":0.80900000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":7.4560000000000004,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.495},{"interface":"front","average":{"1min":1.47,"5min":0.85199999999999998,"15min":0.82699999999999996},"min":{"1min":0.42999999999999999,"5min":0.29399999999999998,"15min":0.27600000000000002},"max":{"1min":12.269,"5min":12.269,"15min":26.940999999999999},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 22:50:55 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":0.88900000000000001,"15min":0.753},"min":{"1min":0.39200000000000002,"5min":0.33600000000000002,"15min":0.29899999999999999},"max":{"1min":12.311,"5min":12.311,"15min":12.311},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.1659999999999999,"5min":0.88200000000000001,"15min":0.72199999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29399999999999998},"max":{"1min":6.7919999999999998,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.53000000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822955,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307732,"kb_used_data":2316,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437420,"statfs":{"total":95995035648,"available":95679918080,"internally_reserved":0,"allocated":2371584,"data_stored":1884516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:50:18 2023","interfaces":[{"interface":"back","average":{"1min":10.109999999999999,"5min":7.133,"15min":3.5830000000000002},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":84.367999999999995,"5min":121.319,"15min":121.319},"last":0.57999999999999996},{"interface":"front","average":{"1min":10.358000000000001,"5min":7.2380000000000004,"15min":3.5099999999999998},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":84.311999999999998,"5min":121.075,"15min":121.075},"last":0.64500000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.5269999999999999,"15min":1.8089999999999999},"min":{"1min":0.29299999999999998,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":6.9189999999999996,"5min":28.658999999999999,"15min":149.203},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.581,"5min":1.9259999999999999,"15min":2.0030000000000001},"min":{"1min":0.34499999999999997,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":7.0119999999999996,"5min":28.664999999999999,"15min":149.554},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8179999999999996,"5min":4.048,"15min":6.4340000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":63.156999999999996,"5min":158.92599999999999,"15min":158.92599999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":4.4349999999999996,"5min":3.8959999999999999,"15min":6.4039999999999999},"min":{"1min":0.318,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":63.277999999999999,"5min":158.91399999999999,"15min":158.91399999999999},"last":0.63300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":10.577999999999999,"5min":9.7460000000000004,"15min":4.899},"min":{"1min":0.31,"5min":0.30499999999999999,"15min":0.25700000000000001},"max":{"1min":98.641000000000005,"5min":257.05000000000001,"15min":257.05000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":11.012,"5min":9.7850000000000001,"15min":4.7750000000000004},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":98.906000000000006,"5min":257.35000000000002,"15min":257.35000000000002},"last":0.623}]},{"osd":4,"last update":"Sat Sep 23 22:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66300000000000003,"15min":0.62},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":5.0309999999999997,"5min":5.0309999999999997,"15min":5.0309999999999997},"last":0.438},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.75700000000000001,"15min":0.624},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":3.4409999999999998,"5min":10.535,"15min":10.535},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.89900000000000002,"15min":1.4099999999999999},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":4.367,"5min":12.615,"15min":140.07499999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.0449999999999999,"15min":1.466},"min":{"1min":0.318,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":6.6219999999999999,"5min":12.74,"15min":139.739},"last":0.45700000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:51:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.67000000000000004,"15min":0.72199999999999998},"min":{"1min":0.36599999999999999,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":2.3359999999999999,"5min":2.738,"15min":7.9809999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.77400000000000002,"15min":0.70299999999999996},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.0459999999999994,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.46600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:51:22.350 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:51:22.351 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:51:22.351 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6] 2023-09-23T22:51:22.351 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T22:51:22.420 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:22.525 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 3 16 7596 7580 157.864 119.75 0.00330865 0.00632568 2023-09-23T22:51:22.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:22.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:22.730 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:22.759 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:8009 9.8 e268/264 9:146250c1:::benchmark_data_smithi052_402718_object8008:head v 268'515, mlcod=268'513)", 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.760940+0000", 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0066856720000000001, 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0067256809999999998, 2023-09-23T22:51:22.775 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760940+0000", 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760940+0000", 2023-09-23T22:51:22.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760939+0000", 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760964+0000", 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5148e-05 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760973+0000", 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.918e-06 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760980+0000", 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4269999999999996e-06 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:22.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764124+0000", 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003143992 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764143+0000", 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9103e-05 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:8010 9.8 e268/264 9:131f4aa1:::benchmark_data_smithi052_402718_object8009:head v 268'516, mlcod=268'513)", 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.760995+0000", 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0066301390000000002, 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0067517310000000004, 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760995+0000", 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760995+0000", 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.760995+0000", 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761019+0000", 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4272000000000001e-05 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761027+0000", 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8029999999999997e-06 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761033+0000", 2023-09-23T22:51:22.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0900000000000001e-06 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764623+0000", 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035901219999999998 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764643+0000", 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9429999999999999e-05 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.783 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:8014 9.8 e268/264 9:163ba97e:::benchmark_data_smithi052_402718_object8013:head v 268'517, mlcod=268'513)", 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.761099+0000", 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0065261329999999999, 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.006717565, 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761099+0000", 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761099+0000", 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761099+0000", 2023-09-23T22:51:22.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761120+0000", 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1409000000000001e-05 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761128+0000", 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5519999999999999e-06 2023-09-23T22:51:22.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.761134+0000", 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6799999999999998e-06 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764970+0000", 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0038363500000000001 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:22.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764989+0000", 2023-09-23T22:51:22.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9221000000000001e-05 2023-09-23T22:51:22.788 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.788 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.788 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:8019 9.8 e268/264)", 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.763106+0000", 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0045193660000000004, 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047732360000000001, 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.763106+0000", 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.763106+0000", 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.763106+0000", 2023-09-23T22:51:22.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.763132+0000", 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6210000000000001e-05 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.763141+0000", 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8140000000000007e-06 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.763147+0000", 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.37e-06 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:8021 9.9 9:942a43b7:::benchmark_data_smithi052_402718_object8020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.764964+0000", 2023-09-23T22:51:22.792 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0026617680000000001, 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029797199999999999, 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 8021 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764964+0000", 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764964+0000", 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764962+0000", 2023-09-23T22:51:22.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764986+0000", 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3331000000000001e-05 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.764999+0000", 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3155e-05 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765008+0000", 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7099999999999996e-06 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765091+0000", 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3418000000000007e-05 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:22.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765227+0000", 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013638800000000001 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765479+0000", 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025182299999999998 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.797 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:8023 9.9 9:9ccc4158:::benchmark_data_smithi052_402718_object8022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.765304+0000", 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0023217559999999999, 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002731561, 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 8023 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.798 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765304+0000", 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765304+0000", 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765303+0000", 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765327+0000", 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4134e-05 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765338+0000", 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0998999999999999e-05 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.765346+0000", 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6310000000000003e-06 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.766075+0000", 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00072951899999999998 2023-09-23T22:51:22.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.766209+0000", 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000133671 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.766363+0000", 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000154439 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.802 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:8024 9.c e268/264 9:3caf37a4:::benchmark_data_smithi052_402718_object8023:head v 268'538, mlcod=268'536)", 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:22.767717+0000", 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4294967295.9999084, 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041372400000000003, 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767717+0000", 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767717+0000", 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:22.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767716+0000", 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767743+0000", 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7429000000000001e-05 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767754+0000", 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1071e-05 2023-09-23T22:51:22.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767762+0000", 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4309999999999997e-06 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767873+0000", 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111227 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:22.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:22.767895+0000", 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2217000000000001e-05 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 7 2023-09-23T22:51:22.807 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:51:22.810 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:22.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:22.915 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:22.948 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:23.006 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:23.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:23.111 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:23.245 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:23.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:51:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:51:23] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-09-23T22:51:23.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 4 16 9870 9854 153.871 142.125 0.00619582 0.00648958 2023-09-23T22:51:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:23 smithi151 ceph-mon[159210]: pgmap v753: 89 pgs: 89 active+clean; 194 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 24 MiB/s wr, 387 op/s 2023-09-23T22:51:23.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:23 smithi052 ceph-mon[180979]: pgmap v753: 89 pgs: 89 active+clean; 194 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 24 MiB/s wr, 387 op/s 2023-09-23T22:51:23.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:23 smithi052 ceph-mon[187743]: pgmap v753: 89 pgs: 89 active+clean; 194 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 24 MiB/s wr, 387 op/s 2023-09-23T22:51:24.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 5 16 11649 11633 145.336 111.188 0.119105 0.00682231 2023-09-23T22:51:24.592 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:24.626 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:51:24.840 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:24.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1452330738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:24.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:24.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:24.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:24.849 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1452330738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:24.849 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1452330738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:24.950 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:24.988 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:25.051 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:25.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:25.158 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:25.194 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:25.257 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:25.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:25.364 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:25.393 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:25.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 6 16 13506 13490 140.457 116.062 0.0015102 0.00699123 2023-09-23T22:51:25.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: pgmap v754: 89 pgs: 89 active+clean; 293 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 30 MiB/s wr, 479 op/s 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[180979]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.741 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: pgmap v754: 89 pgs: 89 active+clean; 293 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 30 MiB/s wr, 479 op/s 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:25 smithi052 ceph-mon[187743]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.937 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:25.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: pgmap v754: 89 pgs: 89 active+clean; 293 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 30 MiB/s wr, 479 op/s 2023-09-23T22:51:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:51:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:51:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T22:51:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: failed to encode map e269 with expected crc 2023-09-23T22:51:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:25 smithi151 ceph-mon[159210]: failed to encode map e269 with expected crc 2023-09-23T22:51:26.043 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:26.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:26.251 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:26.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:26.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:26.357 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:26.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:26.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 7 16 15203 15187 135.543 106.062 0.00668494 0.00736874 2023-09-23T22:51:26.655 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:26.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:26.764 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:26.822 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:26.931 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:26.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:26.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:27.038 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:27.071 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:27.135 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:27.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:27.240 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:27.278 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:27.343 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:27.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:27.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:27.450 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:27.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:27.526 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 8 16 17346 17330 135.341 133.938 0.00283271 0.00738518 2023-09-23T22:51:27.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:27 smithi151 ceph-mon[159210]: pgmap v756: 89 pgs: 89 active+clean; 816 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:51:27.817 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:27.830 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:27 smithi052 ceph-mon[187743]: pgmap v756: 89 pgs: 89 active+clean; 816 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:51:27.831 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:27 smithi052 ceph-mon[180979]: pgmap v756: 89 pgs: 89 active+clean; 816 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:51:27.842 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:27.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:27.949 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:28.029 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:28.059 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:28.224 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:51:28.358 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:28.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:28.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:28.467 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:28.498 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:28.527 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 9 16 19237 19221 133.435 118.188 0.00781504 0.00748725 2023-09-23T22:51:28.554 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:28.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:28.659 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:28.738 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:28.776 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:51:28.819 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:51:28.880 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:51:28.921 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:51:28.928 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:28.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:28.969 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:51:29.012 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:51:29.035 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:29.048 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:51:29.067 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:29.086 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:51:29.092 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.093 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:51:29.124 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:29.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:29.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:29.230 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:29.263 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:29.310 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:29.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:29.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:29.452 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:29.508 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:29.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:29.527 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 10 16 20969 20953 130.912 108.25 0.0144726 0.00763035 2023-09-23T22:51:29.614 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:29.685 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:29 smithi151 ceph-mon[159210]: pgmap v757: 89 pgs: 89 active+clean; 930 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T22:51:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/73480088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:29.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:29 smithi052 ceph-mon[187743]: pgmap v757: 89 pgs: 89 active+clean; 930 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T22:51:29.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/73480088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:29.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:29 smithi052 ceph-mon[180979]: pgmap v757: 89 pgs: 89 active+clean; 930 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T22:51:29.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/73480088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:30.527 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 11 16 22685 22669 128.76 107.25 0.00156113 0.0077209 2023-09-23T22:51:31.331 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:31 smithi052 ceph-mon[180979]: pgmap v758: 89 pgs: 89 active+clean; 966 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:51:31.332 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:31 smithi052 ceph-mon[187743]: pgmap v758: 89 pgs: 89 active+clean; 966 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:51:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:31 smithi151 ceph-mon[159210]: pgmap v758: 89 pgs: 89 active+clean; 966 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:51:31.527 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 12 16 23745 23729 123.551 66.25 0.00164569 0.00804456 2023-09-23T22:51:31.656 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:31.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:31.760 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:31.835 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:32.158 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:32.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:32.272 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:32.303 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:32.359 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:32.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:32.465 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:32.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:32.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 13 16 25352 25336 121.752 100.438 0.00492894 0.00820719 2023-09-23T22:51:32.554 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:32.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:32.608 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:32.660 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:32.691 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:32.746 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:32.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:32.853 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:32.896 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:51:32.896 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:51:32.956 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:33.181 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:33.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:33.299 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:33.335 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:33.392 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:33.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:33.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:33.424 INFO:teuthology.orchestra.run.smithi052.stderr:marked out osd.3. 2023-09-23T22:51:33.449 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:51:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:51:33] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-09-23T22:51:33.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[187743]: pgmap v759: 89 pgs: 89 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T22:51:33.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/55703157' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:51:33.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:51:33.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[187743]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T22:51:33.450 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:33 smithi052 conmon[180954]: 2023-09-23T22:51:33.163+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e269 definitely_dead 0 2023-09-23T22:51:33.451 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[180979]: pgmap v759: 89 pgs: 89 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T22:51:33.451 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/55703157' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:51:33.451 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:51:33.451 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:33 smithi052 ceph-mon[180979]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T22:51:33.497 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:33.527 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:33.529 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 14 16 26699 26683 119.07 84.1875 0.00208051 0.00832102 2023-09-23T22:51:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:33 smithi151 ceph-mon[159210]: pgmap v759: 89 pgs: 89 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T22:51:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/55703157' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:51:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:33 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:51:33.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:33 smithi151 ceph-mon[159210]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T22:51:34.136 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[187743]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:34 smithi052 ceph-mon[180979]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 15 16 26699 26683 111.134 0 - 0.00832102 2023-09-23T22:51:34.630 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:34.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:34 smithi151 ceph-mon[159210]: failed to encode map e270 with expected crc 2023-09-23T22:51:34.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:34.766 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:34.925 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:51:35.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: pgmap v761: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T22:51:35.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: osdmap e271: 8 total, 8 up, 6 in 2023-09-23T22:51:35.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[180979]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.497 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: pgmap v761: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: osdmap e271: 8 total, 8 up, 6 in 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:35.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:35.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:35 smithi052 ceph-mon[187743]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 16 16 26705 26689 104.214 0.1875 2.10012 0.00879322 2023-09-23T22:51:35.582 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:35.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:35.687 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: pgmap v761: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: osdmap e271: 8 total, 8 up, 6 in 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:35 smithi151 ceph-mon[159210]: failed to encode map e271 with expected crc 2023-09-23T22:51:35.726 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:35.831 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:35.840 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:35.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:35.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:35.985 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:36.086 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:36.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:36.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:36.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:36.268 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:36.299 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:36.403 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:36.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:36.418 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.418 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.418 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: osdmap e272: 8 total, 8 up, 6 in 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1487285883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:51:36.419 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: osdmap e272: 8 total, 8 up, 6 in 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.420 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1487285883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:51:36.421 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:51:36.425 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:51:36.505 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:36.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 17 16 26790 26774 98.3976 5.3125 0.0670842 0.00935066 2023-09-23T22:51:36.531 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:36.579 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:36.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:36.684 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: osdmap e272: 8 total, 8 up, 6 in 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: failed to encode map e272 with expected crc 2023-09-23T22:51:36.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1487285883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:51:36.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:51:36.712 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:36.759 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:36.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:36.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:36.864 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:36.890 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:36.932 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:36.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:36.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:37.040 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:37.073 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:37.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 18 16 27224 27208 94.4389 27.125 0.00573951 0.0105704 2023-09-23T22:51:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:37 smithi151 ceph-mon[159210]: pgmap v764: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 85 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s; 3320/74928 objects misplaced (4.431%); 96 KiB/s, 1 objects/s recovering 2023-09-23T22:51:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:37 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:37 smithi151 ceph-mon[159210]: failed to encode map e273 with expected crc 2023-09-23T22:51:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:37 smithi151 ceph-mon[159210]: failed to encode map e273 with expected crc 2023-09-23T22:51:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:51:37.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:37 smithi151 ceph-mon[159210]: osdmap e273: 8 total, 8 up, 6 in 2023-09-23T22:51:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[180979]: pgmap v764: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 85 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s; 3320/74928 objects misplaced (4.431%); 96 KiB/s, 1 objects/s recovering 2023-09-23T22:51:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[180979]: failed to encode map e273 with expected crc 2023-09-23T22:51:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[180979]: failed to encode map e273 with expected crc 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[180979]: osdmap e273: 8 total, 8 up, 6 in 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[187743]: pgmap v764: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 85 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s; 3320/74928 objects misplaced (4.431%); 96 KiB/s, 1 objects/s recovering 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[187743]: failed to encode map e273 with expected crc 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[187743]: failed to encode map e273 with expected crc 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:51:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:37 smithi052 ceph-mon[187743]: osdmap e273: 8 total, 8 up, 6 in 2023-09-23T22:51:38.280 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:38.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:38.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:38.387 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:38.441 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:38.524 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:38.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:38.530 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 19 16 28480 28464 93.5998 78.5 0.0100725 0.0106774 2023-09-23T22:51:38.594 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:38 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 8399/81252 objects degraded (10.337%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:38.596 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:38 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 8399/81252 objects degraded (10.337%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:38.631 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:38.676 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:38.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:38 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 8399/81252 objects degraded (10.337%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:38.759 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:38.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:38.868 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:38.942 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:39.175 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:39.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:39.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:39.284 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:39.320 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:39.360 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:51:39.390 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:39.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:39.404 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:51:39.426 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48261 8.1 8:87871911:::benchmark_data_smithi052_337357_object48260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.49657328000001, 2023-09-23T22:51:39.427 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27316745399999998, 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48261 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.428 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.429 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927089+0000", 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927116+0000", 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7282999999999999e-05 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.430 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927128+0000", 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1987000000000001e-05 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927135+0000", 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4120000000000004e-06 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.431 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927186+0000", 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0982000000000001e-05 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927313+0000", 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012680600000000001 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.432 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927452+0000", 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013846099999999999 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.930719+0000", 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032677600000000002 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.433 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934336+0000", 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036163720000000001 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200180+0000", 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26584448100000002 2023-09-23T22:51:39.434 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200227+0000", 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7108999999999999e-05 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200257+0000", 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0210999999999999e-05 2023-09-23T22:51:39.435 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48276 8.c 8:3458bc4f:::benchmark_data_smithi052_337357_object48275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.48920833900002, 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26596831100000001, 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.436 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48276 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:51:39.437 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934454+0000", 2023-09-23T22:51:39.438 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934485+0000", 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1665999999999998e-05 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934497+0000", 2023-09-23T22:51:39.439 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1136e-05 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934505+0000", 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3399999999999998e-06 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934560+0000", 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4852000000000003e-05 2023-09-23T22:51:39.440 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934713+0000", 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000153129 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934856+0000", 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143199 2023-09-23T22:51:39.441 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.936652+0000", 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001796157 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.936814+0000", 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016158000000000001 2023-09-23T22:51:39.442 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200327+0000", 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26351350400000001 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200382+0000", 2023-09-23T22:51:39.443 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4231000000000003e-05 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200423+0000", 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1641999999999999e-05 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.444 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48281 8.c 8:3ea84d87:::benchmark_data_smithi052_337357_object48280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.48629563600002, 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26513863300000001, 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.445 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48281 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.446 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937366+0000", 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:39.447 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937390+0000", 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3839e-05 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937401+0000", 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1238e-05 2023-09-23T22:51:39.448 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937408+0000", 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.148e-06 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937510+0000", 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101549 2023-09-23T22:51:39.449 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937622+0000", 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111784 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937748+0000", 2023-09-23T22:51:39.450 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012662900000000001 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941281+0000", 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003532355 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941456+0000", 2023-09-23T22:51:39.451 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017523800000000001 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202425+0000", 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26096917400000003 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202471+0000", 2023-09-23T22:51:39.452 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6286999999999999e-05 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202506+0000", 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4978999999999998e-05 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.453 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48284 8.e 8:76fd02f7:::benchmark_data_smithi052_337357_object48283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.48415323099999, 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.264529664, 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.454 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48284 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.455 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939509+0000", 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.456 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939533+0000", 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4029e-05 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939543+0000", 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0251999999999999e-05 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.457 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939550+0000", 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3560000000000004e-06 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939596+0000", 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5862000000000002e-05 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.458 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939720+0000", 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124368 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939922+0000", 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020159799999999999 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.459 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201651+0000", 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.261728827 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201691+0000", 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9750999999999999e-05 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.460 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203971+0000", 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022803329999999998 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204008+0000", 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7373000000000003e-05 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.461 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204040+0000", 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1350999999999998e-05 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48287 8.e 8:77bdb954:::benchmark_data_smithi052_337357_object48286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:51:39.462 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.48218798200003, 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26634659700000002, 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48287 2023-09-23T22:51:39.463 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:51:39.464 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941474+0000", 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941498+0000", 2023-09-23T22:51:39.465 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3611000000000001e-05 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941507+0000", 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8169999999999994e-06 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941519+0000", 2023-09-23T22:51:39.466 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1724e-05 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941642+0000", 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012283800000000001 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941737+0000", 2023-09-23T22:51:39.467 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4828999999999996e-05 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941823+0000", 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6247000000000003e-05 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201726+0000", 2023-09-23T22:51:39.468 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25990260700000001 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204073+0000", 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023471569999999999 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207731+0000", 2023-09-23T22:51:39.469 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036582989999999998 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207785+0000", 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3621e-05 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.470 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207822+0000", 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7115e-05 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48288 8.6 8:6358135e:::benchmark_data_smithi052_337357_object48287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.48213070899999, 2023-09-23T22:51:39.471 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.262402097, 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48288 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.472 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.473 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941532+0000", 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941551+0000", 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9185999999999999e-05 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.474 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941558+0000", 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6479999999999997e-06 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941563+0000", 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3430000000000003e-06 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.475 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941634+0000", 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0407000000000005e-05 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941713+0000", 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9330000000000001e-05 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.476 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941808+0000", 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5241000000000005e-05 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201641+0000", 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25983307300000003 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.477 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201657+0000", 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5184000000000001e-05 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203850+0000", 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021936019999999998 2023-09-23T22:51:39.478 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203898+0000", 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8186999999999998e-05 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203935+0000", 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6375000000000003e-05 2023-09-23T22:51:39.479 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48276 8.c e228/219 ondisk, result = 0)", 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.481745015, 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25851532799999999, 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:39.480 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.481 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941922+0000", 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3780000000000003e-06 2023-09-23T22:51:39.482 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200261+0000", 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25833932399999998 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200269+0000", 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.255e-06 2023-09-23T22:51:39.483 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200312+0000", 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3016999999999998e-05 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200321+0000", 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2790000000000004e-06 2023-09-23T22:51:39.484 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200434+0000", 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000112623 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.485 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48261 8.1 e228/226 ondisk, result = 0)", 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.48150995899999, 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25811125899999998, 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.486 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.487 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942157+0000", 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5770000000000004e-06 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.488 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200098+0000", 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25794090400000003 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200110+0000", 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2119999999999999e-05 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.489 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200163+0000", 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2859999999999999e-05 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200175+0000", 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1765e-05 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.490 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200265+0000", 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9689000000000001e-05 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48284 8.e e228/219 ondisk, result = 0)", 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:51:39.491 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 288.47859727700001, 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25662737099999999, 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945065+0000", 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.493 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945071+0000", 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.088e-06 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200085+0000", 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25501373599999999 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200109+0000", 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3581e-05 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201679+0000", 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015704989999999999 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201685+0000", 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.404e-06 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201693+0000", 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9060000000000002e-06 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.496 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:95424 8.c 8:308751a2:::benchmark_data_smithi052_337357_object95423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e236)", 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:47:41.670881+0000", 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 237.752782596, 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.223511547, 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.497 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 95424 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670881+0000", 2023-09-23T22:51:39.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670881+0000", 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670879+0000", 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670910+0000", 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1600000000000002e-05 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670929+0000", 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8493000000000001e-05 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670939+0000", 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0542e-05 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.670989+0000", 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9926999999999998e-05 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.671110+0000", 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121308 2023-09-23T22:51:39.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.769466+0000", 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.098355572000000002 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.773071+0000", 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036054149999999998 2023-09-23T22:51:39.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.777224+0000", 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041521400000000003 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.894287+0000", 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.117063365 2023-09-23T22:51:39.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.894346+0000", 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9623000000000002e-05 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:47:41.894392+0000", 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5745000000000003e-05 2023-09-23T22:51:39.505 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5820 9.5 9:a2dbc023:::benchmark_data_smithi052_402718_object5819:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617647+0000", 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.806015817999999, 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.230426096, 2023-09-23T22:51:39.506 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5820 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617647+0000", 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617647+0000", 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617646+0000", 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617671+0000", 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4714000000000001e-05 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617683+0000", 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1557e-05 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617691+0000", 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2020000000000004e-06 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617760+0000", 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9345999999999996e-05 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617898+0000", 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013813399999999999 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844639+0000", 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.226740843 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847826+0000", 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031868450000000002 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847890+0000", 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4135999999999997e-05 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847977+0000", 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6452000000000005e-05 2023-09-23T22:51:39.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848024+0000", 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7691999999999997e-05 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848074+0000", 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9237999999999999e-05 2023-09-23T22:51:39.514 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5821 9.5 9:aabc6ac6:::benchmark_data_smithi052_402718_object5820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617707+0000", 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.805956010999999, 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23050403999999999, 2023-09-23T22:51:39.515 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5821 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617707+0000", 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617707+0000", 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617707+0000", 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617731+0000", 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4550999999999999e-05 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.518 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617739+0000", 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9969999999999998e-06 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617746+0000", 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9489999999999999e-06 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.845135+0000", 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.227388479 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.845253+0000", 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118473 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.845468+0000", 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021454699999999999 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847859+0000", 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023915059999999998 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847969+0000", 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110183 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848086+0000", 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000116842 2023-09-23T22:51:39.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848156+0000", 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9461000000000005e-05 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848211+0000", 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5608e-05 2023-09-23T22:51:39.523 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5822 9.b 9:db84e18f:::benchmark_data_smithi052_402718_object5821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617753+0000", 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.805910688000001, 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22947384100000001, 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.524 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5822 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617753+0000", 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617753+0000", 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617752+0000", 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617775+0000", 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2606999999999999e-05 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617782+0000", 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7190000000000001e-06 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617788+0000", 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9410000000000004e-06 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617971+0000", 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000182826 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844708+0000", 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.226736681 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844877+0000", 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168739 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846677+0000", 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001800722 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846713+0000", 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5636000000000002e-05 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847144+0000", 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00043132899999999999 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847191+0000", 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6638000000000003e-05 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847226+0000", 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5481999999999997e-05 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.532 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:5823 9.c e268/264 9:36ec13c5:::benchmark_data_smithi052_402718_object5822:head v 268'412, mlcod=268'410)", 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.618538+0000", 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.805125372999999, 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.227456033, 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.533 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618538+0000", 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618538+0000", 2023-09-23T22:51:39.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618537+0000", 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618569+0000", 2023-09-23T22:51:39.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1646999999999997e-05 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844559+0000", 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225990568 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844580+0000", 2023-09-23T22:51:39.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0801e-05 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844624+0000", 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4252000000000003e-05 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844644+0000", 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0007e-05 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.845948+0000", 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013034850000000001 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.845994+0000", 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6168000000000002e-05 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5828 9.5 9:a486fdb4:::benchmark_data_smithi052_402718_object5827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.619315+0000", 2023-09-23T22:51:39.539 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.804347876000001, 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.229013092, 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5828 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.540 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619315+0000", 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619315+0000", 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619314+0000", 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619368+0000", 2023-09-23T22:51:39.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4088000000000001e-05 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844579+0000", 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22521047899999999 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.844590+0000", 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1681e-05 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.845943+0000", 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001352453 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846072+0000", 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012904 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846210+0000", 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138586 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847929+0000", 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001719071 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848219+0000", 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000289528 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848256+0000", 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7295999999999997e-05 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848296+0000", 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9925000000000001e-05 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.848329+0000", 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2255999999999997e-05 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13476 9.9 9:90c01bcf:::benchmark_data_smithi052_402718_object13475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:51:39.548 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.332745+0000", 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.090918376999999, 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32173884899999999, 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13476 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.549 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332745+0000", 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332745+0000", 2023-09-23T22:51:39.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332744+0000", 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332798+0000", 2023-09-23T22:51:39.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3698e-05 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332809+0000", 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1049e-05 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332824+0000", 2023-09-23T22:51:39.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5863e-05 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332879+0000", 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4376999999999998e-05 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.333001+0000", 2023-09-23T22:51:39.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012225699999999999 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.333144+0000", 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014329499999999999 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.333932+0000", 2023-09-23T22:51:39.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00078775000000000004 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335087+0000", 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011546239999999999 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654409+0000", 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31932254799999998 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654452+0000", 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2259999999999999e-05 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654484+0000", 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2169999999999999e-05 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13478 9.9 9:92c51427:::benchmark_data_smithi052_402718_object13477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.334045+0000", 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.089618525000001, 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.320549849, 2023-09-23T22:51:39.557 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13478 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334045+0000", 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334045+0000", 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334044+0000", 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334078+0000", 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4103000000000002e-05 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334090+0000", 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1783000000000001e-05 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334097+0000", 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6340000000000001e-06 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334145+0000", 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7917000000000002e-05 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334264+0000", 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119301 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334415+0000", 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000150025 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335300+0000", 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088519900000000003 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654326+0000", 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31902618199999999 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654524+0000", 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019771400000000001 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654564+0000", 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0296000000000002e-05 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654595+0000", 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0700000000000001e-05 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.565 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13479 9.5 9:ad6f3e65:::benchmark_data_smithi052_402718_object13478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.334600+0000", 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.089063776, 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31995974199999999, 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:51:39.566 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13479 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334600+0000", 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334600+0000", 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334598+0000", 2023-09-23T22:51:39.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334622+0000", 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4139e-05 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334634+0000", 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2109999999999999e-05 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334642+0000", 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0020000000000006e-06 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.570 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334695+0000", 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2142999999999999e-05 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334823+0000", 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012874199999999999 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334954+0000", 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013098499999999999 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654328+0000", 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31937344499999998 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654413+0000", 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4905000000000001e-05 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654477+0000", 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4732000000000003e-05 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654521+0000", 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4029999999999997e-05 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654559+0000", 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7799999999999997e-05 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.574 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:13479 9.5 e269/264 ondisk, result = 0)", 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.335840+0000", 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.08782349, 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31857624899999998, 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:39.575 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335840+0000", 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335840+0000", 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335839+0000", 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335843+0000", 2023-09-23T22:51:39.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0799999999999999e-06 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654332+0000", 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31848875700000001 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654347+0000", 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4958e-05 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654393+0000", 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5671999999999999e-05 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654403+0000", 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0395e-05 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654416+0000", 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2937e-05 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.580 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:13476 9.9 e269/264 ondisk, result = 0)", 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.336205+0000", 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.087458420000001, 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31828632699999998, 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:51:39.581 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336205+0000", 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336205+0000", 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336204+0000", 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336209+0000", 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2070000000000003e-06 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654332+0000", 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31812279700000001 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654347+0000", 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5012e-05 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654392+0000", 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5139000000000003e-05 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654403+0000", 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0797e-05 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:39.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654491+0000", 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8190999999999996e-05 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:39.586 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:51:39.587 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:39.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:51:39.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:51:39.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:51:39.588 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:51:39.531823+0000 min lat: 0.00119123 max lat: 3.67384 avg lat: 0.0107615 2023-09-23T22:51:39.588 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:51:39.588 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 20 16 29743 29727 92.8613 78.9375 0.00605406 0.0107615 2023-09-23T22:51:39.589 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:51:39.614 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:51:39.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:51:39.656 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:39.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:51:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:39 smithi151 ceph-mon[159210]: pgmap v766: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 79 active+clean; 1.7 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 45 MiB/s wr, 723 op/s; 8399/81252 objects degraded (10.337%); 4936/81252 objects misplaced (6.075%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T22:51:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4128230964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:39 smithi052 ceph-mon[187743]: pgmap v766: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 79 active+clean; 1.7 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 45 MiB/s wr, 723 op/s; 8399/81252 objects degraded (10.337%); 4936/81252 objects misplaced (6.075%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T22:51:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4128230964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:39.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:39 smithi052 ceph-mon[180979]: pgmap v766: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 79 active+clean; 1.7 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 45 MiB/s wr, 723 op/s; 8399/81252 objects degraded (10.337%); 4936/81252 objects misplaced (6.075%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T22:51:39.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4128230964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:39.767 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:39.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:39.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:39.872 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:39.937 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:40.531 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 21 16 31061 31045 92.3613 82.375 0.0133019 0.0108202 2023-09-23T22:51:40.561 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [7, 3] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:51:40.561 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:51:40.641 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:51:41.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:41 smithi151 ceph-mon[159210]: pgmap v767: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 79 active+clean; 1.7 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 44 MiB/s wr, 699 op/s; 11785/83139 objects degraded (14.175%); 4936/83139 objects misplaced (5.937%); 896 KiB/s, 13 objects/s recovering 2023-09-23T22:51:41.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:41 smithi052 ceph-mon[187743]: pgmap v767: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 79 active+clean; 1.7 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 44 MiB/s wr, 699 op/s; 11785/83139 objects degraded (14.175%); 4936/83139 objects misplaced (5.937%); 896 KiB/s, 13 objects/s recovering 2023-09-23T22:51:41.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:41 smithi052 ceph-mon[180979]: pgmap v767: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 79 active+clean; 1.7 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 44 MiB/s wr, 699 op/s; 11785/83139 objects degraded (14.175%); 4936/83139 objects misplaced (5.937%); 896 KiB/s, 13 objects/s recovering 2023-09-23T22:51:41.532 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 22 16 32320 32304 91.7397 78.6875 0.00673537 0.0108946 2023-09-23T22:51:41.698 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:51:41.855 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:41.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:41.962 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:41.996 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:42.066 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:42.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:42.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:42.177 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:42.238 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:42.250 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:42.290 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:42.398 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:42.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:42.506 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:42.533 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 23 16 33447 33431 90.806 70.4375 0.0185641 0.0110069 2023-09-23T22:51:42.582 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:42.790 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:43.464 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:43.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:43.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 24 16 34680 34664 90.2332 77.0625 0.00679618 0.011077 2023-09-23T22:51:43.571 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:43.634 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:43 smithi052 ceph-mon[187743]: pgmap v768: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 57 MiB/s wr, 918 op/s; 11783/92376 objects degraded (12.755%); 4829/92376 objects misplaced (5.228%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:51:43.634 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:43 smithi052 ceph-mon[180979]: pgmap v768: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 57 MiB/s wr, 918 op/s; 11783/92376 objects degraded (12.755%); 4829/92376 objects misplaced (5.228%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:51:43.635 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:51:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:51:43] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.33.4" 2023-09-23T22:51:43.639 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:43 smithi151 ceph-mon[159210]: pgmap v768: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 57 MiB/s wr, 918 op/s; 11783/92376 objects degraded (12.755%); 4829/92376 objects misplaced (5.228%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:51:43.976 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:51:44.482 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:44.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:44.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 25 16 35977 35961 89.8662 81.0625 0.0215895 0.0111208 2023-09-23T22:51:44.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:44.648 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:44.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:44 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/97050 objects degraded (12.141%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:44.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:44 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/97050 objects degraded (12.141%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:44.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:44 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/97050 objects degraded (12.141%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:45.431 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:45.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:45.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 26 16 37163 37147 89.2601 74.125 0.0166641 0.0111964 2023-09-23T22:51:45.536 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:45.555 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:45 smithi052 ceph-mon[187743]: pgmap v769: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 56 MiB/s wr, 901 op/s; 11783/97050 objects degraded (12.141%); 4686/97050 objects misplaced (4.828%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T22:51:45.556 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:45 smithi052 ceph-mon[180979]: pgmap v769: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 56 MiB/s wr, 901 op/s; 11783/97050 objects degraded (12.141%); 4686/97050 objects misplaced (4.828%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T22:51:45.602 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:45.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:45 smithi151 ceph-mon[159210]: pgmap v769: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 56 MiB/s wr, 901 op/s; 11783/97050 objects degraded (12.141%); 4686/97050 objects misplaced (4.828%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T22:51:46.078 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:46.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:46.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:46.188 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:46.257 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:46.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 27 16 38178 38162 88.3037 63.4375 0.003736 0.0112888 2023-09-23T22:51:46.993 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:46.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:46.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:47.100 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:47.154 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:47.248 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:47.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:47.356 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:47.417 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:47.433 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:47.509 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:51:47.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 28 16 39289 39273 87.6299 69.4375 0.0272722 0.0114016 2023-09-23T22:51:47.591 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:47.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:47.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:47.706 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:47.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:47 smithi052 ceph-mon[180979]: pgmap v770: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 61 MiB/s wr, 969 op/s; 11783/104028 objects degraded (11.327%); 4584/104028 objects misplaced (4.407%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:51:47.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2876114650' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:47.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:47.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:47 smithi052 ceph-mon[187743]: pgmap v770: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 61 MiB/s wr, 969 op/s; 11783/104028 objects degraded (11.327%); 4584/104028 objects misplaced (4.407%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:51:47.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2876114650' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:47.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:47.776 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:47.907 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:47.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:47.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:47 smithi151 ceph-mon[159210]: pgmap v770: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 61 MiB/s wr, 969 op/s; 11783/104028 objects degraded (11.327%); 4584/104028 objects misplaced (4.407%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:51:47.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2876114650' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:47.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:51:48.018 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:48.068 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:39818 9.f 9:f80873ac:::benchmark_data_smithi052_402718_object39817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e274)", 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:48.071472+0000", 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0043979620000000001, 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0044342540000000003, 2023-09-23T22:51:48.080 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 39818 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071472+0000", 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:48.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071472+0000", 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071471+0000", 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:51:48.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071497+0000", 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5907000000000001e-05 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071514+0000", 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7560000000000001e-05 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071522+0000", 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2640000000000008e-06 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071590+0000", 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8146999999999997e-05 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071710+0000", 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011959300000000001 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.071807+0000", 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7317000000000004e-05 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.075143+0000", 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033356359999999999 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:48.075153+0000", 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0105000000000001e-05 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:48.086 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:51:48.087 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:48.087 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:51:48.087 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:51:48.087 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 1 2023-09-23T22:51:48.087 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:51:48.158 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:48.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:48.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:48.334 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:48.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:48.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:48.481 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: osdmap e274: 8 total, 8 up, 6 in 2023-09-23T22:51:48.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1687303' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:51:48.512 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:51:48.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 29 16 40345 40329 86.8841 66 0.00746389 0.0115015 2023-09-23T22:51:48.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: osdmap e274: 8 total, 8 up, 6 in 2023-09-23T22:51:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1687303' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: osdmap e274: 8 total, 8 up, 6 in 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: failed to encode map e274 with expected crc 2023-09-23T22:51:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1687303' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:51:49.470 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:49.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:49.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 30 16 41297 41281 85.9712 59.5 0.0583997 0.0116153 2023-09-23T22:51:49.579 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:49.667 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: pgmap v772: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 11783/114912 objects degraded (10.254%); 4427/114912 objects misplaced (3.853%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T22:51:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1687303' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:51:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: osdmap e275: 8 total, 8 up, 6 in 2023-09-23T22:51:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[187743]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: pgmap v772: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 11783/114912 objects degraded (10.254%); 4427/114912 objects misplaced (3.853%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T22:51:49.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1687303' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:51:49.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: osdmap e275: 8 total, 8 up, 6 in 2023-09-23T22:51:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:49 smithi052 ceph-mon[180979]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: pgmap v772: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.3 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 11783/114912 objects degraded (10.254%); 4427/114912 objects misplaced (3.853%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T22:51:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1687303' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:51:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: osdmap e275: 8 total, 8 up, 6 in 2023-09-23T22:51:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: failed to encode map e275 with expected crc 2023-09-23T22:51:49.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:49 smithi151 ceph-mon[159210]: failed to encode map e275 with expected crc 2023-09-23T22:51:50.523 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:50.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:50.534 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 31 16 42324 42308 85.2685 64.1875 0.0119274 0.0117213 2023-09-23T22:51:50.635 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:50.714 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:50.772 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:51:50.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:50.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:50.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/114912 objects degraded (10.254%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:50.773 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:50.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:50.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/114912 objects degraded (10.254%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:51:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:51:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/114912 objects degraded (10.254%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:51.535 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 32 16 43320 43304 84.5491 62.25 0.017051 0.0118209 2023-09-23T22:51:51.558 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:51.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:51.670 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:51.700 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:51 smithi052 ceph-mon[187743]: pgmap v774: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 11783/117609 objects degraded (10.019%); 4427/117609 objects misplaced (3.764%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:51:51.701 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:51:51.703 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:51 smithi052 ceph-mon[180979]: pgmap v774: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 11783/117609 objects degraded (10.019%); 4427/117609 objects misplaced (3.764%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:51:51.761 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:51.930 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:51.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:51 smithi151 ceph-mon[159210]: pgmap v774: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 11783/117609 objects degraded (10.019%); 4427/117609 objects misplaced (3.764%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:51:52.041 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:52.083 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:52.117 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:52.144 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:52.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:52.251 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:52.319 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:52.326 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:51:52.381 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:51:52.432 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:51:52.473 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:51:52.511 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:51:52.535 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 33 16 44285 44269 83.8146 60.3125 0.0175062 0.0119229 2023-09-23T22:51:52.544 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:51:52.579 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:51:52.586 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:52.590 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/337541069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:52.590 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/337541069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:52.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:52.616 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:51:52.624 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.625 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:51:52.696 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:52.742 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:52.771 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:52.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:52.809 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/337541069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:51:52.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:52.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:52.834 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:51:52.911 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:52.952 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:53.022 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:53.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:53.126 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:53.169 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:53.238 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:53.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:53.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:53.344 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:53.410 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:53.495 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:53.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:53.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:53.538 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 34 16 45401 45385 83.3942 69.75 0.016078 0.0119846 2023-09-23T22:51:53.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:53.655 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:53.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:53 smithi052 ceph-mon[180979]: pgmap v775: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 11783/126330 objects degraded (9.327%); 4321/126330 objects misplaced (3.420%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T22:51:53.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:51:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:51:53] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-09-23T22:51:53.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:53 smithi052 ceph-mon[187743]: pgmap v775: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 11783/126330 objects degraded (9.327%); 4321/126330 objects misplaced (3.420%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T22:51:53.750 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:53.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:53.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:53.769 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:53 smithi151 ceph-mon[159210]: pgmap v775: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 11783/126330 objects degraded (9.327%); 4321/126330 objects misplaced (3.420%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T22:51:53.870 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:51:53.929 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:54.012 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:51:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:54.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:51:54.119 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:51:54.193 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:54.538 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 35 16 46319 46303 82.6508 57.375 0.00757754 0.0120728 2023-09-23T22:51:55.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:55.090 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:51:55.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:51:55.191 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:55.280 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:55.480 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:55.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:55.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:55.538 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 36 16 47406 47390 82.2421 67.9375 0.0144469 0.0121526 2023-09-23T22:51:55.589 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:55.642 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:55.727 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:55.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:55.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:55.841 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:51:55.891 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:55 smithi151 ceph-mon[159210]: pgmap v776: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 11783/130200 objects degraded (9.050%); 4184/130200 objects misplaced (3.214%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:51:55.891 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/130200 objects degraded (9.050%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:55.894 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:55.947 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:51:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:55.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:51:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:55 smithi052 ceph-mon[180979]: pgmap v776: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 11783/130200 objects degraded (9.050%); 4184/130200 objects misplaced (3.214%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:51:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/130200 objects degraded (9.050%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:55 smithi052 ceph-mon[187743]: pgmap v776: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 11783/130200 objects degraded (9.050%); 4184/130200 objects misplaced (3.214%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:51:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/130200 objects degraded (9.050%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:51:56.052 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:56.096 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:56.174 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:51:56.174 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:51:56.183 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:56.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:56.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:56.295 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:51:56.358 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:56.538 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 37 16 48413 48397 81.7203 62.9375 0.0111924 0.0122302 2023-09-23T22:51:56.942 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:51:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:56.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:51:56.969 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:51:57.045 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:57.046 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:57.088 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:57.156 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:57.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:57.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:57.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:51:57.298 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:57.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:51:57.365 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:51:57.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:57.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:51:57.472 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:57.545 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 38 16 49270 49254 80.9632 53.5625 0.0100024 0.0123443 2023-09-23T22:51:57.562 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:57 smithi052 ceph-mon[180979]: pgmap v777: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 59 MiB/s wr, 948 op/s; 11783/135816 objects degraded (8.676%); 4083/135816 objects misplaced (3.006%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:51:57.563 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:57 smithi052 ceph-mon[187743]: pgmap v777: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 59 MiB/s wr, 948 op/s; 11783/135816 objects degraded (8.676%); 4083/135816 objects misplaced (3.006%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:51:57.574 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:57 smithi151 ceph-mon[159210]: pgmap v777: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 59 MiB/s wr, 948 op/s; 11783/135816 objects degraded (8.676%); 4083/135816 objects misplaced (3.006%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:51:57.804 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:57.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:57.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:57.912 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:51:57.996 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:58.106 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:51:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:58.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:51:58.216 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:58.278 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:58.546 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 39 16 50122 50106 80.2527 53.25 0.0082283 0.0124376 2023-09-23T22:51:58.559 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:51:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4066575724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:51:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:58 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:51:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4066575724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:51:58.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:58 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:51:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4066575724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:51:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:58 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:51:59.340 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:59.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:51:59.444 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:51:59.494 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:59.546 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:51:59.546317+0000 min lat: 0.00119123 max lat: 3.67384 avg lat: 0.0125298 2023-09-23T22:51:59.546 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:51:59.546 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 40 16 51099 51083 79.773 61.0625 0.0145042 0.0125298 2023-09-23T22:51:59.572 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:51:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:59.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:51:59.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:51:59.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[187743]: pgmap v778: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 11783/145305 objects degraded (8.109%); 3893/145305 objects misplaced (2.679%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:51:59.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[187743]: failed to encode map e276 with expected crc 2023-09-23T22:51:59.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:51:59.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[187743]: osdmap e276: 8 total, 8 up, 6 in 2023-09-23T22:51:59.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[180979]: pgmap v778: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 11783/145305 objects degraded (8.109%); 3893/145305 objects misplaced (2.679%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:51:59.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[180979]: failed to encode map e276 with expected crc 2023-09-23T22:51:59.736 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:51:59.736 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:51:59 smithi052 ceph-mon[180979]: osdmap e276: 8 total, 8 up, 6 in 2023-09-23T22:51:59.739 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:51:59.806 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:51:59.835 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:51:59.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:59 smithi151 ceph-mon[159210]: pgmap v778: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 11783/145305 objects degraded (8.109%); 3893/145305 objects misplaced (2.679%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:51:59.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:59 smithi151 ceph-mon[159210]: failed to encode map e276 with expected crc 2023-09-23T22:51:59.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:51:59.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:51:59 smithi151 ceph-mon[159210]: osdmap e276: 8 total, 8 up, 6 in 2023-09-23T22:51:59.973 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:51:59.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:51:59.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:00.099 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:00.139 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:00.206 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:00.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:00.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:00.314 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:00.326 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:00.418 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:00.546 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 41 16 51971 51955 79.1555 54.5 0.0110668 0.0126261 2023-09-23T22:52:00.549 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:00.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:00.656 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:00.695 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:00.762 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:00.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:00.853 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/145305 objects degraded (8.109%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:00.853 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/145305 objects degraded (8.109%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:00.868 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:00.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/145305 objects degraded (8.109%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:00.961 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:01.546 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 42 16 52896 52880 78.6474 57.8125 0.0164104 0.0127089 2023-09-23T22:52:01.601 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:52:01.633 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:01.641 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:01.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:01.746 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:01.780 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:01 smithi151 ceph-mon[159210]: pgmap v780: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 11783/147954 objects degraded (7.964%); 3893/147954 objects misplaced (2.631%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:52:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1133939606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:01 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:01.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:01 smithi052 ceph-mon[180979]: pgmap v780: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 11783/147954 objects degraded (7.964%); 3893/147954 objects misplaced (2.631%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:52:01.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1133939606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:01.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:01.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:01 smithi052 ceph-mon[187743]: pgmap v780: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 11783/147954 objects degraded (7.964%); 3893/147954 objects misplaced (2.631%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:52:01.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1133939606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:01.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:02.458 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:02.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:02.547 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 43 16 53759 53743 78.0729 53.9375 0.0194953 0.0128032 2023-09-23T22:52:02.572 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:02.633 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:02.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:02.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[187743]: failed to encode map e277 with expected crc 2023-09-23T22:52:02.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[187743]: failed to encode map e277 with expected crc 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[187743]: osdmap e277: 8 total, 8 up, 6 in 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[180979]: failed to encode map e277 with expected crc 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[180979]: failed to encode map e277 with expected crc 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:52:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:02 smithi052 ceph-mon[180979]: osdmap e277: 8 total, 8 up, 6 in 2023-09-23T22:52:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:02 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:02 smithi151 ceph-mon[159210]: failed to encode map e277 with expected crc 2023-09-23T22:52:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:02 smithi151 ceph-mon[159210]: failed to encode map e277 with expected crc 2023-09-23T22:52:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:02 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:52:02.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:02 smithi151 ceph-mon[159210]: osdmap e277: 8 total, 8 up, 6 in 2023-09-23T22:52:03.396 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:03.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:03.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:52:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:52:03] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-09-23T22:52:03.506 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:03.547 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 44 16 54698 54682 77.6311 58.6875 0.00891247 0.0128755 2023-09-23T22:52:03.575 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:03.839 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:03.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:03.846 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:03 smithi052 ceph-mon[187743]: pgmap v782: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 11783/155250 objects degraded (7.590%); 3798/155250 objects misplaced (2.446%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:52:03.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:03 smithi052 ceph-mon[180979]: pgmap v782: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 11783/155250 objects degraded (7.590%); 3798/155250 objects misplaced (2.446%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:52:03.947 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:03.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:03 smithi151 ceph-mon[159210]: pgmap v782: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s; 11783/155250 objects degraded (7.590%); 3798/155250 objects misplaced (2.446%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:52:04.001 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:04.076 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:04.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:04.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:04.187 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:04.235 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:04.311 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:04.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:04.417 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:04.458 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:04.524 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:04.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:04.548 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 45 16 55695 55679 77.2892 62.3125 0.0197654 0.0129312 2023-09-23T22:52:04.572 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:52:04.611 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:52:04.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4013613787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:04.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:04.631 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:04.648 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:52:04.671 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:04.692 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:52:04.727 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:52:04.738 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:04.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:04.760 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:52:04.813 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:52:04.839 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:52:04.847 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.866 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:52:04.867 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:52:04.867 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:52:04.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184397 7.0 7:0017d177:::benchmark_data_smithi052_277165_object91905:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.36004448599999, 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0558009070000001, 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.868 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184397 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:52:04.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495200+0000", 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495205+0000", 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9799999999999998e-06 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495218+0000", 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3602e-05 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495227+0000", 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1780000000000003e-06 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495336+0000", 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000109238 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495345+0000", 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5910000000000002e-06 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500507+0000", 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051612130000000004 2023-09-23T22:52:04.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500517+0000", 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0794e-05 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.550859+0000", 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0503421209999999 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551002+0000", 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014262000000000001 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184406 7.0 7:01b010b6:::benchmark_data_smithi052_277165_object91914:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.36000141300002, 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0749326939999999, 2023-09-23T22:52:04.874 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184406 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495247+0000", 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.853e-06 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495261+0000", 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4344e-05 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495266+0000", 2023-09-23T22:52:04.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3090000000000003e-06 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495397+0000", 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000130827 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495403+0000", 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.682e-06 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500537+0000", 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051338649999999996 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500541+0000", 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3510000000000004e-06 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551486+0000", 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.050945196 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551618+0000", 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013216700000000001 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570084+0000", 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01846515 2023-09-23T22:52:04.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570177+0000", 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3393999999999994e-05 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184408 7.0 7:04875276:::benchmark_data_smithi052_277165_object91916:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:52:04.881 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.35991832399998, 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0894791740000001, 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184408 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:52:04.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495331+0000", 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6959999999999996e-06 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495343+0000", 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.201e-05 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495351+0000", 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0190000000000006e-06 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495428+0000", 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6434999999999996e-05 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:52:04.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495433+0000", 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3870000000000003e-06 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500552+0000", 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051191750000000001 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500556+0000", 2023-09-23T22:52:04.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9040000000000002e-06 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551667+0000", 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.051111047 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551765+0000", 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8115999999999994e-05 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584729+0000", 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.032963237999999999 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584807+0000", 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7855999999999995e-05 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184411 7.0 7:04cf78da:::benchmark_data_smithi052_277165_object91919:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.359888931, 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0680791730000001, 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184411 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.889 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495356+0000", 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495359+0000", 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4090000000000001e-06 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495365+0000", 2023-09-23T22:52:04.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0310000000000004e-06 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495370+0000", 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3120000000000001e-06 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.496599+0000", 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012286339999999999 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.496605+0000", 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1870000000000002e-06 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500568+0000", 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003962559 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500571+0000", 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.698e-06 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551817+0000", 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0512461209999999 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551945+0000", 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012785800000000001 2023-09-23T22:52:04.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.552407+0000", 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000461896 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.561382+0000", 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089752110000000003 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563179+0000", 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017967440000000001 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563267+0000", 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7570000000000004e-05 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563330+0000", 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2973999999999997e-05 2023-09-23T22:52:04.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563436+0000", 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106332 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184468 7.10 7:092ecb73:::benchmark_data_smithi052_277165_object91976:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:52:04.897 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.30706431999999, 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0031411560000001, 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184468 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:52:04.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548184+0000", 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3390000000000001e-06 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548194+0000", 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6420000000000002e-06 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548199+0000", 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6160000000000001e-06 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548264+0000", 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4523999999999994e-05 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548276+0000", 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1939e-05 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534263+0000", 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98598719300000004 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534418+0000", 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015466899999999999 2023-09-23T22:52:04.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534800+0000", 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00038215800000000001 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536718+0000", 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019180110000000001 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551075+0000", 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014356733 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551223+0000", 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014790699999999999 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551276+0000", 2023-09-23T22:52:04.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3312000000000001e-05 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551323+0000", 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6589999999999999e-05 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184490 7.10 7:0fb33c7c:::benchmark_data_smithi052_277165_object91998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.905 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.28493812900001, 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98115489099999997, 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184490 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570305+0000", 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:52:04.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570316+0000", 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0579e-05 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570338+0000", 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2246999999999999e-05 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570347+0000", 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8189999999999998e-06 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570426+0000", 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9136999999999998e-05 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570441+0000", 2023-09-23T22:52:04.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4269e-05 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536742+0000", 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.96630105200000005 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536853+0000", 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111598 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.537044+0000", 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019121900000000001 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551089+0000", 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014044137 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551122+0000", 2023-09-23T22:52:04.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3930000000000002e-05 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551393+0000", 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027049300000000001 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551428+0000", 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4764999999999997e-05 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551462+0000", 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4730000000000001e-05 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184514 7.10 7:08068522:::benchmark_data_smithi052_277165_object92022:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.26318133500001, 2023-09-23T22:52:04.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99227452999999999, 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184514 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:52:04.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592062+0000", 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592072+0000", 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7999999999999993e-06 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592091+0000", 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8834e-05 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592102+0000", 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.079e-05 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592169+0000", 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7201999999999998e-05 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592178+0000", 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.312e-06 2023-09-23T22:52:04.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570201+0000", 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.97802318300000002 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570317+0000", 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115149 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570714+0000", 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039780000000000002 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579033+0000", 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0083185269999999992 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579064+0000", 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1152999999999998e-05 2023-09-23T22:52:04.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584277+0000", 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052129189999999999 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584310+0000", 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3500000000000001e-05 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584339+0000", 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8464999999999999e-05 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184527 7.10 7:0f7ed45d:::benchmark_data_smithi052_277165_object92035:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.25182686699998, 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98101475999999999, 2023-09-23T22:52:04.921 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184527 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603417+0000", 2023-09-23T22:52:04.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603425+0000", 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.294e-06 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603441+0000", 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6146000000000001e-05 2023-09-23T22:52:04.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603450+0000", 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1949999999999992e-06 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603514+0000", 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4456999999999996e-05 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603524+0000", 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0066e-05 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.573937+0000", 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.97041304399999995 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574065+0000", 2023-09-23T22:52:04.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000127327 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574459+0000", 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039406800000000002 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579109+0000", 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046502430000000001 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584224+0000", 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051145239999999996 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584362+0000", 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138436 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584392+0000", 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9927999999999999e-05 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584434+0000", 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1569999999999997e-05 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.929 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184549 7.10 7:0d0c97e5:::benchmark_data_smithi052_277165_object92057:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.22103578100001, 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.95037141400000003, 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184549 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634208+0000", 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634216+0000", 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.123e-06 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634229+0000", 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2289999999999999e-05 2023-09-23T22:52:04.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634235+0000", 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6270000000000002e-06 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634296+0000", 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1091000000000002e-05 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634308+0000", 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1647999999999999e-05 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576583+0000", 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.94227499800000003 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576682+0000", 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9207000000000001e-05 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576998+0000", 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031594400000000002 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579138+0000", 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021399700000000002 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584232+0000", 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050934769999999999 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584502+0000", 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027061099999999997 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584546+0000", 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4187999999999998e-05 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584581+0000", 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4848999999999998e-05 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.937 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184550 7.0 7:012735c8:::benchmark_data_smithi052_277165_object92058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.22099339200003, 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.93521286599999998, 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184550 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634256+0000", 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9559999999999999e-06 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634264+0000", 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9179999999999994e-06 2023-09-23T22:52:04.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634269+0000", 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5249999999999996e-06 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634322+0000", 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3062000000000001e-05 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634332+0000", 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8339999999999994e-06 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.554914+0000", 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.92058223699999997 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.555025+0000", 2023-09-23T22:52:04.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110911 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.555443+0000", 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041786699999999999 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563188+0000", 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0077454409999999996 2023-09-23T22:52:04.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563504+0000", 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031558900000000002 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569298+0000", 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057944709999999998 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569357+0000", 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8242000000000002e-05 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569465+0000", 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010841299999999999 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.945 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184557 7.10 7:0c22b33f:::benchmark_data_smithi052_277165_object92065:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.21089063300002, 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.94057564299999996, 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184557 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644354+0000", 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644361+0000", 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3660000000000003e-06 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644376+0000", 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4778e-05 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644384+0000", 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7519999999999996e-06 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644433+0000", 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9512999999999998e-05 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644441+0000", 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2390000000000002e-06 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579160+0000", 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.93471871500000003 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579275+0000", 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011521799999999999 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579546+0000", 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027091600000000001 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584451+0000", 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049044199999999996 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584842+0000", 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039113700000000001 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584870+0000", 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8340999999999999e-05 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584897+0000", 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6838999999999998e-05 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584931+0000", 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3798999999999999e-05 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184560 7.0 7:00bb5b7f:::benchmark_data_smithi052_277165_object92068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.14208652399998, 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85650294199999999, 2023-09-23T22:52:04.953 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184560 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:52:04.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713158+0000", 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713165+0000", 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.165e-06 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713180+0000", 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5169e-05 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713187+0000", 2023-09-23T22:52:04.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5560000000000002e-06 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713588+0000", 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040024400000000001 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713601+0000", 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3078000000000001e-05 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557158+0000", 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84355768200000003 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557274+0000", 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115092 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557528+0000", 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025477200000000001 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563196+0000", 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056677539999999997 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563652+0000", 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00045571700000000001 2023-09-23T22:52:04.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569510+0000", 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058584919999999999 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569559+0000", 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8903000000000003e-05 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569662+0000", 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000102871 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184562 7.0 7:02f4adfb:::benchmark_data_smithi052_277165_object92070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.13987801799999, 2023-09-23T22:52:04.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85882703400000004, 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184562 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715366+0000", 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:04.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715373+0000", 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0190000000000001e-06 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715389+0000", 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6200000000000001e-05 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715398+0000", 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8440000000000004e-06 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715448+0000", 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9747000000000003e-05 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715458+0000", 2023-09-23T22:52:04.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378e-05 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563719+0000", 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84826098699999997 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563811+0000", 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1298999999999994e-05 2023-09-23T22:52:04.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.564148+0000", 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000337757 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569476+0000", 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0053272930000000003 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.573915+0000", 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004439386 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574045+0000", 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012963800000000001 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574102+0000", 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6929999999999999e-05 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574195+0000", 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3134000000000006e-05 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.969 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184566 7.0 7:034423e1:::benchmark_data_smithi052_277165_object92074:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.13607266700001, 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.86009980500000005, 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184566 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.970 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719172+0000", 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719176+0000", 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5370000000000002e-06 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719184+0000", 2023-09-23T22:52:04.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8159999999999994e-06 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719190+0000", 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4980000000000003e-06 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719260+0000", 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0012000000000007e-05 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719271+0000", 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0862999999999999e-05 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.566596+0000", 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84732440099999995 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.566722+0000", 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012632999999999999 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.567133+0000", 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041073900000000003 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574211+0000", 2023-09-23T22:52:04.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070784969999999996 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576559+0000", 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023475700000000002 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579125+0000", 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025667670000000002 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579184+0000", 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8736999999999997e-05 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579273+0000", 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8580999999999998e-05 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184606 7.0 7:0419327f:::benchmark_data_smithi052_277165_object92114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 374.02732445800001, 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.75649038599999996, 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.978 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184606 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827920+0000", 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827925+0000", 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0579999999999997e-06 2023-09-23T22:52:04.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827937+0000", 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2271e-05 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827943+0000", 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5840000000000002e-06 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.828005+0000", 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2479999999999998e-05 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.828019+0000", 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3534e-05 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571101+0000", 2023-09-23T22:52:04.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.74308152100000002 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571233+0000", 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013277300000000001 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571648+0000", 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041445300000000001 2023-09-23T22:52:04.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579015+0000", 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073670360000000004 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579081+0000", 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5988000000000006e-05 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584275+0000", 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051946539999999999 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584318+0000", 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3046000000000001e-05 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584412+0000", 2023-09-23T22:52:04.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3121999999999994e-05 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184620 7.10 7:0f84e4b0:::benchmark_data_smithi052_277165_object92128:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 373.963534686, 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.69549317099999997, 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.986 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184620 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891709+0000", 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891720+0000", 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0986e-05 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891747+0000", 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7750999999999999e-05 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891758+0000", 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1043000000000001e-05 2023-09-23T22:52:04.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891835+0000", 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6794999999999999e-05 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891850+0000", 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4929999999999999e-05 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.581707+0000", 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.68985722999999999 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.581829+0000", 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012162700000000001 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.582150+0000", 2023-09-23T22:52:04.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032150899999999999 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584591+0000", 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024402400000000002 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584964+0000", 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00037371099999999999 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587111+0000", 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021470249999999999 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587159+0000", 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7295e-05 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587204+0000", 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5411000000000003e-05 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:13486 9.2 e269/264 9:41d2356a:::benchmark_data_smithi052_402718_object13485:head v 269'814, mlcod=269'812)", 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.337390+0000", 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 39.517855308999998, 2023-09-23T22:52:04.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16667193699999999, 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337390+0000", 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337390+0000", 2023-09-23T22:52:04.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337390+0000", 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337416+0000", 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5942e-05 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337426+0000", 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0957e-05 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337434+0000", 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7849999999999992e-06 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:04.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337493+0000", 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9004e-05 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337522+0000", 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8399999999999999e-05 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338508+0000", 2023-09-23T22:52:04.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0009863090000000001 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.504062+0000", 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.165554321 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:13484 9.e e269/264 9:77755ea7:::benchmark_data_smithi052_402718_object13483:head v 269'836, mlcod=269'834)", 2023-09-23T22:52:04.999 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.337478+0000", 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 39.517767861000003, 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16726806399999999, 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337478+0000", 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337478+0000", 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337477+0000", 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338200+0000", 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00072239500000000002 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338211+0000", 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0958000000000001e-05 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:05.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338218+0000", 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.182e-06 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338273+0000", 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4800999999999999e-05 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338291+0000", 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8706999999999999e-05 2023-09-23T22:52:05.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.504696+0000", 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16640424000000001 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.504746+0000", 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0271e-05 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:05.004 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:13487 9.0 e269/264 9:0b3ca2bc:::benchmark_data_smithi052_402718_object13486:head v 269'866, mlcod=269'864)", 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.338231+0000", 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 39.517014199000002, 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16691294200000001, 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:05.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338231+0000", 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338231+0000", 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338231+0000", 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:05.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338275+0000", 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4163999999999999e-05 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338286+0000", 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0888e-05 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338293+0000", 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.985e-06 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338355+0000", 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2232999999999999e-05 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:05.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338375+0000", 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0526e-05 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.505095+0000", 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16671928899999999 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.505144+0000", 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9638000000000001e-05 2023-09-23T22:52:05.009 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:13485 9.d e269/264 9:bbe3e322:::benchmark_data_smithi052_402718_object13484:head v 269'833, mlcod=269'831)", 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.338233+0000", 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 39.517012592999997, 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.166928629, 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:05.010 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338233+0000", 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338233+0000", 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338233+0000", 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338265+0000", 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2833000000000002e-05 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:05.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338273+0000", 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0339999999999997e-06 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338279+0000", 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5759999999999996e-06 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338329+0000", 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0169e-05 2023-09-23T22:52:05.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338350+0000", 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0454000000000001e-05 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.505116+0000", 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.1667661 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:05.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.505162+0000", 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5949999999999999e-05 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:05.015 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:52:05.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4013613787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:05.016 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:05.016 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4013613787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:05.016 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:05.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:05.017 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:05.017 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:05.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:05.118 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:05.147 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:05.237 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:05.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:05.352 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:05.422 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:05.549 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 46 16 56495 56479 76.6949 50 0.00793584 0.0130332 2023-09-23T22:52:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:05 smithi151 ceph-mon[159210]: pgmap v783: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 945 op/s; 11783/158532 objects degraded (7.433%); 3638/158532 objects misplaced (2.295%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:52:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/158532 objects degraded (7.433%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:05.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:05 smithi052 ceph-mon[180979]: pgmap v783: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 945 op/s; 11783/158532 objects degraded (7.433%); 3638/158532 objects misplaced (2.295%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:52:05.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/158532 objects degraded (7.433%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:05.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:05 smithi052 ceph-mon[187743]: pgmap v783: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 945 op/s; 11783/158532 objects degraded (7.433%); 3638/158532 objects misplaced (2.295%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:52:05.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/158532 objects degraded (7.433%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:06.073 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:52:06.244 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:06.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:06.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:06.365 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:06.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:06.471 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:06.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:06.553 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 47 16 57350 57334 76.2001 53.4375 0.0117798 0.0131177 2023-09-23T22:52:06.576 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:06.618 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:06.879 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:52:07.028 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:07.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:07.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:07.135 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:07.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:07.397 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:07.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:07.509 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:07.541 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:07.549 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 48 16 58136 58120 75.6362 49.125 0.0282593 0.0132133 2023-09-23T22:52:07.551 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:07.615 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:07.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:07.722 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:07.790 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:07.825 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:07.909 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:07 smithi052 ceph-mon[187743]: pgmap v784: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 48 MiB/s wr, 769 op/s; 11783/163791 objects degraded (7.194%); 3540/163791 objects misplaced (2.161%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:07.909 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:07 smithi052 ceph-mon[180979]: pgmap v784: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 48 MiB/s wr, 769 op/s; 11783/163791 objects degraded (7.194%); 3540/163791 objects misplaced (2.161%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:07.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:07 smithi151 ceph-mon[159210]: pgmap v784: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 48 MiB/s wr, 769 op/s; 11783/163791 objects degraded (7.194%); 3540/163791 objects misplaced (2.161%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:07.971 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:07.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:07.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:08.078 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:08.133 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:08.224 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:08.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:08.331 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:08.373 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:08.434 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:08.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:08.539 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:08.552 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 49 16 59016 59000 75.211 55 0.0182776 0.0132893 2023-09-23T22:52:08.608 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:08.659 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:52:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3031681710' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:52:08.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:52:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3031681710' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:52:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:52:08.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3031681710' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:52:08.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:52:09.068 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:09.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:09.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:09.178 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:09.206 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:09.218 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:52:09.552 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 50 16 59789 59773 74.6732 48.3125 0.0296636 0.0133846 2023-09-23T22:52:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:09 smithi151 ceph-mon[159210]: pgmap v785: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 11783/172023 objects degraded (6.850%); 3370/172023 objects misplaced (1.959%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:52:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:09 smithi151 ceph-mon[159210]: failed to encode map e278 with expected crc 2023-09-23T22:52:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:09 smithi151 ceph-mon[159210]: failed to encode map e278 with expected crc 2023-09-23T22:52:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:52:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:09 smithi151 ceph-mon[159210]: osdmap e278: 8 total, 8 up, 6 in 2023-09-23T22:52:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[187743]: pgmap v785: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 11783/172023 objects degraded (6.850%); 3370/172023 objects misplaced (1.959%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:52:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[187743]: failed to encode map e278 with expected crc 2023-09-23T22:52:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[187743]: failed to encode map e278 with expected crc 2023-09-23T22:52:09.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:52:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[187743]: osdmap e278: 8 total, 8 up, 6 in 2023-09-23T22:52:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[180979]: pgmap v785: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 11783/172023 objects degraded (6.850%); 3370/172023 objects misplaced (1.959%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T22:52:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[180979]: failed to encode map e278 with expected crc 2023-09-23T22:52:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[180979]: failed to encode map e278 with expected crc 2023-09-23T22:52:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:52:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:09 smithi052 ceph-mon[180979]: osdmap e278: 8 total, 8 up, 6 in 2023-09-23T22:52:10.554 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 51 16 60672 60656 74.2883 55.1875 0.00953651 0.0134551 2023-09-23T22:52:10.899 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:10.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/172023 objects degraded (6.850%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:10.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:10 smithi052 ceph-mon[180979]: pgmap v787: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s; 11783/174282 objects degraded (6.761%); 3370/174282 objects misplaced (1.934%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T22:52:10.908 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/172023 objects degraded (6.850%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:10.908 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:10 smithi052 ceph-mon[187743]: pgmap v787: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s; 11783/174282 objects degraded (6.761%); 3370/174282 objects misplaced (1.934%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T22:52:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:10.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:10.946 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:52:10.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/172023 objects degraded (6.850%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:10 smithi151 ceph-mon[159210]: pgmap v787: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s; 11783/174282 objects degraded (6.761%); 3370/174282 objects misplaced (1.934%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T22:52:11.012 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:11.147 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:11.302 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:11.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:11.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:11.412 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:11.470 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:11.480 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:11.557 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 52 16 61538 61522 73.9006 54.125 0.0189631 0.0135236 2023-09-23T22:52:11.640 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:11.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:11.749 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:11.785 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:11.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:11.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:11.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:11.954 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:11.955 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:52:11.955 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:52:11.955 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T22:52:11.955 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:52:11.992 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:12.050 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:12.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:12.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:12.079 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:52:12.156 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:12.236 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:12.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3899073192' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:12.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:12 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:12.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3899073192' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:12.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:12 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:12.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3899073192' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:12.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:12 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:12.555 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 53 16 62368 62352 73.484 51.875 0.0102832 0.0136022 2023-09-23T22:52:12.715 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:12.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:12.826 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:12.901 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:52:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: osdmap e279: 8 total, 8 up, 6 in 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: failed to encode map e279 with expected crc 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: failed to encode map e279 with expected crc 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: pgmap v789: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s; 11783/181548 objects degraded (6.490%); 3272/181548 objects misplaced (1.802%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: osdmap e279: 8 total, 8 up, 6 in 2023-09-23T22:52:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: failed to encode map e279 with expected crc 2023-09-23T22:52:13.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: failed to encode map e279 with expected crc 2023-09-23T22:52:13.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: pgmap v789: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s; 11783/181548 objects degraded (6.490%); 3272/181548 objects misplaced (1.802%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:52:13.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:52:13.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: osdmap e279: 8 total, 8 up, 6 in 2023-09-23T22:52:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: failed to encode map e279 with expected crc 2023-09-23T22:52:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: failed to encode map e279 with expected crc 2023-09-23T22:52:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: pgmap v789: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s; 11783/181548 objects degraded (6.490%); 3272/181548 objects misplaced (1.802%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:52:13.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:52:13.556 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 54 16 63162 63146 73.0423 49.625 0.0106263 0.0136859 2023-09-23T22:52:13.653 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:13.655 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:52:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:52:13] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:52:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:13.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:13.757 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:13.824 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:13.869 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:13.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:13.979 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:14.008 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:14.419 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:52:14.455 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:14.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:14.539 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:14.557 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 55 16 63935 63919 72.5918 48.3125 0.0225983 0.0137675 2023-09-23T22:52:14.563 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:14.641 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:14.814 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:14.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:14.932 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:15.014 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:15.148 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:15.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:15.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:15.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:15.318 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:15.395 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:15.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:15.511 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:15.561 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 56 16 64738 64722 72.1861 50.1875 0.0147807 0.0138461 2023-09-23T22:52:15.585 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:15 smithi052 ceph-mon[180979]: pgmap v790: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s; 11783/184611 objects degraded (6.383%); 3091/184611 objects misplaced (1.674%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:52:15.585 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/184611 objects degraded (6.383%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:15.585 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:15 smithi052 ceph-mon[187743]: pgmap v790: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s; 11783/184611 objects degraded (6.383%); 3091/184611 objects misplaced (1.674%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:52:15.585 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/184611 objects degraded (6.383%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:15.585 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:52:15.600 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:15.634 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:52:15.686 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:52:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:15 smithi151 ceph-mon[159210]: pgmap v790: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s; 11783/184611 objects degraded (6.383%); 3091/184611 objects misplaced (1.674%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:52:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/184611 objects degraded (6.383%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:15.723 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:52:15.779 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:52:15.830 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:52:15.873 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:52:15.917 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:52:15.928 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:52:16.379 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:52:16.561 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 57 16 65677 65661 71.9493 58.6875 0.0158098 0.0138924 2023-09-23T22:52:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4147512789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:16.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4147512789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:16.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4147512789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:17.108 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:17.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:17.230 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:17.291 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:17.377 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:17.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:17.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:17.487 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:17.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:17.561 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 58 16 66393 66377 71.4805 44.75 0.00347801 0.0139604 2023-09-23T22:52:17.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:17 smithi052 ceph-mon[187743]: pgmap v791: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 45 MiB/s wr, 721 op/s; 11783/189351 objects degraded (6.223%); 2994/189351 objects misplaced (1.581%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:52:17.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:17 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:17.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:17 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:17 smithi052 ceph-mon[180979]: pgmap v791: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 45 MiB/s wr, 721 op/s; 11783/189351 objects degraded (6.223%); 2994/189351 objects misplaced (1.581%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:52:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:17 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:17 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:17 smithi151 ceph-mon[159210]: pgmap v791: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 45 MiB/s wr, 721 op/s; 11783/189351 objects degraded (6.223%); 2994/189351 objects misplaced (1.581%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:52:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:17 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:17.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:17 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:17.811 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:17.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:17.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:17.919 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:17.919 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:18.008 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:18.308 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:18.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:18.419 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:18.456 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:18.523 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:18.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:18.561 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 59 16 67214 67198 71.1388 51.3125 0.0167268 0.0140502 2023-09-23T22:52:18.605 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:18.635 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:18.725 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:18.877 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:18.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:18.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:18.997 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:19.023 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:19.464 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:52:19.561 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:52:19.562015+0000 min lat: 0.00119123 max lat: 3.67384 avg lat: 0.0141518 2023-09-23T22:52:19.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:52:19.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 60 16 67866 67850 70.6325 40.75 0.0209045 0.0141518 2023-09-23T22:52:19.662 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:19.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:19.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:19 smithi052 ceph-mon[187743]: pgmap v792: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s; 11783/196893 objects degraded (5.984%); 2825/196893 objects misplaced (1.435%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:52:19.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1378908574' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:52:19.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:19 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:52:19.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:19 smithi052 ceph-mon[180979]: pgmap v792: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s; 11783/196893 objects degraded (5.984%); 2825/196893 objects misplaced (1.435%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:52:19.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1378908574' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:52:19.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:19 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:52:19.773 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:19.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:19.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:19 smithi151 ceph-mon[159210]: pgmap v792: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s; 11783/196893 objects degraded (5.984%); 2825/196893 objects misplaced (1.435%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:52:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1378908574' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:52:19.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:19 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:52:19.953 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:52:20.428 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:20.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:20.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:20.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:20.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 61 16 68657 68641 70.2851 49.4375 0.0135123 0.0142201 2023-09-23T22:52:20.575 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:20.646 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:20.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:20.656 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: failed to encode map e280 with expected crc 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: failed to encode map e280 with expected crc 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: osdmap e280: 8 total, 8 up, 6 in 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:20.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/196893 objects degraded (5.984%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: failed to encode map e280 with expected crc 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: failed to encode map e280 with expected crc 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: osdmap e280: 8 total, 8 up, 6 in 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:20.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/196893 objects degraded (5.984%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:20.754 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:20.832 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: failed to encode map e280 with expected crc 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: failed to encode map e280 with expected crc 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: osdmap e280: 8 total, 8 up, 6 in 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/196893 objects degraded (5.984%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:21.137 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:52:21.137 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:52:21.178 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:21.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:21.287 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:21.335 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:21.430 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:21.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:21.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:21.475 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:21 smithi052 ceph-mon[180979]: pgmap v794: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 44 MiB/s wr, 707 op/s; 11783/199146 objects degraded (5.917%); 2825/199146 objects misplaced (1.419%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:52:21.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:21.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 62 16 69456 69440 69.9571 49.9375 0.00781411 0.0142888 2023-09-23T22:52:21.613 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:21.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:21 smithi151 ceph-mon[159210]: pgmap v794: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 44 MiB/s wr, 707 op/s; 11783/199146 objects degraded (5.917%); 2825/199146 objects misplaced (1.419%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:52:21.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:21 smithi052 ceph-mon[187743]: pgmap v794: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 44 MiB/s wr, 707 op/s; 11783/199146 objects degraded (5.917%); 2825/199146 objects misplaced (1.419%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:52:22.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 63 16 70318 70302 69.7018 53.875 0.0146826 0.0143418 2023-09-23T22:52:22.717 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:52:22.856 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:22.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:22.975 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:23.010 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:23.069 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:23.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:23.131 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:23.132 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:23.174 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:23.209 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:23.283 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:23.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:23.391 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:23.459 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:23.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 64 16 71067 71051 69.3443 46.8125 0.0104563 0.014416 2023-09-23T22:52:23.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:23 smithi151 ceph-mon[159210]: pgmap v795: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 11783/205755 objects degraded (5.727%); 2725/205755 objects misplaced (1.324%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:52:23.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:23 smithi052 ceph-mon[180979]: pgmap v795: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 11783/205755 objects degraded (5.727%); 2725/205755 objects misplaced (1.324%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:52:23.745 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:52:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:52:23] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:52:23.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:23 smithi052 ceph-mon[187743]: pgmap v795: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 11783/205755 objects degraded (5.727%); 2725/205755 objects misplaced (1.324%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:52:23.790 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:23.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:23.897 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:23.999 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:24.539 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:52:24.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 65 16 71876 71860 69.0554 50.5625 0.0202363 0.0144761 2023-09-23T22:52:24.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1838922733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:24.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:24.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1838922733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:24.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:24.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1838922733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:24.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:25.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 66 16 72488 72472 68.5889 38.25 0.0791307 0.0145609 2023-09-23T22:52:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:25.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:25.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: pgmap v796: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 798 op/s; 11783/208569 objects degraded (5.649%); 2542/208569 objects misplaced (1.219%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:52:25.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:25.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:52:25.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: osdmap e281: 8 total, 8 up, 6 in 2023-09-23T22:52:25.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.730 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 11783/208569 objects degraded (5.649%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:25.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: pgmap v796: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 798 op/s; 11783/208569 objects degraded (5.649%); 2542/208569 objects misplaced (1.219%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: osdmap e281: 8 total, 8 up, 6 in 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.731 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 11783/208569 objects degraded (5.649%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:25.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:25.833 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:25.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: pgmap v796: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 798 op/s; 11783/208569 objects degraded (5.649%); 2542/208569 objects misplaced (1.219%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:52:25.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:25.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:52:25.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: osdmap e281: 8 total, 8 up, 6 in 2023-09-23T22:52:25.886 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.886 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.886 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.886 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: failed to encode map e281 with expected crc 2023-09-23T22:52:25.886 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 11783/208569 objects degraded (5.649%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:25.893 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:25.987 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:25.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:26.089 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:26.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:26.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 67 16 73275 73259 68.2994 49.1875 0.0349634 0.0146354 2023-09-23T22:52:26.726 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:26.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:26.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:26.833 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:26.863 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:26.881 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:26.945 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:26.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:27.050 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:27.079 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:27.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:27.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:27.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:27.337 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:27.364 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:27.411 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:27.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:27.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:27.482 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:52:27.507 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:52:27.516 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:27.533 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:52:27.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 68 16 73442 73426 67.4489 10.4375 0.928153 0.0146744 2023-09-23T22:52:27.562 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:27.578 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:52:27.621 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:52:27.666 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:52:27.698 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:52:27.700 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:27.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: pgmap v798: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 42 MiB/s wr, 665 op/s; 11783/212877 objects degraded (5.535%); 2448/212877 objects misplaced (1.150%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: osdmap e282: 8 total, 8 up, 6 in 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/527131936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:52:27.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:27.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:27.740 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:52:27.763 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:52:27.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: pgmap v798: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 42 MiB/s wr, 665 op/s; 11783/212877 objects degraded (5.535%); 2448/212877 objects misplaced (1.150%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: osdmap e282: 8 total, 8 up, 6 in 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/527131936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: pgmap v798: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 42 MiB/s wr, 665 op/s; 11783/212877 objects degraded (5.535%); 2448/212877 objects misplaced (1.150%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: osdmap e282: 8 total, 8 up, 6 in 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.814 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: failed to encode map e282 with expected crc 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/527131936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:52:27.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:27.870 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:27.972 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:27.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:28.078 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:28.151 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:28.245 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:28.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:28.355 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:28.430 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:28.545 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:52:28.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 69 16 74304 74288 67.2522 53.875 0.0173763 0.0148618 2023-09-23T22:52:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[180979]: osdmap e283: 8 total, 8 up, 6 in 2023-09-23T22:52:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[180979]: failed to encode map e283 with expected crc 2023-09-23T22:52:28.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[180979]: failed to encode map e283 with expected crc 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2058438404' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[187743]: osdmap e283: 8 total, 8 up, 6 in 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[187743]: failed to encode map e283 with expected crc 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[187743]: failed to encode map e283 with expected crc 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2058438404' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:52:28.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:28 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:52:28.805 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:52:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:28 smithi151 ceph-mon[159210]: osdmap e283: 8 total, 8 up, 6 in 2023-09-23T22:52:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:28 smithi151 ceph-mon[159210]: failed to encode map e283 with expected crc 2023-09-23T22:52:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:28 smithi151 ceph-mon[159210]: failed to encode map e283 with expected crc 2023-09-23T22:52:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2058438404' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:52:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:28 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:52:29.362 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:29.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:29.471 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:29.541 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:29.562 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 70 16 75140 75124 67.0379 52.25 0.0120392 0.0149122 2023-09-23T22:52:29.618 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:29.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:29.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:29.722 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:29.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[180979]: pgmap v801: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 764 op/s; 10054/219534 objects degraded (4.580%); 4043/219534 objects misplaced (1.842%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:52:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[180979]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:52:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[180979]: osdmap e284: 8 total, 8 up, 6 in 2023-09-23T22:52:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[180979]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[180979]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[187743]: pgmap v801: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 764 op/s; 10054/219534 objects degraded (4.580%); 4043/219534 objects misplaced (1.842%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:52:29.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[187743]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:52:29.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[187743]: osdmap e284: 8 total, 8 up, 6 in 2023-09-23T22:52:29.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[187743]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:29 smithi052 ceph-mon[187743]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.777 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:29.859 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:29.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:29 smithi151 ceph-mon[159210]: pgmap v801: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 764 op/s; 10054/219534 objects degraded (4.580%); 4043/219534 objects misplaced (1.842%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:52:29.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:29 smithi151 ceph-mon[159210]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:29 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:52:29.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:29 smithi151 ceph-mon[159210]: osdmap e284: 8 total, 8 up, 6 in 2023-09-23T22:52:29.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:29 smithi151 ceph-mon[159210]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:29 smithi151 ceph-mon[159210]: failed to encode map e284 with expected crc 2023-09-23T22:52:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:29.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:29.968 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:30.029 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:30.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:30.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:30.180 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:52:30.215 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:30.258 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:30.345 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:30.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:30.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:30.453 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:30.502 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:30.567 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 71 16 75929 75913 66.7845 49.3125 0.0242846 0.0149675 2023-09-23T22:52:30.569 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/219534 objects degraded (4.580%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:30.575 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:30.585 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:30.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:30.689 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164683 7.f 7:f08671e8:::benchmark_data_smithi052_277165_object72191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.714 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.376581+0000", 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.33410781600003, 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31209462100000002, 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164683 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.715 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376581+0000", 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376581+0000", 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376580+0000", 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:30.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376585+0000", 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6990000000000002e-06 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376593+0000", 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7700000000000007e-06 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376599+0000", 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5210000000000002e-06 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.379215+0000", 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026156759999999999 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for degraded object", 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.382234+0000", 2023-09-23T22:52:30.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030192750000000001 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.393352+0000", 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011118213 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.393458+0000", 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106211 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.393611+0000", 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015228099999999999 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683900+0000", 2023-09-23T22:52:30.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29028897799999998 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683910+0000", 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0356999999999999e-05 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.687485+0000", 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035752599999999998 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.687540+0000", 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4867999999999997e-05 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.688676+0000", 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011357870000000001 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164698 7.3 7:ca17f318:::benchmark_data_smithi052_277165_object72206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396189+0000", 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31449985500001, 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29793002099999999, 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164698 2023-09-23T22:52:30.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396189+0000", 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396189+0000", 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396188+0000", 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396194+0000", 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3960000000000003e-06 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.725 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396206+0000", 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1249e-05 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396214+0000", 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.055e-06 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396255+0000", 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1656999999999998e-05 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396379+0000", 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012328799999999999 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396492+0000", 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113832 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690191+0000", 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29369885699999998 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690228+0000", 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6956000000000003e-05 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694017+0000", 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037884310000000001 2023-09-23T22:52:30.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694080+0000", 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3461999999999996e-05 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694119+0000", 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9007000000000001e-05 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.729 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164699 7.3 7:c3dd2f2c:::benchmark_data_smithi052_277165_object72207:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396243+0000", 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31444590699999, 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.298988266, 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.730 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164699 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396243+0000", 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396243+0000", 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396242+0000", 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396247+0000", 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7140000000000001e-06 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396260+0000", 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2383e-05 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396266+0000", 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5810000000000002e-06 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686458+0000", 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.290191692 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686599+0000", 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000141361 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686735+0000", 2023-09-23T22:52:30.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013546200000000001 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694006+0000", 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072714729999999997 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694126+0000", 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120029 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.695156+0000", 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001029877 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.695195+0000", 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9167e-05 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.695231+0000", 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6157e-05 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164700 7.6 7:61d26359:::benchmark_data_smithi052_277165_object72208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396276+0000", 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31441298800002, 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.302929104, 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.737 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164700 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396276+0000", 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396276+0000", 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396275+0000", 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396279+0000", 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6339999999999999e-06 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396285+0000", 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4080000000000003e-06 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396289+0000", 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2329999999999998e-06 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683153+0000", 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.286864073 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683301+0000", 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014825400000000001 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683436+0000", 2023-09-23T22:52:30.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013492199999999999 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.692875+0000", 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0094391779999999995 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.692912+0000", 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6439999999999997e-05 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.698838+0000", 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059267909999999998 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.698866+0000", 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7628999999999999e-05 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699205+0000", 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033919400000000002 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164703 7.6 7:65560176:::benchmark_data_smithi052_277165_object72211:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.314396746, 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30325594700000003, 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.744 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164703 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396294+0000", 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.688e-06 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396297+0000", 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3079999999999999e-06 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396300+0000", 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8940000000000002e-06 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.688726+0000", 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29242600200000002 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.688873+0000", 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014661899999999999 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:52:30.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.689025+0000", 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015209500000000001 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699257+0000", 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010232194999999999 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699265+0000", 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2779999999999993e-06 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699302+0000", 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6424999999999998e-05 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699367+0000", 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5044999999999998e-05 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699548+0000", 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018163700000000001 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.751 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164706 7.3 7:c61dc2a5:::benchmark_data_smithi052_277165_object72214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31438654999999, 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30120987500000002, 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164706 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396304+0000", 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4750000000000001e-06 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396307+0000", 2023-09-23T22:52:30.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1080000000000001e-06 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396310+0000", 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4809999999999998e-06 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690272+0000", 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29396191900000002 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690388+0000", 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011600899999999999 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690498+0000", 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000109812 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694320+0000", 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0038220609999999999 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697402+0000", 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030819300000000001 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697423+0000", 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.101e-05 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697470+0000", 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7324999999999998e-05 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697512+0000", 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1996000000000002e-05 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164708 7.2 7:4a1cbce6:::benchmark_data_smithi052_277165_object72216:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396315+0000", 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31437435399999, 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29540316500000002, 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164708 2023-09-23T22:52:30.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396315+0000", 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396315+0000", 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396314+0000", 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396316+0000", 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0509999999999999e-06 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396322+0000", 2023-09-23T22:52:30.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4940000000000003e-06 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396326+0000", 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1779999999999996e-06 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396369+0000", 2023-09-23T22:52:30.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2608000000000003e-05 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396441+0000", 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2803000000000001e-05 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396505+0000", 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3466000000000005e-05 2023-09-23T22:52:30.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686583+0000", 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29007853 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686615+0000", 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1557999999999998e-05 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.691629+0000", 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050141949999999999 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.691677+0000", 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7290000000000003e-05 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.691718+0000", 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1288999999999997e-05 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:164704 7.8 e205/196 7:160987c2:::benchmark_data_smithi052_277165_object72212:head v 205'10330, mlcod=167'4547)", 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31399156499998, 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31435047700000002, 2023-09-23T22:52:30.766 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396702+0000", 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1159999999999998e-06 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683041+0000", 2023-09-23T22:52:30.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28633941800000001 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683049+0000", 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7819999999999994e-06 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683120+0000", 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0648000000000005e-05 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683150+0000", 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0161000000000001e-05 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711014+0000", 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.027863675000000001 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711048+0000", 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4217000000000001e-05 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:164705 7.0 e205/146 7:0e09ba28:::benchmark_data_smithi052_277165_object72213:head v 205'10244, mlcod=205'10242)", 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.31013260399999, 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30802488500000003, 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.771 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.772 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400563+0000", 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8009999999999998e-06 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.682993+0000", 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.282430454 2023-09-23T22:52:30.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683007+0000", 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3760999999999999e-05 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683134+0000", 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012747100000000001 2023-09-23T22:52:30.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683157+0000", 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2535e-05 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.708543+0000", 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.025385650999999999 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.708581+0000", 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8730999999999999e-05 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:140973 8.f 8:f33d72b0:::benchmark_data_smithi052_337357_object38522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 211.91558636900001, 2023-09-23T22:52:30.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.329060238, 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140973 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795101+0000", 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795109+0000", 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6429999999999995e-06 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795122+0000", 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3022e-05 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795130+0000", 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7400000000000004e-06 2023-09-23T22:52:30.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810865+0000", 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015735702000000001 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for rw locks", 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810975+0000", 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010935200000000001 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067039+0000", 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.256064085 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067161+0000", 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121952 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067334+0000", 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017327 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.083778+0000", 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016443607999999998 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.084031+0000", 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025363099999999998 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120168+0000", 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036136433000000003 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120220+0000", 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2271000000000001e-05 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.124163+0000", 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039430070000000001 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5823 9.c 9:36ec13c5:::benchmark_data_smithi052_402718_object5822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 69.092909191999993, 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22852289200000001, 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.784 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5823 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617779+0000", 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617800+0000", 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0721999999999999e-05 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617807+0000", 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.878e-06 2023-09-23T22:52:30.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617814+0000", 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8519999999999998e-06 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618047+0000", 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023353299999999999 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618201+0000", 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015408199999999999 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618341+0000", 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139765 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619519+0000", 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011783220000000001 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619626+0000", 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107017 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846195+0000", 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22656818400000001 2023-09-23T22:52:30.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846251+0000", 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6786000000000001e-05 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846303+0000", 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1248999999999997e-05 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.791 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5845 9.c 9:32588725:::benchmark_data_smithi052_402718_object5844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 69.085945769000006, 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.221823086, 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.792 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5845 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:52:30.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624742+0000", 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624765+0000", 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2909e-05 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624775+0000", 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.023e-05 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624783+0000", 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624865+0000", 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1631999999999994e-05 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625069+0000", 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000203685 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625237+0000", 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168426 2023-09-23T22:52:30.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625970+0000", 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00073263699999999996 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.626858+0000", 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088813800000000001 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846472+0000", 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21961366099999999 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846523+0000", 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0986000000000003e-05 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846566+0000", 2023-09-23T22:52:30.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3708000000000002e-05 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5878 9.c 9:34f7d49b:::benchmark_data_smithi052_402718_object5877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 69.071777628999996, 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.208484483, 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.799 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5878 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638910+0000", 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638928+0000", 2023-09-23T22:52:30.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7866000000000001e-05 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638941+0000", 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2618e-05 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638947+0000", 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5639999999999996e-06 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638986+0000", 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9031e-05 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639111+0000", 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124955 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:52:30.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639216+0000", 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105696 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639724+0000", 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00050770100000000005 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.640023+0000", 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029943599999999998 2023-09-23T22:52:30.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847306+0000", 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20728270800000001 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847359+0000", 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3319999999999998e-05 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847396+0000", 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6473999999999997e-05 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:11623 9.8 9:16f5e27e:::benchmark_data_smithi052_402718_object11622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:24.401707+0000", 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 66.308981794000005, 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17655863099999999, 2023-09-23T22:52:30.806 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 11623 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.401707+0000", 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.401707+0000", 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.401707+0000", 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.401729+0000", 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2571000000000001e-05 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.401737+0000", 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6629999999999993e-06 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.401744+0000", 2023-09-23T22:52:30.809 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7789999999999998e-06 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.524196+0000", 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122451979 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.524320+0000", 2023-09-23T22:52:30.810 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012405700000000001 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.524469+0000", 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014964399999999999 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.526086+0000", 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001616588 2023-09-23T22:52:30.811 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.526105+0000", 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9194999999999998e-05 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.578189+0000", 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.052083421999999997 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.578230+0000", 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1295999999999999e-05 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:24.578266+0000", 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6007000000000003e-05 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.813 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13481 9.8 9:1f01482c:::benchmark_data_smithi052_402718_object13480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 65.374685839999998, 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.322004134, 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13481 2023-09-23T22:52:30.814 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.815 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336002+0000", 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336029+0000", 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6545000000000001e-05 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336044+0000", 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5444999999999999e-05 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336063+0000", 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8729e-05 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336119+0000", 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6112000000000001e-05 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336242+0000", 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012232500000000001 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336383+0000", 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014170900000000001 2023-09-23T22:52:30.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337199+0000", 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00081607499999999998 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337730+0000", 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00053117300000000002 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657923+0000", 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32019289400000001 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657967+0000", 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3399999999999998e-05 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.658007+0000", 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0534999999999997e-05 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13489 9.c 9:33ad037a:::benchmark_data_smithi052_402718_object13488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 65.373611155000006, 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32053898800000002, 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.821 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13489 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337077+0000", 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.823 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337098+0000", 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1191000000000001e-05 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337107+0000", 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2179999999999999e-06 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337112+0000", 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337159+0000", 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6615e-05 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337308+0000", 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014894100000000001 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337456+0000", 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148066 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338621+0000", 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011655420000000001 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.339454+0000", 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083201599999999998 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657516+0000", 2023-09-23T22:52:30.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.318062489 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657570+0000", 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4024000000000003e-05 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657617+0000", 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6826000000000002e-05 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.828 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:56102 9.a 9:5f3f5d6a:::benchmark_data_smithi052_402718_object56101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:04.984754+0000", 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 25.725934883000001, 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19523168499999999, 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.829 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 56102 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984754+0000", 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.830 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984754+0000", 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984752+0000", 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984784+0000", 2023-09-23T22:52:30.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1622000000000003e-05 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984802+0000", 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8253e-05 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984813+0000", 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1216999999999999e-05 2023-09-23T22:52:30.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.984879+0000", 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6168999999999996e-05 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.985082+0000", 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000202306 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.985268+0000", 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018585300000000001 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.987618+0000", 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023504889999999999 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.989147+0000", 2023-09-23T22:52:30.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001529117 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:04.989514+0000", 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000367332 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:05.178141+0000", 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18862638000000001 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:05.178206+0000", 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5374000000000001e-05 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:05.179986+0000", 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001779589 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59534 9.a 9:56ae8077:::benchmark_data_smithi052_402718_object59533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 21.616386756000001, 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20214791800000001, 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59534 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.837 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094300+0000", 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094331+0000", 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0930999999999999e-05 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094353+0000", 2023-09-23T22:52:30.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2340999999999998e-05 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094362+0000", 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2519999999999999e-06 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094415+0000", 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2997000000000001e-05 2023-09-23T22:52:30.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094637+0000", 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000222208 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094844+0000", 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000207631 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097382+0000", 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025371360000000002 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097400+0000", 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8714000000000001e-05 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.098411+0000", 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001011131 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294679+0000", 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19626732399999999 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294738+0000", 2023-09-23T22:52:30.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8801000000000002e-05 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296450+0000", 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001712618 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.844 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59542 9.a 9:51ad1323:::benchmark_data_smithi052_402718_object59541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 21.611432566000001, 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19894724899999999, 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59542 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.845 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099254+0000", 2023-09-23T22:52:30.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099282+0000", 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7971999999999999e-05 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099301+0000", 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8593e-05 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099311+0000", 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0074999999999999e-05 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099389+0000", 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7689999999999996e-05 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099595+0000", 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020635099999999999 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099809+0000", 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021421 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.102469+0000", 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002660314 2023-09-23T22:52:30.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104681+0000", 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022113889999999998 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104831+0000", 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015015699999999999 2023-09-23T22:52:30.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296512+0000", 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19168089399999999 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296580+0000", 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8103999999999997e-05 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298204+0000", 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016236810000000001 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59552 9.a 9:5e265662:::benchmark_data_smithi052_402718_object59551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.114161+0000", 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 21.596528347, 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.185672001, 2023-09-23T22:52:30.852 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59552 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114161+0000", 2023-09-23T22:52:30.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114161+0000", 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114159+0000", 2023-09-23T22:52:30.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114215+0000", 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6818999999999999e-05 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114235+0000", 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9661e-05 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114246+0000", 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0661e-05 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114296+0000", 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0149999999999999e-05 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:52:30.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114427+0000", 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013092799999999999 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.114549+0000", 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121889 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.116778+0000", 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022292990000000001 2023-09-23T22:52:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.118306+0000", 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015280960000000001 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.118347+0000", 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0670999999999997e-05 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298285+0000", 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17993811500000001 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298342+0000", 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6813e-05 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:52:30.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.299833+0000", 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014909890000000001 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:52:30.860 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:52:30.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/219534 objects degraded (4.580%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:30.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/219534 objects degraded (4.580%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:30.862 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:30.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:30.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:30.963 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:31.051 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:31.566 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 72 16 76687 76671 66.515 47.375 0.0124595 0.0150282 2023-09-23T22:52:31.697 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:31.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:31.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:31.804 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:31.841 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:31 smithi052 ceph-mon[180979]: pgmap v803: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 726 op/s; 10054/221340 objects degraded (4.542%); 4043/221340 objects misplaced (1.827%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:52:31.842 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:31 smithi052 ceph-mon[187743]: pgmap v803: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 726 op/s; 10054/221340 objects degraded (4.542%); 4043/221340 objects misplaced (1.827%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:52:31.850 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:52:31.889 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:31 smithi151 ceph-mon[159210]: pgmap v803: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 726 op/s; 10054/221340 objects degraded (4.542%); 4043/221340 objects misplaced (1.827%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:52:31.901 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:32.012 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:32.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:32.108 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:32.113 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:52:32.117 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:32.204 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:32.394 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:32.574 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 73 16 77414 77398 66.2196 45.4375 0.0156171 0.0150962 2023-09-23T22:52:32.759 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:32.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:32.865 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:32.909 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:33.036 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:33.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:33.146 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:33.205 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:33.297 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:33.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:33.346 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:52:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:52:33] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:52:33.401 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:33.531 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:33.574 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 74 16 78229 78213 66.0132 50.9375 0.0264125 0.0151431 2023-09-23T22:52:33.586 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:33 smithi151 ceph-mon[159210]: pgmap v804: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 50 MiB/s wr, 797 op/s; 10054/227238 objects degraded (4.424%); 3817/227238 objects misplaced (1.680%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:52:33.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:33 smithi052 ceph-mon[180979]: pgmap v804: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 50 MiB/s wr, 797 op/s; 10054/227238 objects degraded (4.424%); 3817/227238 objects misplaced (1.680%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:52:33.748 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:33 smithi052 ceph-mon[187743]: pgmap v804: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 50 MiB/s wr, 797 op/s; 10054/227238 objects degraded (4.424%); 3817/227238 objects misplaced (1.680%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:52:34.399 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:34.420 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:34.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:34.529 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:34.574 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 75 16 78999 78983 65.7748 48.125 0.00669807 0.0151962 2023-09-23T22:52:34.593 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:34.614 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:52:34.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/332237206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:34.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:34.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:34.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:34.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/332237206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:34.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:34.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:34.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/332237206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:52:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:34.960 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:34.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:35.064 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:35.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:35.574 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 76 16 79733 79717 65.513 45.875 0.0121891 0.0152581 2023-09-23T22:52:35.617 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:35.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:35.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: pgmap v805: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 10054/230211 objects degraded (4.367%); 3817/230211 objects misplaced (1.658%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:52:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: osdmap e285: 8 total, 8 up, 6 in 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/230211 objects degraded (4.367%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:35.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: pgmap v805: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 10054/230211 objects degraded (4.367%); 3817/230211 objects misplaced (1.658%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: osdmap e285: 8 total, 8 up, 6 in 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/230211 objects degraded (4.367%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:35.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: pgmap v805: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 10054/230211 objects degraded (4.367%); 3817/230211 objects misplaced (1.658%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: osdmap e285: 8 total, 8 up, 6 in 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: failed to encode map e285 with expected crc 2023-09-23T22:52:35.721 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/230211 objects degraded (4.367%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:52:35.727 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:35.780 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:35.982 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:52:35.982 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:52:35.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:80038 9.c 9:3eb68ba7:::benchmark_data_smithi052_402718_object80037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e285)", 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:35.967163+0000", 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.011808638, 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011836595, 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 80038 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.983 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967163+0000", 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967163+0000", 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967162+0000", 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4419999999999998e-05 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967206+0000", 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5628999999999999e-05 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:35.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967222+0000", 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8099999999999998e-06 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967230+0000", 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7254000000000004e-05 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967297+0000", 2023-09-23T22:52:35.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012875300000000001 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967426+0000", 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00022503399999999999 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.967651+0000", 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056720659999999999 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.973323+0000", 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002438703 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.975761+0000", 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0085978809999999999 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:35.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:80041 9.c 9:36091df1:::benchmark_data_smithi052_402718_object80040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e285)", 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:35.969278+0000", 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0096944609999999997, 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0097830999999999994, 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:52:35.989 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 80041 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.969278+0000", 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.969278+0000", 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:52:35.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.969276+0000", 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8801000000000007e-05 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.969355+0000", 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.154e-05 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.969376+0000", 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5920000000000001e-06 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.969385+0000", 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097453000000000004 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.970359+0000", 2023-09-23T22:52:35.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000159269 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.970519+0000", 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021991600000000001 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.970738+0000", 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050341919999999998 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:35.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:35.975773+0000", 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064950470000000003 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 2 2023-09-23T22:52:35.994 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:52:36.035 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:36.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:36.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:36.147 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:36.225 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:36.311 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:36.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:36.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:36.472 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:36.575 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 77 16 80404 80388 65.2069 41.9375 0.11779 0.015329 2023-09-23T22:52:36.639 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:52:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2408602934' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:52:36.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:52:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2408602934' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:52:36.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:52:36.773 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:36.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:36.879 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:36.903 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2408602934' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:52:36.904 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:52:36.937 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:52:36.958 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:37.355 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:37.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:37.466 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:37.529 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:52:37.529 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:52:37.573 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:37.574 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 78 16 81158 81142 64.9752 47.125 0.0217187 0.0153833 2023-09-23T22:52:37.577 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:37.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: pgmap v807: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 39 MiB/s wr, 620 op/s; 10054/234438 objects degraded (4.289%); 3706/234438 objects misplaced (1.581%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T22:52:37.630 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:52:37.630 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: osdmap e286: 8 total, 8 up, 6 in 2023-09-23T22:52:37.630 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.630 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.630 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.630 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[180979]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.765 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:37.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:37.841 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:52:37.880 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: pgmap v807: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 39 MiB/s wr, 620 op/s; 10054/234438 objects degraded (4.289%); 3706/234438 objects misplaced (1.581%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: osdmap e286: 8 total, 8 up, 6 in 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.883 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:37 smithi151 ceph-mon[159210]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: pgmap v807: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 39 MiB/s wr, 620 op/s; 10054/234438 objects degraded (4.289%); 3706/234438 objects misplaced (1.581%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T22:52:37.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:52:37.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: osdmap e286: 8 total, 8 up, 6 in 2023-09-23T22:52:37.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: failed to encode map e286 with expected crc 2023-09-23T22:52:37.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:37 smithi052 ceph-mon[187743]: failed to encode map e286 with expected crc 2023-09-23T22:52:38.018 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:38.076 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:52:38.137 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:52:38.169 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:38.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:38.206 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:52:38.269 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:52:38.285 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:38.310 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:52:38.362 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:52:38.400 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:38.407 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:38.409 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:52:38.446 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.467 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:52:38.575 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 79 16 81833 81817 64.6868 42.1875 0.0142905 0.0154513 2023-09-23T22:52:38.677 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:38.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:38.756 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:38 smithi052 ceph-mon[180979]: failed to encode map e286 with expected crc 2023-09-23T22:52:38.756 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3908613026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:38.756 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:38 smithi052 ceph-mon[187743]: failed to encode map e286 with expected crc 2023-09-23T22:52:38.757 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3908613026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:38.780 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:38.860 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:38.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:38 smithi151 ceph-mon[159210]: failed to encode map e286 with expected crc 2023-09-23T22:52:38.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3908613026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:39.427 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:39.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:39.543 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:39.556 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:39.575 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:52:39.575338+0000 min lat: 0.00119123 max lat: 3.67384 avg lat: 0.0155219 2023-09-23T22:52:39.575 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:52:39.575 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 80 16 82507 82491 64.4049 42.125 0.0229257 0.0155219 2023-09-23T22:52:39.613 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:52:39.614 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:52:39.634 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:39 smithi151 ceph-mon[159210]: pgmap v809: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 52 MiB/s wr, 839 op/s; 10054/241482 objects degraded (4.163%); 3596/241482 objects misplaced (1.489%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:52:39.652 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:39.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:39 smithi052 ceph-mon[180979]: pgmap v809: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 52 MiB/s wr, 839 op/s; 10054/241482 objects degraded (4.163%); 3596/241482 objects misplaced (1.489%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:52:39.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:39 smithi052 ceph-mon[187743]: pgmap v809: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 52 MiB/s wr, 839 op/s; 10054/241482 objects degraded (4.163%); 3596/241482 objects misplaced (1.489%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:52:40.144 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:40.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:40.250 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:40.280 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:40.549 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:40.556 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:52:40.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:40.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:40.578 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 81 16 83198 83182 64.1408 43.1875 0.0160047 0.0155864 2023-09-23T22:52:40.660 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:40.735 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/241482 objects degraded (4.163%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:40.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/241482 objects degraded (4.163%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:40.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/241482 objects degraded (4.163%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:41.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:41.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:41.247 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:41.285 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:41.356 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:41.578 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 82 16 84030 84014 63.9922 52 0.0152572 0.015621 2023-09-23T22:52:41.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:52:41.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:41.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:41.735 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:41.779 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:41.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:41 smithi151 ceph-mon[159210]: pgmap v810: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 42 MiB/s wr, 673 op/s; 10054/243408 objects degraded (4.131%); 3596/243408 objects misplaced (1.477%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:52:41.893 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:41 smithi052 ceph-mon[180979]: pgmap v810: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 42 MiB/s wr, 673 op/s; 10054/243408 objects degraded (4.131%); 3596/243408 objects misplaced (1.477%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:52:41.893 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:41 smithi052 ceph-mon[187743]: pgmap v810: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 42 MiB/s wr, 673 op/s; 10054/243408 objects degraded (4.131%); 3596/243408 objects misplaced (1.477%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:52:41.920 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:41.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:41.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:42.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:42.112 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:42.292 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:42.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:42.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:42.397 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:42.445 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:42.498 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:52:42.506 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:52:42.579 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 83 16 84689 84673 63.7176 41.1875 0.0260522 0.0156886 2023-09-23T22:52:42.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:42 smithi151 ceph-mon[159210]: pgmap v811: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 10054/249432 objects degraded (4.031%); 3376/249432 objects misplaced (1.353%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T22:52:42.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:42 smithi151 ceph-mon[159210]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:52:42.725 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:42.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:42.827 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:42.901 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:42.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:42 smithi052 ceph-mon[180979]: pgmap v811: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 10054/249432 objects degraded (4.031%); 3376/249432 objects misplaced (1.353%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T22:52:43.000 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:42 smithi052 ceph-mon[180979]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:52:43.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:42 smithi052 ceph-mon[187743]: pgmap v811: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 10054/249432 objects degraded (4.031%); 3376/249432 objects misplaced (1.353%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T22:52:43.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:42 smithi052 ceph-mon[187743]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:52:43.064 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:43.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:43.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:43.183 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:43.256 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:43.374 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:43.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:43.420 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:52:43.420 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:52:43.483 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:43.577 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:43.578 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 84 16 85524 85508 63.5803 52.1875 0.0165417 0.0157225 2023-09-23T22:52:43.596 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:43.626 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":811,"stamp":"2023-09-23T22:52:42.366140+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5447948192,"num_objects":83144,"num_object_clones":0,"num_object_copies":249432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":3376,"num_objects_unfound":0,"num_objects_dirty":83144,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":83380,"num_write_kb":5322125,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3404,"num_bytes_recovered":221128758,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83014,"ondisk_log_size":83014,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17304752,"kb_used_data":15322816,"kb_used_omap":121,"kb_used_meta":1981766,"kb_avail":545166160,"statfs":{"total":575970213888,"available":558250147840,"internally_reserved":0,"allocated":15690563584,"data_stored":15685730975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2029328501},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,3,6,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":419889152,"num_objects":6407,"num_object_clones":0,"num_object_copies":19221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-441,"num_objects_unfound":0,"num_objects_dirty":6407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6407,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":441,"num_bytes_recovered":28901376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004377"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":283,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617776+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:52:34.617776+0000","last_peered":"2023-09-23T22:52:34.617776+0000","last_clean":"2023-09-23T22:52:34.617776+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:52:34.617776+0000","last_undegraded":"2023-09-23T22:52:34.617776+0000","last_fullsized":"2023-09-23T22:52:34.617776+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":82,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621446+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:52:34.621446+0000","last_peered":"2023-09-23T22:52:34.621446+0000","last_clean":"2023-09-23T22:52:34.621446+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:52:34.621446+0000","last_undegraded":"2023-09-23T22:52:34.621446+0000","last_fullsized":"2023-09-23T22:52:34.621446+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617559+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:52:34.617559+0000","last_peered":"2023-09-23T22:52:34.617559+0000","last_clean":"2023-09-23T22:52:34.617559+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:52:34.617559+0000","last_undegraded":"2023-09-23T22:52:34.617559+0000","last_fullsized":"2023-09-23T22:52:34.617559+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":264,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612392+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:52:34.612392+0000","last_peered":"2023-09-23T22:52:34.612392+0000","last_clean":"2023-09-23T22:52:34.612392+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:52:34.612392+0000","last_undegraded":"2023-09-23T22:52:34.612392+0000","last_fullsized":"2023-09-23T22:52:34.612392+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":268,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.613799+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:52:34.613799+0000","last_peered":"2023-09-23T22:52:34.613799+0000","last_clean":"2023-09-23T22:52:34.613799+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:52:34.613799+0000","last_undegraded":"2023-09-23T22:52:34.613799+0000","last_fullsized":"2023-09-23T22:52:34.613799+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.608017+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:52:34.608017+0000","last_peered":"2023-09-23T22:52:34.608017+0000","last_clean":"2023-09-23T22:52:34.608017+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:52:34.608017+0000","last_undegraded":"2023-09-23T22:52:34.608017+0000","last_fullsized":"2023-09-23T22:52:34.608017+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":195,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612248+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:52:34.612248+0000","last_peered":"2023-09-23T22:52:34.612248+0000","last_clean":"2023-09-23T22:52:34.612248+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:52:34.612248+0000","last_undegraded":"2023-09-23T22:52:34.612248+0000","last_fullsized":"2023-09-23T22:52:34.612248+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":147,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595557+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:52:34.595557+0000","last_peered":"2023-09-23T22:52:34.595557+0000","last_clean":"2023-09-23T22:52:34.595557+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:52:34.595557+0000","last_undegraded":"2023-09-23T22:52:34.595557+0000","last_fullsized":"2023-09-23T22:52:34.595557+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":26,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595418+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:52:34.595418+0000","last_peered":"2023-09-23T22:52:34.595418+0000","last_clean":"2023-09-23T22:52:34.595418+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:52:34.595418+0000","last_undegraded":"2023-09-23T22:52:34.595418+0000","last_fullsized":"2023-09-23T22:52:34.595418+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.608027+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:52:34.608027+0000","last_peered":"2023-09-23T22:52:34.608027+0000","last_clean":"2023-09-23T22:52:34.608027+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:52:34.608027+0000","last_undegraded":"2023-09-23T22:52:34.608027+0000","last_fullsized":"2023-09-23T22:52:34.608027+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":44,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621307+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:52:34.621307+0000","last_peered":"2023-09-23T22:52:34.621307+0000","last_clean":"2023-09-23T22:52:34.621307+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:52:34.621307+0000","last_undegraded":"2023-09-23T22:52:34.621307+0000","last_fullsized":"2023-09-23T22:52:34.621307+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607856+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:52:34.607856+0000","last_peered":"2023-09-23T22:52:34.607856+0000","last_clean":"2023-09-23T22:52:34.607856+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:52:34.607856+0000","last_undegraded":"2023-09-23T22:52:34.607856+0000","last_fullsized":"2023-09-23T22:52:34.607856+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":305,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621662+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:52:34.621662+0000","last_peered":"2023-09-23T22:52:34.621662+0000","last_clean":"2023-09-23T22:52:34.621662+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:52:34.621662+0000","last_undegraded":"2023-09-23T22:52:34.621662+0000","last_fullsized":"2023-09-23T22:52:34.621662+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618020+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:52:34.618020+0000","last_peered":"2023-09-23T22:52:34.618020+0000","last_clean":"2023-09-23T22:52:34.618020+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:52:34.618020+0000","last_undegraded":"2023-09-23T22:52:34.618020+0000","last_fullsized":"2023-09-23T22:52:34.618020+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":144,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618149+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:52:34.618149+0000","last_peered":"2023-09-23T22:52:34.618149+0000","last_clean":"2023-09-23T22:52:34.618149+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:52:34.618149+0000","last_undegraded":"2023-09-23T22:52:34.618149+0000","last_fullsized":"2023-09-23T22:52:34.618149+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":232,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595854+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:52:34.595854+0000","last_peered":"2023-09-23T22:52:34.595854+0000","last_clean":"2023-09-23T22:52:34.595854+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:52:34.595854+0000","last_undegraded":"2023-09-23T22:52:34.595854+0000","last_fullsized":"2023-09-23T22:52:34.595854+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.611667+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:52:34.611667+0000","last_peered":"2023-09-23T22:52:34.611667+0000","last_clean":"2023-09-23T22:52:34.611667+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:52:34.611667+0000","last_undegraded":"2023-09-23T22:52:34.611667+0000","last_fullsized":"2023-09-23T22:52:34.611667+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":209,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607943+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:52:34.607943+0000","last_peered":"2023-09-23T22:52:34.607943+0000","last_clean":"2023-09-23T22:52:34.607943+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:52:34.607943+0000","last_undegraded":"2023-09-23T22:52:34.607943+0000","last_fullsized":"2023-09-23T22:52:34.607943+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617652+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:52:34.617652+0000","last_peered":"2023-09-23T22:52:34.617652+0000","last_clean":"2023-09-23T22:52:34.617652+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:52:34.617652+0000","last_undegraded":"2023-09-23T22:52:34.617652+0000","last_fullsized":"2023-09-23T22:52:34.617652+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612463+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:52:34.612463+0000","last_peered":"2023-09-23T22:52:34.612463+0000","last_clean":"2023-09-23T22:52:34.612463+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:52:34.612463+0000","last_undegraded":"2023-09-23T22:52:34.612463+0000","last_fullsized":"2023-09-23T22:52:34.612463+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":330,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607242+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:52:34.607242+0000","last_peered":"2023-09-23T22:52:34.607242+0000","last_clean":"2023-09-23T22:52:34.607242+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:52:34.607242+0000","last_undegraded":"2023-09-23T22:52:34.607242+0000","last_fullsized":"2023-09-23T22:52:34.607242+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":212,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.598683+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:52:34.598683+0000","last_peered":"2023-09-23T22:52:34.598683+0000","last_clean":"2023-09-23T22:52:34.598683+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:52:34.598683+0000","last_undegraded":"2023-09-23T22:52:34.598683+0000","last_fullsized":"2023-09-23T22:52:34.598683+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"286'5028","reported_seq":5059,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:38.072500+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:52:38.072500+0000","last_peered":"2023-09-23T22:52:38.072500+0000","last_clean":"2023-09-23T22:52:38.072500+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:52:38.072500+0000","last_undegraded":"2023-09-23T22:52:38.072500+0000","last_fullsized":"2023-09-23T22:52:38.072500+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5028,"log_dups_size":0,"ondisk_log_size":5028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329515008,"num_objects":5028,"num_object_clones":0,"num_object_copies":15084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5028,"num_write_kb":321792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":155,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618681+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:52:34.618681+0000","last_peered":"2023-09-23T22:52:34.618681+0000","last_clean":"2023-09-23T22:52:34.618681+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:52:34.618681+0000","last_undegraded":"2023-09-23T22:52:34.618681+0000","last_fullsized":"2023-09-23T22:52:34.618681+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":250,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607297+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:52:34.607297+0000","last_peered":"2023-09-23T22:52:34.607297+0000","last_clean":"2023-09-23T22:52:34.607297+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:52:34.607297+0000","last_undegraded":"2023-09-23T22:52:34.607297+0000","last_fullsized":"2023-09-23T22:52:34.607297+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"286'5402","reported_seq":5438,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:41.596279+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:52:41.596279+0000","last_peered":"2023-09-23T22:52:41.596279+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:52:41.596279+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354025472,"num_objects":5402,"num_object_clones":0,"num_object_copies":16206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5402,"num_write_kb":345728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1661},{"shards":"1,4","objects":3741}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":165,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618725+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:52:34.618725+0000","last_peered":"2023-09-23T22:52:34.618725+0000","last_clean":"2023-09-23T22:52:34.618725+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:52:34.618725+0000","last_undegraded":"2023-09-23T22:52:34.618725+0000","last_fullsized":"2023-09-23T22:52:34.618725+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":275,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617553+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:52:34.617553+0000","last_peered":"2023-09-23T22:52:34.617553+0000","last_clean":"2023-09-23T22:52:34.617553+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:52:34.617553+0000","last_undegraded":"2023-09-23T22:52:34.617553+0000","last_fullsized":"2023-09-23T22:52:34.617553+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"286'5259","reported_seq":5296,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:39.837803+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:52:39.837803+0000","last_peered":"2023-09-23T22:52:39.837803+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:52:39.837803+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5259,"log_dups_size":0,"ondisk_log_size":5259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344653824,"num_objects":5259,"num_object_clones":0,"num_object_copies":15777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5259,"num_write_kb":336576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1771},{"shards":"2,5","objects":3488}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":109,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607862+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:52:34.607862+0000","last_peered":"2023-09-23T22:52:34.607862+0000","last_clean":"2023-09-23T22:52:34.607862+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:52:34.607862+0000","last_undegraded":"2023-09-23T22:52:34.607862+0000","last_fullsized":"2023-09-23T22:52:34.607862+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":278,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607614+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:52:34.607614+0000","last_peered":"2023-09-23T22:52:34.607614+0000","last_clean":"2023-09-23T22:52:34.607614+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:52:34.607614+0000","last_undegraded":"2023-09-23T22:52:34.607614+0000","last_fullsized":"2023-09-23T22:52:34.607614+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"286'5199","reported_seq":5230,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:39.866220+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:52:39.866220+0000","last_peered":"2023-09-23T22:52:39.866220+0000","last_clean":"2023-09-23T22:52:39.866220+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:52:39.866220+0000","last_undegraded":"2023-09-23T22:52:39.866220+0000","last_fullsized":"2023-09-23T22:52:39.866220+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340721664,"num_objects":5199,"num_object_clones":0,"num_object_copies":15597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5199,"num_write_kb":332736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":81,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607641+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:52:34.607641+0000","last_peered":"2023-09-23T22:52:34.607641+0000","last_clean":"2023-09-23T22:52:34.607641+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:52:34.607641+0000","last_undegraded":"2023-09-23T22:52:34.607641+0000","last_fullsized":"2023-09-23T22:52:34.607641+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":179,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612261+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:52:34.612261+0000","last_peered":"2023-09-23T22:52:34.612261+0000","last_clean":"2023-09-23T22:52:34.612261+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:52:34.612261+0000","last_undegraded":"2023-09-23T22:52:34.612261+0000","last_fullsized":"2023-09-23T22:52:34.612261+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"286'5137","reported_seq":7854,"reported_epoch":286,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:52:40.928310+0000","last_change":"2023-09-23T22:51:35.553760+0000","last_active":"2023-09-23T22:52:40.928310+0000","last_peered":"2023-09-23T22:52:40.928310+0000","last_clean":"2023-09-23T22:51:33.374358+0000","last_became_active":"2023-09-23T22:51:35.518836+0000","last_became_peered":"2023-09-23T22:51:35.518836+0000","last_unstale":"2023-09-23T22:52:40.928310+0000","last_undegraded":"2023-09-23T22:52:40.928310+0000","last_fullsized":"2023-09-23T22:51:35.415989+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":335,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1339,"num_bytes_recovered":87752704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":335},{"shards":"4,6","objects":4802}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":51,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.615205+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:52:34.615205+0000","last_peered":"2023-09-23T22:52:34.615205+0000","last_clean":"2023-09-23T22:52:34.615205+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:52:34.615205+0000","last_undegraded":"2023-09-23T22:52:34.615205+0000","last_fullsized":"2023-09-23T22:52:34.615205+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":259,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621571+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:52:34.621571+0000","last_peered":"2023-09-23T22:52:34.621571+0000","last_clean":"2023-09-23T22:52:34.621571+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:52:34.621571+0000","last_undegraded":"2023-09-23T22:52:34.621571+0000","last_fullsized":"2023-09-23T22:52:34.621571+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"286'5138","reported_seq":5168,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:40.895444+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:52:40.895444+0000","last_peered":"2023-09-23T22:52:40.895444+0000","last_clean":"2023-09-23T22:52:40.895444+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:52:40.895444+0000","last_undegraded":"2023-09-23T22:52:40.895444+0000","last_fullsized":"2023-09-23T22:52:40.895444+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5138,"log_dups_size":0,"ondisk_log_size":5138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336723968,"num_objects":5138,"num_object_clones":0,"num_object_copies":15414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5138,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":270,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607442+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:52:34.607442+0000","last_peered":"2023-09-23T22:52:34.607442+0000","last_clean":"2023-09-23T22:52:34.607442+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:52:34.607442+0000","last_undegraded":"2023-09-23T22:52:34.607442+0000","last_fullsized":"2023-09-23T22:52:34.607442+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":281,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621111+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:52:34.621111+0000","last_peered":"2023-09-23T22:52:34.621111+0000","last_clean":"2023-09-23T22:52:34.621111+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:52:34.621111+0000","last_undegraded":"2023-09-23T22:52:34.621111+0000","last_fullsized":"2023-09-23T22:52:34.621111+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"286'5271","reported_seq":5307,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:41.846351+0000","last_change":"2023-09-23T22:51:35.518569+0000","last_active":"2023-09-23T22:52:41.846351+0000","last_peered":"2023-09-23T22:52:41.846351+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:52:41.846351+0000","last_undegraded":"2023-09-23T22:51:35.445687+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":271,"log_start":"268'100","ondisk_log_start":"268'100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5171,"log_dups_size":0,"ondisk_log_size":5171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:39.333773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345440256,"num_objects":5271,"num_object_clones":0,"num_object_copies":15813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5271,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1665},{"shards":"1,5","objects":3606}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":294,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617390+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:52:34.617390+0000","last_peered":"2023-09-23T22:52:34.617390+0000","last_clean":"2023-09-23T22:52:34.617390+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:52:34.617390+0000","last_undegraded":"2023-09-23T22:52:34.617390+0000","last_fullsized":"2023-09-23T22:52:34.617390+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":269,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618800+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:52:34.618800+0000","last_peered":"2023-09-23T22:52:34.618800+0000","last_clean":"2023-09-23T22:52:34.618800+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:52:34.618800+0000","last_undegraded":"2023-09-23T22:52:34.618800+0000","last_fullsized":"2023-09-23T22:52:34.618800+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"286'5083","reported_seq":5120,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:39.850908+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:52:39.850908+0000","last_peered":"2023-09-23T22:52:39.850908+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:52:39.850908+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333119488,"num_objects":5083,"num_object_clones":0,"num_object_copies":15249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5083,"num_write_kb":325312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":3468},{"shards":"1,2,3","objects":1615}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":214,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617136+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:52:34.617136+0000","last_peered":"2023-09-23T22:52:34.617136+0000","last_clean":"2023-09-23T22:52:34.617136+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:52:34.617136+0000","last_undegraded":"2023-09-23T22:52:34.617136+0000","last_fullsized":"2023-09-23T22:52:34.617136+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":223,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.598797+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:52:34.598797+0000","last_peered":"2023-09-23T22:52:34.598797+0000","last_clean":"2023-09-23T22:52:34.598797+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:52:34.598797+0000","last_undegraded":"2023-09-23T22:52:34.598797+0000","last_fullsized":"2023-09-23T22:52:34.598797+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"286'5351","reported_seq":5387,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:41.902513+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:52:41.902513+0000","last_peered":"2023-09-23T22:52:41.902513+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:52:41.902513+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"268'200","ondisk_log_start":"268'200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5151,"log_dups_size":0,"ondisk_log_size":5151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350683136,"num_objects":5351,"num_object_clones":0,"num_object_copies":16053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5351,"num_write_kb":342464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1630},{"shards":"1,5","objects":3721}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":323,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612537+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:52:34.612537+0000","last_peered":"2023-09-23T22:52:34.612537+0000","last_clean":"2023-09-23T22:52:34.612537+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:52:34.612537+0000","last_undegraded":"2023-09-23T22:52:34.612537+0000","last_fullsized":"2023-09-23T22:52:34.612537+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.596305+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:52:34.596305+0000","last_peered":"2023-09-23T22:52:34.596305+0000","last_clean":"2023-09-23T22:52:34.596305+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:52:34.596305+0000","last_undegraded":"2023-09-23T22:52:34.596305+0000","last_fullsized":"2023-09-23T22:52:34.596305+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617697+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:52:34.617697+0000","last_peered":"2023-09-23T22:52:34.617697+0000","last_clean":"2023-09-23T22:52:34.617697+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:52:34.617697+0000","last_undegraded":"2023-09-23T22:52:34.617697+0000","last_fullsized":"2023-09-23T22:52:34.617697+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"286'5114","reported_seq":8547,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:38.088018+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:52:38.088018+0000","last_peered":"2023-09-23T22:52:38.088018+0000","last_clean":"2023-09-23T22:52:38.088018+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:52:38.088018+0000","last_undegraded":"2023-09-23T22:52:38.088018+0000","last_fullsized":"2023-09-23T22:52:38.088018+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5114,"log_dups_size":0,"ondisk_log_size":5114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335151104,"num_objects":5114,"num_object_clones":0,"num_object_copies":15342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5114,"num_write_kb":327296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":120,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607520+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:52:34.607520+0000","last_peered":"2023-09-23T22:52:34.607520+0000","last_clean":"2023-09-23T22:52:34.607520+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:52:34.607520+0000","last_undegraded":"2023-09-23T22:52:34.607520+0000","last_fullsized":"2023-09-23T22:52:34.607520+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":282,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618283+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:52:34.618283+0000","last_peered":"2023-09-23T22:52:34.618283+0000","last_clean":"2023-09-23T22:52:34.618283+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:52:34.618283+0000","last_undegraded":"2023-09-23T22:52:34.618283+0000","last_fullsized":"2023-09-23T22:52:34.618283+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":212,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595921+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:52:34.595921+0000","last_peered":"2023-09-23T22:52:34.595921+0000","last_clean":"2023-09-23T22:52:34.595921+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:52:34.595921+0000","last_undegraded":"2023-09-23T22:52:34.595921+0000","last_fullsized":"2023-09-23T22:52:34.595921+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"286'5249","reported_seq":5289,"reported_epoch":286,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:52:41.104992+0000","last_change":"2023-09-23T22:51:35.528654+0000","last_active":"2023-09-23T22:52:41.104992+0000","last_peered":"2023-09-23T22:52:41.104992+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:52:41.104992+0000","last_undegraded":"2023-09-23T22:52:41.104992+0000","last_fullsized":"2023-09-23T22:52:41.104992+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5249,"log_dups_size":0,"ondisk_log_size":5249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343998464,"num_objects":5249,"num_object_clones":0,"num_object_copies":15747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":5249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5249,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":3595}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":287,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.596075+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:52:34.596075+0000","last_peered":"2023-09-23T22:52:34.596075+0000","last_clean":"2023-09-23T22:52:34.596075+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:52:34.596075+0000","last_undegraded":"2023-09-23T22:52:34.596075+0000","last_fullsized":"2023-09-23T22:52:34.596075+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595425+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:52:34.595425+0000","last_peered":"2023-09-23T22:52:34.595425+0000","last_clean":"2023-09-23T22:52:34.595425+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:52:34.595425+0000","last_undegraded":"2023-09-23T22:52:34.595425+0000","last_fullsized":"2023-09-23T22:52:34.595425+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":292,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612165+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:52:34.612165+0000","last_peered":"2023-09-23T22:52:34.612165+0000","last_clean":"2023-09-23T22:52:34.612165+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:52:34.612165+0000","last_undegraded":"2023-09-23T22:52:34.612165+0000","last_fullsized":"2023-09-23T22:52:34.612165+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"286'5030","reported_seq":5068,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:38.123482+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:52:38.123482+0000","last_peered":"2023-09-23T22:52:38.123482+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:52:38.123482+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5030,"log_dups_size":0,"ondisk_log_size":5030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329646080,"num_objects":5030,"num_object_clones":0,"num_object_copies":15090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5030,"num_write_kb":321920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1712},{"shards":"0,5","objects":3318}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":297,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595703+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:52:34.595703+0000","last_peered":"2023-09-23T22:52:34.595703+0000","last_clean":"2023-09-23T22:52:34.595703+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:52:34.595703+0000","last_undegraded":"2023-09-23T22:52:34.595703+0000","last_fullsized":"2023-09-23T22:52:34.595703+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":374,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.613715+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:52:34.613715+0000","last_peered":"2023-09-23T22:52:34.613715+0000","last_clean":"2023-09-23T22:52:34.613715+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:52:34.613715+0000","last_undegraded":"2023-09-23T22:52:34.613715+0000","last_fullsized":"2023-09-23T22:52:34.613715+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":40,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607781+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:52:34.607781+0000","last_peered":"2023-09-23T22:52:34.607781+0000","last_clean":"2023-09-23T22:52:34.607781+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:52:34.607781+0000","last_undegraded":"2023-09-23T22:52:34.607781+0000","last_fullsized":"2023-09-23T22:52:34.607781+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":278,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.607701+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:52:34.607701+0000","last_peered":"2023-09-23T22:52:34.607701+0000","last_clean":"2023-09-23T22:52:34.607701+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:52:34.607701+0000","last_undegraded":"2023-09-23T22:52:34.607701+0000","last_fullsized":"2023-09-23T22:52:34.607701+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"286'5094","reported_seq":5125,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:38.123061+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:52:38.123061+0000","last_peered":"2023-09-23T22:52:38.123061+0000","last_clean":"2023-09-23T22:52:38.123061+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:52:38.123061+0000","last_undegraded":"2023-09-23T22:52:38.123061+0000","last_fullsized":"2023-09-23T22:52:38.123061+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333840384,"num_objects":5094,"num_object_clones":0,"num_object_copies":15282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5094,"num_write_kb":326016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":261,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.613875+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:52:34.613875+0000","last_peered":"2023-09-23T22:52:34.613875+0000","last_clean":"2023-09-23T22:52:34.613875+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:52:34.613875+0000","last_undegraded":"2023-09-23T22:52:34.613875+0000","last_fullsized":"2023-09-23T22:52:34.613875+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":233,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.611762+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:52:34.611762+0000","last_peered":"2023-09-23T22:52:34.611762+0000","last_clean":"2023-09-23T22:52:34.611762+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:52:34.611762+0000","last_undegraded":"2023-09-23T22:52:34.611762+0000","last_fullsized":"2023-09-23T22:52:34.611762+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":210,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612004+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:52:34.612004+0000","last_peered":"2023-09-23T22:52:34.612004+0000","last_clean":"2023-09-23T22:52:34.612004+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:52:34.612004+0000","last_undegraded":"2023-09-23T22:52:34.612004+0000","last_fullsized":"2023-09-23T22:52:34.612004+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"286'5157","reported_seq":5187,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:41.612017+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:52:41.612017+0000","last_peered":"2023-09-23T22:52:41.612017+0000","last_clean":"2023-09-23T22:52:41.612017+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:52:41.612017+0000","last_undegraded":"2023-09-23T22:52:41.612017+0000","last_fullsized":"2023-09-23T22:52:41.612017+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5157,"log_dups_size":0,"ondisk_log_size":5157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337969152,"num_objects":5157,"num_object_clones":0,"num_object_copies":15471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5157,"num_write_kb":330048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":216,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.614926+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:52:34.614926+0000","last_peered":"2023-09-23T22:52:34.614926+0000","last_clean":"2023-09-23T22:52:34.614926+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:52:34.614926+0000","last_undegraded":"2023-09-23T22:52:34.614926+0000","last_fullsized":"2023-09-23T22:52:34.614926+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":165,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.596067+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:52:34.596067+0000","last_peered":"2023-09-23T22:52:34.596067+0000","last_clean":"2023-09-23T22:52:34.596067+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:52:34.596067+0000","last_undegraded":"2023-09-23T22:52:34.596067+0000","last_fullsized":"2023-09-23T22:52:34.596067+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612978+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:52:34.612978+0000","last_peered":"2023-09-23T22:52:34.612978+0000","last_clean":"2023-09-23T22:52:34.612978+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:52:34.612978+0000","last_undegraded":"2023-09-23T22:52:34.612978+0000","last_fullsized":"2023-09-23T22:52:34.612978+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"286'5328","reported_seq":5358,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:41.580565+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:52:41.580565+0000","last_peered":"2023-09-23T22:52:41.580565+0000","last_clean":"2023-09-23T22:52:41.580565+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:52:41.580565+0000","last_undegraded":"2023-09-23T22:52:41.580565+0000","last_fullsized":"2023-09-23T22:52:41.580565+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5328,"log_dups_size":0,"ondisk_log_size":5328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349175808,"num_objects":5328,"num_object_clones":0,"num_object_copies":15984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5328,"num_write_kb":340992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":42,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.613125+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:52:34.613125+0000","last_peered":"2023-09-23T22:52:34.613125+0000","last_clean":"2023-09-23T22:52:34.613125+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:52:34.613125+0000","last_undegraded":"2023-09-23T22:52:34.613125+0000","last_fullsized":"2023-09-23T22:52:34.613125+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.608298+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:52:34.608298+0000","last_peered":"2023-09-23T22:52:34.608298+0000","last_clean":"2023-09-23T22:52:34.608298+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:52:34.608298+0000","last_undegraded":"2023-09-23T22:52:34.608298+0000","last_fullsized":"2023-09-23T22:52:34.608298+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":262,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.611517+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:52:34.611517+0000","last_peered":"2023-09-23T22:52:34.611517+0000","last_clean":"2023-09-23T22:52:34.611517+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:52:34.611517+0000","last_undegraded":"2023-09-23T22:52:34.611517+0000","last_fullsized":"2023-09-23T22:52:34.611517+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"286'5280","reported_seq":6003,"reported_epoch":286,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:52:41.610267+0000","last_change":"2023-09-23T22:52:26.294673+0000","last_active":"2023-09-23T22:52:41.610267+0000","last_peered":"2023-09-23T22:52:41.610267+0000","last_clean":"2023-09-23T22:51:33.377854+0000","last_became_active":"2023-09-23T22:51:37.033330+0000","last_became_peered":"2023-09-23T22:51:37.033330+0000","last_unstale":"2023-09-23T22:52:41.610267+0000","last_undegraded":"2023-09-23T22:52:41.610267+0000","last_fullsized":"2023-09-23T22:51:35.425861+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5280,"log_dups_size":0,"ondisk_log_size":5280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1387,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":337920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":1387},{"shards":"0,4","objects":3893}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":327,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.608457+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:52:34.608457+0000","last_peered":"2023-09-23T22:52:34.608457+0000","last_clean":"2023-09-23T22:52:34.608457+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:52:34.608457+0000","last_undegraded":"2023-09-23T22:52:34.608457+0000","last_fullsized":"2023-09-23T22:52:34.608457+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":182,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612316+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:52:34.612316+0000","last_peered":"2023-09-23T22:52:34.612316+0000","last_clean":"2023-09-23T22:52:34.612316+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:52:34.612316+0000","last_undegraded":"2023-09-23T22:52:34.612316+0000","last_fullsized":"2023-09-23T22:52:34.612316+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":236,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617446+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:52:34.617446+0000","last_peered":"2023-09-23T22:52:34.617446+0000","last_clean":"2023-09-23T22:52:34.617446+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:52:34.617446+0000","last_undegraded":"2023-09-23T22:52:34.617446+0000","last_fullsized":"2023-09-23T22:52:34.617446+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":218,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.612856+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:52:34.612856+0000","last_peered":"2023-09-23T22:52:34.612856+0000","last_clean":"2023-09-23T22:52:34.612856+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:52:34.612856+0000","last_undegraded":"2023-09-23T22:52:34.612856+0000","last_fullsized":"2023-09-23T22:52:34.612856+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595987+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:52:34.595987+0000","last_peered":"2023-09-23T22:52:34.595987+0000","last_clean":"2023-09-23T22:52:34.595987+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:52:34.595987+0000","last_undegraded":"2023-09-23T22:52:34.595987+0000","last_fullsized":"2023-09-23T22:52:34.595987+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":95,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621490+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:52:34.621490+0000","last_peered":"2023-09-23T22:52:34.621490+0000","last_clean":"2023-09-23T22:52:34.621490+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:52:34.621490+0000","last_undegraded":"2023-09-23T22:52:34.621490+0000","last_fullsized":"2023-09-23T22:52:34.621490+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":294,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.618356+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:52:34.618356+0000","last_peered":"2023-09-23T22:52:34.618356+0000","last_clean":"2023-09-23T22:52:34.618356+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:52:34.618356+0000","last_undegraded":"2023-09-23T22:52:34.618356+0000","last_fullsized":"2023-09-23T22:52:34.618356+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.621211+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:52:34.621211+0000","last_peered":"2023-09-23T22:52:34.621211+0000","last_clean":"2023-09-23T22:52:34.621211+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:52:34.621211+0000","last_undegraded":"2023-09-23T22:52:34.621211+0000","last_fullsized":"2023-09-23T22:52:34.621211+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":166,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617846+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:52:34.617846+0000","last_peered":"2023-09-23T22:52:34.617846+0000","last_clean":"2023-09-23T22:52:34.617846+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:52:34.617846+0000","last_undegraded":"2023-09-23T22:52:34.617846+0000","last_fullsized":"2023-09-23T22:52:34.617846+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":36,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.608168+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:52:34.608168+0000","last_peered":"2023-09-23T22:52:34.608168+0000","last_clean":"2023-09-23T22:52:34.608168+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:52:34.608168+0000","last_undegraded":"2023-09-23T22:52:34.608168+0000","last_fullsized":"2023-09-23T22:52:34.608168+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":268,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.595912+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:52:34.595912+0000","last_peered":"2023-09-23T22:52:34.595912+0000","last_clean":"2023-09-23T22:52:34.595912+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:52:34.595912+0000","last_undegraded":"2023-09-23T22:52:34.595912+0000","last_fullsized":"2023-09-23T22:52:34.595912+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":232,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.615283+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:52:34.615283+0000","last_peered":"2023-09-23T22:52:34.615283+0000","last_clean":"2023-09-23T22:52:34.615283+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:52:34.615283+0000","last_undegraded":"2023-09-23T22:52:34.615283+0000","last_fullsized":"2023-09-23T22:52:34.615283+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":251,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-09-23T22:52:34.617261+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:52:34.617261+0000","last_peered":"2023-09-23T22:52:34.617261+0000","last_clean":"2023-09-23T22:52:34.617261+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:52:34.617261+0000","last_undegraded":"2023-09-23T22:52:34.617261+0000","last_fullsized":"2023-09-23T22:52:34.617261+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5447352320,"num_objects":83120,"num_object_clones":0,"num_object_copies":249360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":3376,"num_objects_unfound":0,"num_objects_dirty":83120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":83120,"num_write_kb":5319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3374,"num_bytes_recovered":221118464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16839999488,"data_stored":16839999488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82820,"ondisk_log_size":82820,"up":48,"acting":40,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430516,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626744,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2536500,"kb_used_data":2239276,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91208652,"statfs":{"total":95995035648,"available":93397659648,"internally_reserved":0,"allocated":2293018624,"data_stored":2292462792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":24.245999999999999,"5min":9.1280000000000001,"15min":3.9020000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":243.35900000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":24.465,"5min":9.5229999999999997,"15min":3.8250000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":243.41499999999999,"5min":243.41499999999999,"15min":243.41499999999999},"last":2.3029999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:51:42 2023","interfaces":[{"interface":"back","average":{"1min":2.8799999999999999,"5min":2.3679999999999999,"15min":2.6899999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":38.055999999999997,"5min":38.055999999999997,"15min":153.78399999999999},"last":6.1849999999999996},{"interface":"front","average":{"1min":2.8359999999999999,"5min":4.3570000000000002,"15min":3.3100000000000001},"min":{"1min":0.32200000000000001,"5min":0.247,"15min":0.247},"max":{"1min":38.204999999999998,"5min":185.411,"15min":185.411},"last":2.3780000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":3.71,"15min":3.367},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":5.4279999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.542,"5min":4.9530000000000003,"15min":3.677},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4390000000000001,"5min":185.21799999999999,"15min":185.21799999999999},"last":1.7370000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:51:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":5.4130000000000003,"15min":3.5510000000000002},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":6.1539999999999999,"5min":185.19399999999999,"15min":185.19399999999999},"last":6.5359999999999996},{"interface":"front","average":{"1min":1.0700000000000001,"5min":4.1669999999999998,"15min":3.23},"min":{"1min":0.39500000000000002,"5min":0.36799999999999999,"15min":0.28899999999999998},"max":{"1min":3.657,"5min":128.19800000000001,"15min":128.19800000000001},"last":2.532}]},{"osd":4,"last update":"Sat Sep 23 22:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.746,"15min":0.69699999999999995},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.198,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.71199999999999997,"15min":0.70599999999999996},"min":{"1min":0.30299999999999999,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":1.431,"5min":6.9089999999999998,"15min":18.597000000000001},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:51:42 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.89600000000000002,"15min":0.79400000000000004},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":0.73599999999999999,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.501},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.77700000000000002,"15min":0.77900000000000003},"min":{"1min":0.35599999999999998,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":3.1629999999999998,"5min":4.9089999999999998,"15min":18.625},"last":0.39600000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:51:55 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.85899999999999999,"15min":0.79100000000000004},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.316},"max":{"1min":2.2349999999999999,"5min":4.3049999999999997,"15min":12.481999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.80700000000000005,"15min":0.77200000000000002},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":5.0049999999999999,"5min":5.3150000000000004,"15min":16.661999999999999},"last":6.5700000000000003}]}]},{"osd":1,"up_from":84,"seq":360777253054,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4228880,"kb_used_data":3812488,"kb_used_omap":9,"kb_used_meta":416374,"kb_avail":89516272,"statfs":{"total":95995035648,"available":91664662528,"internally_reserved":0,"allocated":3903987712,"data_stored":3902943827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":426367977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":7.5039999999999996,"15min":5.2800000000000002},"min":{"1min":0.26800000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.933,"5min":101.605,"15min":101.605},"last":5.3230000000000004},{"interface":"front","average":{"1min":3.1309999999999998,"5min":6.6639999999999997,"15min":5.2309999999999999},"min":{"1min":0.24099999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":8.6620000000000008,"5min":101.892,"15min":101.892},"last":0.20399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.8610000000000002,"15min":2.7989999999999999},"min":{"1min":0.22500000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":12.622999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.214,"5min":4.0629999999999997,"15min":2.8799999999999999},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.887,"5min":37.957000000000001,"15min":37.957000000000001},"last":5.5389999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":3.3170000000000002,"15min":2.3809999999999998},"min":{"1min":0.28999999999999998,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.218,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.9910000000000001,"5min":3.9590000000000001,"15min":2.835},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":4.7859999999999996,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.68999999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.4950000000000001,"15min":2.0950000000000002},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":3.3889999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.4140000000000001,"5min":2.9009999999999998,"15min":2.0920000000000001},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":12.792999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":2.1030000000000002,"15min":1.6459999999999999},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":5.9960000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":1.345},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.6360000000000001,"15min":1.9710000000000001},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":3.347,"5min":87.515000000000001,"15min":87.515000000000001},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.9390000000000001,"15min":1.925},"min":{"1min":0.41799999999999998,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.2539999999999996,"5min":79.311000000000007,"15min":79.311000000000007},"last":5.4800000000000004},{"interface":"front","average":{"1min":1.909,"5min":2.1779999999999999,"15min":1.601},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":5.9340000000000002,"5min":79.328999999999994,"15min":79.328999999999994},"last":3.1579999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.008,"15min":2.3460000000000001},"min":{"1min":0.42499999999999999,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":4.0979999999999999,"5min":11.481999999999999,"15min":159.018},"last":1.4350000000000001},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.4399999999999999,"15min":2.444},"min":{"1min":0.47599999999999998,"5min":0.371,"15min":0.37},"max":{"1min":6.282,"5min":14.032999999999999,"15min":158.96000000000001},"last":5.2489999999999997}]}]},{"osd":0,"up_from":219,"seq":940597837902,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2462796,"kb_used_data":2167684,"kb_used_omap":31,"kb_used_meta":295072,"kb_avail":91282356,"statfs":{"total":95995035648,"available":93473132544,"internally_reserved":0,"allocated":2219708416,"data_stored":2218609451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":302154674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":3.5169999999999999,"15min":3.6469999999999998},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":11.930999999999999,"5min":21.512,"15min":32.344999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.4169999999999998,"5min":3.4249999999999998,"15min":3.5139999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7379999999999995,"5min":20.064,"15min":32.106000000000002},"last":6.9260000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":4.9889999999999999,"15min":3.0609999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.9619999999999997,"5min":111.508,"15min":111.508},"last":3.5899999999999999},{"interface":"front","average":{"1min":2.669,"5min":4.2229999999999999,"15min":2.5790000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.9749999999999996,"5min":111.532,"15min":111.532},"last":6.7969999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":4.9450000000000003,"15min":2.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9269999999999996,"5min":126.73,"15min":126.73},"last":6.9000000000000004},{"interface":"front","average":{"1min":1.177,"5min":3.8519999999999999,"15min":2.0329999999999999},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.8419999999999996,"5min":121.483,"15min":121.483},"last":6.7539999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":2.633,"15min":2.113},"min":{"1min":0.36299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.965999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.5169999999999999,"15min":2.0049999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.9990000000000006,"5min":20.009,"15min":20.009},"last":6.8620000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":2.9180000000000001,"15min":1.863},"min":{"1min":0.41399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.6280000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":3.4540000000000002},{"interface":"front","average":{"1min":2.6379999999999999,"5min":3.3769999999999998,"15min":2.161},"min":{"1min":0.40400000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.0519999999999996,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":2.6909999999999998,"15min":1.8109999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5369999999999999,"5min":59.146999999999998,"15min":59.146999999999998},"last":3.5409999999999999},{"interface":"front","average":{"1min":2.9220000000000002,"5min":3.3679999999999999,"15min":2.246},"min":{"1min":0.48499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":11.882999999999999,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.6800000000000002,"15min":1.9570000000000001},"min":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.0869999999999997,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.5},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.2320000000000002,"15min":1.5289999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.9219999999999997,"5min":21.954999999999998,"15min":21.954999999999998},"last":3.5089999999999999}]}]},{"osd":2,"up_from":131,"seq":562640715915,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2223316,"kb_used_data":1852748,"kb_used_omap":81,"kb_used_meta":370478,"kb_avail":91521836,"statfs":{"total":95995035648,"available":93718360064,"internally_reserved":0,"allocated":1897213952,"data_stored":1896169017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":379369690},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":8.4789999999999992,"5min":6.7309999999999999,"15min":5.7560000000000002},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":114.483,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":8.5,"5min":7.7599999999999998,"15min":6.1799999999999997},"min":{"1min":0.34599999999999997,"5min":0.314,"15min":0.314},"max":{"1min":113.873,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.5310000000000001,"15min":3.0230000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":12.035,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.315},{"interface":"front","average":{"1min":3.5329999999999999,"5min":3.4220000000000002,"15min":3.1589999999999998},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":14.159000000000001,"5min":28.596,"15min":88.436999999999998},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.3250000000000002,"15min":4.1890000000000001},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":6.5419999999999998,"5min":112.895,"15min":112.895},"last":0.437},{"interface":"front","average":{"1min":1.524,"5min":4.617,"15min":3.4169999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.251},"max":{"1min":9.3840000000000003,"5min":113.995,"15min":113.995},"last":0.27100000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.597,"15min":2.4169999999999998},"min":{"1min":0.37,"5min":0.253,"15min":0.253},"max":{"1min":7.1520000000000001,"5min":84.778999999999996,"15min":108.878},"last":7.1520000000000001},{"interface":"front","average":{"1min":1.901,"5min":3.641,"15min":3.056},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.8460000000000001,"5min":84.753,"15min":108.93000000000001},"last":7.8460000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0750000000000002,"15min":2.9009999999999998},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.5970000000000004,"5min":90.082999999999998,"15min":97.322999999999993},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.0049999999999999,"5min":3.3399999999999999,"15min":2.718},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.5380000000000003,"5min":89.986000000000004,"15min":97.301000000000002},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":4.8330000000000002,"15min":2.9390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":4.5789999999999997,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.607,"5min":4.3360000000000003,"15min":2.3180000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.4089999999999998,"5min":98.509,"15min":98.509},"last":0.45700000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0459999999999998,"15min":1.5740000000000001},"min":{"1min":0.42799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.7440000000000002,"5min":10.92,"15min":10.92},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.1240000000000001,"15min":2.323},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":4.2489999999999997,"5min":11.311,"15min":88.584000000000003},"last":0.45000000000000001}]}]},{"osd":3,"up_from":0,"seq":403726926001,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019200,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2209268,"kb_used_data":1912044,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91535884,"statfs":{"total":95995035648,"available":93732745216,"internally_reserved":0,"allocated":1957933056,"data_stored":1957376702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":8.5950000000000006,"5min":3.0579999999999998,"15min":6.3630000000000004},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":113.35599999999999,"5min":113.35599999999999,"15min":113.35599999999999},"last":8.8699999999999992},{"interface":"front","average":{"1min":8.9220000000000006,"5min":3.7679999999999998,"15min":6.9420000000000002},"min":{"1min":0.34000000000000002,"5min":0.217,"15min":0.217},"max":{"1min":113.417,"5min":113.417,"15min":113.417},"last":2.427}]},{"osd":1,"last update":"Sat Sep 23 22:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":1.9379999999999999,"15min":1.7210000000000001},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.24099999999999999},"max":{"1min":6.5289999999999999,"5min":31.888000000000002,"15min":45.320999999999998},"last":2.8109999999999999},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.849,"15min":1.595},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":21.003,"5min":32.5,"15min":45.372999999999998},"last":2.7189999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":2.2240000000000002,"15min":3.1720000000000002},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.308,"5min":20.308,"15min":84.909000000000006},"last":2.3319999999999999},{"interface":"front","average":{"1min":2.8769999999999998,"5min":2.3769999999999998,"15min":3.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.652000000000001,"5min":21.704000000000001,"15min":83.840000000000003},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:51:41 2023","interfaces":[{"interface":"back","average":{"1min":8.2940000000000005,"5min":7.1189999999999998,"15min":4.0300000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":164.453,"5min":164.453,"15min":164.453},"last":1.6970000000000001},{"interface":"front","average":{"1min":9.1509999999999998,"5min":7.2030000000000003,"15min":4.1349999999999998},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":167.05199999999999,"5min":167.05199999999999,"15min":167.05199999999999},"last":2.5259999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.91900000000000004,"15min":0.85699999999999998},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.0030000000000001,"5min":12.238,"15min":43.189999999999998},"last":0.377},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89900000000000002,"15min":0.874},"min":{"1min":0.29499999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.9910000000000001,"5min":7.5460000000000003,"15min":42.450000000000003},"last":2.3969999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.86099999999999999,"15min":0.81499999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3460000000000001,"5min":10.952999999999999,"15min":27.056000000000001},"last":2.3650000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.91800000000000004,"15min":0.83699999999999997},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":4.2969999999999997,"5min":12.269,"15min":26.940999999999999},"last":0.753}]},{"osd":7,"last update":"Sat Sep 23 22:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.91600000000000004,"15min":0.75700000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":2.327,"5min":12.311,"15min":12.311},"last":1.3540000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.92800000000000005,"15min":0.73399999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":2.9660000000000002,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.39100000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822972,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3643992,"kb_used_data":3338576,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90101160,"statfs":{"total":95995035648,"available":92263587840,"internally_reserved":0,"allocated":3418701824,"data_stored":3418169186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":8.0820000000000007,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":74.406000000000006,"5min":121.319,"15min":121.319},"last":2.6720000000000002},{"interface":"front","average":{"1min":7.1799999999999997,"5min":8.2539999999999996,"15min":3.8479999999999999},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":73.899000000000001,"5min":121.075,"15min":121.075},"last":3.2690000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8820000000000001,"5min":2.0649999999999999,"15min":2.056},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":41.463999999999999,"5min":41.463999999999999,"15min":149.203},"last":2.706},{"interface":"front","average":{"1min":4.0869999999999997,"5min":2.5670000000000002,"15min":2.3260000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":41.954000000000001,"5min":41.954000000000001,"15min":149.554},"last":3.2360000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.9660000000000002,"15min":5.6040000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":34.988,"5min":63.156999999999996,"15min":158.92599999999999},"last":3.2919999999999998},{"interface":"front","average":{"1min":4.3799999999999999,"5min":2.7050000000000001,"15min":5.5270000000000001},"min":{"1min":0.38600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":34.872,"5min":63.277999999999999,"15min":158.91399999999999},"last":6.1349999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":5.3380000000000001,"5min":7.6420000000000003,"15min":5.3600000000000003},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":89.995000000000005,"5min":108.58799999999999,"15min":257.05000000000001},"last":4.9710000000000001},{"interface":"front","average":{"1min":5.8029999999999999,"5min":7.8769999999999998,"15min":5.2800000000000002},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":90.117999999999995,"5min":108.15600000000001,"15min":257.35000000000002},"last":2.7370000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.63,"15min":0.64900000000000002},"min":{"1min":0.27400000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":2.024,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":3.3140000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.625},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":1.873,"5min":3.4409999999999998,"15min":10.535},"last":6.3550000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.89500000000000002,"15min":1.4490000000000001},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.7469999999999999,"5min":12.615,"15min":140.07499999999999},"last":3.202},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.85899999999999999,"15min":1.4790000000000001},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":4,"5min":12.74,"15min":139.739},"last":4.3499999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67800000000000005,"15min":0.72299999999999998},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":1.6830000000000001,"5min":2.738,"15min":7.9809999999999999},"last":3.1720000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.78300000000000003,"15min":0.71099999999999997},"min":{"1min":0.374,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":2.988,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":3.335}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2227634176,"data_stored":2227634176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3917217792,"data_stored":3917217792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1899233280,"data_stored":1899233280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1336737792,"data_stored":1336737792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1726021632,"data_stored":1726021632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3426222080,"data_stored":3426222080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2306932736,"data_stored":2306932736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:52:43.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:52:43.633 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:52:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:52:43] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.33.4" 2023-09-23T22:52:43.696 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:43.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:43.724 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:52:43.825 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:43.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:43.952 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:43.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3843841936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:43.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:43 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:43.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:43.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3843841936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:43.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:43 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:43.973 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3843841936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:43.973 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:43 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:52:44.056 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:44.141 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:44.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:52:44.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 85 16 86174 86158 63.3091 40.625 0.0089253 0.0157893 2023-09-23T22:52:44.695 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:44.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:44.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:44.867 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[180979]: failed to encode map e287 with expected crc 2023-09-23T22:52:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:52:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[180979]: osdmap e287: 8 total, 8 up, 6 in 2023-09-23T22:52:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[180979]: failed to encode map e287 with expected crc 2023-09-23T22:52:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[180979]: pgmap v813: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 10054/252144 objects degraded (3.987%); 3376/252144 objects misplaced (1.339%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:52:44.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:44.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[187743]: failed to encode map e287 with expected crc 2023-09-23T22:52:44.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:52:44.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[187743]: osdmap e287: 8 total, 8 up, 6 in 2023-09-23T22:52:44.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[187743]: failed to encode map e287 with expected crc 2023-09-23T22:52:44.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:44 smithi052 ceph-mon[187743]: pgmap v813: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 10054/252144 objects degraded (3.987%); 3376/252144 objects misplaced (1.339%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:52:45.014 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:45.171 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:45.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:45.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:44 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:52:45.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:44 smithi151 ceph-mon[159210]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:52:45.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:44 smithi151 ceph-mon[159210]: osdmap e287: 8 total, 8 up, 6 in 2023-09-23T22:52:45.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:44 smithi151 ceph-mon[159210]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:44 smithi151 ceph-mon[159210]: pgmap v813: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 46 MiB/s wr, 737 op/s; 10054/252144 objects degraded (3.987%); 3376/252144 objects misplaced (1.339%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:52:45.280 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:45.310 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:45.353 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:45.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:45.457 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:45.539 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:45.580 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 86 16 86891 86875 63.0941 44.8125 0.0198148 0.0158443 2023-09-23T22:52:45.771 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:45.800 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:45.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[180979]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[180979]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[180979]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/252144 objects degraded (3.987%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:45.808 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[187743]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[187743]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[187743]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/252144 objects degraded (3.987%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:45.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:45.910 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:45.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:45 smithi151 ceph-mon[159210]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:45 smithi151 ceph-mon[159210]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:45 smithi151 ceph-mon[159210]: failed to encode map e287 with expected crc 2023-09-23T22:52:45.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/252144 objects degraded (3.987%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:45.963 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:46.050 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:46.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:46.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:46.161 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:46.223 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:46.400 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:46.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:46.508 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:46.579 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:46.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 87 16 87697 87681 62.9479 50.375 0.0235731 0.0158805 2023-09-23T22:52:46.618 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:52:46.652 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:52:46.678 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:52:46.697 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:52:46.718 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:46.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:46.733 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:52:46.812 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:52:46.825 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:46.840 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:46 smithi052 ceph-mon[187743]: pgmap v814: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s; 10054/256275 objects degraded (3.923%); 3264/256275 objects misplaced (1.274%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T22:52:46.840 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3033025358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:46.840 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:46 smithi052 ceph-mon[180979]: pgmap v814: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s; 10054/256275 objects degraded (3.923%); 3264/256275 objects misplaced (1.274%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T22:52:46.840 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3033025358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:46.868 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:46.918 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:46.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:46 smithi151 ceph-mon[159210]: pgmap v814: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s; 10054/256275 objects degraded (3.923%); 3264/256275 objects misplaced (1.274%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T22:52:46.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3033025358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:46.976 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:52:46.977 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:52:46.977 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:52:46.977 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:52:46.984 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:52:46.984 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:52:46.984 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:52:46.984 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.984 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.985 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.985 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.985 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.985 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.985 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:52:46.985 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:52:47.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:47.102 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:47.364 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:47.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:47.467 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:47.513 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:47.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 88 16 88213 88197 62.5992 32.25 0.0080158 0.0159686 2023-09-23T22:52:47.821 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:52:47.831 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:47.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:47.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:47.947 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:48.018 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:48.109 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:48.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:48.201 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:48.221 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:48.297 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:48.433 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:48.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:48.459 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:48.543 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:48.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 89 16 88995 88979 62.4451 48.875 0.0285325 0.0160081 2023-09-23T22:52:48.634 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:48.747 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:48.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:48.854 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:48.934 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:49.117 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:49.124 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:52:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:49.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:49.147 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:52:49.234 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:49.305 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:49.410 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:49.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:49.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:49.550 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:49.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total time run: 90.0151 2023-09-23T22:52:49.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total writes made: 89638 2023-09-23T22:52:49.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Write size: 65536 2023-09-23T22:52:49.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object size: 65536 2023-09-23T22:52:49.581 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Bandwidth (MB/sec): 62.2382 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Bandwidth: 31.0257 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max bandwidth (MB/sec): 181.875 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average IOPS: 995 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev IOPS: 496.411 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max IOPS: 2910 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min IOPS: 0 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average Latency(s): 0.0160624 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Latency(s): 0.0426802 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max latency(s): 3.67384 2023-09-23T22:52:49.582 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min latency(s): 0.00119123 2023-09-23T22:52:49.588 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:52:49.612 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:49.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:49 smithi151 ceph-mon[159210]: pgmap v815: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 45 MiB/s wr, 714 op/s; 10054/262941 objects degraded (3.824%); 3152/262941 objects misplaced (1.199%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:49.716 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:49.749 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:49 smithi052 ceph-mon[180979]: pgmap v815: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 45 MiB/s wr, 714 op/s; 10054/262941 objects degraded (3.824%); 3152/262941 objects misplaced (1.199%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:49.749 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:49 smithi052 ceph-mon[187743]: pgmap v815: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 45 MiB/s wr, 714 op/s; 10054/262941 objects degraded (3.824%); 3152/262941 objects misplaced (1.199%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:49.785 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:50.015 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:50.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:50.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:50.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:52:50.126 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:50.224 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:50.422 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:52:50.422 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:52:50.422 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:52:50.422 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:52:50.422 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:52:50.438 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:50 smithi052 ceph-mon[180979]: from='client.55588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:52:50.438 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:50.438 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/262941 objects degraded (3.824%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:50.607 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:50.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:50 smithi151 ceph-mon[159210]: from='client.55588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:52:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/262941 objects degraded (3.824%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:50.713 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:50 smithi052 ceph-mon[187743]: from='client.55588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:52:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:52:50.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/262941 objects degraded (3.824%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:50.768 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:50.806 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":815,"stamp":"2023-09-23T22:52:48.368492+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5743056800,"num_objects":87647,"num_object_clones":0,"num_object_copies":262941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":3152,"num_objects_unfound":0,"num_objects_dirty":87647,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":87883,"num_write_kb":5610317,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3628,"num_bytes_recovered":235808822,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87117,"ondisk_log_size":87117,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18110376,"kb_used_data":16136568,"kb_used_omap":121,"kb_used_meta":1973574,"kb_avail":544360536,"statfs":{"total":575970213888,"available":557425188864,"internally_reserved":0,"allocated":16523845632,"data_stored":16519005539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2020939893},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":468779008,"num_objects":7153,"num_object_clones":0,"num_object_copies":21459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-444,"num_objects_unfound":0,"num_objects_dirty":7153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7153,"num_write_kb":457792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":444,"num_bytes_recovered":29097984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005577"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":284,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636085+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:52:36.636085+0000","last_peered":"2023-09-23T22:52:36.636085+0000","last_clean":"2023-09-23T22:52:36.636085+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:52:36.636085+0000","last_undegraded":"2023-09-23T22:52:36.636085+0000","last_fullsized":"2023-09-23T22:52:36.636085+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637298+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:52:36.637298+0000","last_peered":"2023-09-23T22:52:36.637298+0000","last_clean":"2023-09-23T22:52:36.637298+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:52:36.637298+0000","last_undegraded":"2023-09-23T22:52:36.637298+0000","last_fullsized":"2023-09-23T22:52:36.637298+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637895+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:52:36.637895+0000","last_peered":"2023-09-23T22:52:36.637895+0000","last_clean":"2023-09-23T22:52:36.637895+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:52:36.637895+0000","last_undegraded":"2023-09-23T22:52:36.637895+0000","last_fullsized":"2023-09-23T22:52:36.637895+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":265,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.624220+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:52:36.624220+0000","last_peered":"2023-09-23T22:52:36.624220+0000","last_clean":"2023-09-23T22:52:36.624220+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:52:36.624220+0000","last_undegraded":"2023-09-23T22:52:36.624220+0000","last_fullsized":"2023-09-23T22:52:36.624220+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":269,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.625901+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:52:36.625901+0000","last_peered":"2023-09-23T22:52:36.625901+0000","last_clean":"2023-09-23T22:52:36.625901+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:52:36.625901+0000","last_undegraded":"2023-09-23T22:52:36.625901+0000","last_fullsized":"2023-09-23T22:52:36.625901+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":252,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642493+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:52:36.642493+0000","last_peered":"2023-09-23T22:52:36.642493+0000","last_clean":"2023-09-23T22:52:36.642493+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:52:36.642493+0000","last_undegraded":"2023-09-23T22:52:36.642493+0000","last_fullsized":"2023-09-23T22:52:36.642493+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":196,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623670+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:52:36.623670+0000","last_peered":"2023-09-23T22:52:36.623670+0000","last_clean":"2023-09-23T22:52:36.623670+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:52:36.623670+0000","last_undegraded":"2023-09-23T22:52:36.623670+0000","last_fullsized":"2023-09-23T22:52:36.623670+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":148,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637986+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:52:36.637986+0000","last_peered":"2023-09-23T22:52:36.637986+0000","last_clean":"2023-09-23T22:52:36.637986+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:52:36.637986+0000","last_undegraded":"2023-09-23T22:52:36.637986+0000","last_fullsized":"2023-09-23T22:52:36.637986+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":27,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638918+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:52:36.638918+0000","last_peered":"2023-09-23T22:52:36.638918+0000","last_clean":"2023-09-23T22:52:36.638918+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:52:36.638918+0000","last_undegraded":"2023-09-23T22:52:36.638918+0000","last_fullsized":"2023-09-23T22:52:36.638918+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.641408+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:52:36.641408+0000","last_peered":"2023-09-23T22:52:36.641408+0000","last_clean":"2023-09-23T22:52:36.641408+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:52:36.641408+0000","last_undegraded":"2023-09-23T22:52:36.641408+0000","last_fullsized":"2023-09-23T22:52:36.641408+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":45,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637168+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:52:36.637168+0000","last_peered":"2023-09-23T22:52:36.637168+0000","last_clean":"2023-09-23T22:52:36.637168+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:52:36.637168+0000","last_undegraded":"2023-09-23T22:52:36.637168+0000","last_fullsized":"2023-09-23T22:52:36.637168+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":161,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642631+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:52:36.642631+0000","last_peered":"2023-09-23T22:52:36.642631+0000","last_clean":"2023-09-23T22:52:36.642631+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:52:36.642631+0000","last_undegraded":"2023-09-23T22:52:36.642631+0000","last_fullsized":"2023-09-23T22:52:36.642631+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":306,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638633+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:52:36.638633+0000","last_peered":"2023-09-23T22:52:36.638633+0000","last_clean":"2023-09-23T22:52:36.638633+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:52:36.638633+0000","last_undegraded":"2023-09-23T22:52:36.638633+0000","last_fullsized":"2023-09-23T22:52:36.638633+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":252,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636367+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:52:36.636367+0000","last_peered":"2023-09-23T22:52:36.636367+0000","last_clean":"2023-09-23T22:52:36.636367+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:52:36.636367+0000","last_undegraded":"2023-09-23T22:52:36.636367+0000","last_fullsized":"2023-09-23T22:52:36.636367+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":145,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636450+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:52:36.636450+0000","last_peered":"2023-09-23T22:52:36.636450+0000","last_clean":"2023-09-23T22:52:36.636450+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:52:36.636450+0000","last_undegraded":"2023-09-23T22:52:36.636450+0000","last_fullsized":"2023-09-23T22:52:36.636450+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":233,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638494+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:52:36.638494+0000","last_peered":"2023-09-23T22:52:36.638494+0000","last_clean":"2023-09-23T22:52:36.638494+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:52:36.638494+0000","last_undegraded":"2023-09-23T22:52:36.638494+0000","last_fullsized":"2023-09-23T22:52:36.638494+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623455+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:52:36.623455+0000","last_peered":"2023-09-23T22:52:36.623455+0000","last_clean":"2023-09-23T22:52:36.623455+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:52:36.623455+0000","last_undegraded":"2023-09-23T22:52:36.623455+0000","last_fullsized":"2023-09-23T22:52:36.623455+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":210,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.641915+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:52:36.641915+0000","last_peered":"2023-09-23T22:52:36.641915+0000","last_clean":"2023-09-23T22:52:36.641915+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:52:36.641915+0000","last_undegraded":"2023-09-23T22:52:36.641915+0000","last_fullsized":"2023-09-23T22:52:36.641915+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638020+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:52:36.638020+0000","last_peered":"2023-09-23T22:52:36.638020+0000","last_clean":"2023-09-23T22:52:36.638020+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:52:36.638020+0000","last_undegraded":"2023-09-23T22:52:36.638020+0000","last_fullsized":"2023-09-23T22:52:36.638020+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":253,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.624313+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:52:36.624313+0000","last_peered":"2023-09-23T22:52:36.624313+0000","last_clean":"2023-09-23T22:52:36.624313+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:52:36.624313+0000","last_undegraded":"2023-09-23T22:52:36.624313+0000","last_fullsized":"2023-09-23T22:52:36.624313+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":331,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.641118+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:52:36.641118+0000","last_peered":"2023-09-23T22:52:36.641118+0000","last_clean":"2023-09-23T22:52:36.641118+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:52:36.641118+0000","last_undegraded":"2023-09-23T22:52:36.641118+0000","last_fullsized":"2023-09-23T22:52:36.641118+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.639576+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:52:36.639576+0000","last_peered":"2023-09-23T22:52:36.639576+0000","last_clean":"2023-09-23T22:52:36.639576+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:52:36.639576+0000","last_undegraded":"2023-09-23T22:52:36.639576+0000","last_fullsized":"2023-09-23T22:52:36.639576+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"287'5503","reported_seq":5535,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:48.119378+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:52:48.119378+0000","last_peered":"2023-09-23T22:52:48.119378+0000","last_clean":"2023-09-23T22:52:48.119378+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:52:48.119378+0000","last_undegraded":"2023-09-23T22:52:48.119378+0000","last_fullsized":"2023-09-23T22:52:48.119378+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360644608,"num_objects":5503,"num_object_clones":0,"num_object_copies":16509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5503,"num_write_kb":352192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":156,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637445+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:52:36.637445+0000","last_peered":"2023-09-23T22:52:36.637445+0000","last_clean":"2023-09-23T22:52:36.637445+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:52:36.637445+0000","last_undegraded":"2023-09-23T22:52:36.637445+0000","last_fullsized":"2023-09-23T22:52:36.637445+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.641236+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:52:36.641236+0000","last_peered":"2023-09-23T22:52:36.641236+0000","last_clean":"2023-09-23T22:52:36.641236+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:52:36.641236+0000","last_undegraded":"2023-09-23T22:52:36.641236+0000","last_fullsized":"2023-09-23T22:52:36.641236+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"287'5620","reported_seq":5657,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:46.576357+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:52:46.576357+0000","last_peered":"2023-09-23T22:52:46.576357+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:52:46.576357+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5620,"log_dups_size":0,"ondisk_log_size":5620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368312320,"num_objects":5620,"num_object_clones":0,"num_object_copies":16860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":359680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1661},{"shards":"1,4","objects":3959}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":166,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.639786+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:52:36.639786+0000","last_peered":"2023-09-23T22:52:36.639786+0000","last_clean":"2023-09-23T22:52:36.639786+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:52:36.639786+0000","last_undegraded":"2023-09-23T22:52:36.639786+0000","last_fullsized":"2023-09-23T22:52:36.639786+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":276,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637955+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:52:36.637955+0000","last_peered":"2023-09-23T22:52:36.637955+0000","last_clean":"2023-09-23T22:52:36.637955+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:52:36.637955+0000","last_undegraded":"2023-09-23T22:52:36.637955+0000","last_fullsized":"2023-09-23T22:52:36.637955+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"287'5469","reported_seq":5506,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:44.866451+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:52:44.866451+0000","last_peered":"2023-09-23T22:52:44.866451+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:52:44.866451+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358416384,"num_objects":5469,"num_object_clones":0,"num_object_copies":16407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5469,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1771},{"shards":"2,5","objects":3698}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":110,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.641835+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:52:36.641835+0000","last_peered":"2023-09-23T22:52:36.641835+0000","last_clean":"2023-09-23T22:52:36.641835+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:52:36.641835+0000","last_undegraded":"2023-09-23T22:52:36.641835+0000","last_fullsized":"2023-09-23T22:52:36.641835+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":279,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642553+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:52:36.642553+0000","last_peered":"2023-09-23T22:52:36.642553+0000","last_clean":"2023-09-23T22:52:36.642553+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:52:36.642553+0000","last_undegraded":"2023-09-23T22:52:36.642553+0000","last_fullsized":"2023-09-23T22:52:36.642553+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"287'5432","reported_seq":5464,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:44.776774+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:52:44.776774+0000","last_peered":"2023-09-23T22:52:44.776774+0000","last_clean":"2023-09-23T22:52:44.776774+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:52:44.776774+0000","last_undegraded":"2023-09-23T22:52:44.776774+0000","last_fullsized":"2023-09-23T22:52:44.776774+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355991552,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5432,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.646064+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:52:36.646064+0000","last_peered":"2023-09-23T22:52:36.646064+0000","last_clean":"2023-09-23T22:52:36.646064+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:52:36.646064+0000","last_undegraded":"2023-09-23T22:52:36.646064+0000","last_fullsized":"2023-09-23T22:52:36.646064+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":180,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623857+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:52:36.623857+0000","last_peered":"2023-09-23T22:52:36.623857+0000","last_clean":"2023-09-23T22:52:36.623857+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:52:36.623857+0000","last_undegraded":"2023-09-23T22:52:36.623857+0000","last_fullsized":"2023-09-23T22:52:36.623857+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"287'5354","reported_seq":8296,"reported_epoch":287,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:52:45.939233+0000","last_change":"2023-09-23T22:51:35.553760+0000","last_active":"2023-09-23T22:52:45.939233+0000","last_peered":"2023-09-23T22:52:45.939233+0000","last_clean":"2023-09-23T22:51:33.374358+0000","last_became_active":"2023-09-23T22:51:35.518836+0000","last_became_peered":"2023-09-23T22:51:35.518836+0000","last_unstale":"2023-09-23T22:52:45.939233+0000","last_undegraded":"2023-09-23T22:52:45.939233+0000","last_fullsized":"2023-09-23T22:51:35.415989+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5354,"log_dups_size":0,"ondisk_log_size":5354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350879744,"num_objects":5354,"num_object_clones":0,"num_object_copies":16062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":223,"num_objects_unfound":0,"num_objects_dirty":5354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5354,"num_write_kb":342656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1451,"num_bytes_recovered":95092736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":223},{"shards":"4,6","objects":5131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":52,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623940+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:52:36.623940+0000","last_peered":"2023-09-23T22:52:36.623940+0000","last_clean":"2023-09-23T22:52:36.623940+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:52:36.623940+0000","last_undegraded":"2023-09-23T22:52:36.623940+0000","last_fullsized":"2023-09-23T22:52:36.623940+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":260,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638535+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:52:36.638535+0000","last_peered":"2023-09-23T22:52:36.638535+0000","last_clean":"2023-09-23T22:52:36.638535+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:52:36.638535+0000","last_undegraded":"2023-09-23T22:52:36.638535+0000","last_fullsized":"2023-09-23T22:52:36.638535+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"287'5380","reported_seq":5409,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:45.898865+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:52:45.898865+0000","last_peered":"2023-09-23T22:52:45.898865+0000","last_clean":"2023-09-23T22:52:45.898865+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:52:45.898865+0000","last_undegraded":"2023-09-23T22:52:45.898865+0000","last_fullsized":"2023-09-23T22:52:45.898865+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5380,"log_dups_size":0,"ondisk_log_size":5380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352583680,"num_objects":5380,"num_object_clones":0,"num_object_copies":16140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5380,"num_write_kb":344320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":271,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642404+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:52:36.642404+0000","last_peered":"2023-09-23T22:52:36.642404+0000","last_clean":"2023-09-23T22:52:36.642404+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:52:36.642404+0000","last_undegraded":"2023-09-23T22:52:36.642404+0000","last_fullsized":"2023-09-23T22:52:36.642404+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":282,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636992+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:52:36.636992+0000","last_peered":"2023-09-23T22:52:36.636992+0000","last_clean":"2023-09-23T22:52:36.636992+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:52:36.636992+0000","last_undegraded":"2023-09-23T22:52:36.636992+0000","last_fullsized":"2023-09-23T22:52:36.636992+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"287'5479","reported_seq":5516,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:46.783414+0000","last_change":"2023-09-23T22:51:35.518569+0000","last_active":"2023-09-23T22:52:46.783414+0000","last_peered":"2023-09-23T22:52:46.783414+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:52:46.783414+0000","last_undegraded":"2023-09-23T22:51:35.445687+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":271,"log_start":"268'300","ondisk_log_start":"268'300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:39.333773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359071744,"num_objects":5479,"num_object_clones":0,"num_object_copies":16437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5479,"num_write_kb":350656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1665},{"shards":"1,5","objects":3814}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":295,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637771+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:52:36.637771+0000","last_peered":"2023-09-23T22:52:36.637771+0000","last_clean":"2023-09-23T22:52:36.637771+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:52:36.637771+0000","last_undegraded":"2023-09-23T22:52:36.637771+0000","last_fullsized":"2023-09-23T22:52:36.637771+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":270,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.639786+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:52:36.639786+0000","last_peered":"2023-09-23T22:52:36.639786+0000","last_clean":"2023-09-23T22:52:36.639786+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:52:36.639786+0000","last_undegraded":"2023-09-23T22:52:36.639786+0000","last_fullsized":"2023-09-23T22:52:36.639786+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"287'5332","reported_seq":5370,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:44.759480+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:52:44.759480+0000","last_peered":"2023-09-23T22:52:44.759480+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:52:44.759480+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349437952,"num_objects":5332,"num_object_clones":0,"num_object_copies":15996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5332,"num_write_kb":341248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":3717},{"shards":"1,2,3","objects":1615}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":215,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.635646+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:52:36.635646+0000","last_peered":"2023-09-23T22:52:36.635646+0000","last_clean":"2023-09-23T22:52:36.635646+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:52:36.635646+0000","last_undegraded":"2023-09-23T22:52:36.635646+0000","last_fullsized":"2023-09-23T22:52:36.635646+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":224,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.639673+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:52:36.639673+0000","last_peered":"2023-09-23T22:52:36.639673+0000","last_clean":"2023-09-23T22:52:36.639673+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:52:36.639673+0000","last_undegraded":"2023-09-23T22:52:36.639673+0000","last_fullsized":"2023-09-23T22:52:36.639673+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"287'5579","reported_seq":5615,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:46.787417+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:52:46.787417+0000","last_peered":"2023-09-23T22:52:46.787417+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:52:46.787417+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"268'400","ondisk_log_start":"268'400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365625344,"num_objects":5579,"num_object_clones":0,"num_object_copies":16737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1630},{"shards":"1,5","objects":3949}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":324,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623920+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:52:36.623920+0000","last_peered":"2023-09-23T22:52:36.623920+0000","last_clean":"2023-09-23T22:52:36.623920+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:52:36.623920+0000","last_undegraded":"2023-09-23T22:52:36.623920+0000","last_fullsized":"2023-09-23T22:52:36.623920+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":149,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638344+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:52:36.638344+0000","last_peered":"2023-09-23T22:52:36.638344+0000","last_clean":"2023-09-23T22:52:36.638344+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:52:36.638344+0000","last_undegraded":"2023-09-23T22:52:36.638344+0000","last_fullsized":"2023-09-23T22:52:36.638344+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636003+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:52:36.636003+0000","last_peered":"2023-09-23T22:52:36.636003+0000","last_clean":"2023-09-23T22:52:36.636003+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:52:36.636003+0000","last_undegraded":"2023-09-23T22:52:36.636003+0000","last_fullsized":"2023-09-23T22:52:36.636003+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"287'5521","reported_seq":8954,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:48.123720+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:52:48.123720+0000","last_peered":"2023-09-23T22:52:48.123720+0000","last_clean":"2023-09-23T22:52:48.123720+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:52:48.123720+0000","last_undegraded":"2023-09-23T22:52:48.123720+0000","last_fullsized":"2023-09-23T22:52:48.123720+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361824256,"num_objects":5521,"num_object_clones":0,"num_object_copies":16563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":353344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":121,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642480+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:52:36.642480+0000","last_peered":"2023-09-23T22:52:36.642480+0000","last_clean":"2023-09-23T22:52:36.642480+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:52:36.642480+0000","last_undegraded":"2023-09-23T22:52:36.642480+0000","last_fullsized":"2023-09-23T22:52:36.642480+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":283,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636523+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:52:36.636523+0000","last_peered":"2023-09-23T22:52:36.636523+0000","last_clean":"2023-09-23T22:52:36.636523+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:52:36.636523+0000","last_undegraded":"2023-09-23T22:52:36.636523+0000","last_fullsized":"2023-09-23T22:52:36.636523+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":213,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638561+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:52:36.638561+0000","last_peered":"2023-09-23T22:52:36.638561+0000","last_clean":"2023-09-23T22:52:36.638561+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:52:36.638561+0000","last_undegraded":"2023-09-23T22:52:36.638561+0000","last_fullsized":"2023-09-23T22:52:36.638561+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"287'5475","reported_seq":5515,"reported_epoch":287,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:52:46.148454+0000","last_change":"2023-09-23T22:51:35.528654+0000","last_active":"2023-09-23T22:52:46.148454+0000","last_peered":"2023-09-23T22:52:46.148454+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:52:46.148454+0000","last_undegraded":"2023-09-23T22:52:46.148454+0000","last_fullsized":"2023-09-23T22:52:46.148454+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":3821}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":288,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638037+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:52:36.638037+0000","last_peered":"2023-09-23T22:52:36.638037+0000","last_clean":"2023-09-23T22:52:36.638037+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:52:36.638037+0000","last_undegraded":"2023-09-23T22:52:36.638037+0000","last_fullsized":"2023-09-23T22:52:36.638037+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:43.722673+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:52:43.722673+0000","last_peered":"2023-09-23T22:52:43.722673+0000","last_clean":"2023-09-23T22:52:43.722673+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:52:43.722673+0000","last_undegraded":"2023-09-23T22:52:43.722673+0000","last_fullsized":"2023-09-23T22:52:43.722673+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":293,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623593+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:52:36.623593+0000","last_peered":"2023-09-23T22:52:36.623593+0000","last_clean":"2023-09-23T22:52:36.623593+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:52:36.623593+0000","last_undegraded":"2023-09-23T22:52:36.623593+0000","last_fullsized":"2023-09-23T22:52:36.623593+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"287'5472","reported_seq":5511,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:48.099087+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:52:48.099087+0000","last_peered":"2023-09-23T22:52:48.099087+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:52:48.099087+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358612992,"num_objects":5472,"num_object_clones":0,"num_object_copies":16416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5472,"num_write_kb":350208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1712},{"shards":"0,5","objects":3760}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":298,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638345+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:52:36.638345+0000","last_peered":"2023-09-23T22:52:36.638345+0000","last_clean":"2023-09-23T22:52:36.638345+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:52:36.638345+0000","last_undegraded":"2023-09-23T22:52:36.638345+0000","last_fullsized":"2023-09-23T22:52:36.638345+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":375,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.625831+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:52:36.625831+0000","last_peered":"2023-09-23T22:52:36.625831+0000","last_clean":"2023-09-23T22:52:36.625831+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:52:36.625831+0000","last_undegraded":"2023-09-23T22:52:36.625831+0000","last_fullsized":"2023-09-23T22:52:36.625831+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":41,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642595+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:52:36.642595+0000","last_peered":"2023-09-23T22:52:36.642595+0000","last_clean":"2023-09-23T22:52:36.642595+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:52:36.642595+0000","last_undegraded":"2023-09-23T22:52:36.642595+0000","last_fullsized":"2023-09-23T22:52:36.642595+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":279,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.641683+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:52:36.641683+0000","last_peered":"2023-09-23T22:52:36.641683+0000","last_clean":"2023-09-23T22:52:36.641683+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:52:36.641683+0000","last_undegraded":"2023-09-23T22:52:36.641683+0000","last_fullsized":"2023-09-23T22:52:36.641683+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"287'5553","reported_seq":5584,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:48.140892+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:52:48.140892+0000","last_peered":"2023-09-23T22:52:48.140892+0000","last_clean":"2023-09-23T22:52:48.140892+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:52:48.140892+0000","last_undegraded":"2023-09-23T22:52:48.140892+0000","last_fullsized":"2023-09-23T22:52:48.140892+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363921408,"num_objects":5553,"num_object_clones":0,"num_object_copies":16659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5553,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":263,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:43.702371+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:52:43.702371+0000","last_peered":"2023-09-23T22:52:43.702371+0000","last_clean":"2023-09-23T22:52:43.702371+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:52:43.702371+0000","last_undegraded":"2023-09-23T22:52:43.702371+0000","last_fullsized":"2023-09-23T22:52:43.702371+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":234,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623544+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:52:36.623544+0000","last_peered":"2023-09-23T22:52:36.623544+0000","last_clean":"2023-09-23T22:52:36.623544+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:52:36.623544+0000","last_undegraded":"2023-09-23T22:52:36.623544+0000","last_fullsized":"2023-09-23T22:52:36.623544+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":212,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:43.158840+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:52:43.158840+0000","last_peered":"2023-09-23T22:52:43.158840+0000","last_clean":"2023-09-23T22:52:43.158840+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:52:43.158840+0000","last_undegraded":"2023-09-23T22:52:43.158840+0000","last_fullsized":"2023-09-23T22:52:43.158840+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"287'5364","reported_seq":5395,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:46.575997+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:52:46.575997+0000","last_peered":"2023-09-23T22:52:46.575997+0000","last_clean":"2023-09-23T22:52:46.575997+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:52:46.575997+0000","last_undegraded":"2023-09-23T22:52:46.575997+0000","last_fullsized":"2023-09-23T22:52:46.575997+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5364,"log_dups_size":0,"ondisk_log_size":5364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351535104,"num_objects":5364,"num_object_clones":0,"num_object_copies":16092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5364,"num_write_kb":343296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":217,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.625441+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:52:36.625441+0000","last_peered":"2023-09-23T22:52:36.625441+0000","last_clean":"2023-09-23T22:52:36.625441+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:52:36.625441+0000","last_undegraded":"2023-09-23T22:52:36.625441+0000","last_fullsized":"2023-09-23T22:52:36.625441+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":166,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638706+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:52:36.638706+0000","last_peered":"2023-09-23T22:52:36.638706+0000","last_clean":"2023-09-23T22:52:36.638706+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:52:36.638706+0000","last_undegraded":"2023-09-23T22:52:36.638706+0000","last_fullsized":"2023-09-23T22:52:36.638706+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":217,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.624150+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:52:36.624150+0000","last_peered":"2023-09-23T22:52:36.624150+0000","last_clean":"2023-09-23T22:52:36.624150+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:52:36.624150+0000","last_undegraded":"2023-09-23T22:52:36.624150+0000","last_fullsized":"2023-09-23T22:52:36.624150+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"287'5567","reported_seq":5598,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T22:52:46.579389+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:52:46.579389+0000","last_peered":"2023-09-23T22:52:46.579389+0000","last_clean":"2023-09-23T22:52:46.579389+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:52:46.579389+0000","last_undegraded":"2023-09-23T22:52:46.579389+0000","last_fullsized":"2023-09-23T22:52:46.579389+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364838912,"num_objects":5567,"num_object_clones":0,"num_object_copies":16701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5567,"num_write_kb":356288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":43,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623903+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:52:36.623903+0000","last_peered":"2023-09-23T22:52:36.623903+0000","last_clean":"2023-09-23T22:52:36.623903+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:52:36.623903+0000","last_undegraded":"2023-09-23T22:52:36.623903+0000","last_fullsized":"2023-09-23T22:52:36.623903+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":85,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642204+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:52:36.642204+0000","last_peered":"2023-09-23T22:52:36.642204+0000","last_clean":"2023-09-23T22:52:36.642204+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:52:36.642204+0000","last_undegraded":"2023-09-23T22:52:36.642204+0000","last_fullsized":"2023-09-23T22:52:36.642204+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":263,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.625531+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:52:36.625531+0000","last_peered":"2023-09-23T22:52:36.625531+0000","last_clean":"2023-09-23T22:52:36.625531+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:52:36.625531+0000","last_undegraded":"2023-09-23T22:52:36.625531+0000","last_fullsized":"2023-09-23T22:52:36.625531+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"287'5523","reported_seq":6470,"reported_epoch":287,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:52:46.600585+0000","last_change":"2023-09-23T22:52:26.294673+0000","last_active":"2023-09-23T22:52:46.600585+0000","last_peered":"2023-09-23T22:52:46.600585+0000","last_clean":"2023-09-23T22:51:33.377854+0000","last_became_active":"2023-09-23T22:51:37.033330+0000","last_became_peered":"2023-09-23T22:51:37.033330+0000","last_unstale":"2023-09-23T22:52:46.600585+0000","last_undegraded":"2023-09-23T22:52:46.600585+0000","last_fullsized":"2023-09-23T22:51:35.425861+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361955328,"num_objects":5523,"num_object_clones":0,"num_object_copies":16569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1275,"num_objects_unfound":0,"num_objects_dirty":5523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5523,"num_write_kb":353472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":454,"num_bytes_recovered":29753344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":1275},{"shards":"0,4","objects":4248}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":328,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642354+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:52:36.642354+0000","last_peered":"2023-09-23T22:52:36.642354+0000","last_clean":"2023-09-23T22:52:36.642354+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:52:36.642354+0000","last_undegraded":"2023-09-23T22:52:36.642354+0000","last_fullsized":"2023-09-23T22:52:36.642354+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":184,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:43.158851+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:52:43.158851+0000","last_peered":"2023-09-23T22:52:43.158851+0000","last_clean":"2023-09-23T22:52:43.158851+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:52:43.158851+0000","last_undegraded":"2023-09-23T22:52:43.158851+0000","last_fullsized":"2023-09-23T22:52:43.158851+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":237,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637831+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:52:36.637831+0000","last_peered":"2023-09-23T22:52:36.637831+0000","last_clean":"2023-09-23T22:52:36.637831+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:52:36.637831+0000","last_undegraded":"2023-09-23T22:52:36.637831+0000","last_fullsized":"2023-09-23T22:52:36.637831+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":219,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.624019+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:52:36.624019+0000","last_peered":"2023-09-23T22:52:36.624019+0000","last_clean":"2023-09-23T22:52:36.624019+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:52:36.624019+0000","last_undegraded":"2023-09-23T22:52:36.624019+0000","last_fullsized":"2023-09-23T22:52:36.624019+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":69,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638626+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:52:36.638626+0000","last_peered":"2023-09-23T22:52:36.638626+0000","last_clean":"2023-09-23T22:52:36.638626+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:52:36.638626+0000","last_undegraded":"2023-09-23T22:52:36.638626+0000","last_fullsized":"2023-09-23T22:52:36.638626+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":96,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.638464+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:52:36.638464+0000","last_peered":"2023-09-23T22:52:36.638464+0000","last_clean":"2023-09-23T22:52:36.638464+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:52:36.638464+0000","last_undegraded":"2023-09-23T22:52:36.638464+0000","last_fullsized":"2023-09-23T22:52:36.638464+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":295,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636598+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:52:36.636598+0000","last_peered":"2023-09-23T22:52:36.636598+0000","last_clean":"2023-09-23T22:52:36.636598+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:52:36.636598+0000","last_undegraded":"2023-09-23T22:52:36.636598+0000","last_fullsized":"2023-09-23T22:52:36.636598+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":97,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637072+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:52:36.637072+0000","last_peered":"2023-09-23T22:52:36.637072+0000","last_clean":"2023-09-23T22:52:36.637072+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:52:36.637072+0000","last_undegraded":"2023-09-23T22:52:36.637072+0000","last_fullsized":"2023-09-23T22:52:36.637072+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":167,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636154+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:52:36.636154+0000","last_peered":"2023-09-23T22:52:36.636154+0000","last_clean":"2023-09-23T22:52:36.636154+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:52:36.636154+0000","last_undegraded":"2023-09-23T22:52:36.636154+0000","last_fullsized":"2023-09-23T22:52:36.636154+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":37,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.642068+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:52:36.642068+0000","last_peered":"2023-09-23T22:52:36.642068+0000","last_clean":"2023-09-23T22:52:36.642068+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:52:36.642068+0000","last_undegraded":"2023-09-23T22:52:36.642068+0000","last_fullsized":"2023-09-23T22:52:36.642068+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":269,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.637960+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:52:36.637960+0000","last_peered":"2023-09-23T22:52:36.637960+0000","last_clean":"2023-09-23T22:52:36.637960+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:52:36.637960+0000","last_undegraded":"2023-09-23T22:52:36.637960+0000","last_fullsized":"2023-09-23T22:52:36.637960+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":233,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.623751+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:52:36.623751+0000","last_peered":"2023-09-23T22:52:36.623751+0000","last_clean":"2023-09-23T22:52:36.623751+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:52:36.623751+0000","last_undegraded":"2023-09-23T22:52:36.623751+0000","last_fullsized":"2023-09-23T22:52:36.623751+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":252,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T22:52:36.636011+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:52:36.636011+0000","last_peered":"2023-09-23T22:52:36.636011+0000","last_clean":"2023-09-23T22:52:36.636011+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:52:36.636011+0000","last_undegraded":"2023-09-23T22:52:36.636011+0000","last_fullsized":"2023-09-23T22:52:36.636011+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5742460928,"num_objects":87623,"num_object_clones":0,"num_object_copies":262869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":3152,"num_objects_unfound":0,"num_objects_dirty":87623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87623,"num_write_kb":5607872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3598,"num_bytes_recovered":235798528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17689214976,"data_stored":17689214976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86923,"ondisk_log_size":86923,"up":48,"acting":40,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430517,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626745,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2667580,"kb_used_data":2366260,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91077572,"statfs":{"total":95995035648,"available":93263433728,"internally_reserved":0,"allocated":2423050240,"data_stored":2422494406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":24.245999999999999,"5min":9.1280000000000001,"15min":3.9020000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":243.35900000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.442},{"interface":"front","average":{"1min":24.465,"5min":9.5229999999999997,"15min":3.8250000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":243.41499999999999,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.72599999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":14.752000000000001,"5min":5.0700000000000003,"15min":3.6389999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":254.62100000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.81499999999999995},{"interface":"front","average":{"1min":20.399999999999999,"5min":7.9690000000000003,"15min":4.6319999999999997},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":255.20099999999999,"5min":255.20099999999999,"15min":255.20099999999999},"last":0.27000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":3.71,"15min":3.367},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":5.4279999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.542,"5min":4.9530000000000003,"15min":3.677},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4390000000000001,"5min":185.21799999999999,"15min":185.21799999999999},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":5.6150000000000002,"15min":3.706},"min":{"1min":0.36199999999999999,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":7.734,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.919,"5min":3.919,"15min":3.3170000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.7770000000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.76000000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.875,"15min":0.753},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.7290000000000001,"5min":6.5309999999999997,"15min":18.085999999999999},"last":1.5589999999999999},{"interface":"front","average":{"1min":1.3320000000000001,"5min":0.78900000000000003,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4800000000000004,"5min":7.4800000000000004,"15min":18.597000000000001},"last":0.84699999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.0529999999999999,"15min":0.85299999999999998},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.7560000000000002,"5min":10.148999999999999,"15min":18.972000000000001},"last":2.5720000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.83799999999999997,"15min":0.80900000000000005},"min":{"1min":0.316,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":2.782,"5min":4.9089999999999998,"15min":18.625},"last":2.782}]},{"osd":7,"last update":"Sat Sep 23 22:51:55 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.85899999999999999,"15min":0.79100000000000004},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.316},"max":{"1min":2.2349999999999999,"5min":4.3049999999999997,"15min":12.481999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.80700000000000005,"15min":0.77200000000000002},"min":{"1min":0.377,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":5.0049999999999999,"5min":5.3150000000000004,"15min":16.661999999999999},"last":0.38200000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253055,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4413528,"kb_used_data":3988944,"kb_used_omap":9,"kb_used_meta":424566,"kb_avail":89331624,"statfs":{"total":95995035648,"available":91475582976,"internally_reserved":0,"allocated":4084678656,"data_stored":4083634769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":434756585},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":7.5039999999999996,"15min":5.2800000000000002},"min":{"1min":0.26800000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.933,"5min":101.605,"15min":101.605},"last":13.691000000000001},{"interface":"front","average":{"1min":3.1309999999999998,"5min":6.6639999999999997,"15min":5.2309999999999999},"min":{"1min":0.24099999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":8.6620000000000008,"5min":101.892,"15min":101.892},"last":2.0110000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.8610000000000002,"15min":2.7989999999999999},"min":{"1min":0.22500000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":12.622999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":10.952999999999999},{"interface":"front","average":{"1min":2.214,"5min":4.0629999999999997,"15min":2.8799999999999999},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.887,"5min":37.957000000000001,"15min":37.957000000000001},"last":10.823}]},{"osd":3,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":3.3170000000000002,"15min":2.3809999999999998},"min":{"1min":0.28999999999999998,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.218,"5min":87.787999999999997,"15min":87.787999999999997},"last":1.9810000000000001},{"interface":"front","average":{"1min":1.9910000000000001,"5min":3.9590000000000001,"15min":2.835},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":4.7859999999999996,"5min":89.861999999999995,"15min":89.861999999999995},"last":8.7720000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.4950000000000001,"15min":2.0950000000000002},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":3.3889999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":8.7469999999999999},{"interface":"front","average":{"1min":2.4140000000000001,"5min":2.9009999999999998,"15min":2.0920000000000001},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":12.792999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":10.894}]},{"osd":5,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":2.1030000000000002,"15min":1.6459999999999999},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":5.9960000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":1.9419999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.6360000000000001,"15min":1.9710000000000001},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":3.347,"5min":87.515000000000001,"15min":87.515000000000001},"last":2.5179999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.9390000000000001,"15min":1.925},"min":{"1min":0.41799999999999998,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.2539999999999996,"5min":79.311000000000007,"15min":79.311000000000007},"last":10.853999999999999},{"interface":"front","average":{"1min":1.909,"5min":2.1779999999999999,"15min":1.601},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":5.9340000000000002,"5min":79.328999999999994,"15min":79.328999999999994},"last":4.0389999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.008,"15min":2.3460000000000001},"min":{"1min":0.42499999999999999,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":4.0979999999999999,"5min":11.481999999999999,"15min":159.018},"last":8.702},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.4399999999999999,"15min":2.444},"min":{"1min":0.47599999999999998,"5min":0.371,"15min":0.37},"max":{"1min":6.282,"5min":14.032999999999999,"15min":158.96000000000001},"last":10.923}]}]},{"osd":0,"up_from":219,"seq":940597837904,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2657432,"kb_used_data":2366352,"kb_used_omap":31,"kb_used_meta":290976,"kb_avail":91087720,"statfs":{"total":95995035648,"available":93273825280,"internally_reserved":0,"allocated":2423144448,"data_stored":2422041745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":297960370},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":3.5169999999999999,"15min":3.6469999999999998},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":11.930999999999999,"5min":21.512,"15min":32.344999999999999},"last":3.8340000000000001},{"interface":"front","average":{"1min":2.4169999999999998,"5min":3.4249999999999998,"15min":3.5139999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7379999999999995,"5min":20.064,"15min":32.106000000000002},"last":1.9890000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":4.9889999999999999,"15min":3.0609999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.9619999999999997,"5min":111.508,"15min":111.508},"last":2.0179999999999998},{"interface":"front","average":{"1min":2.669,"5min":4.2229999999999999,"15min":2.5790000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.9749999999999996,"5min":111.532,"15min":111.532},"last":0.68899999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":4.9450000000000003,"15min":2.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9269999999999996,"5min":126.73,"15min":126.73},"last":2.343},{"interface":"front","average":{"1min":1.177,"5min":3.8519999999999999,"15min":2.0329999999999999},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.8419999999999996,"5min":121.483,"15min":121.483},"last":0.82799999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":2.633,"15min":2.113},"min":{"1min":0.36299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.965999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.745},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.5169999999999999,"15min":2.0049999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.9990000000000006,"5min":20.009,"15min":20.009},"last":1.9379999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":2.9180000000000001,"15min":1.863},"min":{"1min":0.41399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.6280000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":2.6379999999999999,"5min":3.3769999999999998,"15min":2.161},"min":{"1min":0.40400000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.0519999999999996,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.79600000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":2.6909999999999998,"15min":1.8109999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5369999999999999,"5min":59.146999999999998,"15min":59.146999999999998},"last":1.8999999999999999},{"interface":"front","average":{"1min":2.9220000000000002,"5min":3.3679999999999999,"15min":2.246},"min":{"1min":0.48499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":11.882999999999999,"5min":59.164000000000001,"15min":59.164000000000001},"last":3.7839999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.6800000000000002,"15min":1.9570000000000001},"min":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.0869999999999997,"5min":18.640000000000001,"15min":18.640000000000001},"last":4.0369999999999999},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.2320000000000002,"15min":1.5289999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.9219999999999997,"5min":21.954999999999998,"15min":21.954999999999998},"last":2.8239999999999998}]}]},{"osd":2,"up_from":131,"seq":562640715916,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307168,"kb_used_data":1948888,"kb_used_omap":81,"kb_used_meta":358190,"kb_avail":91437984,"statfs":{"total":95995035648,"available":93632495616,"internally_reserved":0,"allocated":1995661312,"data_stored":1994612639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":366786778},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":8.4789999999999992,"5min":6.7309999999999999,"15min":5.7560000000000002},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":114.483,"5min":132.79499999999999,"15min":132.79499999999999},"last":3.5019999999999998},{"interface":"front","average":{"1min":8.5,"5min":7.7599999999999998,"15min":6.1799999999999997},"min":{"1min":0.34599999999999997,"5min":0.314,"15min":0.314},"max":{"1min":113.873,"5min":132.58099999999999,"15min":132.58099999999999},"last":1.9850000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.5310000000000001,"15min":3.0230000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":12.035,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.5329999999999999,"5min":3.4220000000000002,"15min":3.1589999999999998},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":14.159000000000001,"5min":28.596,"15min":88.436999999999998},"last":3.4670000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.3250000000000002,"15min":4.1890000000000001},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":6.5419999999999998,"5min":112.895,"15min":112.895},"last":3.641},{"interface":"front","average":{"1min":1.524,"5min":4.617,"15min":3.4169999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.251},"max":{"1min":9.3840000000000003,"5min":113.995,"15min":113.995},"last":3.5270000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.597,"15min":2.4169999999999998},"min":{"1min":0.37,"5min":0.253,"15min":0.253},"max":{"1min":7.1520000000000001,"5min":84.778999999999996,"15min":108.878},"last":3.5539999999999998},{"interface":"front","average":{"1min":1.901,"5min":3.641,"15min":3.056},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.8460000000000001,"5min":84.753,"15min":108.93000000000001},"last":0.36899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0750000000000002,"15min":2.9009999999999998},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.5970000000000004,"5min":90.082999999999998,"15min":97.322999999999993},"last":1.3420000000000001},{"interface":"front","average":{"1min":2.0049999999999999,"5min":3.3399999999999999,"15min":2.718},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.5380000000000003,"5min":89.986000000000004,"15min":97.301000000000002},"last":3.407}]},{"osd":6,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":4.8330000000000002,"15min":2.9390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":4.5789999999999997,"5min":98.597999999999999,"15min":98.597999999999999},"last":3.4460000000000002},{"interface":"front","average":{"1min":1.607,"5min":4.3360000000000003,"15min":2.3180000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.4089999999999998,"5min":98.509,"15min":98.509},"last":4.5609999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0459999999999998,"15min":1.5740000000000001},"min":{"1min":0.42799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.7440000000000002,"5min":10.92,"15min":10.92},"last":3.5779999999999998},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.1240000000000001,"15min":2.323},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":4.2489999999999997,"5min":11.311,"15min":88.584000000000003},"last":4.7270000000000003}]}]},{"osd":3,"up_from":0,"seq":403726926002,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019201,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2277052,"kb_used_data":1975732,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91468100,"statfs":{"total":95995035648,"available":93663334400,"internally_reserved":0,"allocated":2023149568,"data_stored":2022593212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":8.5950000000000006,"5min":3.0579999999999998,"15min":6.3630000000000004},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":113.35599999999999,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":8.9220000000000006,"5min":3.7679999999999998,"15min":6.9420000000000002},"min":{"1min":0.34000000000000002,"5min":0.217,"15min":0.217},"max":{"1min":113.417,"5min":113.417,"15min":113.417},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":11.753,"5min":3.7679999999999998,"15min":2.4649999999999999},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":132.304,"5min":132.304,"15min":132.304},"last":2.1299999999999999},{"interface":"front","average":{"1min":11.4,"5min":3.6560000000000001,"15min":2.3180000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":132.453,"5min":132.453,"15min":132.453},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:52:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":2.2240000000000002,"15min":3.1720000000000002},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.308,"5min":20.308,"15min":84.909000000000006},"last":4.9189999999999996},{"interface":"front","average":{"1min":2.8769999999999998,"5min":2.3769999999999998,"15min":3.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.652000000000001,"5min":21.704000000000001,"15min":83.840000000000003},"last":5.2640000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":5.3339999999999996,"5min":5.3849999999999998,"15min":4.3410000000000002},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":74.138999999999996,"5min":164.453,"15min":164.453},"last":0.47899999999999998},{"interface":"front","average":{"1min":4.9329999999999998,"5min":5.3520000000000003,"15min":4.4249999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":75.099999999999994,"5min":167.05199999999999,"15min":167.05199999999999},"last":2.02}]},{"osd":5,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.91900000000000004,"15min":0.85699999999999998},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.0030000000000001,"5min":12.238,"15min":43.189999999999998},"last":2.1659999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89900000000000002,"15min":0.874},"min":{"1min":0.29499999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.9910000000000001,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.95199999999999996,"15min":0.84399999999999997},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.758,"5min":10.952999999999999,"15min":27.056000000000001},"last":2.4350000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.88600000000000001,"15min":0.84299999999999997},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":1.978,"5min":12.269,"15min":26.940999999999999},"last":2.0659999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.91600000000000004,"15min":0.75700000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":2.327,"5min":12.311,"15min":12.311},"last":2.1000000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.92800000000000005,"15min":0.73399999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":2.9660000000000002,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.45500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822973,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3787616,"kb_used_data":3490392,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":89957536,"statfs":{"total":95995035648,"available":92116516864,"internally_reserved":0,"allocated":3574161408,"data_stored":3573628768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":8.0820000000000007,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":74.406000000000006,"5min":121.319,"15min":121.319},"last":7.7919999999999998},{"interface":"front","average":{"1min":7.1799999999999997,"5min":8.2539999999999996,"15min":3.8479999999999999},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":73.899000000000001,"5min":121.075,"15min":121.075},"last":0.72199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8820000000000001,"5min":2.0649999999999999,"15min":2.056},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":41.463999999999999,"5min":41.463999999999999,"15min":149.203},"last":7.7400000000000002},{"interface":"front","average":{"1min":4.0869999999999997,"5min":2.5670000000000002,"15min":2.3260000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":41.954000000000001,"5min":41.954000000000001,"15min":149.554},"last":6.0389999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.9660000000000002,"15min":5.6040000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":34.988,"5min":63.156999999999996,"15min":158.92599999999999},"last":0.753},{"interface":"front","average":{"1min":4.3799999999999999,"5min":2.7050000000000001,"15min":5.5270000000000001},"min":{"1min":0.38600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":34.872,"5min":63.277999999999999,"15min":158.91399999999999},"last":2.5630000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":5.3380000000000001,"5min":7.6420000000000003,"15min":5.3600000000000003},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":89.995000000000005,"5min":108.58799999999999,"15min":257.05000000000001},"last":2.601},{"interface":"front","average":{"1min":5.8029999999999999,"5min":7.8769999999999998,"15min":5.2800000000000002},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":90.117999999999995,"5min":108.15600000000001,"15min":257.35000000000002},"last":3.77}]},{"osd":4,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.63,"15min":0.64900000000000002},"min":{"1min":0.27400000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":2.024,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":6.0730000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.625},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":1.873,"5min":3.4409999999999998,"15min":10.535},"last":2.5089999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.89500000000000002,"15min":1.4490000000000001},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.7469999999999999,"5min":12.615,"15min":140.07499999999999},"last":1.2869999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.85899999999999999,"15min":1.4790000000000001},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":4,"5min":12.74,"15min":139.739},"last":7.9669999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67800000000000005,"15min":0.72299999999999998},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":1.6830000000000001,"5min":2.738,"15min":7.9809999999999999},"last":7.9089999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.78300000000000003,"15min":0.71099999999999997},"min":{"1min":0.374,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":2.988,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.77500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2430074880,"data_stored":2430074880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4094296064,"data_stored":4094296064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1997733888,"data_stored":1997733888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1351548928,"data_stored":1351548928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1799684096,"data_stored":1799684096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3579510784,"data_stored":3579510784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2436366336,"data_stored":2436366336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:52:50.812 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:52:50.830 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:50.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:50.936 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:50.992 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:51.064 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:51.311 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:52:51.320 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:51.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:51.431 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:51.479 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:51.623 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:51.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:51.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:51.636 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:51 smithi151 ceph-mon[159210]: pgmap v816: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 715 op/s; 10054/264627 objects degraded (3.799%); 3152/264627 objects misplaced (1.191%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:51.737 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:51.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:51 smithi052 ceph-mon[180979]: pgmap v816: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 715 op/s; 10054/264627 objects degraded (3.799%); 3152/264627 objects misplaced (1.191%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:51.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:51 smithi052 ceph-mon[187743]: pgmap v816: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 715 op/s; 10054/264627 objects degraded (3.799%); 3152/264627 objects misplaced (1.191%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:51.824 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:51.913 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:51.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:51.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:51.927 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:52.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:52.066 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:52.110 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:52.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:52.214 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:52.276 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:52.456 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3888682475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:52.456 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:52 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:52.457 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3888682475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:52.457 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:52 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:52.507 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:52:52.630 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:52.648 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:52.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3888682475' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:52.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:52 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:52:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:52.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:52.762 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:52.816 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:53.407 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:53.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:53.517 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:53.588 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:53.617 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: pgmap v817: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 39 MiB/s wr, 706 op/s; 10054/265884 objects degraded (3.781%); 2930/265884 objects misplaced (1.102%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:53.617 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:52:53.617 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.617 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:53.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: osdmap e288: 8 total, 8 up, 6 in 2023-09-23T22:52:53.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:53 smithi151 ceph-mon[159210]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: pgmap v817: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 39 MiB/s wr, 706 op/s; 10054/265884 objects degraded (3.781%); 2930/265884 objects misplaced (1.102%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: osdmap e288: 8 total, 8 up, 6 in 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[187743]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:52:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:52:53] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.33.4" 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: pgmap v817: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 39 MiB/s wr, 706 op/s; 10054/265884 objects degraded (3.781%); 2930/265884 objects misplaced (1.102%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: osdmap e288: 8 total, 8 up, 6 in 2023-09-23T22:52:53.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.689 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.689 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:53 smithi052 ceph-mon[180979]: failed to encode map e288 with expected crc 2023-09-23T22:52:53.755 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:53.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:53.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:53.866 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:53.914 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:52:53.914 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:52:53.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:52:53.914 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101957 7.0 7:0053af50:::benchmark_data_smithi052_277165_object9465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.526400+0000", 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 561.38245874500001, 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226700761, 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:53.915 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101957 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526400+0000", 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526400+0000", 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:53.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526398+0000", 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7970000000000001e-06 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526406+0000", 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4841000000000001e-05 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526421+0000", 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9680000000000003e-06 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526427+0000", 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2902000000000003e-05 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526480+0000", 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014171499999999999 2023-09-23T22:52:53.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526622+0000", 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012000100000000001 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526742+0000", 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006025875 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.532768+0000", 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6030000000000004e-06 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.532773+0000", 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22024379699999999 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753017+0000", 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7750000000000003e-05 2023-09-23T22:52:53.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753055+0000", 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5847999999999999e-05 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753101+0000", 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226700761 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.921 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101967 7.0 7:0d54f750:::benchmark_data_smithi052_277165_object9475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 561.38176111999996, 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226169548, 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101967 2023-09-23T22:52:53.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527097+0000", 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5950000000000001e-06 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527104+0000", 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1766000000000001e-05 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527116+0000", 2023-09-23T22:52:53.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7259999999999996e-06 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527123+0000", 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028034370000000002 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.529926+0000", 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000130782 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530057+0000", 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012912700000000001 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530186+0000", 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002686627 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.532873+0000", 2023-09-23T22:52:53.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00060950499999999999 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.533482+0000", 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.219671964 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753154+0000", 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6359000000000006e-05 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753221+0000", 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6399000000000001e-05 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.753267+0000", 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.226169548 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164705 7.0 7:0e09ba28:::benchmark_data_smithi052_277165_object72213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 452.51246190799998, 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31253646400000001, 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164705 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396395+0000", 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4019999999999999e-06 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396402+0000", 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1674999999999999e-05 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396414+0000", 2023-09-23T22:52:53.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6180000000000003e-06 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396419+0000", 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4564e-05 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396494+0000", 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013145799999999999 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396625+0000", 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000161607 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396787+0000", 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013774191999999999 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410561+0000", 2023-09-23T22:52:53.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0606999999999998e-05 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410612+0000", 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29822478200000002 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708836+0000", 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1789999999999998e-05 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708878+0000", 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4976999999999999e-05 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708933+0000", 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31253646400000001 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164707 7.0 7:017e1e7c:::benchmark_data_smithi052_277165_object72215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 452.51242589100002, 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31516750300000002, 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164707 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.936 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396432+0000", 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.994e-06 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396435+0000", 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0390000000000001e-06 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396441+0000", 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.155e-06 2023-09-23T22:52:53.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396445+0000", 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052570480000000003 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401702+0000", 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000104561 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401807+0000", 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012558299999999999 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401933+0000", 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0087188879999999993 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410651+0000", 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2320000000000004e-06 2023-09-23T22:52:53.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410659+0000", 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30079543800000003 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711454+0000", 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100597 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711555+0000", 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5464999999999999e-05 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711600+0000", 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31516750300000002 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.942 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164711 7.0 7:0f1a7674:::benchmark_data_smithi052_277165_object72219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 452.51240867500002, 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31527841600000001, 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164711 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396449+0000", 2023-09-23T22:52:53.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6979999999999998e-06 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396452+0000", 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0100000000000001e-06 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396458+0000", 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3000000000000002e-06 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396461+0000", 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099721069999999992 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406434+0000", 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012163399999999999 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406555+0000", 2023-09-23T22:52:53.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000215744 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406771+0000", 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039129949999999998 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410684+0000", 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8912999999999998e-05 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410723+0000", 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30092187100000001 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711645+0000", 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9745e-05 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711685+0000", 2023-09-23T22:52:53.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3800000000000001e-05 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711728+0000", 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31527841600000001 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:80359 9.7 e285/271 9:ebab4714:::benchmark_data_smithi052_402718_object80358:head v 285'5134, mlcod=0'0)", 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:52:53.949 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.450410226999999, 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056466570000000001, 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9662e-05 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458497+0000", 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4939999999999997e-06 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458505+0000", 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.817e-06 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458509+0000", 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5979000000000001e-05 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458565+0000", 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2603999999999999e-05 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458587+0000", 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056246605999999998 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514834+0000", 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0945999999999994e-05 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514915+0000", 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056466570000000001 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:80360 9.8 e285/271 9:1e29dae4:::benchmark_data_smithi052_402718_object80359:head v 285'4950, mlcod=0'0)", 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:36.459432+0000", 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.449427003, 2023-09-23T22:52:53.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055518602, 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459432+0000", 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459432+0000", 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:53.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459431+0000", 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9472999999999999e-05 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459481+0000", 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7770000000000002e-06 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459488+0000", 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.746e-06 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459493+0000", 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3930000000000001e-05 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459537+0000", 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2201999999999999e-05 2023-09-23T22:52:53.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459559+0000", 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055311410999999998 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514871+0000", 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9516000000000002e-05 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514950+0000", 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055518602 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:80363 9.f e285/271 9:fb73b1ae:::benchmark_data_smithi052_402718_object80362:head v 285'5075, mlcod=0'0)", 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:36.459469+0000", 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.449390008000002, 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055513920000000001, 2023-09-23T22:52:53.959 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459469+0000", 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459469+0000", 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459468+0000", 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0589e-05 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459518+0000", 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6420000000000002e-06 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.961 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459528+0000", 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7760000000000003e-06 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459534+0000", 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3977999999999999e-05 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459568+0000", 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6143999999999999e-05 2023-09-23T22:52:53.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459594+0000", 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055326382 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514920+0000", 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2236000000000005e-05 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514982+0000", 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055513920000000001 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:80366 9.a e285/271 9:5bfe2620:::benchmark_data_smithi052_402718_object80365:head v 285'5051, mlcod=0'0)", 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:36.465226+0000", 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.443632228999999, 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.052161121999999997, 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.964 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.465226+0000", 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.465226+0000", 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.465225+0000", 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013641400000000001 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.465362+0000", 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.049473175000000001 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514835+0000", 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7306e-05 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514852+0000", 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000109416 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514962+0000", 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0323e-05 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.515002+0000", 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023550440000000001 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.517357+0000", 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0366999999999999e-05 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.517387+0000", 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.052161121999999997 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:88058 9.8 e287/271 9:1ecd12c4:::benchmark_data_smithi052_402718_object88057:head v 287'5436, mlcod=0'0)", 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:47.275265+0000", 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 6.6335937439999997, 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030706041, 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.969 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.275265+0000", 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.275265+0000", 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.275264+0000", 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7404999999999998e-05 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.275301+0000", 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1011e-05 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.275312+0000", 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.083e-06 2023-09-23T22:52:53.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.275318+0000", 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001332682 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276651+0000", 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5681999999999999e-05 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276677+0000", 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029258718 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.305935+0000", 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5380999999999999e-05 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.305971+0000", 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030706041 2023-09-23T22:52:53.973 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:88055 9.b e287/264 9:d5ed1dca:::benchmark_data_smithi052_402718_object88054:head v 287'5511, mlcod=287'5508)", 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:47.276622+0000", 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 6.6322365039999998, 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040371325, 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276622+0000", 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276622+0000", 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276621+0000", 2023-09-23T22:52:53.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1794e-05 2023-09-23T22:52:53.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276663+0000", 2023-09-23T22:52:53.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8940000000000005e-06 2023-09-23T22:52:53.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276673+0000", 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3680000000000001e-06 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.276679+0000", 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029991139 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.306670+0000", 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9153000000000001e-05 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.978 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.306689+0000", 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010275960000000001 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.316965+0000", 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8620999999999999e-05 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.316993+0000", 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040371325 2023-09-23T22:52:53.979 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:88064 9.d e287/264 9:baf48f03:::benchmark_data_smithi052_402718_object88063:head v 287'5586, mlcod=287'5583)", 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:47.278091+0000", 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 6.6307678389999998, 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.032537350999999999, 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.278091+0000", 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.278091+0000", 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.278090+0000", 2023-09-23T22:52:53.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.423e-05 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.278114+0000", 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024654941999999999 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.302769+0000", 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0369e-05 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.302779+0000", 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2280000000000002e-05 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.302822+0000", 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3504999999999999e-05 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.302845+0000", 2023-09-23T22:52:53.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077394250000000003 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.310585+0000", 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3466e-05 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:47.310628+0000", 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.032537350999999999 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.984 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:88964 9.d e287/264 9:b075a35b:::benchmark_data_smithi052_402718_object88963:head v 287'5638, mlcod=287'5634)", 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:48.549637+0000", 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 5.3592212610000001, 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030038017, 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.549637+0000", 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.549637+0000", 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.549637+0000", 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4842999999999999e-05 2023-09-23T22:52:53.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.549662+0000", 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1760000000000004e-06 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.549669+0000", 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0069999999999999e-06 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.549673+0000", 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055920930000000002 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.555265+0000", 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8040999999999994e-05 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.555333+0000", 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024305060999999999 2023-09-23T22:52:53.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.579638+0000", 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7147000000000002e-05 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:48.579675+0000", 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030038017 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89608 9.3 e287/271 9:c63bb180:::benchmark_data_smithi052_402718_object89607:head v 287'5589, mlcod=0'0)", 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.395287+0000", 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.5135717880000001, 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11530741, 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395287+0000", 2023-09-23T22:52:53.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395287+0000", 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395286+0000", 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5213999999999998e-05 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395331+0000", 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3318e-05 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395344+0000", 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5339999999999996e-06 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395351+0000", 2023-09-23T22:52:53.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7935000000000001e-05 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395399+0000", 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5122000000000001e-05 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395424+0000", 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11509657299999999 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510520+0000", 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3898000000000003e-05 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510594+0000", 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11530741 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89611 9.6 e287/264 9:6fa34574:::benchmark_data_smithi052_402718_object89610:head v 287'5657, mlcod=287'5654)", 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.5118180900000002, 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11885746699999999, 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:53.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0765000000000004e-05 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397110+0000", 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4840000000000004e-06 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397119+0000", 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397123+0000", 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3129000000000001e-05 2023-09-23T22:52:53.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397157+0000", 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2439999999999999e-05 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397169+0000", 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.118685622 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515855+0000", 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3374999999999997e-05 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515898+0000", 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11885746699999999 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:53.999 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89612 9.8 e287/271 9:161009cf:::benchmark_data_smithi052_402718_object89611:head v 287'5522, mlcod=0'0)", 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.400046+0000", 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.5088126229999999, 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.115819034, 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400046+0000", 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400046+0000", 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400045+0000", 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4580999999999998e-05 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400100+0000", 2023-09-23T22:52:54.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.110411013 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510511+0000", 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.218e-05 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510523+0000", 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8108000000000001e-05 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510571+0000", 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5058999999999998e-05 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510606+0000", 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052171370000000002 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515823+0000", 2023-09-23T22:52:54.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1754e-05 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515865+0000", 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.115819034 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89619 9.6 e287/264 9:69e57cf2:::benchmark_data_smithi052_402718_object89618:head v 287'5658, mlcod=287'5654)", 2023-09-23T22:52:54.004 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.400616+0000", 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.5082428160000001, 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11554326400000001, 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400616+0000", 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:54.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400616+0000", 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400615+0000", 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4066000000000002e-05 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400649+0000", 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10986414999999999 2023-09-23T22:52:54.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510513+0000", 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0089999999999996e-06 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510522+0000", 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002766569 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:54.007 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.513289+0000", 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9255000000000001e-05 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.513308+0000", 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028208360000000002 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.516129+0000", 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0105000000000001e-05 2023-09-23T22:52:54.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.516159+0000", 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11554326400000001 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89623 9.f e287/271 9:faa01721:::benchmark_data_smithi052_402718_object89622:head v 287'5673, mlcod=0'0)", 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.406207+0000", 2023-09-23T22:52:54.009 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.5026512900000002, 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10779472499999999, 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406207+0000", 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:54.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406207+0000", 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406206+0000", 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7041999999999999e-05 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406284+0000", 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10422577399999999 2023-09-23T22:52:54.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510509+0000", 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4123e-05 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510523+0000", 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3522999999999995e-05 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:54.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510587+0000", 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7676000000000001e-05 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510615+0000", 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033438729999999998 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.513958+0000", 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3506e-05 2023-09-23T22:52:54.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.514002+0000", 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10779472499999999 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:91885 9.6 e288/264 9:62cc27a6:::benchmark_data_smithi052_402718_object2244:head v 288'5791, mlcod=288'5789)", 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:52.759688+0000", 2023-09-23T22:52:54.014 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 1.1491702269999999, 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029472654000000001, 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759688+0000", 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:54.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759688+0000", 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759687+0000", 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.189e-05 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759699+0000", 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0862e-05 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759710+0000", 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4809999999999997e-06 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759716+0000", 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2753000000000002e-05 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:54.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759758+0000", 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1084999999999999e-05 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.759780+0000", 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.02934637 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.789126+0000", 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5104999999999997e-05 2023-09-23T22:52:54.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.789161+0000", 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029472654000000001 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:91886 9.a e288/271 9:5007df7a:::benchmark_data_smithi052_402718_object2245:head v 288'5781, mlcod=0'0)", 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:52.760277+0000", 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 1.148581595, 2023-09-23T22:52:54.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.032170802999999998, 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.760277+0000", 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.760277+0000", 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:52:54.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.760276+0000", 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0960000000000007e-06 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.760285+0000", 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5340000000000005e-06 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.760295+0000", 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7080000000000002e-06 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.760301+0000", 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023720008000000001 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.784021+0000", 2023-09-23T22:52:54.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8676000000000001e-05 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.784049+0000", 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00836691 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.792416+0000", 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1575000000000002e-05 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:52:54.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.792448+0000", 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.032170802999999998 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:52:54.024 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:52:54.099 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:52:54.137 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:52:54.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:54.180 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:52:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:54.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:54.225 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:52:54.258 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:52:54.288 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:52:54.290 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:54.319 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:54.362 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:54.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:54.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:54.438 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:52:54.465 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:52:54.497 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:52:54.538 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:54.639 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:52:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:54.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:52:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/199392772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:54.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/199392772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:54.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/199392772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:52:54.746 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:54.804 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:52:54.824 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:55.324 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:55.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:55 smithi052 ceph-mon[187743]: pgmap v819: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 702 op/s; 10054/264741 objects degraded (3.798%); 2930/264741 objects misplaced (1.107%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:55.482 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/264741 objects degraded (3.798%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:55.482 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:55 smithi052 ceph-mon[180979]: pgmap v819: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 702 op/s; 10054/264741 objects degraded (3.798%); 2930/264741 objects misplaced (1.107%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:55.483 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/264741 objects degraded (3.798%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:55.529 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:52:55.542 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:55.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:55.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:55.647 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:55.710 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:55.808 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:55 smithi151 ceph-mon[159210]: pgmap v819: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 702 op/s; 10054/264741 objects degraded (3.798%); 2930/264741 objects misplaced (1.107%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:55.809 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/264741 objects degraded (3.798%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:52:55.963 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:55.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:56.070 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:56.177 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:56.235 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:52:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:56.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:52:56.282 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:56.339 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:56.384 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:56.612 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:52:56.612 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 10054/260832 objects degraded (3.855%), 6 pgs degraded, 8 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1109,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":289,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":6,"osd_in_since":1695509493,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":80},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+remapped","count":2},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":86944,"data_bytes":5696919480,"bytes_used":18621292544,"bytes_avail":557348921344,"bytes_total":575970213888,"degraded_objects":10054,"degraded_total":260832,"degraded_ratio":0.038545883940620786,"misplaced_objects":2820,"misplaced_total":260832,"misplaced_ratio":0.010811556864188443,"recovering_objects_per_sec":44,"recovering_bytes_per_sec":2908516,"recovering_keys_per_sec":0,"num_objects_recovered":444,"num_bytes_recovered":29097984,"num_keys_recovered":0,"read_bytes_sec":102,"write_bytes_sec":27755476,"read_op_per_sec":0,"write_op_per_sec":695},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":27,"modified":"2023-09-23T22:50:10.280818+0000","services":{}},"progress_events":{"6b27ada5-699f-4815-a812-68a0dbe12316":{"message":"Global Recovery Event (75s)\n [=========================...] (remaining: 8s)","progress":0.89887642860412598,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:52:56.639 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:56.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:56.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:56.746 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:52:56.801 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:57.058 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:52:57.058 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:52:57.321 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:52:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:57.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:52:57.424 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:57.452 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:57.501 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:57.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:57.604 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:52:57.648 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:52:57.665 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:57.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[180979]: pgmap v820: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 26 MiB/s wr, 695 op/s; 10054/260832 objects degraded (3.855%); 2820/260832 objects misplaced (1.081%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:57.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[180979]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[180979]: osdmap e289: 8 total, 8 up, 6 in 2023-09-23T22:52:57.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[180979]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[180979]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2442471290' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[187743]: pgmap v820: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 26 MiB/s wr, 695 op/s; 10054/260832 objects degraded (3.855%); 2820/260832 objects misplaced (1.081%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[187743]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[187743]: osdmap e289: 8 total, 8 up, 6 in 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[187743]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[187743]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2442471290' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:52:57.854 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:52:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:57.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:52:57.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:57 smithi151 ceph-mon[159210]: pgmap v820: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 26 MiB/s wr, 695 op/s; 10054/260832 objects degraded (3.855%); 2820/260832 objects misplaced (1.081%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T22:52:57.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:57 smithi151 ceph-mon[159210]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:57 smithi151 ceph-mon[159210]: osdmap e289: 8 total, 8 up, 6 in 2023-09-23T22:52:57.866 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:57 smithi151 ceph-mon[159210]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:57 smithi151 ceph-mon[159210]: failed to encode map e289 with expected crc 2023-09-23T22:52:57.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2442471290' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:52:57.880 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:52:57.963 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:57.991 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:58.099 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:52:58.099 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:52:58.121 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:58.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:58.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:58.234 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:52:58.306 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:58.399 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:52:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:58.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:52:58.468 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:52:58.506 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:58.557 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:58.610 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:58.665 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:58.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:58 smithi151 ceph-mon[159210]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:58 smithi151 ceph-mon[159210]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:58 smithi151 ceph-mon[159210]: osdmap e290: 8 total, 8 up, 6 in 2023-09-23T22:52:58.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:58 smithi151 ceph-mon[159210]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:58.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:58.768 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:52:58.776 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[180979]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.776 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[180979]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.776 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[180979]: osdmap e290: 8 total, 8 up, 6 in 2023-09-23T22:52:58.776 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[180979]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.779 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[187743]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.779 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[187743]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.779 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[187743]: osdmap e290: 8 total, 8 up, 6 in 2023-09-23T22:52:58.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:58 smithi052 ceph-mon[187743]: failed to encode map e290 with expected crc 2023-09-23T22:52:58.826 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:58.913 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:52:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:58.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:52:59.023 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:52:59.132 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:59.294 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:52:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:59.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:52:59.385 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:52:59.406 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:52:59.433 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:59.474 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:52:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:52:59.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:52:59.512 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:52:59.577 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:52:59.651 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:59 smithi052 ceph-mon[187743]: pgmap v823: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 2.5 MiB/s wr, 711 op/s; 10054/254526 objects degraded (3.950%); 2710/254526 objects misplaced (1.065%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:52:59.651 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:52:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3811910882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:59.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:59 smithi052 ceph-mon[180979]: pgmap v823: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 2.5 MiB/s wr, 711 op/s; 10054/254526 objects degraded (3.950%); 2710/254526 objects misplaced (1.065%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:52:59.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:52:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3811910882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:52:59.665 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:52:59.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:59 smithi151 ceph-mon[159210]: pgmap v823: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 2.5 MiB/s wr, 711 op/s; 10054/254526 objects degraded (3.950%); 2710/254526 objects misplaced (1.065%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:52:59.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:52:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3811910882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:00.279 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:00.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:00.383 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:00.414 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:53:00.414 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:53:00.471 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:00.631 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:00.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:00.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:00.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:00 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:00 smithi151 ceph-mon[159210]: failed to encode map e291 with expected crc 2023-09-23T22:53:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3811910882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:53:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:00 smithi151 ceph-mon[159210]: osdmap e291: 8 total, 8 up, 6 in 2023-09-23T22:53:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:00 smithi151 ceph-mon[159210]: failed to encode map e291 with expected crc 2023-09-23T22:53:00.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/254526 objects degraded (3.950%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:00.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:00.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[180979]: failed to encode map e291 with expected crc 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3811910882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[180979]: osdmap e291: 8 total, 8 up, 6 in 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[180979]: failed to encode map e291 with expected crc 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/254526 objects degraded (3.950%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[187743]: failed to encode map e291 with expected crc 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3811910882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:53:00.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[187743]: osdmap e291: 8 total, 8 up, 6 in 2023-09-23T22:53:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[187743]: failed to encode map e291 with expected crc 2023-09-23T22:53:00.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/254526 objects degraded (3.950%), 6 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:00.775 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:00.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:00.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:00.940 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:00.988 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:01.359 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:01.361 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:53:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:01.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:01.469 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:01.513 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:01 smithi052 ceph-mon[180979]: pgmap v825: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 10054/252873 objects degraded (3.976%); 2710/252873 objects misplaced (1.072%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:53:01.514 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:01 smithi052 ceph-mon[187743]: pgmap v825: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 10054/252873 objects degraded (3.976%); 2710/252873 objects misplaced (1.072%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:53:01.541 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:01.696 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:01.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:01 smithi151 ceph-mon[159210]: pgmap v825: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 10054/252873 objects degraded (3.976%); 2710/252873 objects misplaced (1.072%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:53:01.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:01.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:01.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:01.834 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:01.912 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:53:02.192 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:02.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:02.303 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:02.333 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:02.367 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:02.372 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:02.417 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:02.473 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:02.502 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:02.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:02.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:02.522 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:02.559 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:02.613 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:02.614 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:02.656 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:53:02.667 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:02.684 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.684 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.685 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.685 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.685 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.685 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.685 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.685 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:53:02.713 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1402904424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:02.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1402904424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:02.737 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:02.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:02.845 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:02.868 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1402904424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:02.870 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:02.961 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:02.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:03.055 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:53:03.060 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:53:03.062 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:53:03.066 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:03.128 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:03.177 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:03.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:03.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:03.281 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:03.350 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:53:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:53:03] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.33.4" 2023-09-23T22:53:03.352 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:03.541 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:03 smithi151 ceph-mon[159210]: pgmap v826: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 10054/247884 objects degraded (4.056%); 2601/247884 objects misplaced (1.049%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:53:03.598 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:03.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:03.645 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:03 smithi052 ceph-mon[180979]: pgmap v826: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 10054/247884 objects degraded (4.056%); 2601/247884 objects misplaced (1.049%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:53:03.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:03 smithi052 ceph-mon[187743]: pgmap v826: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 10054/247884 objects degraded (4.056%); 2601/247884 objects misplaced (1.049%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:53:03.705 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:03.738 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:03.831 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100776 7.c 7:350d58b3:::benchmark_data_smithi052_277165_object8284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 573.25234702299997, 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999, 2023-09-23T22:53:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100776 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576705+0000", 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576703+0000", 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0799999999999995e-06 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576712+0000", 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2679e-05 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576725+0000", 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6329999999999999e-06 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.576731+0000", 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18361677800000001 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760348+0000", 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000140829 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760489+0000", 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016031800000000001 2023-09-23T22:53:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.760649+0000", 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010853742 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.771503+0000", 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023432639000000002 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.794936+0000", 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035400660000000001 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798476+0000", 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5828999999999999e-05 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.798511+0000", 2023-09-23T22:53:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012960090000000001 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.799807+0000", 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22310252799999999 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:100781 7.c 7:39965936:::benchmark_data_smithi052_277165_object8289:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-09-23T22:53:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 573.24506837900003, 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999, 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100781 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583984+0000", 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583981+0000", 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.59e-05 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.583996+0000", 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.174193922 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758190+0000", 2023-09-23T22:53:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2031000000000001e-05 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.758202+0000", 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049640919999999998 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763167+0000", 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143024 2023-09-23T22:53:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763310+0000", 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016401100000000001 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.763474+0000", 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012004406 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.775478+0000", 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019532911 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.795011+0000", 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072752770000000001 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802286+0000", 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0192000000000002e-05 2023-09-23T22:53:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.802346+0000", 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018569999999999999 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:30.804203+0000", 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22021975599999999 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:101958 7.7 7:e819df4a:::benchmark_data_smithi052_277165_object9466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 571.30231891999995, 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001, 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 101958 2023-09-23T22:53:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:53:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526733+0000", 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526731+0000", 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.396e-06 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526739+0000", 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3361e-05 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526752+0000", 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2920000000000003e-06 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526757+0000", 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1291000000000002e-05 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526799+0000", 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000121802 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.526920+0000", 2023-09-23T22:53:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014018900000000001 2023-09-23T22:53:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:53:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.527061+0000", 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033778649999999999 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530439+0000", 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000232978 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.530671+0000", 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21847482400000001 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749146+0000", 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749200+0000", 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5618000000000003e-05 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:43:32.749245+0000", 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22251238300000001 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162201 7.c 7:3f19fea2:::benchmark_data_smithi052_277165_object69709:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 466.57379928699999, 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.27262330099999998, 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162201 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255253+0000", 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255252+0000", 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.712e-06 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255255+0000", 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8510000000000001e-06 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255259+0000", 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5349999999999999e-06 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255261+0000", 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1329999999999999e-05 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255303+0000", 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000132652 2023-09-23T22:53:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255435+0000", 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012794600000000001 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.255563+0000", 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040626919999999997 2023-09-23T22:53:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.259626+0000", 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060452930000000002 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.265671+0000", 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.261175247 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.526847+0000", 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4647000000000005e-05 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.526921+0000", 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00095477599999999995 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.527876+0000", 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.27262330099999998 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162214 7.c 7:33fa1cb0:::benchmark_data_smithi052_277165_object69722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 466.55777804299998, 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25716776299999999, 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162214 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271274+0000", 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271272+0000", 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9519999999999994e-06 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271280+0000", 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2356e-05 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271292+0000", 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9319999999999999e-06 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271299+0000", 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8116999999999998e-05 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271377+0000", 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014364799999999999 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271521+0000", 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014726999999999999 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.271668+0000", 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089242190000000006 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280593+0000", 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6284000000000004e-05 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280659+0000", 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24732958699999999 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.527989+0000", 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4438999999999998e-05 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528043+0000", 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000398756 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528442+0000", 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25716776299999999 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:162220 7.c 7:33dba7f1:::benchmark_data_smithi052_277165_object69728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 466.55302079799998, 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25290671399999998, 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 162220 2023-09-23T22:53:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276031+0000", 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276030+0000", 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2030000000000003e-06 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276037+0000", 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4914e-05 2023-09-23T22:53:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276052+0000", 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3910000000000003e-06 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276058+0000", 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010739300000000001 2023-09-23T22:53:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276166+0000", 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018878300000000001 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276355+0000", 2023-09-23T22:53:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000217147 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.276572+0000", 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041362409999999997 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280708+0000", 2023-09-23T22:53:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2092e-05 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.280720+0000", 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24777231399999999 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528492+0000", 2023-09-23T22:53:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9494999999999999e-05 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528552+0000", 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00038610499999999998 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:17.528938+0000", 2023-09-23T22:53:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25290671399999998 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164712 7.7 7:e35a7399:::benchmark_data_smithi052_277165_object72220:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 462.43263447800001, 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29594756100000003, 2023-09-23T22:53:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164712 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:53:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396418+0000", 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396416+0000", 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6749999999999997e-06 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396425+0000", 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1613000000000001e-05 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396447+0000", 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6890000000000004e-06 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396455+0000", 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021833180000000001 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398638+0000", 2023-09-23T22:53:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139655 2023-09-23T22:53:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398778+0000", 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015481100000000001 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3", 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.398932+0000", 2023-09-23T22:53:03.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045151879999999998 2023-09-23T22:53:03.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.403448+0000", 2023-09-23T22:53:03.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.288777124 2023-09-23T22:53:03.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692225+0000", 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9673999999999999e-05 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692274+0000", 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0642000000000004e-05 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.692365+0000", 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29594756100000003 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:168558 7.c 7:3f3c3116:::benchmark_data_smithi052_277165_object76066:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 456.48914438000003, 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.244409458, 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.894 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 168558 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339908+0000", 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339906+0000", 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0329999999999997e-06 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339915+0000", 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3118999999999999e-05 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339928+0000", 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8990000000000001e-06 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339936+0000", 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8579999999999998e-05 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.339994+0000", 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000148231 2023-09-23T22:53:03.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.340142+0000", 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017634200000000001 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.340319+0000", 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037420520000000001 2023-09-23T22:53:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344061+0000", 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056251469999999996 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.349686+0000", 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23382455599999999 2023-09-23T22:53:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.583510+0000", 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4010999999999993e-05 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.583584+0000", 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073256099999999995 2023-09-23T22:53:03.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584317+0000", 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.244409458 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:168562 7.c 7:3c1c89f1:::benchmark_data_smithi052_277165_object76070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 456.48451255800001, 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.240143207, 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:03.903 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 168562 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344539+0000", 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4640000000000002e-06 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344542+0000", 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0480000000000004e-06 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344548+0000", 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1690000000000002e-06 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344553+0000", 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3362999999999996e-05 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344616+0000", 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014696300000000001 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344763+0000", 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018186 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.344945+0000", 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039275380000000004 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.348872+0000", 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00096992300000000003 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.349842+0000", 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.234521651 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584364+0000", 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3619999999999998e-05 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584418+0000", 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00026506700000000002 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:27.584683+0000", 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.240143207 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45206.0:48297 8.9 8:9a4ce8fa:::benchmark_data_smithi052_337357_object48296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 372.88112660199999, 2023-09-23T22:53:03.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25518236, 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45206", 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 48297 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947925+0000", 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947924+0000", 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9459000000000002e-05 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947974+0000", 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9210000000000004e-06 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947984+0000", 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2860000000000001e-06 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.947988+0000", 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4461000000000001e-05 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948022+0000", 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6387999999999999e-05 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948119+0000", 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9592999999999993e-05 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:53:03.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.948218+0000", 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001431501 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.949650+0000", 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00027939799999999998 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:50.949929+0000", 2023-09-23T22:53:03.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25307412600000001 2023-09-23T22:53:03.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203003+0000", 2023-09-23T22:53:03.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0455000000000003e-05 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203064+0000", 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3832999999999999e-05 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:46:51.203108+0000", 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25518236 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:13487 9.0 9:0b3ca2bc:::benchmark_data_smithi052_402718_object13486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:51:25.336942+0000", 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 98.492110281999999, 2023-09-23T22:53:03.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16835860699999999, 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 13487 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.336942+0000", 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.336942+0000", 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.336940+0000", 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5560999999999999e-05 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.336996+0000", 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4386000000000001e-05 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.923 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.337010+0000", 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.337016+0000", 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4807e-05 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.337061+0000", 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015471800000000001 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.337215+0000", 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016772099999999999 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.337383+0000", 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00092832199999999998 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.338311+0000", 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026620989999999998 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.340973+0000", 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.164240212 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.505214+0000", 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2722000000000002e-05 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.505256+0000", 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3884999999999997e-05 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:51:25.505300+0000", 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16835860699999999 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:56099 9.0 9:0819b3ed:::benchmark_data_smithi052_402718_object56098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:04.978541+0000", 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 58.850511112, 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.197984889, 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.929 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 56099 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978541+0000", 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978541+0000", 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978540+0000", 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0374999999999998e-05 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978590+0000", 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.261e-05 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978603+0000", 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4240000000000001e-06 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978608+0000", 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0899999999999998e-05 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978649+0000", 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8744000000000002e-05 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978748+0000", 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012340699999999999 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.978871+0000", 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020475549999999999 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.980919+0000", 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0058862510000000003 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.986805+0000", 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18965290600000001 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:05.176458+0000", 2023-09-23T22:53:03.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7098999999999998e-05 2023-09-23T22:53:03.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:05.176495+0000", 2023-09-23T22:53:03.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0923999999999997e-05 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:05.176526+0000", 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.197984889 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.938 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:56101 9.0 9:070711fb:::benchmark_data_smithi052_402718_object56100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-09-23T22:53:03.938 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:04.984652+0000", 2023-09-23T22:53:03.938 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 58.844399838000001, 2023-09-23T22:53:03.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.193888334, 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 56101 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984652+0000", 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984652+0000", 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:03.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984651+0000", 2023-09-23T22:53:03.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8010000000000003e-05 2023-09-23T22:53:03.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984699+0000", 2023-09-23T22:53:03.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4647000000000001e-05 2023-09-23T22:53:03.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984713+0000", 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5530000000000004e-06 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984721+0000", 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9348999999999998e-05 2023-09-23T22:53:03.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984770+0000", 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012118 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.984892+0000", 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011507500000000001 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.985007+0000", 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019117240000000001 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.986918+0000", 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000832623 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:04.987751+0000", 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.190726329 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:05.178477+0000", 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3701999999999997e-05 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:05.178511+0000", 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9502999999999999e-05 2023-09-23T22:53:03.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:05.178540+0000", 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.193888334 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:59548 9.0 9:02173e18:::benchmark_data_smithi052_402718_object59547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:09.106390+0000", 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 54.722662309999997, 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18330722599999999, 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.949 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 59548 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106390+0000", 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106390+0000", 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106388+0000", 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9231999999999999e-05 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106437+0000", 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7136999999999999e-05 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.952 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106454+0000", 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0260000000000004e-06 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106460+0000", 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1898999999999998e-05 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106522+0000", 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017070599999999999 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106693+0000", 2023-09-23T22:53:03.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019482500000000001 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.106888+0000", 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023154170000000002 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.109203+0000", 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055956998000000001 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.165160+0000", 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12447504600000001 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.289635+0000", 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4996999999999997e-05 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.289670+0000", 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6787e-05 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:09.289697+0000", 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18330722599999999 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:77301 9.1 9:85af518f:::benchmark_data_smithi052_402718_object77300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e284)", 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:32.326069+0000", 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.502983455999999, 2023-09-23T22:53:03.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17298981899999999, 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 77301 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.961 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326069+0000", 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326069+0000", 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326066+0000", 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1100999999999994e-05 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326157+0000", 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9545999999999999e-05 2023-09-23T22:53:03.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326177+0000", 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8070000000000002e-06 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326184+0000", 2023-09-23T22:53:03.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5284000000000002e-05 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326229+0000", 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000190681 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326420+0000", 2023-09-23T22:53:03.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024637299999999998 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.326666+0000", 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002828971 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.329495+0000", 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0051070059999999999 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.334602+0000", 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.163949133 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.498551+0000", 2023-09-23T22:53:03.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5652000000000001e-05 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.498617+0000", 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00044150999999999999 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.499058+0000", 2023-09-23T22:53:03.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17298981899999999 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:77302 9.0 9:055fb83f:::benchmark_data_smithi052_402718_object77301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e284)", 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:32.331121+0000", 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.497930616000001, 2023-09-23T22:53:03.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16745597400000001, 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 77302 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.970 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331121+0000", 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331121+0000", 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331119+0000", 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4665000000000003e-05 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331204+0000", 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0672000000000001e-05 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331225+0000", 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6070000000000008e-06 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331233+0000", 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9029000000000006e-05 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331302+0000", 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000160421 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331463+0000", 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000209218 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.331672+0000", 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029988369999999999 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.334671+0000", 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2125000000000002e-05 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.334703+0000", 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16377692099999999 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.498480+0000", 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8772e-05 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.498529+0000", 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8584000000000001e-05 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.498577+0000", 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16745597400000001 2023-09-23T22:53:03.977 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:77321 9.1 9:8300db6b:::benchmark_data_smithi052_402718_object77320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e284)", 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:32.352009+0000", 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 31.477042998999998, 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15500339099999999, 2023-09-23T22:53:03.978 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 77321 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352009+0000", 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352009+0000", 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352007+0000", 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3742999999999998e-05 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352051+0000", 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4324999999999999e-05 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352066+0000", 2023-09-23T22:53:03.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0869999999999997e-06 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352071+0000", 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4716999999999999e-05 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352115+0000", 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000115717 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352231+0000", 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000142783 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:53:03.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.352374+0000", 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032835189999999999 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.355657+0000", 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.213e-05 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.355679+0000", 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15082110000000001 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.506501+0000", 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0907e-05 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.985 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.506541+0000", 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047090599999999999 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:32.507012+0000", 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15500339099999999 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:03.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.987 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:91973 9.0 9:00589706:::benchmark_data_smithi052_402718_object2332:head [delete] snapc 0=[] ondisk+write+known_if_redirected e288)", 2023-09-23T22:53:03.987 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:52.875072+0000", 2023-09-23T22:53:03.987 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 10.953980313000001, 2023-09-23T22:53:03.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16771124200000001, 2023-09-23T22:53:03.987 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 91973 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.988 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875072+0000", 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:03.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875072+0000", 2023-09-23T22:53:03.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:03.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:03.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875070+0000", 2023-09-23T22:53:03.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4510000000000003e-06 2023-09-23T22:53:03.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:03.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875078+0000", 2023-09-23T22:53:03.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0890000000000001e-05 2023-09-23T22:53:03.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.992 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:03.992 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875089+0000", 2023-09-23T22:53:03.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8080000000000001e-06 2023-09-23T22:53:03.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:03.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875094+0000", 2023-09-23T22:53:03.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1761000000000002e-05 2023-09-23T22:53:03.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:03.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875136+0000", 2023-09-23T22:53:03.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112805 2023-09-23T22:53:03.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:03.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875249+0000", 2023-09-23T22:53:03.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012251699999999999 2023-09-23T22:53:03.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:03.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.875371+0000", 2023-09-23T22:53:03.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075087269999999998 2023-09-23T22:53:03.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:03.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:52.882880+0000", 2023-09-23T22:53:03.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.157093337 2023-09-23T22:53:03.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:53.039973+0000", 2023-09-23T22:53:03.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027124440000000001 2023-09-23T22:53:03.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:53.042686+0000", 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4308000000000002e-05 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:53.042740+0000", 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2694999999999997e-05 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:03.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:53.042783+0000", 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16771124200000001 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:92971 9.a e288/271 9:5c995b08:::benchmark_data_smithi052_402718_object3330:head v 288'5848, mlcod=0'0)", 2023-09-23T22:53:04.000 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:54.271747+0000", 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 9.5573046080000008, 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.14508960500000001, 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.271747+0000", 2023-09-23T22:53:04.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.271747+0000", 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:04.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.271747+0000", 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4420000000000001e-06 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.271754+0000", 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0282000000000001e-05 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:04.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.271764+0000", 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6749999999999999e-06 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.271770+0000", 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00244422 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:04.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.274214+0000", 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0721999999999999e-05 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.274235+0000", 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.14255546999999999 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:04.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.416790+0000", 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6499999999999999e-05 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:54.416837+0000", 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.14508960500000001 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.006 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:04.007 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.007 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:98498 9.0 9:0a7efb48:::benchmark_data_smithi052_402718_object8857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:04.007 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:02.968400+0000", 2023-09-23T22:53:04.007 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.86065195299999997, 2023-09-23T22:53:04.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21991888900000001, 2023-09-23T22:53:04.008 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:53:04.008 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:04.008 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:53:04.008 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:53:04.008 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:04.009 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 98498 2023-09-23T22:53:04.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.009 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:53:04.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:53:04.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968400+0000", 2023-09-23T22:53:04.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:53:04.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:53:04.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968400+0000", 2023-09-23T22:53:04.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:04.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:53:04.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968398+0000", 2023-09-23T22:53:04.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0780000000000002e-06 2023-09-23T22:53:04.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:53:04.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968407+0000", 2023-09-23T22:53:04.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7113e-05 2023-09-23T22:53:04.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:53:04.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968424+0000", 2023-09-23T22:53:04.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3889999999999999e-06 2023-09-23T22:53:04.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:53:04.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968432+0000", 2023-09-23T22:53:04.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9778999999999999e-05 2023-09-23T22:53:04.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:53:04.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968482+0000", 2023-09-23T22:53:04.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143781 2023-09-23T22:53:04.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:53:04.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968625+0000", 2023-09-23T22:53:04.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013883400000000001 2023-09-23T22:53:04.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:04.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.968764+0000", 2023-09-23T22:53:04.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002556937 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.971321+0000", 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048252900000000003 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:04.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:02.976146+0000", 2023-09-23T22:53:04.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21210306100000001 2023-09-23T22:53:04.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:04.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:03.188249+0000", 2023-09-23T22:53:04.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0849999999999997e-05 2023-09-23T22:53:04.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:53:04.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:03.188290+0000", 2023-09-23T22:53:04.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8602000000000002e-05 2023-09-23T22:53:04.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:53:04.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:53:04.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:53:04.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:03.188319+0000", 2023-09-23T22:53:04.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.21991888900000001 2023-09-23T22:53:04.022 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.022 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:04.022 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.023 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:53:04.023 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:53:04.023 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:53:04.137 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:04.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:04.241 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":826,"stamp":"2023-09-23T22:53:02.380892+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5414066104,"num_objects":82628,"num_object_clones":0,"num_object_copies":247884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":2601,"num_objects_unfound":0,"num_objects_dirty":82628,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":96934,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4179,"num_bytes_recovered":271919158,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94868,"ondisk_log_size":94868,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17509932,"kb_used_data":15460924,"kb_used_omap":121,"kb_used_meta":2048646,"kb_avail":544960980,"statfs":{"total":575970213888,"available":558040043520,"internally_reserved":0,"allocated":15831986176,"data_stored":15827125887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2097813621},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-282853376,"num_objects":-4316,"num_object_clones":0,"num_object_copies":-12948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-219,"num_objects_unfound":0,"num_objects_dirty":-4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4316,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":219,"num_bytes_recovered":14352384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.008719"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":288,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498038+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:52:57.498038+0000","last_peered":"2023-09-23T22:52:57.498038+0000","last_clean":"2023-09-23T22:52:57.498038+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:52:57.498038+0000","last_undegraded":"2023-09-23T22:52:57.498038+0000","last_fullsized":"2023-09-23T22:52:57.498038+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500341+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:52:57.500341+0000","last_peered":"2023-09-23T22:52:57.500341+0000","last_clean":"2023-09-23T22:52:57.500341+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:52:57.500341+0000","last_undegraded":"2023-09-23T22:52:57.500341+0000","last_fullsized":"2023-09-23T22:52:57.500341+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498071+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:52:57.498071+0000","last_peered":"2023-09-23T22:52:57.498071+0000","last_clean":"2023-09-23T22:52:57.498071+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:52:57.498071+0000","last_undegraded":"2023-09-23T22:52:57.498071+0000","last_fullsized":"2023-09-23T22:52:57.498071+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":268,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507665+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:52:56.507665+0000","last_peered":"2023-09-23T22:52:56.507665+0000","last_clean":"2023-09-23T22:52:56.507665+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:52:56.507665+0000","last_undegraded":"2023-09-23T22:52:56.507665+0000","last_fullsized":"2023-09-23T22:52:56.507665+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.508735+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:52:56.508735+0000","last_peered":"2023-09-23T22:52:56.508735+0000","last_clean":"2023-09-23T22:52:56.508735+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:52:56.508735+0000","last_undegraded":"2023-09-23T22:52:56.508735+0000","last_fullsized":"2023-09-23T22:52:56.508735+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500734+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:52:57.500734+0000","last_peered":"2023-09-23T22:52:57.500734+0000","last_clean":"2023-09-23T22:52:57.500734+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:52:57.500734+0000","last_undegraded":"2023-09-23T22:52:57.500734+0000","last_fullsized":"2023-09-23T22:52:57.500734+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":199,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507138+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:52:56.507138+0000","last_peered":"2023-09-23T22:52:56.507138+0000","last_clean":"2023-09-23T22:52:56.507138+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:52:56.507138+0000","last_undegraded":"2023-09-23T22:52:56.507138+0000","last_fullsized":"2023-09-23T22:52:56.507138+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":152,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524362+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:52:57.524362+0000","last_peered":"2023-09-23T22:52:57.524362+0000","last_clean":"2023-09-23T22:52:57.524362+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:52:57.524362+0000","last_undegraded":"2023-09-23T22:52:57.524362+0000","last_fullsized":"2023-09-23T22:52:57.524362+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":31,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524831+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:52:57.524831+0000","last_peered":"2023-09-23T22:52:57.524831+0000","last_clean":"2023-09-23T22:52:57.524831+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:52:57.524831+0000","last_undegraded":"2023-09-23T22:52:57.524831+0000","last_fullsized":"2023-09-23T22:52:57.524831+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.523215+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:52:57.523215+0000","last_peered":"2023-09-23T22:52:57.523215+0000","last_clean":"2023-09-23T22:52:57.523215+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:52:57.523215+0000","last_undegraded":"2023-09-23T22:52:57.523215+0000","last_fullsized":"2023-09-23T22:52:57.523215+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":49,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500224+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:52:57.500224+0000","last_peered":"2023-09-23T22:52:57.500224+0000","last_clean":"2023-09-23T22:52:57.500224+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:52:57.500224+0000","last_undegraded":"2023-09-23T22:52:57.500224+0000","last_fullsized":"2023-09-23T22:52:57.500224+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500544+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:52:57.500544+0000","last_peered":"2023-09-23T22:52:57.500544+0000","last_clean":"2023-09-23T22:52:57.500544+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:52:57.500544+0000","last_undegraded":"2023-09-23T22:52:57.500544+0000","last_fullsized":"2023-09-23T22:52:57.500544+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":310,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525079+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:52:57.525079+0000","last_peered":"2023-09-23T22:52:57.525079+0000","last_clean":"2023-09-23T22:52:57.525079+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:52:57.525079+0000","last_undegraded":"2023-09-23T22:52:57.525079+0000","last_fullsized":"2023-09-23T22:52:57.525079+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499173+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:52:57.499173+0000","last_peered":"2023-09-23T22:52:57.499173+0000","last_clean":"2023-09-23T22:52:57.499173+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:52:57.499173+0000","last_undegraded":"2023-09-23T22:52:57.499173+0000","last_fullsized":"2023-09-23T22:52:57.499173+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":149,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499311+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:52:57.499311+0000","last_peered":"2023-09-23T22:52:57.499311+0000","last_clean":"2023-09-23T22:52:57.499311+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:52:57.499311+0000","last_undegraded":"2023-09-23T22:52:57.499311+0000","last_fullsized":"2023-09-23T22:52:57.499311+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524537+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:52:57.524537+0000","last_peered":"2023-09-23T22:52:57.524537+0000","last_clean":"2023-09-23T22:52:57.524537+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:52:57.524537+0000","last_undegraded":"2023-09-23T22:52:57.524537+0000","last_fullsized":"2023-09-23T22:52:57.524537+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.506907+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:52:56.506907+0000","last_peered":"2023-09-23T22:52:56.506907+0000","last_clean":"2023-09-23T22:52:56.506907+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:52:56.506907+0000","last_undegraded":"2023-09-23T22:52:56.506907+0000","last_fullsized":"2023-09-23T22:52:56.506907+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":214,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500728+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:52:57.500728+0000","last_peered":"2023-09-23T22:52:57.500728+0000","last_clean":"2023-09-23T22:52:57.500728+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:52:57.500728+0000","last_undegraded":"2023-09-23T22:52:57.500728+0000","last_fullsized":"2023-09-23T22:52:57.500728+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":141,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498151+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:52:57.498151+0000","last_peered":"2023-09-23T22:52:57.498151+0000","last_clean":"2023-09-23T22:52:57.498151+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:52:57.498151+0000","last_undegraded":"2023-09-23T22:52:57.498151+0000","last_fullsized":"2023-09-23T22:52:57.498151+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":256,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507733+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:52:56.507733+0000","last_peered":"2023-09-23T22:52:56.507733+0000","last_clean":"2023-09-23T22:52:56.507733+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:52:56.507733+0000","last_undegraded":"2023-09-23T22:52:56.507733+0000","last_fullsized":"2023-09-23T22:52:56.507733+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":335,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500069+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:52:57.500069+0000","last_peered":"2023-09-23T22:52:57.500069+0000","last_clean":"2023-09-23T22:52:57.500069+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:52:57.500069+0000","last_undegraded":"2023-09-23T22:52:57.500069+0000","last_fullsized":"2023-09-23T22:52:57.500069+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":217,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524997+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:52:57.524997+0000","last_peered":"2023-09-23T22:52:57.524997+0000","last_clean":"2023-09-23T22:52:57.524997+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:52:57.524997+0000","last_undegraded":"2023-09-23T22:52:57.524997+0000","last_fullsized":"2023-09-23T22:52:57.524997+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"290'5911","reported_seq":5944,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:58.146450+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:52:58.146450+0000","last_peered":"2023-09-23T22:52:58.146450+0000","last_clean":"2023-09-23T22:52:58.146450+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:52:58.146450+0000","last_undegraded":"2023-09-23T22:52:58.146450+0000","last_fullsized":"2023-09-23T22:52:58.146450+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342163456,"num_objects":5221,"num_object_clones":0,"num_object_copies":15663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5911,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":160,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524900+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:52:57.524900+0000","last_peered":"2023-09-23T22:52:57.524900+0000","last_clean":"2023-09-23T22:52:57.524900+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:52:57.524900+0000","last_undegraded":"2023-09-23T22:52:57.524900+0000","last_fullsized":"2023-09-23T22:52:57.524900+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500130+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:52:57.500130+0000","last_peered":"2023-09-23T22:52:57.500130+0000","last_clean":"2023-09-23T22:52:57.500130+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:52:57.500130+0000","last_undegraded":"2023-09-23T22:52:57.500130+0000","last_fullsized":"2023-09-23T22:52:57.500130+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"291'6230","reported_seq":6272,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:01.580572+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:53:01.580572+0000","last_peered":"2023-09-23T22:53:01.580572+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:53:01.580572+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6230,"log_dups_size":0,"ondisk_log_size":6230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344850432,"num_objects":5262,"num_object_clones":0,"num_object_copies":15786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6230,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1177},{"shards":"1,4","objects":3601}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":170,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525008+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:52:57.525008+0000","last_peered":"2023-09-23T22:52:57.525008+0000","last_clean":"2023-09-23T22:52:57.525008+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:52:57.525008+0000","last_undegraded":"2023-09-23T22:52:57.525008+0000","last_fullsized":"2023-09-23T22:52:57.525008+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":280,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498090+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:52:57.498090+0000","last_peered":"2023-09-23T22:52:57.498090+0000","last_clean":"2023-09-23T22:52:57.498090+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:52:57.498090+0000","last_undegraded":"2023-09-23T22:52:57.498090+0000","last_fullsized":"2023-09-23T22:52:57.498090+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"291'6107","reported_seq":6149,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:59.891403+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:52:59.891403+0000","last_peered":"2023-09-23T22:52:59.891403+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:52:59.891403+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6107,"log_dups_size":0,"ondisk_log_size":6107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339542016,"num_objects":5181,"num_object_clones":0,"num_object_copies":15543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6107,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1308},{"shards":"2,5","objects":3410}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":114,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500706+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:52:57.500706+0000","last_peered":"2023-09-23T22:52:57.500706+0000","last_clean":"2023-09-23T22:52:57.500706+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:52:57.500706+0000","last_undegraded":"2023-09-23T22:52:57.500706+0000","last_fullsized":"2023-09-23T22:52:57.500706+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":283,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500383+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:52:57.500383+0000","last_peered":"2023-09-23T22:52:57.500383+0000","last_clean":"2023-09-23T22:52:57.500383+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:52:57.500383+0000","last_undegraded":"2023-09-23T22:52:57.500383+0000","last_fullsized":"2023-09-23T22:52:57.500383+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"291'6061","reported_seq":6097,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.835266+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:52:59.835266+0000","last_peered":"2023-09-23T22:52:59.835266+0000","last_clean":"2023-09-23T22:52:59.835266+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:52:59.835266+0000","last_undegraded":"2023-09-23T22:52:59.835266+0000","last_fullsized":"2023-09-23T22:52:59.835266+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344391680,"num_objects":5255,"num_object_clones":0,"num_object_copies":15765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6061,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":86,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500901+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:52:57.500901+0000","last_peered":"2023-09-23T22:52:57.500901+0000","last_clean":"2023-09-23T22:52:57.500901+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:52:57.500901+0000","last_undegraded":"2023-09-23T22:52:57.500901+0000","last_fullsized":"2023-09-23T22:52:57.500901+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":184,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.539003+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:52:57.539003+0000","last_peered":"2023-09-23T22:52:57.539003+0000","last_clean":"2023-09-23T22:52:57.539003+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:52:57.539003+0000","last_undegraded":"2023-09-23T22:52:57.539003+0000","last_fullsized":"2023-09-23T22:52:57.539003+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"291'5955","reported_seq":9356,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:00.752553+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:53:00.752553+0000","last_peered":"2023-09-23T22:53:00.752553+0000","last_clean":"2023-09-23T22:53:00.752553+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:53:00.752553+0000","last_undegraded":"2023-09-23T22:53:00.752553+0000","last_fullsized":"2023-09-23T22:53:00.752553+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5955,"log_dups_size":0,"ondisk_log_size":5955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331153408,"num_objects":5053,"num_object_clones":0,"num_object_copies":15159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5955,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":55,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.513472+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:52:56.513472+0000","last_peered":"2023-09-23T22:52:56.513472+0000","last_clean":"2023-09-23T22:52:56.513472+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:52:56.513472+0000","last_undegraded":"2023-09-23T22:52:56.513472+0000","last_fullsized":"2023-09-23T22:52:56.513472+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":264,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524995+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:52:57.524995+0000","last_peered":"2023-09-23T22:52:57.524995+0000","last_clean":"2023-09-23T22:52:57.524995+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:52:57.524995+0000","last_undegraded":"2023-09-23T22:52:57.524995+0000","last_fullsized":"2023-09-23T22:52:57.524995+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"291'5999","reported_seq":6034,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:00.821748+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:53:00.821748+0000","last_peered":"2023-09-23T22:53:00.821748+0000","last_clean":"2023-09-23T22:53:00.821748+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:53:00.821748+0000","last_undegraded":"2023-09-23T22:53:00.821748+0000","last_fullsized":"2023-09-23T22:53:00.821748+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5999,"log_dups_size":0,"ondisk_log_size":5999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333119488,"num_objects":5083,"num_object_clones":0,"num_object_copies":15249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5999,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500650+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:52:57.500650+0000","last_peered":"2023-09-23T22:52:57.500650+0000","last_clean":"2023-09-23T22:52:57.500650+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:52:57.500650+0000","last_undegraded":"2023-09-23T22:52:57.500650+0000","last_fullsized":"2023-09-23T22:52:57.500650+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":286,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500059+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:52:57.500059+0000","last_peered":"2023-09-23T22:52:57.500059+0000","last_clean":"2023-09-23T22:52:57.500059+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:52:57.500059+0000","last_undegraded":"2023-09-23T22:52:57.500059+0000","last_fullsized":"2023-09-23T22:52:57.500059+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"291'6093","reported_seq":6134,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:01.998065+0000","last_change":"2023-09-23T22:51:35.518569+0000","last_active":"2023-09-23T22:53:01.998065+0000","last_peered":"2023-09-23T22:53:01.998065+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:53:01.998065+0000","last_undegraded":"2023-09-23T22:51:35.445687+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":271,"log_start":"269'1000","ondisk_log_start":"269'1000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5093,"log_dups_size":0,"ondisk_log_size":5093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:39.333773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333381632,"num_objects":5087,"num_object_clones":0,"num_object_copies":15261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6093,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1162},{"shards":"1,5","objects":3422}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":299,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497985+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:52:57.497985+0000","last_peered":"2023-09-23T22:52:57.497985+0000","last_clean":"2023-09-23T22:52:57.497985+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:52:57.497985+0000","last_undegraded":"2023-09-23T22:52:57.497985+0000","last_fullsized":"2023-09-23T22:52:57.497985+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":274,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525100+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:52:57.525100+0000","last_peered":"2023-09-23T22:52:57.525100+0000","last_clean":"2023-09-23T22:52:57.525100+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:52:57.525100+0000","last_undegraded":"2023-09-23T22:52:57.525100+0000","last_fullsized":"2023-09-23T22:52:57.525100+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"291'5928","reported_seq":5970,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:59.878182+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:52:59.878182+0000","last_peered":"2023-09-23T22:52:59.878182+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:52:59.878182+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5928,"log_dups_size":0,"ondisk_log_size":5928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336986112,"num_objects":5142,"num_object_clones":0,"num_object_copies":15426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5928,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":3527},{"shards":"1,2,3","objects":1222}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":219,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497828+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:52:57.497828+0000","last_peered":"2023-09-23T22:52:57.497828+0000","last_clean":"2023-09-23T22:52:57.497828+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:52:57.497828+0000","last_undegraded":"2023-09-23T22:52:57.497828+0000","last_fullsized":"2023-09-23T22:52:57.497828+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":228,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525079+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:52:57.525079+0000","last_peered":"2023-09-23T22:52:57.525079+0000","last_clean":"2023-09-23T22:52:57.525079+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:52:57.525079+0000","last_undegraded":"2023-09-23T22:52:57.525079+0000","last_fullsized":"2023-09-23T22:52:57.525079+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"291'6167","reported_seq":6208,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:01.943505+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:53:01.943505+0000","last_peered":"2023-09-23T22:53:01.943505+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:53:01.943505+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"269'1000","ondisk_log_start":"269'1000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339804160,"num_objects":5185,"num_object_clones":0,"num_object_copies":15555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6167,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1139},{"shards":"1,5","objects":3555}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":327,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.509457+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:52:56.509457+0000","last_peered":"2023-09-23T22:52:56.509457+0000","last_clean":"2023-09-23T22:52:56.509457+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:52:56.509457+0000","last_undegraded":"2023-09-23T22:52:56.509457+0000","last_fullsized":"2023-09-23T22:52:56.509457+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":153,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524843+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:52:57.524843+0000","last_peered":"2023-09-23T22:52:57.524843+0000","last_clean":"2023-09-23T22:52:57.524843+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:52:57.524843+0000","last_undegraded":"2023-09-23T22:52:57.524843+0000","last_fullsized":"2023-09-23T22:52:57.524843+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497978+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:52:57.497978+0000","last_peered":"2023-09-23T22:52:57.497978+0000","last_clean":"2023-09-23T22:52:57.497978+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:52:57.497978+0000","last_undegraded":"2023-09-23T22:52:57.497978+0000","last_fullsized":"2023-09-23T22:52:57.497978+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"290'5941","reported_seq":9378,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:58.127995+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:52:58.127995+0000","last_peered":"2023-09-23T22:52:58.127995+0000","last_clean":"2023-09-23T22:52:58.127995+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:52:58.127995+0000","last_undegraded":"2023-09-23T22:52:58.127995+0000","last_fullsized":"2023-09-23T22:52:58.127995+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5941,"log_dups_size":0,"ondisk_log_size":5941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343212032,"num_objects":5237,"num_object_clones":0,"num_object_copies":15711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5941,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":125,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500298+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:52:57.500298+0000","last_peered":"2023-09-23T22:52:57.500298+0000","last_clean":"2023-09-23T22:52:57.500298+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:52:57.500298+0000","last_undegraded":"2023-09-23T22:52:57.500298+0000","last_fullsized":"2023-09-23T22:52:57.500298+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":287,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499392+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:52:57.499392+0000","last_peered":"2023-09-23T22:52:57.499392+0000","last_clean":"2023-09-23T22:52:57.499392+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:52:57.499392+0000","last_undegraded":"2023-09-23T22:52:57.499392+0000","last_fullsized":"2023-09-23T22:52:57.499392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":217,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524572+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:52:57.524572+0000","last_peered":"2023-09-23T22:52:57.524572+0000","last_clean":"2023-09-23T22:52:57.524572+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:52:57.524572+0000","last_undegraded":"2023-09-23T22:52:57.524572+0000","last_fullsized":"2023-09-23T22:52:57.524572+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"291'6079","reported_seq":6124,"reported_epoch":291,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:53:01.144505+0000","last_change":"2023-09-23T22:51:35.528654+0000","last_active":"2023-09-23T22:53:01.144505+0000","last_peered":"2023-09-23T22:53:01.144505+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:53:01.144505+0000","last_undegraded":"2023-09-23T22:53:01.144505+0000","last_fullsized":"2023-09-23T22:53:01.144505+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340590592,"num_objects":5197,"num_object_clones":0,"num_object_copies":15591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":5197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6079,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":3543}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":292,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524609+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:52:57.524609+0000","last_peered":"2023-09-23T22:52:57.524609+0000","last_clean":"2023-09-23T22:52:57.524609+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:52:57.524609+0000","last_undegraded":"2023-09-23T22:52:57.524609+0000","last_fullsized":"2023-09-23T22:52:57.524609+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":172,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524256+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:52:57.524256+0000","last_peered":"2023-09-23T22:52:57.524256+0000","last_clean":"2023-09-23T22:52:57.524256+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:52:57.524256+0000","last_undegraded":"2023-09-23T22:52:57.524256+0000","last_fullsized":"2023-09-23T22:52:57.524256+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":296,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507048+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:52:56.507048+0000","last_peered":"2023-09-23T22:52:56.507048+0000","last_clean":"2023-09-23T22:52:56.507048+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:52:56.507048+0000","last_undegraded":"2023-09-23T22:52:56.507048+0000","last_fullsized":"2023-09-23T22:52:56.507048+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"290'5890","reported_seq":5932,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:52:58.140085+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:52:58.140085+0000","last_peered":"2023-09-23T22:52:58.140085+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:52:58.140085+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338165760,"num_objects":5160,"num_object_clones":0,"num_object_copies":15480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5890,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1347},{"shards":"0,5","objects":3448}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":302,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524460+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:52:57.524460+0000","last_peered":"2023-09-23T22:52:57.524460+0000","last_clean":"2023-09-23T22:52:57.524460+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:52:57.524460+0000","last_undegraded":"2023-09-23T22:52:57.524460+0000","last_fullsized":"2023-09-23T22:52:57.524460+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":378,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.508642+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:52:56.508642+0000","last_peered":"2023-09-23T22:52:56.508642+0000","last_clean":"2023-09-23T22:52:56.508642+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:52:56.508642+0000","last_undegraded":"2023-09-23T22:52:56.508642+0000","last_fullsized":"2023-09-23T22:52:56.508642+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":45,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500469+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:52:57.500469+0000","last_peered":"2023-09-23T22:52:57.500469+0000","last_clean":"2023-09-23T22:52:57.500469+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:52:57.500469+0000","last_undegraded":"2023-09-23T22:52:57.500469+0000","last_fullsized":"2023-09-23T22:52:57.500469+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500518+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:52:57.500518+0000","last_peered":"2023-09-23T22:52:57.500518+0000","last_clean":"2023-09-23T22:52:57.500518+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:52:57.500518+0000","last_undegraded":"2023-09-23T22:52:57.500518+0000","last_fullsized":"2023-09-23T22:52:57.500518+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"290'5966","reported_seq":6001,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:58.121884+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:52:58.121884+0000","last_peered":"2023-09-23T22:52:58.121884+0000","last_clean":"2023-09-23T22:52:58.121884+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:52:58.121884+0000","last_undegraded":"2023-09-23T22:52:58.121884+0000","last_fullsized":"2023-09-23T22:52:58.121884+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344719360,"num_objects":5260,"num_object_clones":0,"num_object_copies":15780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5966,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.508822+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:52:56.508822+0000","last_peered":"2023-09-23T22:52:56.508822+0000","last_clean":"2023-09-23T22:52:56.508822+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:52:56.508822+0000","last_undegraded":"2023-09-23T22:52:56.508822+0000","last_fullsized":"2023-09-23T22:52:56.508822+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":237,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.508726+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:52:56.508726+0000","last_peered":"2023-09-23T22:52:56.508726+0000","last_clean":"2023-09-23T22:52:56.508726+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:52:56.508726+0000","last_undegraded":"2023-09-23T22:52:56.508726+0000","last_fullsized":"2023-09-23T22:52:56.508726+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":215,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507519+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:52:56.507519+0000","last_peered":"2023-09-23T22:52:56.507519+0000","last_clean":"2023-09-23T22:52:56.507519+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:52:56.507519+0000","last_undegraded":"2023-09-23T22:52:56.507519+0000","last_fullsized":"2023-09-23T22:52:56.507519+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"291'5979","reported_seq":6014,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:01.558829+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:53:01.558829+0000","last_peered":"2023-09-23T22:53:01.558829+0000","last_clean":"2023-09-23T22:53:01.558829+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:53:01.558829+0000","last_undegraded":"2023-09-23T22:53:01.558829+0000","last_fullsized":"2023-09-23T22:53:01.558829+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327352320,"num_objects":4995,"num_object_clones":0,"num_object_copies":14985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5979,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":220,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.508442+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:52:56.508442+0000","last_peered":"2023-09-23T22:52:56.508442+0000","last_clean":"2023-09-23T22:52:56.508442+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:52:56.508442+0000","last_undegraded":"2023-09-23T22:52:56.508442+0000","last_fullsized":"2023-09-23T22:52:56.508442+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":171,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.525306+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:52:59.525306+0000","last_peered":"2023-09-23T22:52:59.525306+0000","last_clean":"2023-09-23T22:52:59.525306+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:52:59.525306+0000","last_undegraded":"2023-09-23T22:52:59.525306+0000","last_fullsized":"2023-09-23T22:52:59.525306+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":220,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.513675+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:52:56.513675+0000","last_peered":"2023-09-23T22:52:56.513675+0000","last_clean":"2023-09-23T22:52:56.513675+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:52:56.513675+0000","last_undegraded":"2023-09-23T22:52:56.513675+0000","last_fullsized":"2023-09-23T22:52:56.513675+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"291'6193","reported_seq":6229,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:01.596364+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:53:01.596364+0000","last_peered":"2023-09-23T22:53:01.596364+0000","last_clean":"2023-09-23T22:53:01.596364+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:53:01.596364+0000","last_undegraded":"2023-09-23T22:53:01.596364+0000","last_fullsized":"2023-09-23T22:53:01.596364+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6193,"log_dups_size":0,"ondisk_log_size":6193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338690072,"num_objects":5169,"num_object_clones":0,"num_object_copies":15507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5169,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6193,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":46,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507371+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:52:56.507371+0000","last_peered":"2023-09-23T22:52:56.507371+0000","last_clean":"2023-09-23T22:52:56.507371+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:52:56.507371+0000","last_undegraded":"2023-09-23T22:52:56.507371+0000","last_fullsized":"2023-09-23T22:52:56.507371+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.522969+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:52:57.522969+0000","last_peered":"2023-09-23T22:52:57.522969+0000","last_clean":"2023-09-23T22:52:57.522969+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:52:57.522969+0000","last_undegraded":"2023-09-23T22:52:57.522969+0000","last_fullsized":"2023-09-23T22:52:57.522969+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.506713+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:52:56.506713+0000","last_peered":"2023-09-23T22:52:56.506713+0000","last_clean":"2023-09-23T22:52:56.506713+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:52:56.506713+0000","last_undegraded":"2023-09-23T22:52:56.506713+0000","last_fullsized":"2023-09-23T22:52:56.506713+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"291'6175","reported_seq":7783,"reported_epoch":291,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:53:01.584716+0000","last_change":"2023-09-23T22:52:26.294673+0000","last_active":"2023-09-23T22:53:01.584716+0000","last_peered":"2023-09-23T22:53:01.584716+0000","last_clean":"2023-09-23T22:51:33.377854+0000","last_became_active":"2023-09-23T22:51:37.033330+0000","last_became_peered":"2023-09-23T22:51:37.033330+0000","last_unstale":"2023-09-23T22:53:01.584716+0000","last_undegraded":"2023-09-23T22:53:01.584716+0000","last_fullsized":"2023-09-23T22:51:35.425861+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6175,"log_dups_size":0,"ondisk_log_size":6175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335347712,"num_objects":5117,"num_object_clones":0,"num_object_copies":15351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":947,"num_objects_unfound":0,"num_objects_dirty":5117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6175,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":782,"num_bytes_recovered":51249152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":947},{"shards":"0,4","objects":4170}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500604+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:52:57.500604+0000","last_peered":"2023-09-23T22:52:57.500604+0000","last_clean":"2023-09-23T22:52:57.500604+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:52:57.500604+0000","last_undegraded":"2023-09-23T22:52:57.500604+0000","last_fullsized":"2023-09-23T22:52:57.500604+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":187,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507594+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:52:56.507594+0000","last_peered":"2023-09-23T22:52:56.507594+0000","last_clean":"2023-09-23T22:52:56.507594+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:52:56.507594+0000","last_undegraded":"2023-09-23T22:52:56.507594+0000","last_fullsized":"2023-09-23T22:52:56.507594+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":241,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498025+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:52:57.498025+0000","last_peered":"2023-09-23T22:52:57.498025+0000","last_clean":"2023-09-23T22:52:57.498025+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:52:57.498025+0000","last_undegraded":"2023-09-23T22:52:57.498025+0000","last_fullsized":"2023-09-23T22:52:57.498025+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":222,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.513550+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:52:56.513550+0000","last_peered":"2023-09-23T22:52:56.513550+0000","last_clean":"2023-09-23T22:52:56.513550+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:52:56.513550+0000","last_undegraded":"2023-09-23T22:52:56.513550+0000","last_fullsized":"2023-09-23T22:52:56.513550+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":74,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524620+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:52:57.524620+0000","last_peered":"2023-09-23T22:52:57.524620+0000","last_clean":"2023-09-23T22:52:57.524620+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:52:57.524620+0000","last_undegraded":"2023-09-23T22:52:57.524620+0000","last_fullsized":"2023-09-23T22:52:57.524620+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":100,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524905+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:52:57.524905+0000","last_peered":"2023-09-23T22:52:57.524905+0000","last_clean":"2023-09-23T22:52:57.524905+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:52:57.524905+0000","last_undegraded":"2023-09-23T22:52:57.524905+0000","last_fullsized":"2023-09-23T22:52:57.524905+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":299,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499458+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:52:57.499458+0000","last_peered":"2023-09-23T22:52:57.499458+0000","last_clean":"2023-09-23T22:52:57.499458+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:52:57.499458+0000","last_undegraded":"2023-09-23T22:52:57.499458+0000","last_fullsized":"2023-09-23T22:52:57.499458+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":101,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500139+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:52:57.500139+0000","last_peered":"2023-09-23T22:52:57.500139+0000","last_clean":"2023-09-23T22:52:57.500139+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:52:57.500139+0000","last_undegraded":"2023-09-23T22:52:57.500139+0000","last_fullsized":"2023-09-23T22:52:57.500139+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499604+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:52:57.499604+0000","last_peered":"2023-09-23T22:52:57.499604+0000","last_clean":"2023-09-23T22:52:57.499604+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:52:57.499604+0000","last_undegraded":"2023-09-23T22:52:57.499604+0000","last_fullsized":"2023-09-23T22:52:57.499604+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":41,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.522897+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:52:57.522897+0000","last_peered":"2023-09-23T22:52:57.522897+0000","last_clean":"2023-09-23T22:52:57.522897+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:52:57.522897+0000","last_undegraded":"2023-09-23T22:52:57.522897+0000","last_fullsized":"2023-09-23T22:52:57.522897+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":273,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524448+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:52:57.524448+0000","last_peered":"2023-09-23T22:52:57.524448+0000","last_clean":"2023-09-23T22:52:57.524448+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:52:57.524448+0000","last_undegraded":"2023-09-23T22:52:57.524448+0000","last_fullsized":"2023-09-23T22:52:57.524448+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":236,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T22:52:56.507231+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:52:56.507231+0000","last_peered":"2023-09-23T22:52:56.507231+0000","last_clean":"2023-09-23T22:52:56.507231+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:52:56.507231+0000","last_undegraded":"2023-09-23T22:52:56.507231+0000","last_fullsized":"2023-09-23T22:52:56.507231+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497919+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:52:57.497919+0000","last_peered":"2023-09-23T22:52:57.497919+0000","last_clean":"2023-09-23T22:52:57.497919+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:52:57.497919+0000","last_undegraded":"2023-09-23T22:52:57.497919+0000","last_fullsized":"2023-09-23T22:52:57.497919+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5413470232,"num_objects":82604,"num_object_clones":0,"num_object_copies":247812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":2601,"num_objects_unfound":0,"num_objects_dirty":82604,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":96674,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4149,"num_bytes_recovered":271908864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16899780608,"data_stored":16899768392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94674,"ondisk_log_size":94674,"up":48,"acting":41,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430520,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626748,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2646108,"kb_used_data":2340692,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91099044,"statfs":{"total":95995035648,"available":93285421056,"internally_reserved":0,"allocated":2396868608,"data_stored":2396312679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":24.245999999999999,"5min":9.1280000000000001,"15min":3.9020000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":243.35900000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":2.5049999999999999},{"interface":"front","average":{"1min":24.465,"5min":9.5229999999999997,"15min":3.8250000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":243.41499999999999,"5min":243.41499999999999,"15min":243.41499999999999},"last":4.476}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":14.752000000000001,"5min":5.0700000000000003,"15min":3.6389999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":254.62100000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":20.399999999999999,"5min":7.9690000000000003,"15min":4.6319999999999997},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":255.20099999999999,"5min":255.20099999999999,"15min":255.20099999999999},"last":2.7519999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":3.71,"15min":3.367},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":5.4279999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":2.7879999999999998},{"interface":"front","average":{"1min":1.542,"5min":4.9530000000000003,"15min":3.677},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4390000000000001,"5min":185.21799999999999,"15min":185.21799999999999},"last":2.536}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":5.6150000000000002,"15min":3.706},"min":{"1min":0.36199999999999999,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":7.734,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.309},{"interface":"front","average":{"1min":1.919,"5min":3.919,"15min":3.3170000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.7770000000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":5.0970000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.875,"15min":0.753},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.7290000000000001,"5min":6.5309999999999997,"15min":18.085999999999999},"last":5.5659999999999998},{"interface":"front","average":{"1min":1.3320000000000001,"5min":0.78900000000000003,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4800000000000004,"5min":7.4800000000000004,"15min":18.597000000000001},"last":2.879}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.0529999999999999,"15min":0.85299999999999998},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.7560000000000002,"5min":10.148999999999999,"15min":18.972000000000001},"last":5.9299999999999997},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.83799999999999997,"15min":0.80900000000000005},"min":{"1min":0.316,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":2.782,"5min":4.9089999999999998,"15min":18.625},"last":6.0469999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":0.86499999999999999,"15min":0.82999999999999996},"min":{"1min":0.35799999999999998,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":4.2869999999999999,"5min":4.3049999999999997,"15min":12.481999999999999},"last":2.5600000000000001},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.03,"15min":0.86199999999999999},"min":{"1min":0.38200000000000001,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":6.6269999999999998,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.33200000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253058,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4249856,"kb_used_data":3778872,"kb_used_omap":9,"kb_used_meta":470966,"kb_avail":89495296,"statfs":{"total":95995035648,"available":91643183104,"internally_reserved":0,"allocated":3869564928,"data_stored":3868513138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":482270185},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":7.5039999999999996,"15min":5.2800000000000002},"min":{"1min":0.26800000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.933,"5min":101.605,"15min":101.605},"last":1.5169999999999999},{"interface":"front","average":{"1min":3.1309999999999998,"5min":6.6639999999999997,"15min":5.2309999999999999},"min":{"1min":0.24099999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":8.6620000000000008,"5min":101.892,"15min":101.892},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.8610000000000002,"15min":2.7989999999999999},"min":{"1min":0.22500000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":12.622999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":2.2320000000000002},{"interface":"front","average":{"1min":2.214,"5min":4.0629999999999997,"15min":2.8799999999999999},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.887,"5min":37.957000000000001,"15min":37.957000000000001},"last":1.417}]},{"osd":3,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":3.3170000000000002,"15min":2.3809999999999998},"min":{"1min":0.28999999999999998,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.218,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.9910000000000001,"5min":3.9590000000000001,"15min":2.835},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":4.7859999999999996,"5min":89.861999999999995,"15min":89.861999999999995},"last":1.71}]},{"osd":4,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.4950000000000001,"15min":2.0950000000000002},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":3.3889999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":2.2069999999999999},{"interface":"front","average":{"1min":2.4140000000000001,"5min":2.9009999999999998,"15min":2.0920000000000001},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":12.792999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":2.3250000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":2.1030000000000002,"15min":1.6459999999999999},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":5.9960000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":5.0049999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.6360000000000001,"15min":1.9710000000000001},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":3.347,"5min":87.515000000000001,"15min":87.515000000000001},"last":6.4630000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.9390000000000001,"15min":1.925},"min":{"1min":0.41799999999999998,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.2539999999999996,"5min":79.311000000000007,"15min":79.311000000000007},"last":2.2770000000000001},{"interface":"front","average":{"1min":1.909,"5min":2.1779999999999999,"15min":1.601},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":5.9340000000000002,"5min":79.328999999999994,"15min":79.328999999999994},"last":1.25}]},{"osd":7,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.008,"15min":2.3460000000000001},"min":{"1min":0.42499999999999999,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":4.0979999999999999,"5min":11.481999999999999,"15min":159.018},"last":1.8440000000000001},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.4399999999999999,"15min":2.444},"min":{"1min":0.47599999999999998,"5min":0.371,"15min":0.37},"max":{"1min":6.282,"5min":14.032999999999999,"15min":158.96000000000001},"last":2.3039999999999998}]}]},{"osd":0,"up_from":219,"seq":940597837906,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2577832,"kb_used_data":2278624,"kb_used_omap":31,"kb_used_meta":299168,"kb_avail":91167320,"statfs":{"total":95995035648,"available":93355335680,"internally_reserved":0,"allocated":2333310976,"data_stored":2332208240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":306348978},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":3.5169999999999999,"15min":3.6469999999999998},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":11.930999999999999,"5min":21.512,"15min":32.344999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.4169999999999998,"5min":3.4249999999999998,"15min":3.5139999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7379999999999995,"5min":20.064,"15min":32.106000000000002},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":4.9889999999999999,"15min":3.0609999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.9619999999999997,"5min":111.508,"15min":111.508},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.669,"5min":4.2229999999999999,"15min":2.5790000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.9749999999999996,"5min":111.532,"15min":111.532},"last":0.62}]},{"osd":3,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":4.9450000000000003,"15min":2.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9269999999999996,"5min":126.73,"15min":126.73},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.177,"5min":3.8519999999999999,"15min":2.0329999999999999},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.8419999999999996,"5min":121.483,"15min":121.483},"last":0.67500000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":2.633,"15min":2.113},"min":{"1min":0.36299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.965999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.5169999999999999,"15min":2.0049999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.9990000000000006,"5min":20.009,"15min":20.009},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":2.9180000000000001,"15min":1.863},"min":{"1min":0.41399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.6280000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":1.9970000000000001},{"interface":"front","average":{"1min":2.6379999999999999,"5min":3.3769999999999998,"15min":2.161},"min":{"1min":0.40400000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.0519999999999996,"5min":61.777000000000001,"15min":61.777000000000001},"last":8.8770000000000007}]},{"osd":6,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":2.6909999999999998,"15min":1.8109999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5369999999999999,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.9220000000000002,"5min":3.3679999999999999,"15min":2.246},"min":{"1min":0.48499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":11.882999999999999,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.6800000000000002,"15min":1.9570000000000001},"min":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.0869999999999997,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.2320000000000002,"15min":1.5289999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.9219999999999997,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.747}]}]},{"osd":2,"up_from":131,"seq":562640715919,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2273732,"kb_used_data":1915452,"kb_used_omap":81,"kb_used_meta":358190,"kb_avail":91471420,"statfs":{"total":95995035648,"available":93666734080,"internally_reserved":0,"allocated":1961422848,"data_stored":1960370344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":366786778},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":8.4789999999999992,"5min":6.7309999999999999,"15min":5.7560000000000002},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":114.483,"5min":132.79499999999999,"15min":132.79499999999999},"last":9.6120000000000001},{"interface":"front","average":{"1min":8.5,"5min":7.7599999999999998,"15min":6.1799999999999997},"min":{"1min":0.34599999999999997,"5min":0.314,"15min":0.314},"max":{"1min":113.873,"5min":132.58099999999999,"15min":132.58099999999999},"last":10.413}]},{"osd":1,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.5310000000000001,"15min":3.0230000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":12.035,"5min":15.622999999999999,"15min":88.316000000000003},"last":1.544},{"interface":"front","average":{"1min":3.5329999999999999,"5min":3.4220000000000002,"15min":3.1589999999999998},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":14.159000000000001,"5min":28.596,"15min":88.436999999999998},"last":3.7610000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.3250000000000002,"15min":4.1890000000000001},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":6.5419999999999998,"5min":112.895,"15min":112.895},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.524,"5min":4.617,"15min":3.4169999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.251},"max":{"1min":9.3840000000000003,"5min":113.995,"15min":113.995},"last":1.1699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.597,"15min":2.4169999999999998},"min":{"1min":0.37,"5min":0.253,"15min":0.253},"max":{"1min":7.1520000000000001,"5min":84.778999999999996,"15min":108.878},"last":2.0960000000000001},{"interface":"front","average":{"1min":1.901,"5min":3.641,"15min":3.056},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.8460000000000001,"5min":84.753,"15min":108.93000000000001},"last":2.7589999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0750000000000002,"15min":2.9009999999999998},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.5970000000000004,"5min":90.082999999999998,"15min":97.322999999999993},"last":2.4409999999999998},{"interface":"front","average":{"1min":2.0049999999999999,"5min":3.3399999999999999,"15min":2.718},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.5380000000000003,"5min":89.986000000000004,"15min":97.301000000000002},"last":3.802}]},{"osd":6,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":4.8330000000000002,"15min":2.9390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":4.5789999999999997,"5min":98.597999999999999,"15min":98.597999999999999},"last":4.5099999999999998},{"interface":"front","average":{"1min":1.607,"5min":4.3360000000000003,"15min":2.3180000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.4089999999999998,"5min":98.509,"15min":98.509},"last":2.6389999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0459999999999998,"15min":1.5740000000000001},"min":{"1min":0.42799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.7440000000000002,"5min":10.92,"15min":10.92},"last":1.645},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.1240000000000001,"15min":2.323},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":4.2489999999999997,"5min":11.311,"15min":88.584000000000003},"last":1.6180000000000001}]}]},{"osd":3,"up_from":0,"seq":403726926005,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019204,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2152864,"kb_used_data":1839256,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91592288,"statfs":{"total":95995035648,"available":93790502912,"internally_reserved":0,"allocated":1883398144,"data_stored":1882837621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":8.5950000000000006,"5min":3.0579999999999998,"15min":6.3630000000000004},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":113.35599999999999,"5min":113.35599999999999,"15min":113.35599999999999},"last":2.8159999999999998},{"interface":"front","average":{"1min":8.9220000000000006,"5min":3.7679999999999998,"15min":6.9420000000000002},"min":{"1min":0.34000000000000002,"5min":0.217,"15min":0.217},"max":{"1min":113.417,"5min":113.417,"15min":113.417},"last":6.3609999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":11.753,"5min":3.7679999999999998,"15min":2.4649999999999999},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":132.304,"5min":132.304,"15min":132.304},"last":2.6059999999999999},{"interface":"front","average":{"1min":11.4,"5min":3.6560000000000001,"15min":2.3180000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":132.453,"5min":132.453,"15min":132.453},"last":2.839}]},{"osd":2,"last update":"Sat Sep 23 22:52:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":2.2240000000000002,"15min":3.1720000000000002},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.308,"5min":20.308,"15min":84.909000000000006},"last":6.5499999999999998},{"interface":"front","average":{"1min":2.8769999999999998,"5min":2.3769999999999998,"15min":3.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.652000000000001,"5min":21.704000000000001,"15min":83.840000000000003},"last":2.8769999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":5.3339999999999996,"5min":5.3849999999999998,"15min":4.3410000000000002},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":74.138999999999996,"5min":164.453,"15min":164.453},"last":2.988},{"interface":"front","average":{"1min":4.9329999999999998,"5min":5.3520000000000003,"15min":4.4249999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":75.099999999999994,"5min":167.05199999999999,"15min":167.05199999999999},"last":2.6829999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.91900000000000004,"15min":0.85699999999999998},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.0030000000000001,"5min":12.238,"15min":43.189999999999998},"last":3.6499999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89900000000000002,"15min":0.874},"min":{"1min":0.29499999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.9910000000000001,"5min":7.5460000000000003,"15min":42.450000000000003},"last":6.4349999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.95199999999999996,"15min":0.84399999999999997},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.758,"5min":10.952999999999999,"15min":27.056000000000001},"last":6.4039999999999999},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.88600000000000001,"15min":0.84299999999999997},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":1.978,"5min":12.269,"15min":26.940999999999999},"last":2.6520000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:51:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.91600000000000004,"15min":0.75700000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":2.327,"5min":12.311,"15min":12.311},"last":2.7469999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.92800000000000005,"15min":0.73399999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":2.9660000000000002,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":2.899}]}]},{"osd":5,"up_from":36,"seq":154618822976,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3609540,"kb_used_data":3308028,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90135612,"statfs":{"total":95995035648,"available":92298866688,"internally_reserved":0,"allocated":3387420672,"data_stored":3386883865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":8.0820000000000007,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":74.406000000000006,"5min":121.319,"15min":121.319},"last":19.716999999999999},{"interface":"front","average":{"1min":7.1799999999999997,"5min":8.2539999999999996,"15min":3.8479999999999999},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":73.899000000000001,"5min":121.075,"15min":121.075},"last":20.146999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8820000000000001,"5min":2.0649999999999999,"15min":2.056},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":41.463999999999999,"5min":41.463999999999999,"15min":149.203},"last":17.658999999999999},{"interface":"front","average":{"1min":4.0869999999999997,"5min":2.5670000000000002,"15min":2.3260000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":41.954000000000001,"5min":41.954000000000001,"15min":149.554},"last":17.84}]},{"osd":2,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.9660000000000002,"15min":5.6040000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":34.988,"5min":63.156999999999996,"15min":158.92599999999999},"last":15.574999999999999},{"interface":"front","average":{"1min":4.3799999999999999,"5min":2.7050000000000001,"15min":5.5270000000000001},"min":{"1min":0.38600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":34.872,"5min":63.277999999999999,"15min":158.91399999999999},"last":15.542999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":5.3380000000000001,"5min":7.6420000000000003,"15min":5.3600000000000003},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":89.995000000000005,"5min":108.58799999999999,"15min":257.05000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":5.8029999999999999,"5min":7.8769999999999998,"15min":5.2800000000000002},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":90.117999999999995,"5min":108.15600000000001,"15min":257.35000000000002},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.63,"15min":0.64900000000000002},"min":{"1min":0.27400000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":2.024,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.625},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":1.873,"5min":3.4409999999999998,"15min":10.535},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.89500000000000002,"15min":1.4490000000000001},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.7469999999999999,"5min":12.615,"15min":140.07499999999999},"last":0.436},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.85899999999999999,"15min":1.4790000000000001},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":4,"5min":12.74,"15min":139.739},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67800000000000005,"15min":0.72299999999999998},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":1.6830000000000001,"5min":2.738,"15min":7.9809999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.78300000000000003,"15min":0.71099999999999997},"min":{"1min":0.374,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":2.988,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.46700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2318663680,"data_stored":2318663680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3852603392,"data_stored":3852599320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1955856384,"data_stored":1955856384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1333329920,"data_stored":1333329920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1674711040,"data_stored":1674706968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3377598464,"data_stored":3377594392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2387017728,"data_stored":2387017728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:53:04.248 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:04.277 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:04.324 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:04.330 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:04.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:04.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:04.485 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:04.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:04 smithi151 ceph-mon[159210]: from='client.45716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:04.564 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:04.673 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:04.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:04.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:04.715 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:53:04.804 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:04.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:04 smithi052 ceph-mon[180979]: from='client.45716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:04.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:04.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:04 smithi052 ceph-mon[187743]: from='client.45716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:04.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:04.877 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:05.100 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:05.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:05.205 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:05.234 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:05.258 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:53:05.279 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:05.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:05.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:05.432 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:05.543 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:05.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:05.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:05.649 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:05 smithi151 ceph-mon[159210]: pgmap v827: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 10054/245409 objects degraded (4.097%); 2601/245409 objects misplaced (1.060%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:53:05.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/245409 objects degraded (4.097%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:05.723 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:05.933 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:05.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:05 smithi052 ceph-mon[180979]: pgmap v827: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 10054/245409 objects degraded (4.097%); 2601/245409 objects misplaced (1.060%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:53:05.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/245409 objects degraded (4.097%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:06.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:05 smithi052 ceph-mon[187743]: pgmap v827: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 10054/245409 objects degraded (4.097%); 2601/245409 objects misplaced (1.060%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:53:06.001 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/245409 objects degraded (4.097%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:06.040 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:06.048 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:53:06.101 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:06.206 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:06.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:06.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:06.408 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:06.519 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:06.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:06.634 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:06.736 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:06.745 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:06.854 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:06.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:06.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:07.042 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:07.145 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:53:07.145 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:53:07.145 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:53:07.145 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:53:07.146 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:07.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:07.249 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:53:07.254 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:07.319 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:07.442 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:07.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:07.545 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:07.575 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:07 smithi052 ceph-mon[180979]: pgmap v828: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 10054/241515 objects degraded (4.163%); 2601/241515 objects misplaced (1.077%); 785 KiB/s, 12 objects/s recovering 2023-09-23T22:53:07.575 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3167151659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:07.576 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:07.595 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:53:07.606 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:07.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:07 smithi151 ceph-mon[159210]: pgmap v828: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 10054/241515 objects degraded (4.163%); 2601/241515 objects misplaced (1.077%); 785 KiB/s, 12 objects/s recovering 2023-09-23T22:53:07.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3167151659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:07.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:07.971 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:53:07.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:07 smithi052 ceph-mon[187743]: pgmap v828: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 576 op/s; 10054/241515 objects degraded (4.163%); 2601/241515 objects misplaced (1.077%); 785 KiB/s, 12 objects/s recovering 2023-09-23T22:53:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3167151659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:08.129 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:08.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:08.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:08.234 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:08.281 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:08.396 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:08.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:08.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:08.499 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:08.535 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:08.539 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:53:08.585 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:08.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:08.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:08.691 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:08.748 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:08.815 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:08 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:08.815 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:08 smithi151 ceph-mon[159210]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:08.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:08 smithi151 ceph-mon[159210]: osdmap e292: 8 total, 8 up, 6 in 2023-09-23T22:53:08.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:08 smithi151 ceph-mon[159210]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:08 smithi151 ceph-mon[159210]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.832 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:08.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:08.938 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:08.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[180979]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[180979]: osdmap e292: 8 total, 8 up, 6 in 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[180979]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[180979]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[187743]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:08.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[187743]: osdmap e292: 8 total, 8 up, 6 in 2023-09-23T22:53:08.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[187743]: failed to encode map e292 with expected crc 2023-09-23T22:53:08.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:08 smithi052 ceph-mon[187743]: failed to encode map e292 with expected crc 2023-09-23T22:53:09.046 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:09.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:09.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:09.286 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:09.338 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:09.381 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:09.395 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:09.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:09.498 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:09.499 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:09.545 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:09.562 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:09.591 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:09.633 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:09.666 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:09.695 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:09 smithi151 ceph-mon[159210]: pgmap v830: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 10054/235449 objects degraded (4.270%); 2491/235449 objects misplaced (1.058%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:53:09.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3211352595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:09.708 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:09.724 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:09.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:09.764 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:53:09.812 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:09 smithi052 ceph-mon[180979]: pgmap v830: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 10054/235449 objects degraded (4.270%); 2491/235449 objects misplaced (1.058%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:53:09.812 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3211352595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:09.812 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:09 smithi052 ceph-mon[187743]: pgmap v830: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 10054/235449 objects degraded (4.270%); 2491/235449 objects misplaced (1.058%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:53:09.812 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3211352595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:09.827 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:09.911 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:10.018 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:10.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:10.124 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:10.169 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:10.258 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:10.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:10.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:10.368 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:10.368 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:53:10.369 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:53:10.428 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:10.575 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:10.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:10.681 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:10.707 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:10.795 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:10.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:10.904 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:10.961 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:10.988 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:53:11.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:11 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/235449 objects degraded (4.270%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:11.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:11 smithi052 ceph-mon[180979]: pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 638 op/s; 10054/233721 objects degraded (4.302%); 2491/233721 objects misplaced (1.066%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:53:11.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:11 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/235449 objects degraded (4.270%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:11.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:11 smithi052 ceph-mon[187743]: pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 638 op/s; 10054/233721 objects degraded (4.302%); 2491/233721 objects misplaced (1.066%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:53:11.247 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:11 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/235449 objects degraded (4.270%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:11.247 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:11 smithi151 ceph-mon[159210]: pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 638 op/s; 10054/233721 objects degraded (4.302%); 2491/233721 objects misplaced (1.066%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:53:11.267 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:11.274 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:53:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:11.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:11.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:11.473 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:11.758 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:11.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:11.841 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:53:11.865 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:11.966 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:12.015 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:12.086 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:12.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:12.197 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:12.260 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:12.345 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:12.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:12.392 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:53:12.400 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:53:12.454 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:12.535 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:12.635 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:12.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:12.744 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:12.811 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:12.925 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:12.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:13.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:13.079 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:13.338 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:53:13.346 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:53:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:53:13] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.33.4" 2023-09-23T22:53:13.683 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:53:13.684 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:53:13.693 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:13 smithi151 ceph-mon[159210]: from='client.35676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:13.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:13 smithi151 ceph-mon[159210]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 10054/228114 objects degraded (4.407%); 2380/228114 objects misplaced (1.043%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:13.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:13 smithi052 ceph-mon[187743]: from='client.35676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:13.703 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:13 smithi052 ceph-mon[187743]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 10054/228114 objects degraded (4.407%); 2380/228114 objects misplaced (1.043%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:13.704 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:13 smithi052 ceph-mon[180979]: from='client.35676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:13.704 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:13 smithi052 ceph-mon[180979]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 10054/228114 objects degraded (4.407%); 2380/228114 objects misplaced (1.043%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:13.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:13.805 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:13.841 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:13.915 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:13.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:14.024 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:14.105 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":831,"stamp":"2023-09-23T22:53:10.385948+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5104670648,"num_objects":77907,"num_object_clones":0,"num_object_copies":233721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":2491,"num_objects_unfound":0,"num_objects_dirty":77907,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":101655,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4289,"num_bytes_recovered":279128118,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99189,"ondisk_log_size":99189,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16841544,"kb_used_data":14765528,"kb_used_omap":121,"kb_used_meta":2074886,"kb_avail":545629368,"statfs":{"total":575970213888,"available":558724472832,"internally_reserved":0,"allocated":15119900672,"data_stored":15115036562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2124683381},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,12,3,6,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-418381824,"num_objects":-6384,"num_object_clones":0,"num_object_copies":-19152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-219,"num_objects_unfound":0,"num_objects_dirty":-6384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6384,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":219,"num_bytes_recovered":14352384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005649"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":288,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498038+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:52:57.498038+0000","last_peered":"2023-09-23T22:52:57.498038+0000","last_clean":"2023-09-23T22:52:57.498038+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:52:57.498038+0000","last_undegraded":"2023-09-23T22:52:57.498038+0000","last_fullsized":"2023-09-23T22:52:57.498038+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500341+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:52:57.500341+0000","last_peered":"2023-09-23T22:52:57.500341+0000","last_clean":"2023-09-23T22:52:57.500341+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:52:57.500341+0000","last_undegraded":"2023-09-23T22:52:57.500341+0000","last_fullsized":"2023-09-23T22:52:57.500341+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498071+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:52:57.498071+0000","last_peered":"2023-09-23T22:52:57.498071+0000","last_clean":"2023-09-23T22:52:57.498071+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:52:57.498071+0000","last_undegraded":"2023-09-23T22:52:57.498071+0000","last_fullsized":"2023-09-23T22:52:57.498071+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":270,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534581+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:52:59.534581+0000","last_peered":"2023-09-23T22:52:59.534581+0000","last_clean":"2023-09-23T22:52:59.534581+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:52:59.534581+0000","last_undegraded":"2023-09-23T22:52:59.534581+0000","last_fullsized":"2023-09-23T22:52:59.534581+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":274,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.535959+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:52:59.535959+0000","last_peered":"2023-09-23T22:52:59.535959+0000","last_clean":"2023-09-23T22:52:59.535959+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:52:59.535959+0000","last_undegraded":"2023-09-23T22:52:59.535959+0000","last_fullsized":"2023-09-23T22:52:59.535959+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500734+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:52:57.500734+0000","last_peered":"2023-09-23T22:52:57.500734+0000","last_clean":"2023-09-23T22:52:57.500734+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:52:57.500734+0000","last_undegraded":"2023-09-23T22:52:57.500734+0000","last_fullsized":"2023-09-23T22:52:57.500734+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":201,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534810+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:52:59.534810+0000","last_peered":"2023-09-23T22:52:59.534810+0000","last_clean":"2023-09-23T22:52:59.534810+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:52:59.534810+0000","last_undegraded":"2023-09-23T22:52:59.534810+0000","last_fullsized":"2023-09-23T22:52:59.534810+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":153,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.524759+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:52:59.524759+0000","last_peered":"2023-09-23T22:52:59.524759+0000","last_clean":"2023-09-23T22:52:59.524759+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:52:59.524759+0000","last_undegraded":"2023-09-23T22:52:59.524759+0000","last_fullsized":"2023-09-23T22:52:59.524759+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":32,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.524820+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:52:59.524820+0000","last_peered":"2023-09-23T22:52:59.524820+0000","last_clean":"2023-09-23T22:52:59.524820+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:52:59.524820+0000","last_undegraded":"2023-09-23T22:52:59.524820+0000","last_fullsized":"2023-09-23T22:52:59.524820+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.523215+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:52:57.523215+0000","last_peered":"2023-09-23T22:52:57.523215+0000","last_clean":"2023-09-23T22:52:57.523215+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:52:57.523215+0000","last_undegraded":"2023-09-23T22:52:57.523215+0000","last_fullsized":"2023-09-23T22:52:57.523215+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":49,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500224+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:52:57.500224+0000","last_peered":"2023-09-23T22:52:57.500224+0000","last_clean":"2023-09-23T22:52:57.500224+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:52:57.500224+0000","last_undegraded":"2023-09-23T22:52:57.500224+0000","last_fullsized":"2023-09-23T22:52:57.500224+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500544+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:52:57.500544+0000","last_peered":"2023-09-23T22:52:57.500544+0000","last_clean":"2023-09-23T22:52:57.500544+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:52:57.500544+0000","last_undegraded":"2023-09-23T22:52:57.500544+0000","last_fullsized":"2023-09-23T22:52:57.500544+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":310,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525079+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:52:57.525079+0000","last_peered":"2023-09-23T22:52:57.525079+0000","last_clean":"2023-09-23T22:52:57.525079+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:52:57.525079+0000","last_undegraded":"2023-09-23T22:52:57.525079+0000","last_fullsized":"2023-09-23T22:52:57.525079+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499173+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:52:57.499173+0000","last_peered":"2023-09-23T22:52:57.499173+0000","last_clean":"2023-09-23T22:52:57.499173+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:52:57.499173+0000","last_undegraded":"2023-09-23T22:52:57.499173+0000","last_fullsized":"2023-09-23T22:52:57.499173+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":149,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499311+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:52:57.499311+0000","last_peered":"2023-09-23T22:52:57.499311+0000","last_clean":"2023-09-23T22:52:57.499311+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:52:57.499311+0000","last_undegraded":"2023-09-23T22:52:57.499311+0000","last_fullsized":"2023-09-23T22:52:57.499311+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":238,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.525066+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:52:59.525066+0000","last_peered":"2023-09-23T22:52:59.525066+0000","last_clean":"2023-09-23T22:52:59.525066+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:52:59.525066+0000","last_undegraded":"2023-09-23T22:52:59.525066+0000","last_fullsized":"2023-09-23T22:52:59.525066+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":257,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534234+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:52:59.534234+0000","last_peered":"2023-09-23T22:52:59.534234+0000","last_clean":"2023-09-23T22:52:59.534234+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:52:59.534234+0000","last_undegraded":"2023-09-23T22:52:59.534234+0000","last_fullsized":"2023-09-23T22:52:59.534234+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":214,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500728+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:52:57.500728+0000","last_peered":"2023-09-23T22:52:57.500728+0000","last_clean":"2023-09-23T22:52:57.500728+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:52:57.500728+0000","last_undegraded":"2023-09-23T22:52:57.500728+0000","last_fullsized":"2023-09-23T22:52:57.500728+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":141,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498151+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:52:57.498151+0000","last_peered":"2023-09-23T22:52:57.498151+0000","last_clean":"2023-09-23T22:52:57.498151+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:52:57.498151+0000","last_undegraded":"2023-09-23T22:52:57.498151+0000","last_fullsized":"2023-09-23T22:52:57.498151+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":258,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534662+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:52:59.534662+0000","last_peered":"2023-09-23T22:52:59.534662+0000","last_clean":"2023-09-23T22:52:59.534662+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:52:59.534662+0000","last_undegraded":"2023-09-23T22:52:59.534662+0000","last_fullsized":"2023-09-23T22:52:59.534662+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":335,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500069+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:52:57.500069+0000","last_peered":"2023-09-23T22:52:57.500069+0000","last_clean":"2023-09-23T22:52:57.500069+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:52:57.500069+0000","last_undegraded":"2023-09-23T22:52:57.500069+0000","last_fullsized":"2023-09-23T22:52:57.500069+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":218,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.527321+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:52:59.527321+0000","last_peered":"2023-09-23T22:52:59.527321+0000","last_clean":"2023-09-23T22:52:59.527321+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:52:59.527321+0000","last_undegraded":"2023-09-23T22:52:59.527321+0000","last_fullsized":"2023-09-23T22:52:59.527321+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"292'6310","reported_seq":6347,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:08.100332+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:53:08.100332+0000","last_peered":"2023-09-23T22:53:08.100332+0000","last_clean":"2023-09-23T22:53:08.100332+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:53:08.100332+0000","last_undegraded":"2023-09-23T22:53:08.100332+0000","last_fullsized":"2023-09-23T22:53:08.100332+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6310,"log_dups_size":0,"ondisk_log_size":6310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316014592,"num_objects":4822,"num_object_clones":0,"num_object_copies":14466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6310,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":160,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524900+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:52:57.524900+0000","last_peered":"2023-09-23T22:52:57.524900+0000","last_clean":"2023-09-23T22:52:57.524900+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:52:57.524900+0000","last_undegraded":"2023-09-23T22:52:57.524900+0000","last_fullsized":"2023-09-23T22:52:57.524900+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500130+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:52:57.500130+0000","last_peered":"2023-09-23T22:52:57.500130+0000","last_clean":"2023-09-23T22:52:57.500130+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:52:57.500130+0000","last_undegraded":"2023-09-23T22:52:57.500130+0000","last_fullsized":"2023-09-23T22:52:57.500130+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"291'6427","reported_seq":6469,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:06.602255+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:53:06.602255+0000","last_peered":"2023-09-23T22:53:06.602255+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:53:06.602255+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6427,"log_dups_size":0,"ondisk_log_size":6427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331939840,"num_objects":5065,"num_object_clones":0,"num_object_copies":15195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6427,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":980},{"shards":"1,4","objects":3404}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":170,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525008+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:52:57.525008+0000","last_peered":"2023-09-23T22:52:57.525008+0000","last_clean":"2023-09-23T22:52:57.525008+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:52:57.525008+0000","last_undegraded":"2023-09-23T22:52:57.525008+0000","last_fullsized":"2023-09-23T22:52:57.525008+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":280,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498090+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:52:57.498090+0000","last_peered":"2023-09-23T22:52:57.498090+0000","last_clean":"2023-09-23T22:52:57.498090+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:52:57.498090+0000","last_undegraded":"2023-09-23T22:52:57.498090+0000","last_fullsized":"2023-09-23T22:52:57.498090+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"292'6521","reported_seq":6564,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:09.845398+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:53:09.845398+0000","last_peered":"2023-09-23T22:53:09.845398+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:53:09.845398+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6521,"log_dups_size":0,"ondisk_log_size":6521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312410112,"num_objects":4767,"num_object_clones":0,"num_object_copies":14301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6521,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":894},{"shards":"2,5","objects":2996}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":114,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500706+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:52:57.500706+0000","last_peered":"2023-09-23T22:52:57.500706+0000","last_clean":"2023-09-23T22:52:57.500706+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:52:57.500706+0000","last_undegraded":"2023-09-23T22:52:57.500706+0000","last_fullsized":"2023-09-23T22:52:57.500706+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":283,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500383+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:52:57.500383+0000","last_peered":"2023-09-23T22:52:57.500383+0000","last_clean":"2023-09-23T22:52:57.500383+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:52:57.500383+0000","last_undegraded":"2023-09-23T22:52:57.500383+0000","last_fullsized":"2023-09-23T22:52:57.500383+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"292'6468","reported_seq":6505,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:09.860125+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:53:09.860125+0000","last_peered":"2023-09-23T22:53:09.860125+0000","last_clean":"2023-09-23T22:53:09.860125+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:53:09.860125+0000","last_undegraded":"2023-09-23T22:53:09.860125+0000","last_fullsized":"2023-09-23T22:53:09.860125+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317718528,"num_objects":4848,"num_object_clones":0,"num_object_copies":14544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6468,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":86,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500901+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:52:57.500901+0000","last_peered":"2023-09-23T22:52:57.500901+0000","last_clean":"2023-09-23T22:52:57.500901+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:52:57.500901+0000","last_undegraded":"2023-09-23T22:52:57.500901+0000","last_fullsized":"2023-09-23T22:52:57.500901+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":186,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534406+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:52:59.534406+0000","last_peered":"2023-09-23T22:52:59.534406+0000","last_clean":"2023-09-23T22:52:59.534406+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:52:59.534406+0000","last_undegraded":"2023-09-23T22:52:59.534406+0000","last_fullsized":"2023-09-23T22:52:59.534406+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"291'6181","reported_seq":9582,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:05.916366+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:53:05.916366+0000","last_peered":"2023-09-23T22:53:05.916366+0000","last_clean":"2023-09-23T22:53:05.916366+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:53:05.916366+0000","last_undegraded":"2023-09-23T22:53:05.916366+0000","last_fullsized":"2023-09-23T22:53:05.916366+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6181,"log_dups_size":0,"ondisk_log_size":6181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6181,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":57,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534512+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:52:59.534512+0000","last_peered":"2023-09-23T22:52:59.534512+0000","last_clean":"2023-09-23T22:52:59.534512+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:52:59.534512+0000","last_undegraded":"2023-09-23T22:52:59.534512+0000","last_fullsized":"2023-09-23T22:52:59.534512+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":264,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524995+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:52:57.524995+0000","last_peered":"2023-09-23T22:52:57.524995+0000","last_clean":"2023-09-23T22:52:57.524995+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:52:57.524995+0000","last_undegraded":"2023-09-23T22:52:57.524995+0000","last_fullsized":"2023-09-23T22:52:57.524995+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"291'6210","reported_seq":6245,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:05.931895+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:53:05.931895+0000","last_peered":"2023-09-23T22:53:05.931895+0000","last_clean":"2023-09-23T22:53:05.931895+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:53:05.931895+0000","last_undegraded":"2023-09-23T22:53:05.931895+0000","last_fullsized":"2023-09-23T22:53:05.931895+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6210,"log_dups_size":0,"ondisk_log_size":6210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319291392,"num_objects":4872,"num_object_clones":0,"num_object_copies":14616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500650+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:52:57.500650+0000","last_peered":"2023-09-23T22:52:57.500650+0000","last_clean":"2023-09-23T22:52:57.500650+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:52:57.500650+0000","last_undegraded":"2023-09-23T22:52:57.500650+0000","last_fullsized":"2023-09-23T22:52:57.500650+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":286,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500059+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:52:57.500059+0000","last_peered":"2023-09-23T22:52:57.500059+0000","last_clean":"2023-09-23T22:52:57.500059+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:52:57.500059+0000","last_undegraded":"2023-09-23T22:52:57.500059+0000","last_fullsized":"2023-09-23T22:52:57.500059+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"291'6309","reported_seq":6349,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:06.998764+0000","last_change":"2023-09-23T22:51:35.518569+0000","last_active":"2023-09-23T22:53:06.998764+0000","last_peered":"2023-09-23T22:53:06.998764+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:53:06.998764+0000","last_undegraded":"2023-09-23T22:51:35.445687+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":271,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5109,"log_dups_size":0,"ondisk_log_size":5109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:39.333773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319225856,"num_objects":4871,"num_object_clones":0,"num_object_copies":14613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6309,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":946},{"shards":"1,5","objects":3206}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":299,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497985+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:52:57.497985+0000","last_peered":"2023-09-23T22:52:57.497985+0000","last_clean":"2023-09-23T22:52:57.497985+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:52:57.497985+0000","last_undegraded":"2023-09-23T22:52:57.497985+0000","last_fullsized":"2023-09-23T22:52:57.497985+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":274,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.525100+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:52:57.525100+0000","last_peered":"2023-09-23T22:52:57.525100+0000","last_clean":"2023-09-23T22:52:57.525100+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:52:57.525100+0000","last_undegraded":"2023-09-23T22:52:57.525100+0000","last_fullsized":"2023-09-23T22:52:57.525100+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"292'6333","reported_seq":6376,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:09.878905+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:53:09.878905+0000","last_peered":"2023-09-23T22:53:09.878905+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:53:09.878905+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310444032,"num_objects":4737,"num_object_clones":0,"num_object_copies":14211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6333,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":3122},{"shards":"1,2,3","objects":817}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":219,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497828+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:52:57.497828+0000","last_peered":"2023-09-23T22:52:57.497828+0000","last_clean":"2023-09-23T22:52:57.497828+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:52:57.497828+0000","last_undegraded":"2023-09-23T22:52:57.497828+0000","last_fullsized":"2023-09-23T22:52:57.497828+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":229,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.536409+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:52:59.536409+0000","last_peered":"2023-09-23T22:52:59.536409+0000","last_clean":"2023-09-23T22:52:59.536409+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:52:59.536409+0000","last_undegraded":"2023-09-23T22:52:59.536409+0000","last_fullsized":"2023-09-23T22:52:59.536409+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"291'6353","reported_seq":6394,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:06.938226+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:53:06.938226+0000","last_peered":"2023-09-23T22:53:06.938226+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:53:06.938226+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5153,"log_dups_size":0,"ondisk_log_size":5153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327614464,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6353,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":953},{"shards":"1,5","objects":3369}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":330,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:03.163914+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:53:03.163914+0000","last_peered":"2023-09-23T22:53:03.163914+0000","last_clean":"2023-09-23T22:53:03.163914+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:53:03.163914+0000","last_undegraded":"2023-09-23T22:53:03.163914+0000","last_fullsized":"2023-09-23T22:53:03.163914+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.525555+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:52:59.525555+0000","last_peered":"2023-09-23T22:52:59.525555+0000","last_clean":"2023-09-23T22:52:59.525555+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:52:59.525555+0000","last_undegraded":"2023-09-23T22:52:59.525555+0000","last_fullsized":"2023-09-23T22:52:59.525555+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497978+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:52:57.497978+0000","last_peered":"2023-09-23T22:52:57.497978+0000","last_clean":"2023-09-23T22:52:57.497978+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:52:57.497978+0000","last_undegraded":"2023-09-23T22:52:57.497978+0000","last_fullsized":"2023-09-23T22:52:57.497978+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"292'6383","reported_seq":9822,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:08.152508+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:53:08.152508+0000","last_peered":"2023-09-23T22:53:08.152508+0000","last_clean":"2023-09-23T22:53:08.152508+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:53:08.152508+0000","last_undegraded":"2023-09-23T22:53:08.152508+0000","last_fullsized":"2023-09-23T22:53:08.152508+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6383,"log_dups_size":0,"ondisk_log_size":6383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314245120,"num_objects":4795,"num_object_clones":0,"num_object_copies":14385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6383,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":125,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500298+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:52:57.500298+0000","last_peered":"2023-09-23T22:52:57.500298+0000","last_clean":"2023-09-23T22:52:57.500298+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:52:57.500298+0000","last_undegraded":"2023-09-23T22:52:57.500298+0000","last_fullsized":"2023-09-23T22:52:57.500298+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":287,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499392+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:52:57.499392+0000","last_peered":"2023-09-23T22:52:57.499392+0000","last_clean":"2023-09-23T22:52:57.499392+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:52:57.499392+0000","last_undegraded":"2023-09-23T22:52:57.499392+0000","last_fullsized":"2023-09-23T22:52:57.499392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.525148+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:52:59.525148+0000","last_peered":"2023-09-23T22:52:59.525148+0000","last_clean":"2023-09-23T22:52:59.525148+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:52:59.525148+0000","last_undegraded":"2023-09-23T22:52:59.525148+0000","last_fullsized":"2023-09-23T22:52:59.525148+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"291'6290","reported_seq":6335,"reported_epoch":291,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:53:06.069350+0000","last_change":"2023-09-23T22:51:35.528654+0000","last_active":"2023-09-23T22:53:06.069350+0000","last_peered":"2023-09-23T22:53:06.069350+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:53:06.069350+0000","last_undegraded":"2023-09-23T22:53:06.069350+0000","last_fullsized":"2023-09-23T22:53:06.069350+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6290,"log_dups_size":0,"ondisk_log_size":6290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326762496,"num_objects":4986,"num_object_clones":0,"num_object_copies":14958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":4986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6290,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":3332}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":293,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.524920+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:52:59.524920+0000","last_peered":"2023-09-23T22:52:59.524920+0000","last_clean":"2023-09-23T22:52:59.524920+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:52:59.524920+0000","last_undegraded":"2023-09-23T22:52:59.524920+0000","last_fullsized":"2023-09-23T22:52:59.524920+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":173,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.524669+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:52:59.524669+0000","last_peered":"2023-09-23T22:52:59.524669+0000","last_clean":"2023-09-23T22:52:59.524669+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:52:59.524669+0000","last_undegraded":"2023-09-23T22:52:59.524669+0000","last_fullsized":"2023-09-23T22:52:59.524669+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":298,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534794+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:52:59.534794+0000","last_peered":"2023-09-23T22:52:59.534794+0000","last_clean":"2023-09-23T22:52:59.534794+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:52:59.534794+0000","last_undegraded":"2023-09-23T22:52:59.534794+0000","last_fullsized":"2023-09-23T22:52:59.534794+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"292'6300","reported_seq":6342,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:08.126615+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:53:08.126615+0000","last_peered":"2023-09-23T22:53:08.126615+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:53:08.126615+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6300,"log_dups_size":0,"ondisk_log_size":6300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311296000,"num_objects":4750,"num_object_clones":0,"num_object_copies":14250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6300,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":937},{"shards":"0,5","objects":3038}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":303,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.524888+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:52:59.524888+0000","last_peered":"2023-09-23T22:52:59.524888+0000","last_clean":"2023-09-23T22:52:59.524888+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:52:59.524888+0000","last_undegraded":"2023-09-23T22:52:59.524888+0000","last_fullsized":"2023-09-23T22:52:59.524888+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":380,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.535874+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:52:59.535874+0000","last_peered":"2023-09-23T22:52:59.535874+0000","last_clean":"2023-09-23T22:52:59.535874+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:52:59.535874+0000","last_undegraded":"2023-09-23T22:52:59.535874+0000","last_fullsized":"2023-09-23T22:52:59.535874+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":45,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500469+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:52:57.500469+0000","last_peered":"2023-09-23T22:52:57.500469+0000","last_clean":"2023-09-23T22:52:57.500469+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:52:57.500469+0000","last_undegraded":"2023-09-23T22:52:57.500469+0000","last_fullsized":"2023-09-23T22:52:57.500469+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500518+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:52:57.500518+0000","last_peered":"2023-09-23T22:52:57.500518+0000","last_clean":"2023-09-23T22:52:57.500518+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:52:57.500518+0000","last_undegraded":"2023-09-23T22:52:57.500518+0000","last_fullsized":"2023-09-23T22:52:57.500518+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"292'6346","reported_seq":6383,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:08.126314+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:53:08.126314+0000","last_peered":"2023-09-23T22:53:08.126314+0000","last_clean":"2023-09-23T22:53:08.126314+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:53:08.126314+0000","last_undegraded":"2023-09-23T22:53:08.126314+0000","last_fullsized":"2023-09-23T22:53:08.126314+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6346,"log_dups_size":0,"ondisk_log_size":6346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319815680,"num_objects":4880,"num_object_clones":0,"num_object_copies":14640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6346,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.536029+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:52:59.536029+0000","last_peered":"2023-09-23T22:52:59.536029+0000","last_clean":"2023-09-23T22:52:59.536029+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:52:59.536029+0000","last_undegraded":"2023-09-23T22:52:59.536029+0000","last_fullsized":"2023-09-23T22:52:59.536029+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":239,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534281+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:52:59.534281+0000","last_peered":"2023-09-23T22:52:59.534281+0000","last_clean":"2023-09-23T22:52:59.534281+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:52:59.534281+0000","last_undegraded":"2023-09-23T22:52:59.534281+0000","last_fullsized":"2023-09-23T22:52:59.534281+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":217,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534497+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:52:59.534497+0000","last_peered":"2023-09-23T22:52:59.534497+0000","last_clean":"2023-09-23T22:52:59.534497+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:52:59.534497+0000","last_undegraded":"2023-09-23T22:52:59.534497+0000","last_fullsized":"2023-09-23T22:52:59.534497+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"291'6177","reported_seq":6212,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:06.562781+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:53:06.562781+0000","last_peered":"2023-09-23T22:53:06.562781+0000","last_clean":"2023-09-23T22:53:06.562781+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:53:06.562781+0000","last_undegraded":"2023-09-23T22:53:06.562781+0000","last_fullsized":"2023-09-23T22:53:06.562781+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314376192,"num_objects":4797,"num_object_clones":0,"num_object_copies":14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6177,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":222,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534480+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:52:59.534480+0000","last_peered":"2023-09-23T22:52:59.534480+0000","last_clean":"2023-09-23T22:52:59.534480+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:52:59.534480+0000","last_undegraded":"2023-09-23T22:52:59.534480+0000","last_fullsized":"2023-09-23T22:52:59.534480+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":172,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.903475+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:52:59.903475+0000","last_peered":"2023-09-23T22:52:59.903475+0000","last_clean":"2023-09-23T22:52:59.903475+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:52:59.903475+0000","last_undegraded":"2023-09-23T22:52:59.903475+0000","last_fullsized":"2023-09-23T22:52:59.903475+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":222,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534731+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:52:59.534731+0000","last_peered":"2023-09-23T22:52:59.534731+0000","last_clean":"2023-09-23T22:52:59.534731+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:52:59.534731+0000","last_undegraded":"2023-09-23T22:52:59.534731+0000","last_fullsized":"2023-09-23T22:52:59.534731+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"291'6397","reported_seq":6433,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:53:06.524148+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:53:06.524148+0000","last_peered":"2023-09-23T22:53:06.524148+0000","last_clean":"2023-09-23T22:53:06.524148+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:53:06.524148+0000","last_undegraded":"2023-09-23T22:53:06.524148+0000","last_fullsized":"2023-09-23T22:53:06.524148+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6397,"log_dups_size":0,"ondisk_log_size":6397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325320728,"num_objects":4965,"num_object_clones":0,"num_object_copies":14895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4965,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6397,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":48,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534934+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:52:59.534934+0000","last_peered":"2023-09-23T22:52:59.534934+0000","last_clean":"2023-09-23T22:52:59.534934+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:52:59.534934+0000","last_undegraded":"2023-09-23T22:52:59.534934+0000","last_fullsized":"2023-09-23T22:52:59.534934+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.522969+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:52:57.522969+0000","last_peered":"2023-09-23T22:52:57.522969+0000","last_clean":"2023-09-23T22:52:57.522969+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:52:57.522969+0000","last_undegraded":"2023-09-23T22:52:57.522969+0000","last_fullsized":"2023-09-23T22:52:57.522969+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534168+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:52:59.534168+0000","last_peered":"2023-09-23T22:52:59.534168+0000","last_clean":"2023-09-23T22:52:59.534168+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:52:59.534168+0000","last_undegraded":"2023-09-23T22:52:59.534168+0000","last_fullsized":"2023-09-23T22:52:59.534168+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"291'6390","reported_seq":8217,"reported_epoch":291,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:53:06.624761+0000","last_change":"2023-09-23T22:52:26.294673+0000","last_active":"2023-09-23T22:53:06.624761+0000","last_peered":"2023-09-23T22:53:06.624761+0000","last_clean":"2023-09-23T22:51:33.377854+0000","last_became_active":"2023-09-23T22:51:37.033330+0000","last_became_peered":"2023-09-23T22:51:37.033330+0000","last_unstale":"2023-09-23T22:53:06.624761+0000","last_undegraded":"2023-09-23T22:53:06.624761+0000","last_fullsized":"2023-09-23T22:51:35.425861+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321257472,"num_objects":4902,"num_object_clones":0,"num_object_copies":14706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":837,"num_objects_unfound":0,"num_objects_dirty":4902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6390,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":892,"num_bytes_recovered":58458112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":837},{"shards":"0,4","objects":4065}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500604+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:52:57.500604+0000","last_peered":"2023-09-23T22:52:57.500604+0000","last_clean":"2023-09-23T22:52:57.500604+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:52:57.500604+0000","last_undegraded":"2023-09-23T22:52:57.500604+0000","last_fullsized":"2023-09-23T22:52:57.500604+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":189,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534583+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:52:59.534583+0000","last_peered":"2023-09-23T22:52:59.534583+0000","last_clean":"2023-09-23T22:52:59.534583+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:52:59.534583+0000","last_undegraded":"2023-09-23T22:52:59.534583+0000","last_fullsized":"2023-09-23T22:52:59.534583+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":241,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.498025+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:52:57.498025+0000","last_peered":"2023-09-23T22:52:57.498025+0000","last_clean":"2023-09-23T22:52:57.498025+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:52:57.498025+0000","last_undegraded":"2023-09-23T22:52:57.498025+0000","last_fullsized":"2023-09-23T22:52:57.498025+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":224,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534596+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:52:59.534596+0000","last_peered":"2023-09-23T22:52:59.534596+0000","last_clean":"2023-09-23T22:52:59.534596+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:52:59.534596+0000","last_undegraded":"2023-09-23T22:52:59.534596+0000","last_fullsized":"2023-09-23T22:52:59.534596+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":75,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.525227+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:52:59.525227+0000","last_peered":"2023-09-23T22:52:59.525227+0000","last_clean":"2023-09-23T22:52:59.525227+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:52:59.525227+0000","last_undegraded":"2023-09-23T22:52:59.525227+0000","last_fullsized":"2023-09-23T22:52:59.525227+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":100,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.524905+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:52:57.524905+0000","last_peered":"2023-09-23T22:52:57.524905+0000","last_clean":"2023-09-23T22:52:57.524905+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:52:57.524905+0000","last_undegraded":"2023-09-23T22:52:57.524905+0000","last_fullsized":"2023-09-23T22:52:57.524905+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":299,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499458+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:52:57.499458+0000","last_peered":"2023-09-23T22:52:57.499458+0000","last_clean":"2023-09-23T22:52:57.499458+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:52:57.499458+0000","last_undegraded":"2023-09-23T22:52:57.499458+0000","last_fullsized":"2023-09-23T22:52:57.499458+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":101,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.500139+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:52:57.500139+0000","last_peered":"2023-09-23T22:52:57.500139+0000","last_clean":"2023-09-23T22:52:57.500139+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:52:57.500139+0000","last_undegraded":"2023-09-23T22:52:57.500139+0000","last_fullsized":"2023-09-23T22:52:57.500139+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.499604+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:52:57.499604+0000","last_peered":"2023-09-23T22:52:57.499604+0000","last_clean":"2023-09-23T22:52:57.499604+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:52:57.499604+0000","last_undegraded":"2023-09-23T22:52:57.499604+0000","last_fullsized":"2023-09-23T22:52:57.499604+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":41,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.522897+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:52:57.522897+0000","last_peered":"2023-09-23T22:52:57.522897+0000","last_clean":"2023-09-23T22:52:57.522897+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:52:57.522897+0000","last_undegraded":"2023-09-23T22:52:57.522897+0000","last_fullsized":"2023-09-23T22:52:57.522897+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.524764+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:52:59.524764+0000","last_peered":"2023-09-23T22:52:59.524764+0000","last_clean":"2023-09-23T22:52:59.524764+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:52:59.524764+0000","last_undegraded":"2023-09-23T22:52:59.524764+0000","last_fullsized":"2023-09-23T22:52:59.524764+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":238,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T22:52:59.534852+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:52:59.534852+0000","last_peered":"2023-09-23T22:52:59.534852+0000","last_clean":"2023-09-23T22:52:59.534852+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:52:59.534852+0000","last_undegraded":"2023-09-23T22:52:59.534852+0000","last_fullsized":"2023-09-23T22:52:59.534852+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T22:52:57.497919+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:52:57.497919+0000","last_peered":"2023-09-23T22:52:57.497919+0000","last_clean":"2023-09-23T22:52:57.497919+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:52:57.497919+0000","last_undegraded":"2023-09-23T22:52:57.497919+0000","last_fullsized":"2023-09-23T22:52:57.497919+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5104074776,"num_objects":77883,"num_object_clones":0,"num_object_copies":233649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10054,"num_objects_misplaced":2491,"num_objects_unfound":0,"num_objects_dirty":77883,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101395,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4259,"num_bytes_recovered":279117824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16182751232,"data_stored":16182739016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98995,"ondisk_log_size":98995,"up":48,"acting":41,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430521,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626749,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2590428,"kb_used_data":2280916,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91154724,"statfs":{"total":95995035648,"available":93342437376,"internally_reserved":0,"allocated":2335657984,"data_stored":2335102055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":24.245999999999999,"5min":9.1280000000000001,"15min":3.9020000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":243.35900000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":4.8010000000000002},{"interface":"front","average":{"1min":24.465,"5min":9.5229999999999997,"15min":3.8250000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":243.41499999999999,"5min":243.41499999999999,"15min":243.41499999999999},"last":2.2130000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":14.752000000000001,"5min":5.0700000000000003,"15min":3.6389999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":254.62100000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":7.9080000000000004},{"interface":"front","average":{"1min":20.399999999999999,"5min":7.9690000000000003,"15min":4.6319999999999997},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":255.20099999999999,"5min":255.20099999999999,"15min":255.20099999999999},"last":3.2730000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":3.71,"15min":3.367},"min":{"1min":0.314,"5min":0.314,"15min":0.28299999999999997},"max":{"1min":5.4279999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.542,"5min":4.9530000000000003,"15min":3.677},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":5.4390000000000001,"5min":185.21799999999999,"15min":185.21799999999999},"last":4.9180000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":5.6150000000000002,"15min":3.706},"min":{"1min":0.36199999999999999,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":7.734,"5min":185.19399999999999,"15min":185.19399999999999},"last":5.8899999999999997},{"interface":"front","average":{"1min":1.919,"5min":3.919,"15min":3.3170000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.7770000000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.82299999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.875,"15min":0.753},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.7290000000000001,"5min":6.5309999999999997,"15min":18.085999999999999},"last":2.1890000000000001},{"interface":"front","average":{"1min":1.3320000000000001,"5min":0.78900000000000003,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4800000000000004,"5min":7.4800000000000004,"15min":18.597000000000001},"last":4.8550000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.0529999999999999,"15min":0.85299999999999998},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.7560000000000002,"5min":10.148999999999999,"15min":18.972000000000001},"last":4.5570000000000004},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.83799999999999997,"15min":0.80900000000000005},"min":{"1min":0.316,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":2.782,"5min":4.9089999999999998,"15min":18.625},"last":4.8840000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":0.86499999999999999,"15min":0.82999999999999996},"min":{"1min":0.35799999999999998,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":4.2869999999999999,"5min":4.3049999999999997,"15min":12.481999999999999},"last":4.8360000000000003},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.03,"15min":0.86199999999999999},"min":{"1min":0.38200000000000001,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":6.6269999999999998,"5min":6.6269999999999998,"15min":16.661999999999999},"last":5.2930000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253059,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4125568,"kb_used_data":3646392,"kb_used_omap":9,"kb_used_meta":479158,"kb_avail":89619584,"statfs":{"total":95995035648,"available":91770454016,"internally_reserved":0,"allocated":3733905408,"data_stored":3732853618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":490658793},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":7.5039999999999996,"15min":5.2800000000000002},"min":{"1min":0.26800000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.933,"5min":101.605,"15min":101.605},"last":0.20899999999999999},{"interface":"front","average":{"1min":3.1309999999999998,"5min":6.6639999999999997,"15min":5.2309999999999999},"min":{"1min":0.24099999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":8.6620000000000008,"5min":101.892,"15min":101.892},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.8610000000000002,"15min":2.7989999999999999},"min":{"1min":0.22500000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":12.622999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":0.311},{"interface":"front","average":{"1min":2.214,"5min":4.0629999999999997,"15min":2.8799999999999999},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.887,"5min":37.957000000000001,"15min":37.957000000000001},"last":0.66500000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":3.3170000000000002,"15min":2.3809999999999998},"min":{"1min":0.28999999999999998,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.218,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.9910000000000001,"5min":3.9590000000000001,"15min":2.835},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":4.7859999999999996,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.4950000000000001,"15min":2.0950000000000002},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":3.3889999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.4140000000000001,"5min":2.9009999999999998,"15min":2.0920000000000001},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":12.792999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":2.1030000000000002,"15min":1.6459999999999999},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":5.9960000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.6360000000000001,"15min":1.9710000000000001},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":3.347,"5min":87.515000000000001,"15min":87.515000000000001},"last":6.2889999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.9390000000000001,"15min":1.925},"min":{"1min":0.41799999999999998,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.2539999999999996,"5min":79.311000000000007,"15min":79.311000000000007},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.909,"5min":2.1779999999999999,"15min":1.601},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":5.9340000000000002,"5min":79.328999999999994,"15min":79.328999999999994},"last":5.1849999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.008,"15min":2.3460000000000001},"min":{"1min":0.42499999999999999,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":4.0979999999999999,"5min":11.481999999999999,"15min":159.018},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.4399999999999999,"15min":2.444},"min":{"1min":0.47599999999999998,"5min":0.371,"15min":0.37},"max":{"1min":6.282,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.49199999999999999}]}]},{"osd":0,"up_from":219,"seq":940597837908,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2411324,"kb_used_data":2116212,"kb_used_omap":31,"kb_used_meta":295072,"kb_avail":91333828,"statfs":{"total":95995035648,"available":93525839872,"internally_reserved":0,"allocated":2167001088,"data_stored":2165894562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":302154674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":3.5169999999999999,"15min":3.6469999999999998},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":11.930999999999999,"5min":21.512,"15min":32.344999999999999},"last":0.19},{"interface":"front","average":{"1min":2.4169999999999998,"5min":3.4249999999999998,"15min":3.5139999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7379999999999995,"5min":20.064,"15min":32.106000000000002},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":4.9889999999999999,"15min":3.0609999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.9619999999999997,"5min":111.508,"15min":111.508},"last":1.022},{"interface":"front","average":{"1min":2.669,"5min":4.2229999999999999,"15min":2.5790000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.9749999999999996,"5min":111.532,"15min":111.532},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":4.9450000000000003,"15min":2.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9269999999999996,"5min":126.73,"15min":126.73},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.177,"5min":3.8519999999999999,"15min":2.0329999999999999},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.8419999999999996,"5min":121.483,"15min":121.483},"last":0.45300000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":2.633,"15min":2.113},"min":{"1min":0.36299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.965999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":2.0049999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.5169999999999999,"15min":2.0049999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.9990000000000006,"5min":20.009,"15min":20.009},"last":3.468}]},{"osd":5,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":2.9180000000000001,"15min":1.863},"min":{"1min":0.41399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.6280000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":2.9129999999999998},{"interface":"front","average":{"1min":2.6379999999999999,"5min":3.3769999999999998,"15min":2.161},"min":{"1min":0.40400000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.0519999999999996,"5min":61.777000000000001,"15min":61.777000000000001},"last":6.0309999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":2.6909999999999998,"15min":1.8109999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5369999999999999,"5min":59.146999999999998,"15min":59.146999999999998},"last":2.6240000000000001},{"interface":"front","average":{"1min":2.9220000000000002,"5min":3.3679999999999999,"15min":2.246},"min":{"1min":0.48499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":11.882999999999999,"5min":59.164000000000001,"15min":59.164000000000001},"last":5.9770000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.6800000000000002,"15min":1.9570000000000001},"min":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.0869999999999997,"5min":18.640000000000001,"15min":18.640000000000001},"last":1.7509999999999999},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.2320000000000002,"15min":1.5289999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.9219999999999997,"5min":21.954999999999998,"15min":21.954999999999998},"last":1.7290000000000001}]}]},{"osd":2,"up_from":131,"seq":562640715921,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2165516,"kb_used_data":1780804,"kb_used_omap":81,"kb_used_meta":384430,"kb_avail":91579636,"statfs":{"total":95995035648,"available":93777547264,"internally_reserved":0,"allocated":1823543296,"data_stored":1822490761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":393656538},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":8.4789999999999992,"5min":6.7309999999999999,"15min":5.7560000000000002},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":114.483,"5min":132.79499999999999,"15min":132.79499999999999},"last":2.8799999999999999},{"interface":"front","average":{"1min":8.5,"5min":7.7599999999999998,"15min":6.1799999999999997},"min":{"1min":0.34599999999999997,"5min":0.314,"15min":0.314},"max":{"1min":113.873,"5min":132.58099999999999,"15min":132.58099999999999},"last":3.1560000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.5310000000000001,"15min":3.0230000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":12.035,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.53100000000000003},{"interface":"front","average":{"1min":3.5329999999999999,"5min":3.4220000000000002,"15min":3.1589999999999998},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":14.159000000000001,"5min":28.596,"15min":88.436999999999998},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.3250000000000002,"15min":4.1890000000000001},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":6.5419999999999998,"5min":112.895,"15min":112.895},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.524,"5min":4.617,"15min":3.4169999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.251},"max":{"1min":9.3840000000000003,"5min":113.995,"15min":113.995},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.597,"15min":2.4169999999999998},"min":{"1min":0.37,"5min":0.253,"15min":0.253},"max":{"1min":7.1520000000000001,"5min":84.778999999999996,"15min":108.878},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.901,"5min":3.641,"15min":3.056},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.8460000000000001,"5min":84.753,"15min":108.93000000000001},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0750000000000002,"15min":2.9009999999999998},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.5970000000000004,"5min":90.082999999999998,"15min":97.322999999999993},"last":0.442},{"interface":"front","average":{"1min":2.0049999999999999,"5min":3.3399999999999999,"15min":2.718},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.5380000000000003,"5min":89.986000000000004,"15min":97.301000000000002},"last":0.876}]},{"osd":6,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":4.8330000000000002,"15min":2.9390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":4.5789999999999997,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.607,"5min":4.3360000000000003,"15min":2.3180000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.4089999999999998,"5min":98.509,"15min":98.509},"last":0.77500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0459999999999998,"15min":1.5740000000000001},"min":{"1min":0.42799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.7440000000000002,"5min":10.92,"15min":10.92},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.1240000000000001,"15min":2.323},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":4.2489999999999997,"5min":11.311,"15min":88.584000000000003},"last":0.59999999999999998}]}]},{"osd":3,"up_from":0,"seq":403726926006,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019205,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2061344,"kb_used_data":1764120,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91683808,"statfs":{"total":95995035648,"available":93884219392,"internally_reserved":0,"allocated":1806458880,"data_stored":1805898357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":8.5950000000000006,"5min":3.0579999999999998,"15min":6.3630000000000004},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":113.35599999999999,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":8.9220000000000006,"5min":3.7679999999999998,"15min":6.9420000000000002},"min":{"1min":0.34000000000000002,"5min":0.217,"15min":0.217},"max":{"1min":113.417,"5min":113.417,"15min":113.417},"last":0.76900000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":11.753,"5min":3.7679999999999998,"15min":2.4649999999999999},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":132.304,"5min":132.304,"15min":132.304},"last":1.46},{"interface":"front","average":{"1min":11.4,"5min":3.6560000000000001,"15min":2.3180000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":132.453,"5min":132.453,"15min":132.453},"last":1.661}]},{"osd":2,"last update":"Sat Sep 23 22:52:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":2.2240000000000002,"15min":3.1720000000000002},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.308,"5min":20.308,"15min":84.909000000000006},"last":0.92100000000000004},{"interface":"front","average":{"1min":2.8769999999999998,"5min":2.3769999999999998,"15min":3.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.652000000000001,"5min":21.704000000000001,"15min":83.840000000000003},"last":1.282}]},{"osd":3,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":5.3339999999999996,"5min":5.3849999999999998,"15min":4.3410000000000002},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":74.138999999999996,"5min":164.453,"15min":164.453},"last":0.39500000000000002},{"interface":"front","average":{"1min":4.9329999999999998,"5min":5.3520000000000003,"15min":4.4249999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":75.099999999999994,"5min":167.05199999999999,"15min":167.05199999999999},"last":3.7069999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.91900000000000004,"15min":0.85699999999999998},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.0030000000000001,"5min":12.238,"15min":43.189999999999998},"last":1.3979999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89900000000000002,"15min":0.874},"min":{"1min":0.29499999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.9910000000000001,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.35199999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.95199999999999996,"15min":0.84399999999999997},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.758,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.378},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.88600000000000001,"15min":0.84299999999999997},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":1.978,"5min":12.269,"15min":26.940999999999999},"last":1.423}]},{"osd":7,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.0629999999999999,"15min":0.82199999999999995},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":10.278,"5min":12.311,"15min":12.311},"last":1.4419999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.98099999999999998,"15min":0.76100000000000001},"min":{"1min":0.38800000000000001,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":3.7909999999999999,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.503}]}]},{"osd":5,"up_from":36,"seq":154618822977,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3487364,"kb_used_data":3177084,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90257788,"statfs":{"total":95995035648,"available":92423974912,"internally_reserved":0,"allocated":3253334016,"data_stored":3252797209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":7.0720000000000001,"5min":8.0820000000000007,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":74.406000000000006,"5min":121.319,"15min":121.319},"last":185.62},{"interface":"front","average":{"1min":7.1799999999999997,"5min":8.2539999999999996,"15min":3.8479999999999999},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":73.899000000000001,"5min":121.075,"15min":121.075},"last":185.499}]},{"osd":1,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8820000000000001,"5min":2.0649999999999999,"15min":2.056},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":41.463999999999999,"5min":41.463999999999999,"15min":149.203},"last":0.41999999999999998},{"interface":"front","average":{"1min":4.0869999999999997,"5min":2.5670000000000002,"15min":2.3260000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":41.954000000000001,"5min":41.954000000000001,"15min":149.554},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.9660000000000002,"15min":5.6040000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":34.988,"5min":63.156999999999996,"15min":158.92599999999999},"last":189.126},{"interface":"front","average":{"1min":4.3799999999999999,"5min":2.7050000000000001,"15min":5.5270000000000001},"min":{"1min":0.38600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":34.872,"5min":63.277999999999999,"15min":158.91399999999999},"last":188.74799999999999}]},{"osd":3,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":5.3380000000000001,"5min":7.6420000000000003,"15min":5.3600000000000003},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":89.995000000000005,"5min":108.58799999999999,"15min":257.05000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":5.8029999999999999,"5min":7.8769999999999998,"15min":5.2800000000000002},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":90.117999999999995,"5min":108.15600000000001,"15min":257.35000000000002},"last":0.441}]},{"osd":4,"last update":"Sat Sep 23 22:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.63,"15min":0.64900000000000002},"min":{"1min":0.27400000000000002,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":2.024,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.625},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":1.873,"5min":3.4409999999999998,"15min":10.535},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.89500000000000002,"15min":1.4490000000000001},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.7469999999999999,"5min":12.615,"15min":140.07499999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.85899999999999999,"15min":1.4790000000000001},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":4,"5min":12.74,"15min":139.739},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:53:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":0.97899999999999998,"15min":0.82799999999999996},"min":{"1min":0.40100000000000002,"5min":0.33400000000000002,"15min":0.29499999999999998},"max":{"1min":10.367000000000001,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.353,"5min":0.89400000000000002,"15min":0.76100000000000001},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":6.1120000000000001,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2156265472,"data_stored":2156265472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3718189056,"data_stored":3718184984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1818624000,"data_stored":1818624000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1319436288,"data_stored":1319436288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1608257536,"data_stored":1608253464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3239251968,"data_stored":3239247896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2322726912,"data_stored":2322726912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:53:14.112 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:53:14.137 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:14.353 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:14.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:14.462 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:14.509 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:14.608 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:14.619 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:53:14.642 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:14.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:14.752 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:14.798 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:14.881 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:14.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:14.988 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:15.063 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:15.199 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:15.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:15.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:15.318 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:15.414 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:15.433 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:15.598 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:15.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:15.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:15 smithi052 ceph-mon[180979]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10054/225414 objects degraded (4.460%); 2380/225414 objects misplaced (1.056%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:15.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10054/225414 objects degraded (4.460%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:15.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:15 smithi052 ceph-mon[187743]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10054/225414 objects degraded (4.460%); 2380/225414 objects misplaced (1.056%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:15.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10054/225414 objects degraded (4.460%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:15.712 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:15.740 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:15 smithi151 ceph-mon[159210]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10054/225414 objects degraded (4.460%); 2380/225414 objects misplaced (1.056%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:15.741 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10054/225414 objects degraded (4.460%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:15.743 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:15.820 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:15.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:15.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:15.927 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:16.002 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:16.345 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:16.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:16.459 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:16.524 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:53:16.576 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:16.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/152310121' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:53:16.613 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:53:16.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/152310121' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:53:16.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:53:16.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/152310121' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:53:16.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:53:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: pgmap v834: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10054/221505 objects degraded (4.539%); 2380/221505 objects misplaced (1.074%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: osdmap e293: 8 total, 8 up, 6 in 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[180979]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: pgmap v834: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10054/221505 objects degraded (4.539%); 2380/221505 objects misplaced (1.074%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: osdmap e293: 8 total, 8 up, 6 in 2023-09-23T22:53:17.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:17 smithi052 ceph-mon[187743]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: pgmap v834: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 10054/221505 objects degraded (4.539%); 2380/221505 objects misplaced (1.074%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: osdmap e293: 8 total, 8 up, 6 in 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: failed to encode map e293 with expected crc 2023-09-23T22:53:17.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:17 smithi151 ceph-mon[159210]: failed to encode map e293 with expected crc 2023-09-23T22:53:18.144 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:18.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:18.173 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:18.206 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:18.252 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:18.253 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:18.294 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:18.295 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:18.309 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:18.348 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:18.368 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:18.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:18.386 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:18.418 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:18.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:53:18.476 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:18.495 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.496 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:53:18.516 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:18.605 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:18.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:18.617 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4181730261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:18.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:18.718 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:18.787 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4181730261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:18.787 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4181730261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:18.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:19.439 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:19.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:19.545 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:19.599 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:19.695 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:19.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:19.731 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:19 smithi052 ceph-mon[180979]: pgmap v836: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 10123/214815 objects degraded (4.712%); 2196/214815 objects misplaced (1.022%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:19.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:19.732 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:19 smithi052 ceph-mon[187743]: pgmap v836: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 10123/214815 objects degraded (4.712%); 2196/214815 objects misplaced (1.022%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:19.732 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:19.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:19.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:19 smithi151 ceph-mon[159210]: pgmap v836: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 10123/214815 objects degraded (4.712%); 2196/214815 objects misplaced (1.022%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:19.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:19.863 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:19.968 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:19.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:19.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:20.077 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:20.142 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:20.359 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:53:20.454 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:20.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:20.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:20.569 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:20.680 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:53:20.681 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 10123/214815 objects degraded (4.712%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1133,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":293,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":6,"osd_in_since":1695509493,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":71605,"data_bytes":4691662776,"bytes_used":16172068864,"bytes_avail":559798145024,"bytes_total":575970213888,"degraded_objects":10123,"degraded_total":214815,"degraded_ratio":0.047124269720457142,"misplaced_objects":2196,"misplaced_total":214815,"misplaced_ratio":0.010222749807974304,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1480349,"recovering_keys_per_sec":0,"num_objects_recovered":226,"num_bytes_recovered":14811136,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":687},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T22:53:02.381180+0000","services":{}},"progress_events":{"6b27ada5-699f-4815-a812-68a0dbe12316":{"message":"Global Recovery Event (95s)\n [=========================...] (remaining: 9s)","progress":0.91011238098144531,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:53:20.702 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10123/214815 objects degraded (4.712%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:20.702 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10123/214815 objects degraded (4.712%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:20.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:20.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10123/214815 objects degraded (4.712%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:21.719 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:21.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:21.733 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:53:21.733 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:53:21.824 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:21.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:53:21.841 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:21 smithi052 ceph-mon[187743]: pgmap v837: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 700 op/s; 10123/212682 objects degraded (4.760%); 2196/212682 objects misplaced (1.033%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:21.841 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1863610820' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:53:21.843 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:21 smithi052 ceph-mon[180979]: pgmap v837: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 700 op/s; 10123/212682 objects degraded (4.760%); 2196/212682 objects misplaced (1.033%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:21.844 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1863610820' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:53:21.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:21 smithi151 ceph-mon[159210]: pgmap v837: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 700 op/s; 10123/212682 objects degraded (4.760%); 2196/212682 objects misplaced (1.033%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:53:21.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1863610820' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:53:21.866 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:21.930 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:21.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:111366 9.2 9:40667c51:::benchmark_data_smithi052_402718_object21725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.973048+0000", 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.002988344, 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030269199999999998, 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:21.992 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 111366 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973048+0000", 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973048+0000", 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973046+0000", 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:21.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973060+0000", 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3631999999999999e-05 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973072+0000", 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2099999999999999e-05 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973080+0000", 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6229999999999997e-06 2023-09-23T22:53:21.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973153+0000", 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3484000000000003e-05 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973312+0000", 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015895300000000001 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973464+0000", 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015150600000000001 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:111367 9.2 9.485501d2 (undecoded) ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.973095+0000", 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0029411440000000001, 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003066095, 2023-09-23T22:53:21.997 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 111367 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973095+0000", 2023-09-23T22:53:21.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973095+0000", 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973095+0000", 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:21.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973098+0000", 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9409999999999999e-06 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973103+0000", 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5659999999999997e-06 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:22.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973108+0000", 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2229999999999998e-06 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:111368 9.0 e293/264)", 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.973921+0000", 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0021148550000000001, 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002294773, 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:22.001 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973921+0000", 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973921+0000", 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973921+0000", 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973929+0000", 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9080000000000001e-06 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973944+0000", 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4976e-05 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.973952+0000", 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3159999999999999e-06 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:111356 9.7 9:e00c5203:::benchmark_data_smithi052_402718_object21715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.941227+0000", 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.034809338000000002, 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.035042984999999999, 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 111356 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.005 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941227+0000", 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941227+0000", 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941227+0000", 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941229+0000", 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.768e-06 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941235+0000", 2023-09-23T22:53:22.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6899999999999997e-06 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941240+0000", 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7480000000000001e-06 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.948451+0000", 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072112119999999998 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.948625+0000", 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017436 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.948780+0000", 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015503899999999999 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.958259+0000", 2023-09-23T22:53:22.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0094781919999999999 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.958485+0000", 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000226648 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.010 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:111357 9.7 9:e51bbd6d:::benchmark_data_smithi052_402718_object21716:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.941244+0000", 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.034792219999999999, 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.035114380000000001, 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 111357 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:22.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941244+0000", 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941244+0000", 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941244+0000", 2023-09-23T22:53:22.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941246+0000", 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.339e-06 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941252+0000", 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5470000000000004e-06 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.941256+0000", 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2919999999999997e-06 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.953735+0000", 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012479273000000001 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:22.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.953932+0000", 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019691700000000001 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.954098+0000", 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016622699999999999 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.958531+0000", 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0044329520000000004 2023-09-23T22:53:22.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.968404+0000", 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098730080000000008 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 5 2023-09-23T22:53:22.016 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:53:22.035 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:22.172 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:22.744 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:22.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:22.858 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:22.888 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:22.910 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:53:22.943 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:23.095 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:23.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:23.155 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:23.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:23.321 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:23.596 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:23 smithi052 ceph-mon[180979]: pgmap v838: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 711 op/s; 10242/206745 objects degraded (4.954%); 1961/206745 objects misplaced (0.949%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:23.596 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:53:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:53:23] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-09-23T22:53:23.596 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:23 smithi052 ceph-mon[187743]: pgmap v838: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 711 op/s; 10242/206745 objects degraded (4.954%); 1961/206745 objects misplaced (0.949%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:23.677 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:23 smithi151 ceph-mon[159210]: pgmap v838: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 711 op/s; 10242/206745 objects degraded (4.954%); 1961/206745 objects misplaced (0.949%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:24.091 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:24.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:24.201 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:24.288 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:24.838 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:24.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:24.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:24.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:24.994 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:25.385 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:25.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:25.492 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:25.523 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:25.528 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:25.607 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:25.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:25.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:25 smithi052 ceph-mon[187743]: pgmap v839: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 10242/204414 objects degraded (5.010%); 1961/204414 objects misplaced (0.959%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:25.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10242/204414 objects degraded (5.010%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:25.656 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:25 smithi052 ceph-mon[180979]: pgmap v839: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 10242/204414 objects degraded (5.010%); 1961/204414 objects misplaced (0.959%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:25.656 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10242/204414 objects degraded (5.010%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:25.720 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:25.762 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:25.840 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:25.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:25.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:25.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:25 smithi151 ceph-mon[159210]: pgmap v839: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 10242/204414 objects degraded (5.010%); 1961/204414 objects misplaced (0.959%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:25.861 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10242/204414 objects degraded (5.010%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:25.962 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:26.015 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:26.114 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:26.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:26.220 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:26.262 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:26.350 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:26.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:26.462 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:26.555 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:26.605 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:53:26.628 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1138916934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:26.628 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:26.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1138916934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:26.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1138916934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:26.952 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:26.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:27.071 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:27.121 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:27.189 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:27.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:27.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:27.303 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:27.348 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:27.351 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:27.394 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:27.433 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:27.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:27.442 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:27.486 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:27.534 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:27.542 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:27.566 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:27.571 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: pgmap v840: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10242/200712 objects degraded (5.103%); 1961/200712 objects misplaced (0.977%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:27.572 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:53:27.572 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: failed to encode map e294 with expected crc 2023-09-23T22:53:27.572 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: failed to encode map e294 with expected crc 2023-09-23T22:53:27.572 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:27.572 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: osdmap e294: 8 total, 8 up, 6 in 2023-09-23T22:53:27.572 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3626467359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:27.604 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:27.619 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:27.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: pgmap v840: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10242/200712 objects degraded (5.103%); 1961/200712 objects misplaced (0.977%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: failed to encode map e294 with expected crc 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: failed to encode map e294 with expected crc 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: osdmap e294: 8 total, 8 up, 6 in 2023-09-23T22:53:27.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3626467359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:27.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: pgmap v840: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 10242/200712 objects degraded (5.103%); 1961/200712 objects misplaced (0.977%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:27.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:53:27.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: failed to encode map e294 with expected crc 2023-09-23T22:53:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: failed to encode map e294 with expected crc 2023-09-23T22:53:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: osdmap e294: 8 total, 8 up, 6 in 2023-09-23T22:53:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3626467359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:28.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:28.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:53:29.204 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:29.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:29.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:29.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:29.367 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:29.472 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:29.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:29.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:29.581 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:29.660 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:29.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:29 smithi151 ceph-mon[159210]: pgmap v842: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 10325/194424 objects degraded (5.311%); 1762/194424 objects misplaced (0.906%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:29.837 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:29 smithi052 ceph-mon[180979]: pgmap v842: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 10325/194424 objects degraded (5.311%); 1762/194424 objects misplaced (0.906%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:29.838 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:29 smithi052 ceph-mon[187743]: pgmap v842: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 10325/194424 objects degraded (5.311%); 1762/194424 objects misplaced (0.906%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:29.859 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:29.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:29.969 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:29.994 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:53:30.005 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:53:30.020 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:30.104 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:30.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:30.202 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:53:30.202 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:53:30.219 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:30.292 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10325/194424 objects degraded (5.311%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:30.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10325/194424 objects degraded (5.311%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:30.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10325/194424 objects degraded (5.311%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:31.052 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:31.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:31.159 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:31.234 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:31.297 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:53:31.607 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:53:31.610 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:31.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:31 smithi052 ceph-mon[180979]: from='client.35685 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:31.614 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:31 smithi052 ceph-mon[180979]: pgmap v843: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 10325/192432 objects degraded (5.366%); 1762/192432 objects misplaced (0.916%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:31.614 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:31 smithi052 ceph-mon[187743]: from='client.35685 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:31.614 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:31 smithi052 ceph-mon[187743]: pgmap v843: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 10325/192432 objects degraded (5.366%); 1762/192432 objects misplaced (0.916%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:31.615 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":842,"stamp":"2023-09-23T22:53:28.394942+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4246214584,"num_objects":64808,"num_object_clones":0,"num_object_copies":194424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10325,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":64808,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":114754,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4747,"num_bytes_recovered":309143606,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107488,"ondisk_log_size":107488,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14713344,"kb_used_data":12605072,"kb_used_omap":121,"kb_used_meta":2107974,"kb_avail":547757568,"statfs":{"total":575970213888,"available":560903749632,"internally_reserved":0,"allocated":12907593728,"data_stored":12902714271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2158565493},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,4,6,8,7,0,0,13],"upper_bound":512},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-445448192,"num_objects":-6797,"num_object_clones":0,"num_object_copies":-20391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":202,"num_objects_misplaced":-434,"num_objects_unfound":0,"num_objects_dirty":-6797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6797,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":15204352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004467"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":291,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564348+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:53:16.564348+0000","last_peered":"2023-09-23T22:53:16.564348+0000","last_clean":"2023-09-23T22:53:16.564348+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:53:16.564348+0000","last_undegraded":"2023-09-23T22:53:16.564348+0000","last_fullsized":"2023-09-23T22:53:16.564348+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565179+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:53:16.565179+0000","last_peered":"2023-09-23T22:53:16.565179+0000","last_clean":"2023-09-23T22:53:16.565179+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:53:16.565179+0000","last_undegraded":"2023-09-23T22:53:16.565179+0000","last_fullsized":"2023-09-23T22:53:16.565179+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":76,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564422+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:53:16.564422+0000","last_peered":"2023-09-23T22:53:16.564422+0000","last_clean":"2023-09-23T22:53:16.564422+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:53:16.564422+0000","last_undegraded":"2023-09-23T22:53:16.564422+0000","last_fullsized":"2023-09-23T22:53:16.564422+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":272,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.572205+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:53:16.572205+0000","last_peered":"2023-09-23T22:53:16.572205+0000","last_clean":"2023-09-23T22:53:16.572205+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:53:16.572205+0000","last_undegraded":"2023-09-23T22:53:16.572205+0000","last_fullsized":"2023-09-23T22:53:16.572205+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":276,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.567512+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:53:16.567512+0000","last_peered":"2023-09-23T22:53:16.567512+0000","last_clean":"2023-09-23T22:53:16.567512+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:53:16.567512+0000","last_undegraded":"2023-09-23T22:53:16.567512+0000","last_fullsized":"2023-09-23T22:53:16.567512+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":258,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.678829+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:53:07.678829+0000","last_peered":"2023-09-23T22:53:07.678829+0000","last_clean":"2023-09-23T22:53:07.678829+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:53:07.678829+0000","last_undegraded":"2023-09-23T22:53:07.678829+0000","last_fullsized":"2023-09-23T22:53:07.678829+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":204,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584271+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:53:26.584271+0000","last_peered":"2023-09-23T22:53:26.584271+0000","last_clean":"2023-09-23T22:53:26.584271+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:53:26.584271+0000","last_undegraded":"2023-09-23T22:53:26.584271+0000","last_fullsized":"2023-09-23T22:53:26.584271+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":154,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682273+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:53:07.682273+0000","last_peered":"2023-09-23T22:53:07.682273+0000","last_clean":"2023-09-23T22:53:07.682273+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:53:07.682273+0000","last_undegraded":"2023-09-23T22:53:07.682273+0000","last_fullsized":"2023-09-23T22:53:07.682273+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":33,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682778+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:53:07.682778+0000","last_peered":"2023-09-23T22:53:07.682778+0000","last_clean":"2023-09-23T22:53:07.682778+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:53:07.682778+0000","last_undegraded":"2023-09-23T22:53:07.682778+0000","last_fullsized":"2023-09-23T22:53:07.682778+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":177,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.688376+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:53:07.688376+0000","last_peered":"2023-09-23T22:53:07.688376+0000","last_clean":"2023-09-23T22:53:07.688376+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:53:07.688376+0000","last_undegraded":"2023-09-23T22:53:07.688376+0000","last_fullsized":"2023-09-23T22:53:07.688376+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":52,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565051+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:53:16.565051+0000","last_peered":"2023-09-23T22:53:16.565051+0000","last_clean":"2023-09-23T22:53:16.565051+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:53:16.565051+0000","last_undegraded":"2023-09-23T22:53:16.565051+0000","last_fullsized":"2023-09-23T22:53:16.565051+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.692783+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:53:07.692783+0000","last_peered":"2023-09-23T22:53:07.692783+0000","last_clean":"2023-09-23T22:53:07.692783+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:53:07.692783+0000","last_undegraded":"2023-09-23T22:53:07.692783+0000","last_fullsized":"2023-09-23T22:53:07.692783+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":313,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565801+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:53:16.565801+0000","last_peered":"2023-09-23T22:53:16.565801+0000","last_clean":"2023-09-23T22:53:16.565801+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:53:16.565801+0000","last_undegraded":"2023-09-23T22:53:16.565801+0000","last_fullsized":"2023-09-23T22:53:16.565801+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":259,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564256+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:53:16.564256+0000","last_peered":"2023-09-23T22:53:16.564256+0000","last_clean":"2023-09-23T22:53:16.564256+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:53:16.564256+0000","last_undegraded":"2023-09-23T22:53:16.564256+0000","last_fullsized":"2023-09-23T22:53:16.564256+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":152,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564615+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:53:16.564615+0000","last_peered":"2023-09-23T22:53:16.564615+0000","last_clean":"2023-09-23T22:53:16.564615+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:53:16.564615+0000","last_undegraded":"2023-09-23T22:53:16.564615+0000","last_fullsized":"2023-09-23T22:53:16.564615+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682559+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:53:07.682559+0000","last_peered":"2023-09-23T22:53:07.682559+0000","last_clean":"2023-09-23T22:53:07.682559+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:53:07.682559+0000","last_undegraded":"2023-09-23T22:53:07.682559+0000","last_fullsized":"2023-09-23T22:53:07.682559+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":260,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564602+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:53:16.564602+0000","last_peered":"2023-09-23T22:53:16.564602+0000","last_clean":"2023-09-23T22:53:16.564602+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:53:16.564602+0000","last_undegraded":"2023-09-23T22:53:16.564602+0000","last_fullsized":"2023-09-23T22:53:16.564602+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":216,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.677750+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:53:07.677750+0000","last_peered":"2023-09-23T22:53:07.677750+0000","last_clean":"2023-09-23T22:53:07.677750+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:53:07.677750+0000","last_undegraded":"2023-09-23T22:53:07.677750+0000","last_fullsized":"2023-09-23T22:53:07.677750+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564583+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:53:16.564583+0000","last_peered":"2023-09-23T22:53:16.564583+0000","last_clean":"2023-09-23T22:53:16.564583+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:53:16.564583+0000","last_undegraded":"2023-09-23T22:53:16.564583+0000","last_fullsized":"2023-09-23T22:53:16.564583+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":261,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.586258+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:53:26.586258+0000","last_peered":"2023-09-23T22:53:26.586258+0000","last_clean":"2023-09-23T22:53:26.586258+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:53:26.586258+0000","last_undegraded":"2023-09-23T22:53:26.586258+0000","last_fullsized":"2023-09-23T22:53:26.586258+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":337,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.679989+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:53:07.679989+0000","last_peered":"2023-09-23T22:53:07.679989+0000","last_clean":"2023-09-23T22:53:07.679989+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:53:07.679989+0000","last_undegraded":"2023-09-23T22:53:07.679989+0000","last_fullsized":"2023-09-23T22:53:07.679989+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":219,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.683307+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:53:07.683307+0000","last_peered":"2023-09-23T22:53:07.683307+0000","last_clean":"2023-09-23T22:53:07.683307+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:53:07.683307+0000","last_undegraded":"2023-09-23T22:53:07.683307+0000","last_fullsized":"2023-09-23T22:53:07.683307+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"294'7138","reported_seq":7178,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:28.061709+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:53:28.061709+0000","last_peered":"2023-09-23T22:53:28.061709+0000","last_clean":"2023-09-23T22:53:28.061709+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:53:28.061709+0000","last_undegraded":"2023-09-23T22:53:28.061709+0000","last_fullsized":"2023-09-23T22:53:28.061709+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7138,"log_dups_size":0,"ondisk_log_size":7138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261750784,"num_objects":3994,"num_object_clones":0,"num_object_copies":11982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7138,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":163,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564846+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:53:16.564846+0000","last_peered":"2023-09-23T22:53:16.564846+0000","last_clean":"2023-09-23T22:53:16.564846+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:53:16.564846+0000","last_undegraded":"2023-09-23T22:53:16.564846+0000","last_fullsized":"2023-09-23T22:53:16.564846+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.680078+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:53:07.680078+0000","last_peered":"2023-09-23T22:53:07.680078+0000","last_clean":"2023-09-23T22:53:07.680078+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:53:07.680078+0000","last_undegraded":"2023-09-23T22:53:07.680078+0000","last_fullsized":"2023-09-23T22:53:07.680078+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"294'7287","reported_seq":7331,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:26.632659+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:53:26.632659+0000","last_peered":"2023-09-23T22:53:26.632659+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:53:26.632659+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"268'600","ondisk_log_start":"268'600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275578880,"num_objects":4205,"num_object_clones":0,"num_object_copies":12615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7287,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":120},{"shards":"1,4","objects":2544}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":173,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564962+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:53:16.564962+0000","last_peered":"2023-09-23T22:53:16.564962+0000","last_clean":"2023-09-23T22:53:16.564962+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:53:16.564962+0000","last_undegraded":"2023-09-23T22:53:16.564962+0000","last_fullsized":"2023-09-23T22:53:16.564962+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":283,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564498+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:53:16.564498+0000","last_peered":"2023-09-23T22:53:16.564498+0000","last_clean":"2023-09-23T22:53:16.564498+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:53:16.564498+0000","last_undegraded":"2023-09-23T22:53:16.564498+0000","last_fullsized":"2023-09-23T22:53:16.564498+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"293'7221","reported_seq":7265,"reported_epoch":293,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:24.818675+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:53:24.818675+0000","last_peered":"2023-09-23T22:53:24.818675+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:53:24.818675+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7221,"log_dups_size":0,"ondisk_log_size":7221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266534912,"num_objects":4067,"num_object_clones":0,"num_object_copies":12201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7221,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":194},{"shards":"2,5","objects":2296}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":116,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.677676+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:53:07.677676+0000","last_peered":"2023-09-23T22:53:07.677676+0000","last_clean":"2023-09-23T22:53:07.677676+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:53:07.677676+0000","last_undegraded":"2023-09-23T22:53:07.677676+0000","last_fullsized":"2023-09-23T22:53:07.677676+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":285,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.692739+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:53:07.692739+0000","last_peered":"2023-09-23T22:53:07.692739+0000","last_clean":"2023-09-23T22:53:07.692739+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:53:07.692739+0000","last_undegraded":"2023-09-23T22:53:07.692739+0000","last_fullsized":"2023-09-23T22:53:07.692739+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"293'7135","reported_seq":7174,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:24.959814+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:53:24.959814+0000","last_peered":"2023-09-23T22:53:24.959814+0000","last_clean":"2023-09-23T22:53:24.959814+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:53:24.959814+0000","last_undegraded":"2023-09-23T22:53:24.959814+0000","last_fullsized":"2023-09-23T22:53:24.959814+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7135,"log_dups_size":0,"ondisk_log_size":7135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7135,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":88,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.678989+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:53:07.678989+0000","last_peered":"2023-09-23T22:53:07.678989+0000","last_clean":"2023-09-23T22:53:07.678989+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:53:07.678989+0000","last_undegraded":"2023-09-23T22:53:07.678989+0000","last_fullsized":"2023-09-23T22:53:07.678989+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":188,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564730+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:53:16.564730+0000","last_peered":"2023-09-23T22:53:16.564730+0000","last_clean":"2023-09-23T22:53:16.564730+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:53:16.564730+0000","last_undegraded":"2023-09-23T22:53:16.564730+0000","last_fullsized":"2023-09-23T22:53:16.564730+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"293'7043","reported_seq":10446,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:25.955393+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:53:25.955393+0000","last_peered":"2023-09-23T22:53:25.955393+0000","last_clean":"2023-09-23T22:53:25.955393+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:53:25.955393+0000","last_undegraded":"2023-09-23T22:53:25.955393+0000","last_fullsized":"2023-09-23T22:53:25.955393+0000","mapping_epoch":289,"log_start":"268'600","ondisk_log_start":"268'600","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6443,"log_dups_size":0,"ondisk_log_size":6443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259850240,"num_objects":3965,"num_object_clones":0,"num_object_copies":11895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7043,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":59,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564837+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:53:16.564837+0000","last_peered":"2023-09-23T22:53:16.564837+0000","last_clean":"2023-09-23T22:53:16.564837+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:53:16.564837+0000","last_undegraded":"2023-09-23T22:53:16.564837+0000","last_fullsized":"2023-09-23T22:53:16.564837+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":267,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565734+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:53:16.565734+0000","last_peered":"2023-09-23T22:53:16.565734+0000","last_clean":"2023-09-23T22:53:16.565734+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:53:16.565734+0000","last_undegraded":"2023-09-23T22:53:16.565734+0000","last_fullsized":"2023-09-23T22:53:16.565734+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"293'7066","reported_seq":7103,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:25.903295+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:53:25.903295+0000","last_peered":"2023-09-23T22:53:25.903295+0000","last_clean":"2023-09-23T22:53:25.903295+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:53:25.903295+0000","last_undegraded":"2023-09-23T22:53:25.903295+0000","last_fullsized":"2023-09-23T22:53:25.903295+0000","mapping_epoch":264,"log_start":"268'600","ondisk_log_start":"268'600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6466,"log_dups_size":0,"ondisk_log_size":6466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263192576,"num_objects":4016,"num_object_clones":0,"num_object_copies":12048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7066,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":277,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.692636+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:53:07.692636+0000","last_peered":"2023-09-23T22:53:07.692636+0000","last_clean":"2023-09-23T22:53:07.692636+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:53:07.692636+0000","last_undegraded":"2023-09-23T22:53:07.692636+0000","last_fullsized":"2023-09-23T22:53:07.692636+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":289,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564889+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:53:16.564889+0000","last_peered":"2023-09-23T22:53:16.564889+0000","last_clean":"2023-09-23T22:53:16.564889+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:53:16.564889+0000","last_undegraded":"2023-09-23T22:53:16.564889+0000","last_fullsized":"2023-09-23T22:53:16.564889+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"294'7170","reported_seq":7214,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:27.036442+0000","last_change":"2023-09-23T22:51:35.518569+0000","last_active":"2023-09-23T22:53:27.036442+0000","last_peered":"2023-09-23T22:53:27.036442+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:53:27.036442+0000","last_undegraded":"2023-09-23T22:51:35.445687+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":271,"log_start":"273'2000","ondisk_log_start":"273'2000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5170,"log_dups_size":0,"ondisk_log_size":5170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:39.333773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262799360,"num_objects":4010,"num_object_clones":0,"num_object_copies":12030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7170,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":85},{"shards":"1,5","objects":2345}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":302,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564266+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:53:16.564266+0000","last_peered":"2023-09-23T22:53:16.564266+0000","last_clean":"2023-09-23T22:53:16.564266+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:53:16.564266+0000","last_undegraded":"2023-09-23T22:53:16.564266+0000","last_fullsized":"2023-09-23T22:53:16.564266+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":277,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564997+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:53:16.564997+0000","last_peered":"2023-09-23T22:53:16.564997+0000","last_clean":"2023-09-23T22:53:16.564997+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:53:16.564997+0000","last_undegraded":"2023-09-23T22:53:16.564997+0000","last_fullsized":"2023-09-23T22:53:16.564997+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"293'6946","reported_seq":6990,"reported_epoch":293,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:24.829137+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:53:24.829137+0000","last_peered":"2023-09-23T22:53:24.829137+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:53:24.829137+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6946,"log_dups_size":0,"ondisk_log_size":6946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270270464,"num_objects":4124,"num_object_clones":0,"num_object_copies":12372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6946,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":2509},{"shards":"1,2,3","objects":204}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":222,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564094+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:53:16.564094+0000","last_peered":"2023-09-23T22:53:16.564094+0000","last_clean":"2023-09-23T22:53:16.564094+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:53:16.564094+0000","last_undegraded":"2023-09-23T22:53:16.564094+0000","last_fullsized":"2023-09-23T22:53:16.564094+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":230,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.683416+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:53:07.683416+0000","last_peered":"2023-09-23T22:53:07.683416+0000","last_clean":"2023-09-23T22:53:07.683416+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:53:07.683416+0000","last_undegraded":"2023-09-23T22:53:07.683416+0000","last_fullsized":"2023-09-23T22:53:07.683416+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"294'7220","reported_seq":7264,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:27.001058+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:53:27.001058+0000","last_peered":"2023-09-23T22:53:27.001058+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:53:27.001058+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"273'2100","ondisk_log_start":"273'2100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7220,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3,5","objects":86},{"shards":"1,5","objects":2502}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":332,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.567180+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:53:16.567180+0000","last_peered":"2023-09-23T22:53:16.567180+0000","last_clean":"2023-09-23T22:53:16.567180+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:53:16.567180+0000","last_undegraded":"2023-09-23T22:53:16.567180+0000","last_fullsized":"2023-09-23T22:53:16.567180+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":155,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682685+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:53:07.682685+0000","last_peered":"2023-09-23T22:53:07.682685+0000","last_clean":"2023-09-23T22:53:07.682685+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:53:07.682685+0000","last_undegraded":"2023-09-23T22:53:07.682685+0000","last_fullsized":"2023-09-23T22:53:07.682685+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564268+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:53:16.564268+0000","last_peered":"2023-09-23T22:53:16.564268+0000","last_clean":"2023-09-23T22:53:16.564268+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:53:16.564268+0000","last_undegraded":"2023-09-23T22:53:16.564268+0000","last_fullsized":"2023-09-23T22:53:16.564268+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"294'7231","reported_seq":10673,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:28.085183+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:53:28.085183+0000","last_peered":"2023-09-23T22:53:28.085183+0000","last_clean":"2023-09-23T22:53:28.085183+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:53:28.085183+0000","last_undegraded":"2023-09-23T22:53:28.085183+0000","last_fullsized":"2023-09-23T22:53:28.085183+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7231,"log_dups_size":0,"ondisk_log_size":7231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258670592,"num_objects":3947,"num_object_clones":0,"num_object_copies":11841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7231,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.692692+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:53:07.692692+0000","last_peered":"2023-09-23T22:53:07.692692+0000","last_clean":"2023-09-23T22:53:07.692692+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:53:07.692692+0000","last_undegraded":"2023-09-23T22:53:07.692692+0000","last_fullsized":"2023-09-23T22:53:07.692692+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":290,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564618+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:53:16.564618+0000","last_peered":"2023-09-23T22:53:16.564618+0000","last_clean":"2023-09-23T22:53:16.564618+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:53:16.564618+0000","last_undegraded":"2023-09-23T22:53:16.564618+0000","last_fullsized":"2023-09-23T22:53:16.564618+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":219,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682613+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:53:07.682613+0000","last_peered":"2023-09-23T22:53:07.682613+0000","last_clean":"2023-09-23T22:53:07.682613+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:53:07.682613+0000","last_undegraded":"2023-09-23T22:53:07.682613+0000","last_fullsized":"2023-09-23T22:53:07.682613+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"293'7145","reported_seq":7193,"reported_epoch":293,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:53:26.152096+0000","last_change":"2023-09-23T22:51:35.528654+0000","last_active":"2023-09-23T22:53:26.152096+0000","last_peered":"2023-09-23T22:53:26.152096+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:53:26.152096+0000","last_undegraded":"2023-09-23T22:53:26.152096+0000","last_fullsized":"2023-09-23T22:53:26.152096+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7145,"log_dups_size":0,"ondisk_log_size":7145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270729216,"num_objects":4131,"num_object_clones":0,"num_object_copies":12393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":4131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7145,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":2477}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":294,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682490+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:53:07.682490+0000","last_peered":"2023-09-23T22:53:07.682490+0000","last_clean":"2023-09-23T22:53:07.682490+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:53:07.682490+0000","last_undegraded":"2023-09-23T22:53:07.682490+0000","last_fullsized":"2023-09-23T22:53:07.682490+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":174,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682189+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:53:07.682189+0000","last_peered":"2023-09-23T22:53:07.682189+0000","last_clean":"2023-09-23T22:53:07.682189+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:53:07.682189+0000","last_undegraded":"2023-09-23T22:53:07.682189+0000","last_fullsized":"2023-09-23T22:53:07.682189+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":301,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:18.186599+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:53:18.186599+0000","last_peered":"2023-09-23T22:53:18.186599+0000","last_clean":"2023-09-23T22:53:18.186599+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:53:18.186599+0000","last_undegraded":"2023-09-23T22:53:18.186599+0000","last_fullsized":"2023-09-23T22:53:18.186599+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"294'7200","reported_seq":7246,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:28.070459+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:53:28.070459+0000","last_peered":"2023-09-23T22:53:28.070459+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:53:28.070459+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252313600,"num_objects":3850,"num_object_clones":0,"num_object_copies":11550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7200,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":37},{"shards":"0,5","objects":2138}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682428+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:53:07.682428+0000","last_peered":"2023-09-23T22:53:07.682428+0000","last_clean":"2023-09-23T22:53:07.682428+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:53:07.682428+0000","last_undegraded":"2023-09-23T22:53:07.682428+0000","last_fullsized":"2023-09-23T22:53:07.682428+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":382,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.567423+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:53:16.567423+0000","last_peered":"2023-09-23T22:53:16.567423+0000","last_clean":"2023-09-23T22:53:16.567423+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:53:16.567423+0000","last_undegraded":"2023-09-23T22:53:16.567423+0000","last_fullsized":"2023-09-23T22:53:16.567423+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":47,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.692776+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:53:07.692776+0000","last_peered":"2023-09-23T22:53:07.692776+0000","last_clean":"2023-09-23T22:53:07.692776+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:53:07.692776+0000","last_undegraded":"2023-09-23T22:53:07.692776+0000","last_fullsized":"2023-09-23T22:53:07.692776+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":285,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.677524+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:53:07.677524+0000","last_peered":"2023-09-23T22:53:07.677524+0000","last_clean":"2023-09-23T22:53:07.677524+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:53:07.677524+0000","last_undegraded":"2023-09-23T22:53:07.677524+0000","last_fullsized":"2023-09-23T22:53:07.677524+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"294'7193","reported_seq":7233,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:28.189996+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:53:28.189996+0000","last_peered":"2023-09-23T22:53:28.189996+0000","last_clean":"2023-09-23T22:53:28.189996+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:53:28.189996+0000","last_undegraded":"2023-09-23T22:53:28.189996+0000","last_fullsized":"2023-09-23T22:53:28.189996+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7193,"log_dups_size":0,"ondisk_log_size":7193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264306688,"num_objects":4033,"num_object_clones":0,"num_object_copies":12099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7193,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":270,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.567589+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:53:16.567589+0000","last_peered":"2023-09-23T22:53:16.567589+0000","last_clean":"2023-09-23T22:53:16.567589+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:53:16.567589+0000","last_undegraded":"2023-09-23T22:53:16.567589+0000","last_fullsized":"2023-09-23T22:53:16.567589+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":241,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565400+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:53:16.565400+0000","last_peered":"2023-09-23T22:53:16.565400+0000","last_clean":"2023-09-23T22:53:16.565400+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:53:16.565400+0000","last_undegraded":"2023-09-23T22:53:16.565400+0000","last_fullsized":"2023-09-23T22:53:16.565400+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":219,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.572080+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:53:16.572080+0000","last_peered":"2023-09-23T22:53:16.572080+0000","last_clean":"2023-09-23T22:53:16.572080+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:53:16.572080+0000","last_undegraded":"2023-09-23T22:53:16.572080+0000","last_fullsized":"2023-09-23T22:53:16.572080+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"294'7025","reported_seq":7063,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.598808+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:53:26.598808+0000","last_peered":"2023-09-23T22:53:26.598808+0000","last_clean":"2023-09-23T22:53:26.598808+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:53:26.598808+0000","last_undegraded":"2023-09-23T22:53:26.598808+0000","last_fullsized":"2023-09-23T22:53:26.598808+0000","mapping_epoch":264,"log_start":"268'300","ondisk_log_start":"268'300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258801664,"num_objects":3949,"num_object_clones":0,"num_object_copies":11847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7025,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":224,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.566549+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:53:16.566549+0000","last_peered":"2023-09-23T22:53:16.566549+0000","last_clean":"2023-09-23T22:53:16.566549+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:53:16.566549+0000","last_undegraded":"2023-09-23T22:53:16.566549+0000","last_fullsized":"2023-09-23T22:53:16.566549+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":173,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682700+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:53:07.682700+0000","last_peered":"2023-09-23T22:53:07.682700+0000","last_clean":"2023-09-23T22:53:07.682700+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:53:07.682700+0000","last_undegraded":"2023-09-23T22:53:07.682700+0000","last_fullsized":"2023-09-23T22:53:07.682700+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565100+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:53:16.565100+0000","last_peered":"2023-09-23T22:53:16.565100+0000","last_clean":"2023-09-23T22:53:16.565100+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:53:16.565100+0000","last_undegraded":"2023-09-23T22:53:16.565100+0000","last_fullsized":"2023-09-23T22:53:16.565100+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"294'7209","reported_seq":7247,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.606100+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:53:26.606100+0000","last_peered":"2023-09-23T22:53:26.606100+0000","last_clean":"2023-09-23T22:53:26.606100+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:53:26.606100+0000","last_undegraded":"2023-09-23T22:53:26.606100+0000","last_fullsized":"2023-09-23T22:53:26.606100+0000","mapping_epoch":264,"log_start":"268'500","ondisk_log_start":"268'500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272105496,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7209,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":50,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565180+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:53:16.565180+0000","last_peered":"2023-09-23T22:53:16.565180+0000","last_clean":"2023-09-23T22:53:16.565180+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:53:16.565180+0000","last_undegraded":"2023-09-23T22:53:16.565180+0000","last_fullsized":"2023-09-23T22:53:16.565180+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.677987+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:53:07.677987+0000","last_peered":"2023-09-23T22:53:07.677987+0000","last_clean":"2023-09-23T22:53:07.677987+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:53:07.677987+0000","last_undegraded":"2023-09-23T22:53:07.677987+0000","last_fullsized":"2023-09-23T22:53:07.677987+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":270,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565624+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:53:16.565624+0000","last_peered":"2023-09-23T22:53:16.565624+0000","last_clean":"2023-09-23T22:53:16.565624+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:53:16.565624+0000","last_undegraded":"2023-09-23T22:53:16.565624+0000","last_fullsized":"2023-09-23T22:53:16.565624+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"293'7265","reported_seq":10012,"reported_epoch":294,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:26.598950+0000","last_change":"2023-09-23T22:53:13.718777+0000","last_active":"2023-09-23T22:53:26.598950+0000","last_peered":"2023-09-23T22:53:26.598950+0000","last_clean":"2023-09-23T22:51:33.377854+0000","last_became_active":"2023-09-23T22:51:37.033330+0000","last_became_peered":"2023-09-23T22:51:37.033330+0000","last_unstale":"2023-09-23T22:53:26.598950+0000","last_undegraded":"2023-09-23T22:53:13.714740+0000","last_fullsized":"2023-09-23T22:51:35.425861+0000","mapping_epoch":271,"log_start":"268'500","ondisk_log_start":"268'500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263913472,"num_objects":4027,"num_object_clones":0,"num_object_copies":12081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":271,"num_objects_misplaced":108,"num_objects_unfound":0,"num_objects_dirty":4027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7265,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1350,"num_bytes_recovered":88473600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":108},{"shards":"0,4","objects":3648}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":334,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.678706+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:53:07.678706+0000","last_peered":"2023-09-23T22:53:07.678706+0000","last_clean":"2023-09-23T22:53:07.678706+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:53:07.678706+0000","last_undegraded":"2023-09-23T22:53:07.678706+0000","last_fullsized":"2023-09-23T22:53:07.678706+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":191,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.572177+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:53:16.572177+0000","last_peered":"2023-09-23T22:53:16.572177+0000","last_clean":"2023-09-23T22:53:16.572177+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:53:16.572177+0000","last_undegraded":"2023-09-23T22:53:16.572177+0000","last_fullsized":"2023-09-23T22:53:16.572177+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":244,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564345+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:53:16.564345+0000","last_peered":"2023-09-23T22:53:16.564345+0000","last_clean":"2023-09-23T22:53:16.564345+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:53:16.564345+0000","last_undegraded":"2023-09-23T22:53:16.564345+0000","last_fullsized":"2023-09-23T22:53:16.564345+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":226,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564943+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:53:16.564943+0000","last_peered":"2023-09-23T22:53:16.564943+0000","last_clean":"2023-09-23T22:53:16.564943+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:53:16.564943+0000","last_undegraded":"2023-09-23T22:53:16.564943+0000","last_fullsized":"2023-09-23T22:53:16.564943+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":76,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682651+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:53:07.682651+0000","last_peered":"2023-09-23T22:53:07.682651+0000","last_clean":"2023-09-23T22:53:07.682651+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:53:07.682651+0000","last_undegraded":"2023-09-23T22:53:07.682651+0000","last_fullsized":"2023-09-23T22:53:07.682651+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":103,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565661+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:53:16.565661+0000","last_peered":"2023-09-23T22:53:16.565661+0000","last_clean":"2023-09-23T22:53:16.565661+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:53:16.565661+0000","last_undegraded":"2023-09-23T22:53:16.565661+0000","last_fullsized":"2023-09-23T22:53:16.565661+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":302,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564700+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:53:16.564700+0000","last_peered":"2023-09-23T22:53:16.564700+0000","last_clean":"2023-09-23T22:53:16.564700+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:53:16.564700+0000","last_undegraded":"2023-09-23T22:53:16.564700+0000","last_fullsized":"2023-09-23T22:53:16.564700+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":104,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564966+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:53:16.564966+0000","last_peered":"2023-09-23T22:53:16.564966+0000","last_clean":"2023-09-23T22:53:16.564966+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:53:16.564966+0000","last_undegraded":"2023-09-23T22:53:16.564966+0000","last_fullsized":"2023-09-23T22:53:16.564966+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":174,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564422+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:53:16.564422+0000","last_peered":"2023-09-23T22:53:16.564422+0000","last_clean":"2023-09-23T22:53:16.564422+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:53:16.564422+0000","last_undegraded":"2023-09-23T22:53:16.564422+0000","last_fullsized":"2023-09-23T22:53:16.564422+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":43,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.677865+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:53:07.677865+0000","last_peered":"2023-09-23T22:53:07.677865+0000","last_clean":"2023-09-23T22:53:07.677865+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:53:07.677865+0000","last_undegraded":"2023-09-23T22:53:07.677865+0000","last_fullsized":"2023-09-23T22:53:07.677865+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":275,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T22:53:07.682364+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:53:07.682364+0000","last_peered":"2023-09-23T22:53:07.682364+0000","last_clean":"2023-09-23T22:53:07.682364+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:53:07.682364+0000","last_undegraded":"2023-09-23T22:53:07.682364+0000","last_fullsized":"2023-09-23T22:53:07.682364+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":240,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.565024+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:53:16.565024+0000","last_peered":"2023-09-23T22:53:16.565024+0000","last_clean":"2023-09-23T22:53:16.565024+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:53:16.565024+0000","last_undegraded":"2023-09-23T22:53:16.565024+0000","last_fullsized":"2023-09-23T22:53:16.565024+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":259,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T22:53:16.564178+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:53:16.564178+0000","last_peered":"2023-09-23T22:53:16.564178+0000","last_clean":"2023-09-23T22:53:16.564178+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:53:16.564178+0000","last_undegraded":"2023-09-23T22:53:16.564178+0000","last_fullsized":"2023-09-23T22:53:16.564178+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4245618712,"num_objects":64784,"num_object_clones":0,"num_object_copies":194352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10325,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":64784,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114494,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4717,"num_bytes_recovered":309133312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13950267392,"data_stored":13950255176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107294,"ondisk_log_size":107294,"up":48,"acting":41,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430525,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626753,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2359148,"kb_used_data":2053732,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91386004,"statfs":{"total":95995035648,"available":93579268096,"internally_reserved":0,"allocated":2103021568,"data_stored":2102465577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":6.9699999999999998,"5min":9.5869999999999997,"15min":4.2800000000000002},"min":{"1min":0.24199999999999999,"5min":0.218,"15min":0.218},"max":{"1min":112.51000000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.379},{"interface":"front","average":{"1min":7.6239999999999997,"5min":9.9469999999999992,"15min":4.2679999999999998},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":112.154,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":14.752000000000001,"5min":5.0700000000000003,"15min":3.6389999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":254.62100000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":20.399999999999999,"5min":7.9690000000000003,"15min":4.6319999999999997},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":255.20099999999999,"5min":255.20099999999999,"15min":255.20099999999999},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":3.867,"15min":3.4199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":38.527999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":38.527999999999999},{"interface":"front","average":{"1min":4.3499999999999996,"5min":3.4969999999999999,"15min":3.786},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":38.776000000000003,"5min":101.13,"15min":185.21799999999999},"last":38.776000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":5.6150000000000002,"15min":3.706},"min":{"1min":0.36199999999999999,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":7.734,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.374},{"interface":"front","average":{"1min":1.919,"5min":3.919,"15min":3.3170000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.7770000000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.875,"15min":0.753},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.7290000000000001,"5min":6.5309999999999997,"15min":18.085999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.3320000000000001,"5min":0.78900000000000003,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4800000000000004,"5min":7.4800000000000004,"15min":18.597000000000001},"last":0.32900000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.0529999999999999,"15min":0.85299999999999998},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.7560000000000002,"5min":10.148999999999999,"15min":18.972000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.83799999999999997,"15min":0.80900000000000005},"min":{"1min":0.316,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":2.782,"5min":4.9089999999999998,"15min":18.625},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":0.86499999999999999,"15min":0.82999999999999996},"min":{"1min":0.35799999999999998,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":4.2869999999999999,"5min":4.3049999999999997,"15min":12.481999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.03,"15min":0.86199999999999999},"min":{"1min":0.38200000000000001,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":6.6269999999999998,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.46400000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253063,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3580384,"kb_used_data":3105176,"kb_used_omap":9,"kb_used_meta":475062,"kb_avail":90164768,"statfs":{"total":95995035648,"available":92328722432,"internally_reserved":0,"allocated":3179700224,"data_stored":3178640869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":486464489},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,13],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":7.5039999999999996,"15min":5.2800000000000002},"min":{"1min":0.26800000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.933,"5min":101.605,"15min":101.605},"last":1.595},{"interface":"front","average":{"1min":3.1309999999999998,"5min":6.6639999999999997,"15min":5.2309999999999999},"min":{"1min":0.24099999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":8.6620000000000008,"5min":101.892,"15min":101.892},"last":0.70699999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.8610000000000002,"15min":2.7989999999999999},"min":{"1min":0.22500000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":12.622999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":0.997},{"interface":"front","average":{"1min":2.214,"5min":4.0629999999999997,"15min":2.8799999999999999},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.887,"5min":37.957000000000001,"15min":37.957000000000001},"last":1.633}]},{"osd":3,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":3.3170000000000002,"15min":2.3809999999999998},"min":{"1min":0.28999999999999998,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.218,"5min":87.787999999999997,"15min":87.787999999999997},"last":1.101},{"interface":"front","average":{"1min":1.9910000000000001,"5min":3.9590000000000001,"15min":2.835},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":4.7859999999999996,"5min":89.861999999999995,"15min":89.861999999999995},"last":1.5349999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.4950000000000001,"15min":2.0950000000000002},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":3.3889999999999998,"5min":78.948999999999998,"15min":78.948999999999998},"last":1.4670000000000001},{"interface":"front","average":{"1min":2.4140000000000001,"5min":2.9009999999999998,"15min":2.0920000000000001},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":12.792999999999999,"5min":78.963999999999999,"15min":78.963999999999999},"last":0.81699999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":2.1030000000000002,"15min":1.6459999999999999},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.27900000000000003},"max":{"1min":5.9960000000000004,"5min":84.254999999999995,"15min":84.254999999999995},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.4930000000000001,"5min":2.6360000000000001,"15min":1.9710000000000001},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":3.347,"5min":87.515000000000001,"15min":87.515000000000001},"last":1.4970000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.9390000000000001,"15min":1.925},"min":{"1min":0.41799999999999998,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.2539999999999996,"5min":79.311000000000007,"15min":79.311000000000007},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.909,"5min":2.1779999999999999,"15min":1.601},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":5.9340000000000002,"5min":79.328999999999994,"15min":79.328999999999994},"last":0.74299999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.008,"15min":2.3460000000000001},"min":{"1min":0.42499999999999999,"5min":0.39400000000000002,"15min":0.32800000000000001},"max":{"1min":4.0979999999999999,"5min":11.481999999999999,"15min":159.018},"last":1.569},{"interface":"front","average":{"1min":2.3210000000000002,"5min":2.4399999999999999,"15min":2.444},"min":{"1min":0.47599999999999998,"5min":0.371,"15min":0.37},"max":{"1min":6.282,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.874}]}]},{"osd":0,"up_from":219,"seq":940597837912,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2103384,"kb_used_data":1786128,"kb_used_omap":31,"kb_used_meta":317216,"kb_avail":91641768,"statfs":{"total":95995035648,"available":93841170432,"internally_reserved":0,"allocated":1828995072,"data_stored":1827884717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":324830130},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":3.5169999999999999,"15min":3.6469999999999998},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":11.930999999999999,"5min":21.512,"15min":32.344999999999999},"last":7.2519999999999998},{"interface":"front","average":{"1min":2.4169999999999998,"5min":3.4249999999999998,"15min":3.5139999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.7379999999999995,"5min":20.064,"15min":32.106000000000002},"last":7.133}]},{"osd":2,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":4.9889999999999999,"15min":3.0609999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.9619999999999997,"5min":111.508,"15min":111.508},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.669,"5min":4.2229999999999999,"15min":2.5790000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.9749999999999996,"5min":111.532,"15min":111.532},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":4.9450000000000003,"15min":2.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9269999999999996,"5min":126.73,"15min":126.73},"last":1.5429999999999999},{"interface":"front","average":{"1min":1.177,"5min":3.8519999999999999,"15min":2.0329999999999999},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.8419999999999996,"5min":121.483,"15min":121.483},"last":0.91300000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":2.633,"15min":2.113},"min":{"1min":0.36299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.965999999999999,"5min":18.670999999999999,"15min":18.670999999999999},"last":1.7869999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.5169999999999999,"15min":2.0049999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.9990000000000006,"5min":20.009,"15min":20.009},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":2.9180000000000001,"15min":1.863},"min":{"1min":0.41399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.6280000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.497},{"interface":"front","average":{"1min":2.6379999999999999,"5min":3.3769999999999998,"15min":2.161},"min":{"1min":0.40400000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.0519999999999996,"5min":61.777000000000001,"15min":61.777000000000001},"last":1.819}]},{"osd":6,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":2.6909999999999998,"15min":1.8109999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5369999999999999,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.9220000000000002,"5min":3.3679999999999999,"15min":2.246},"min":{"1min":0.48499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":11.882999999999999,"5min":59.164000000000001,"15min":59.164000000000001},"last":1.7410000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.6800000000000002,"15min":1.9570000000000001},"min":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.0869999999999997,"5min":18.640000000000001,"15min":18.640000000000001},"last":1.849},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.2320000000000002,"15min":1.5289999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.9219999999999997,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.52500000000000002}]}]},{"osd":2,"up_from":131,"seq":562640715924,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1978328,"kb_used_data":1570576,"kb_used_omap":81,"kb_used_meta":407662,"kb_avail":91766824,"statfs":{"total":95995035648,"available":93969227776,"internally_reserved":0,"allocated":1608269824,"data_stored":1607213522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":417446106},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:52:31 2023","interfaces":[{"interface":"back","average":{"1min":8.4789999999999992,"5min":6.7309999999999999,"15min":5.7560000000000002},"min":{"1min":0.255,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":114.483,"5min":132.79499999999999,"15min":132.79499999999999},"last":3.3370000000000002},{"interface":"front","average":{"1min":8.5,"5min":7.7599999999999998,"15min":6.1799999999999997},"min":{"1min":0.34599999999999997,"5min":0.314,"15min":0.314},"max":{"1min":113.873,"5min":132.58099999999999,"15min":132.58099999999999},"last":3.4329999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.5310000000000001,"15min":3.0230000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":12.035,"5min":15.622999999999999,"15min":88.316000000000003},"last":2.246},{"interface":"front","average":{"1min":3.5329999999999999,"5min":3.4220000000000002,"15min":3.1589999999999998},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":14.159000000000001,"5min":28.596,"15min":88.436999999999998},"last":3.8050000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.3250000000000002,"15min":4.1890000000000001},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":6.5419999999999998,"5min":112.895,"15min":112.895},"last":3.7330000000000001},{"interface":"front","average":{"1min":1.524,"5min":4.617,"15min":3.4169999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.251},"max":{"1min":9.3840000000000003,"5min":113.995,"15min":113.995},"last":1.4730000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.597,"15min":2.4169999999999998},"min":{"1min":0.37,"5min":0.253,"15min":0.253},"max":{"1min":7.1520000000000001,"5min":84.778999999999996,"15min":108.878},"last":1.4179999999999999},{"interface":"front","average":{"1min":1.901,"5min":3.641,"15min":3.056},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.8460000000000001,"5min":84.753,"15min":108.93000000000001},"last":2.2679999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0750000000000002,"15min":2.9009999999999998},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.5970000000000004,"5min":90.082999999999998,"15min":97.322999999999993},"last":2.29},{"interface":"front","average":{"1min":2.0049999999999999,"5min":3.3399999999999999,"15min":2.718},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.5380000000000003,"5min":89.986000000000004,"15min":97.301000000000002},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":4.8330000000000002,"15min":2.9390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":4.5789999999999997,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.607,"5min":4.3360000000000003,"15min":2.3180000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.4089999999999998,"5min":98.509,"15min":98.509},"last":1.4390000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0459999999999998,"15min":1.5740000000000001},"min":{"1min":0.42799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.7440000000000002,"5min":10.92,"15min":10.92},"last":1.46},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.1240000000000001,"15min":2.323},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":4.2489999999999997,"5min":11.311,"15min":88.584000000000003},"last":2.3100000000000001}]}]},{"osd":3,"up_from":0,"seq":403726926010,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019209,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1749360,"kb_used_data":1452136,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91995792,"statfs":{"total":95995035648,"available":94203691008,"internally_reserved":0,"allocated":1486987264,"data_stored":1486426679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":8.0700000000000003,"5min":4.423,"15min":6.367},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":82.376000000000005,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":8.1430000000000007,"5min":4.9420000000000002,"15min":6.9160000000000004},"min":{"1min":0.34100000000000003,"5min":0.217,"15min":0.217},"max":{"1min":79.403000000000006,"5min":113.417,"15min":113.417},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":11.753,"5min":3.7679999999999998,"15min":2.4649999999999999},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":132.304,"5min":132.304,"15min":132.304},"last":0.98999999999999999},{"interface":"front","average":{"1min":11.4,"5min":3.6560000000000001,"15min":2.3180000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":132.453,"5min":132.453,"15min":132.453},"last":0.31}]},{"osd":2,"last update":"Sat Sep 23 22:52:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1179999999999999,"5min":2.2240000000000002,"15min":3.1720000000000002},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":20.308,"5min":20.308,"15min":84.909000000000006},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.8769999999999998,"5min":2.3769999999999998,"15min":3.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":20.652000000000001,"5min":21.704000000000001,"15min":83.840000000000003},"last":2.3370000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":5.3339999999999996,"5min":5.3849999999999998,"15min":4.3410000000000002},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":74.138999999999996,"5min":164.453,"15min":164.453},"last":0.70199999999999996},{"interface":"front","average":{"1min":4.9329999999999998,"5min":5.3520000000000003,"15min":4.4249999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":75.099999999999994,"5min":167.05199999999999,"15min":167.05199999999999},"last":1.0369999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.097,"15min":0.93799999999999994},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":10.321999999999999,"5min":12.238,"15min":43.189999999999998},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.079,"15min":0.93999999999999995},"min":{"1min":0.34200000000000003,"5min":0.248,"15min":0.248},"max":{"1min":6.4349999999999996,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.95199999999999996,"15min":0.84399999999999997},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.758,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.88600000000000001,"15min":0.84299999999999997},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":1.978,"5min":12.269,"15min":26.940999999999999},"last":0.96399999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.0629999999999999,"15min":0.82199999999999995},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":10.278,"5min":12.311,"15min":12.311},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.98099999999999998,"15min":0.76100000000000001},"min":{"1min":0.38800000000000001,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":3.7909999999999999,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.73199999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822981,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2942740,"kb_used_data":2637324,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90802412,"statfs":{"total":95995035648,"available":92981669888,"internally_reserved":0,"allocated":2700619776,"data_stored":2700082907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":12.433,"5min":6.899,"15min":4.6070000000000002},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":185.62,"5min":185.62,"15min":185.62},"last":0.35199999999999998},{"interface":"front","average":{"1min":12.449999999999999,"5min":7.0140000000000002,"15min":4.569},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":185.499,"5min":185.499,"15min":185.499},"last":2.7959999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8820000000000001,"5min":2.0649999999999999,"15min":2.056},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":41.463999999999999,"5min":41.463999999999999,"15min":149.203},"last":11.821},{"interface":"front","average":{"1min":4.0869999999999997,"5min":2.5670000000000002,"15min":2.3260000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":41.954000000000001,"5min":41.954000000000001,"15min":149.554},"last":9.0950000000000006}]},{"osd":2,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.9660000000000002,"15min":5.6040000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":34.988,"5min":63.156999999999996,"15min":158.92599999999999},"last":2.891},{"interface":"front","average":{"1min":4.3799999999999999,"5min":2.7050000000000001,"15min":5.5270000000000001},"min":{"1min":0.38600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":34.872,"5min":63.277999999999999,"15min":158.91399999999999},"last":0.69099999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":5.3380000000000001,"5min":7.6420000000000003,"15min":5.3600000000000003},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":89.995000000000005,"5min":108.58799999999999,"15min":257.05000000000001},"last":1.5},{"interface":"front","average":{"1min":5.8029999999999999,"5min":7.8769999999999998,"15min":5.2800000000000002},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":90.117999999999995,"5min":108.15600000000001,"15min":257.35000000000002},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":0.84499999999999997,"15min":0.72199999999999998},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":6.3949999999999996,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":1.7789999999999999},{"interface":"front","average":{"1min":1.956,"5min":0.83799999999999997,"15min":0.72299999999999998},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":6.3550000000000004,"5min":6.3550000000000004,"15min":10.535},"last":0.37}]},{"osd":6,"last update":"Sat Sep 23 22:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.89500000000000002,"15min":1.4490000000000001},"min":{"1min":0.307,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.7469999999999999,"5min":12.615,"15min":140.07499999999999},"last":1.8100000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.85899999999999999,"15min":1.4790000000000001},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":4,"5min":12.74,"15min":139.739},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:53:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":0.97899999999999998,"15min":0.82799999999999996},"min":{"1min":0.40100000000000002,"5min":0.33400000000000002,"15min":0.29499999999999998},"max":{"1min":10.367000000000001,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.353,"5min":0.89400000000000002,"15min":0.76100000000000001},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":6.1120000000000001,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":1.8320000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1819279360,"data_stored":1819279360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3161919488,"data_stored":3161915416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1604976640,"data_stored":1604976640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1263403008,"data_stored":1263403008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1328222208,"data_stored":1328218136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2682458112,"data_stored":2682454040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2090008576,"data_stored":2090008576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:53:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:31.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:31.647 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:53:31.725 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:31.768 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:31.836 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:31.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:31 smithi151 ceph-mon[159210]: from='client.35685 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:31.848 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:31 smithi151 ceph-mon[159210]: pgmap v843: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 10325/192432 objects degraded (5.366%); 1762/192432 objects misplaced (0.916%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:53:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:31.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:31.950 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:31.993 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:32.073 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:32.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:32.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:32.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:32.242 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:32.305 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:32.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:32.376 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:32.415 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:32.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:32.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:32 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:32.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:32.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:32 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:32.764 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:32.886 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:32.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:32 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:32.998 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:33.078 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:33.292 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:33.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:33.376 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:33.408 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:33.454 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:33.479 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:53:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:53:33] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-09-23T22:53:33.515 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:33.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:33.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:33.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:33.718 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:33.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:33 smithi052 ceph-mon[180979]: pgmap v844: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 10317/186840 objects degraded (5.522%); 1654/186840 objects misplaced (0.885%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:33.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:33 smithi052 ceph-mon[187743]: pgmap v844: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 10317/186840 objects degraded (5.522%); 1654/186840 objects misplaced (0.885%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:33.927 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:33.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:34.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:34.105 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:33 smithi151 ceph-mon[159210]: pgmap v844: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 10317/186840 objects degraded (5.522%); 1654/186840 objects misplaced (0.885%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:34.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:53:34.920 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:34.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:35.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:35.106 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:35.265 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:35.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:35.381 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:35.473 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[180979]: pgmap v845: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 671 op/s; 10317/184263 objects degraded (5.599%); 1654/184263 objects misplaced (0.898%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10317/184263 objects degraded (5.599%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[187743]: pgmap v845: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 671 op/s; 10317/184263 objects degraded (5.599%); 1654/184263 objects misplaced (0.898%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10317/184263 objects degraded (5.599%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:35.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:35 smithi151 ceph-mon[159210]: pgmap v845: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 671 op/s; 10317/184263 objects degraded (5.599%); 1654/184263 objects misplaced (0.898%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:35.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:35.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:35.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10317/184263 objects degraded (5.599%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:36.044 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:36.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:36.155 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:36.180 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:36.279 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:36.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:36.389 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:36.431 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:36.524 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:36.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:36.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:36.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:36.743 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:37.008 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:37.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:37.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:37.074 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:37.117 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:37.134 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:37.151 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:37.201 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:37.209 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:37.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:37.245 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:37.280 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:37.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:37.333 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:37.353 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:37.374 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:37.411 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:37.424 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:53:37.426 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:37.432 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.432 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.432 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.432 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.432 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.432 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.433 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.433 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:53:37.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:37.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:37.527 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161600 7.8 7:101b9260:::benchmark_data_smithi052_277165_object69108:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:53:37.528 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 501.27630657200001, 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28372276299999999, 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161600 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246434+0000", 2023-09-23T22:53:37.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246437+0000", 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.666e-06 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246443+0000", 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2940000000000002e-06 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246448+0000", 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5199999999999997e-06 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251386+0000", 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00493751 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251523+0000", 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013705799999999999 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.251637+0000", 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000114521 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263851+0000", 2023-09-23T22:53:37.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012213869 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263860+0000", 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0440000000000001e-06 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.529403+0000", 2023-09-23T22:53:37.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26554292099999999 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.529465+0000", 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1758000000000001e-05 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530157+0000", 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069202700000000001 2023-09-23T22:53:37.535 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161606 7.8 7:189adb79:::benchmark_data_smithi052_277165_object69114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 501.27626279700002, 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28428864599999998, 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.536 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161606 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246478+0000", 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246480+0000", 2023-09-23T22:53:37.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6749999999999998e-06 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246488+0000", 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0269999999999995e-06 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.246494+0000", 2023-09-23T22:53:37.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6579999999999999e-06 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255454+0000", 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089600449999999998 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255609+0000", 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015489900000000001 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.255761+0000", 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015231500000000001 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263866+0000", 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081051539999999998 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.264798+0000", 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000931608 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530292+0000", 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26549436100000001 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530346+0000", 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3328000000000002e-05 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530767+0000", 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00042096800000000003 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:161607 7.8 7:17f6cc07:::benchmark_data_smithi052_277165_object69115:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 501.270379558, 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27881840099999999, 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 161607 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.544 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252361+0000", 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252360+0000", 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252365+0000", 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4920000000000001e-06 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252374+0000", 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.49e-06 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.252379+0000", 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2750000000000003e-06 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259765+0000", 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073862559999999999 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259893+0000", 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012820900000000001 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.259999+0000", 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105535 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.263874+0000", 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003875295 2023-09-23T22:53:37.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.264894+0000", 2023-09-23T22:53:37.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010201089999999999 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530815+0000", 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26592050099999998 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.530865+0000", 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9648999999999998e-05 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:16.531180+0000", 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031517999999999997 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.552 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162195 7.2 e205/146 7:4bcbd502:::benchmark_data_smithi052_277165_object69703:head v 205'10032, mlcod=205'10027)", 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 500.27430521000002, 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.263093829, 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248436+0000", 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248435+0000", 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248443+0000", 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7219999999999999e-06 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248449+0000", 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8039999999999998e-06 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.248453+0000", 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4100000000000001e-06 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.252722+0000", 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004268679 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.252744+0000", 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1648e-05 2023-09-23T22:53:37.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.261335+0000", 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085910190000000001 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511529+0000", 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25019480999999999 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162201 7.c e205/197 7:3f19fea2:::benchmark_data_smithi052_277165_object69709:head v 205'9991, mlcod=164'3559)", 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 500.26717190199997, 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27094163999999998, 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255569+0000", 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255568+0000", 2023-09-23T22:53:37.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255577+0000", 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1099999999999992e-06 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255589+0000", 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2707999999999999e-05 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255597+0000", 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7239999999999992e-06 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255657+0000", 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0096000000000002e-05 2023-09-23T22:53:37.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.255685+0000", 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7429000000000001e-05 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526472+0000", 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.270787481 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526511+0000", 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8352000000000001e-05 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:162200 7.8 7:1ffee83e:::benchmark_data_smithi052_277165_object69708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 500.26655636999999, 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28005922700000002, 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.563 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 162200 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256184+0000", 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256183+0000", 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256190+0000", 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8719999999999996e-06 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256200+0000", 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7520000000000008e-06 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.256207+0000", 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4189999999999996e-06 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259499+0000", 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032920950000000001 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259640+0000", 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000141433 2023-09-23T22:53:37.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.259790+0000", 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014928000000000001 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516611+0000", 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25682131899999999 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.516650+0000", 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8952000000000002e-05 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.535505+0000", 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018855212999999999 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.535560+0000", 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5226999999999998e-05 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.536244+0000", 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00068324599999999998 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162202 7.2 e205/146 7:4736e59a:::benchmark_data_smithi052_277165_object69710:head v 205'10033, mlcod=205'10030)", 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:53:37.570 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 500.25955551700002, 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.255326575, 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263185+0000", 2023-09-23T22:53:37.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263184+0000", 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.263192+0000", 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3390000000000004e-06 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511467+0000", 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24827568999999999 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511483+0000", 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5209e-05 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511557+0000", 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4202999999999994e-05 2023-09-23T22:53:37.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511576+0000", 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8870000000000001e-05 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.518481+0000", 2023-09-23T22:53:37.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0069055719999999996 2023-09-23T22:53:37.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.518512+0000", 2023-09-23T22:53:37.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0574999999999998e-05 2023-09-23T22:53:37.578 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.578 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.578 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.578 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:162214 7.c e205/197 7:33fa1cb0:::benchmark_data_smithi052_277165_object69722:head v 205'9992, mlcod=164'3559)", 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 500.25109778400002, 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25489373100000001, 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271643+0000", 2023-09-23T22:53:37.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271642+0000", 2023-09-23T22:53:37.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.271650+0000", 2023-09-23T22:53:37.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1620000000000002e-06 2023-09-23T22:53:37.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511482+0000", 2023-09-23T22:53:37.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239832029 2023-09-23T22:53:37.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511494+0000", 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511527+0000", 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2679999999999999e-05 2023-09-23T22:53:37.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.511557+0000", 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0304e-05 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526516+0000", 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014958767 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:17.526537+0000", 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.12e-05 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164704 7.8 7:160987c2:::benchmark_data_smithi052_277165_object72212:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:53:37.586 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 496.12650995000001, 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31601756800000003, 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164704 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.587 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396231+0000", 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396230+0000", 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:37.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396236+0000", 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4389999999999997e-06 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396252+0000", 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6412000000000001e-05 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396260+0000", 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1160000000000001e-06 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396306+0000", 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6183999999999997e-05 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396424+0000", 2023-09-23T22:53:37.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118237 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396585+0000", 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016076799999999999 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.401552+0000", 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049672090000000002 2023-09-23T22:53:37.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.405610+0000", 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040580670000000003 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711571+0000", 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30596056300000002 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711627+0000", 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6586999999999999e-05 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.712248+0000", 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00062132300000000001 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168547 7.e 7:7817ab24:::benchmark_data_smithi052_277165_object76055:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 490.19681686400003, 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248803264, 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.594 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168547 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:53:37.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325924+0000", 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325922+0000", 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325931+0000", 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4120000000000007e-06 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325949+0000", 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7567999999999998e-05 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.597 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.325959+0000", 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8749999999999995e-06 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326592+0000", 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00063299999999999999 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326706+0000", 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113925 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.326821+0000", 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115184 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341186+0000", 2023-09-23T22:53:37.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014364745 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574591+0000", 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.233404956 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574635+0000", 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4357000000000002e-05 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574727+0000", 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2262999999999994e-05 2023-09-23T22:53:37.601 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.602 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.602 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.602 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168550 7.e 7:7960d031:::benchmark_data_smithi052_277165_object76058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.602 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:53:37.603 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 490.19153045100001, 2023-09-23T22:53:37.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24916470700000001, 2023-09-23T22:53:37.603 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.603 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.603 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.604 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.604 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.604 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168550 2023-09-23T22:53:37.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.604 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.604 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331210+0000", 2023-09-23T22:53:37.605 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331209+0000", 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.606 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331217+0000", 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0609999999999996e-06 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331230+0000", 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3159000000000001e-05 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.607 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331249+0000", 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9205999999999999e-05 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332625+0000", 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001375498 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332749+0000", 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012410500000000001 2023-09-23T22:53:37.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.332864+0000", 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115203 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574774+0000", 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.241910027 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580246+0000", 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054718789999999998 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580287+0000", 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0991000000000001e-05 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580375+0000", 2023-09-23T22:53:37.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8368999999999994e-05 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:168552 7.e 7:7af651e0:::benchmark_data_smithi052_277165_object76060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 490.19147413500002, 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24921663199999999, 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.611 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168552 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331267+0000", 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331266+0000", 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331269+0000", 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2389999999999998e-06 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331276+0000", 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8900000000000001e-06 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.331281+0000", 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0660000000000003e-06 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341213+0000", 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0099318059999999996 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341329+0000", 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115771 2023-09-23T22:53:37.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5", 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.341443+0000", 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113826 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.574809+0000", 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23336596900000001 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580383+0000", 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055746950000000002 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580417+0000", 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3581999999999998e-05 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:27.580483+0000", 2023-09-23T22:53:37.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6353999999999995e-05 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48295 8.d 8:ba748e4c:::benchmark_data_smithi052_337357_object48294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 406.575893087, 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29879999400000001, 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.618 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48295 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946847+0000", 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946865+0000", 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8448999999999999e-05 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946874+0000", 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3960000000000006e-06 2023-09-23T22:53:37.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946879+0000", 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946921+0000", 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2422000000000001e-05 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.947042+0000", 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120188 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.947180+0000", 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013842699999999999 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948502+0000", 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001321979 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948841+0000", 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033914399999999999 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245568+0000", 2023-09-23T22:53:37.624 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29672640900000002 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245607+0000", 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9313000000000001e-05 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245648+0000", 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0911999999999997e-05 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.625 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48295 8.d e228/217 ondisk, result = 0)", 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 406.57386816899998, 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29678497500000001, 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.626 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.627 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948872+0000", 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948876+0000", 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1010000000000002e-06 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245491+0000", 2023-09-23T22:53:37.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29661482700000003 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245512+0000", 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1078e-05 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245551+0000", 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9286999999999998e-05 2023-09-23T22:53:37.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245561+0000", 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7019999999999996e-06 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245658+0000", 2023-09-23T22:53:37.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6485999999999997e-05 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:87840 9.6 9:6275a8cc:::benchmark_data_smithi052_402718_object87839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:46.793102+0000", 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 50.729638925000003, 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21918126199999999, 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.631 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 87840 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.632 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793102+0000", 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793102+0000", 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793100+0000", 2023-09-23T22:53:37.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793129+0000", 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8666e-05 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793148+0000", 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9463e-05 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793157+0000", 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2470000000000008e-06 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793222+0000", 2023-09-23T22:53:37.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5100999999999999e-05 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793359+0000", 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013759500000000001 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.793484+0000", 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124843 2023-09-23T22:53:37.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802477+0000", 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089924640000000004 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802484+0000", 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5989999999999996e-06 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:47.012191+0000", 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20970696899999999 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:47.012245+0000", 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3715999999999999e-05 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:47.012283+0000", 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8374999999999998e-05 2023-09-23T22:53:37.638 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:87845 9.6 9:608779b0:::benchmark_data_smithi052_402718_object87844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:46.802201+0000", 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 50.720539700000003, 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21712906900000001, 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.639 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 87845 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802201+0000", 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802201+0000", 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802200+0000", 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802225+0000", 2023-09-23T22:53:37.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.525e-05 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802239+0000", 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4042999999999999e-05 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802251+0000", 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1269e-05 2023-09-23T22:53:37.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802507+0000", 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025601200000000001 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802628+0000", 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121305 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.802760+0000", 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013227200000000001 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.804984+0000", 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002223769 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:46.808804+0000", 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0038196990000000002 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:47.019185+0000", 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21038148300000001 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:47.019237+0000", 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1795000000000001e-05 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:47.019330+0000", 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3313000000000006e-05 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.646 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98496 9.6 9:6ce8dd16:::benchmark_data_smithi052_402718_object8855:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.961519+0000", 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 34.561221875000001, 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23376190999999999, 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:37.647 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98496 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961519+0000", 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961519+0000", 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961517+0000", 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961526+0000", 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0510000000000002e-06 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961543+0000", 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6424000000000001e-05 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961553+0000", 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0069e-05 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961608+0000", 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5439000000000002e-05 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961746+0000", 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013826499999999999 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.961867+0000", 2023-09-23T22:53:37.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012058899999999999 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.965429+0000", 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035619979999999998 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967114+0000", 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016853580000000001 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.195196+0000", 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22808204200000001 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.195241+0000", 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4749000000000001e-05 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.195281+0000", 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9780000000000002e-05 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98503 9.7 9:e79d5792:::benchmark_data_smithi052_402718_object8862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:37.654 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.976208+0000", 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 34.546532434, 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23617806699999999, 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:37.655 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98503 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.976208+0000", 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.656 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.976208+0000", 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.658 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.976207+0000", 2023-09-23T22:53:37.658 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:37.658 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.658 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.976214+0000", 2023-09-23T22:53:37.659 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.861e-06 2023-09-23T22:53:37.659 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.659 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.659 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.976227+0000", 2023-09-23T22:53:37.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3284e-05 2023-09-23T22:53:37.660 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.660 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.660 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.660 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.976235+0000", 2023-09-23T22:53:37.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1159999999999999e-06 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977493+0000", 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001258054 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.661 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.662 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.662 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977699+0000", 2023-09-23T22:53:37.662 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020624 2023-09-23T22:53:37.662 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.662 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.663 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:37.663 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977868+0000", 2023-09-23T22:53:37.663 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168235 2023-09-23T22:53:37.663 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.663 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.664 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186615+0000", 2023-09-23T22:53:37.664 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20874699599999999 2023-09-23T22:53:37.664 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186627+0000", 2023-09-23T22:53:37.665 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2767e-05 2023-09-23T22:53:37.665 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.665 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.665 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.211296+0000", 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.024668840000000001 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.211345+0000", 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9147999999999998e-05 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.666 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.212386+0000", 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001041015 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98513 9.7 9:e026439f:::benchmark_data_smithi052_402718_object8872:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.983988+0000", 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 34.538752959999997, 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.229480869, 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.667 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98513 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.983988+0000", 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.668 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.983988+0000", 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.983987+0000", 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.669 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.983991+0000", 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3379999999999998e-06 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186496+0000", 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20250453500000001 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186508+0000", 2023-09-23T22:53:37.670 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1695000000000001e-05 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186648+0000", 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014002400000000001 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186783+0000", 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000135341 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.186887+0000", 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010411799999999999 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.194795+0000", 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079081459999999996 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.194805+0000", 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.01e-05 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.212461+0000", 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017655095999999999 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.673 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.212513+0000", 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2611999999999997e-05 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.213469+0000", 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00095560700000000003 2023-09-23T22:53:37.674 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110911 9.2 9:4f121c65:::benchmark_data_smithi052_402718_object21270:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.078288+0000", 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.444453307, 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23945135300000001, 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:37.675 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110911 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:37.676 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078288+0000", 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078288+0000", 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078286+0000", 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:37.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078293+0000", 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4429999999999997e-06 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078306+0000", 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2908000000000001e-05 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078315+0000", 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1829999999999994e-06 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078362+0000", 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7488999999999999e-05 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078471+0000", 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000108894 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.078558+0000", 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7373000000000005e-05 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.081392+0000", 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028341780000000001 2023-09-23T22:53:37.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.082382+0000", 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000989773 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.316803+0000", 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23442054100000001 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.681 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.316852+0000", 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8860999999999999e-05 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.317739+0000", 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088725100000000003 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.682 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:37.683 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:37.683 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:53:37.683 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:37.684 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:53:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:37 smithi151 ceph-mon[159210]: pgmap v846: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 10317/180438 objects degraded (5.718%); 1654/180438 objects misplaced (0.917%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/857045616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/756919851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:37.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:37.735 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[187743]: pgmap v846: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 10317/180438 objects degraded (5.718%); 1654/180438 objects misplaced (0.917%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/857045616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/756919851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:37.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[180979]: pgmap v846: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 10317/180438 objects degraded (5.718%); 1654/180438 objects misplaced (0.917%); 1.1 MiB/s, 23 objects/s recovering 2023-09-23T22:53:37.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/857045616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:37.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/756919851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:37.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:53:37.813 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:37.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:37.919 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:37.984 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:38.078 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:38.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:38.190 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:38.215 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:38.277 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:38.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:38.390 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:38.431 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:38 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:38 smithi151 ceph-mon[159210]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:53:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:38 smithi151 ceph-mon[159210]: osdmap e295: 8 total, 8 up, 6 in 2023-09-23T22:53:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:38 smithi151 ceph-mon[159210]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:38 smithi151 ceph-mon[159210]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[180979]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:53:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[180979]: osdmap e295: 8 total, 8 up, 6 in 2023-09-23T22:53:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[180979]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[180979]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:38.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[187743]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:53:38.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[187743]: osdmap e295: 8 total, 8 up, 6 in 2023-09-23T22:53:38.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[187743]: failed to encode map e295 with expected crc 2023-09-23T22:53:38.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:38 smithi052 ceph-mon[187743]: failed to encode map e295 with expected crc 2023-09-23T22:53:39.539 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:39 smithi052 ceph-mon[180979]: pgmap v848: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 10201/173853 objects degraded (5.868%); 1654/173853 objects misplaced (0.951%); 397 KiB/s, 23 objects/s recovering 2023-09-23T22:53:39.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:39 smithi151 ceph-mon[159210]: pgmap v848: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 10201/173853 objects degraded (5.868%); 1654/173853 objects misplaced (0.951%); 397 KiB/s, 23 objects/s recovering 2023-09-23T22:53:39.961 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:39 smithi052 ceph-mon[187743]: pgmap v848: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 10201/173853 objects degraded (5.868%); 1654/173853 objects misplaced (0.951%); 397 KiB/s, 23 objects/s recovering 2023-09-23T22:53:39.985 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:53:40.028 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:40.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:40.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:40.144 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:40.180 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:40.257 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:40.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:40.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:40.363 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:40.418 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:40.490 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:40.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:40.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:40.597 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:40.683 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10201/173853 objects degraded (5.868%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:40.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10201/173853 objects degraded (5.868%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:40.694 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:40.841 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:40.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:40.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:40.948 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:40.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10201/173853 objects degraded (5.868%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:41.043 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:41.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:53:41.322 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:41.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:41.428 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:41.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:41.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:41 smithi052 ceph-mon[187743]: pgmap v849: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 10201/171867 objects degraded (5.935%); 1654/171867 objects misplaced (0.962%); 397 KiB/s, 23 objects/s recovering 2023-09-23T22:53:41.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:41 smithi052 ceph-mon[180979]: pgmap v849: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 10201/171867 objects degraded (5.935%); 1654/171867 objects misplaced (0.962%); 397 KiB/s, 23 objects/s recovering 2023-09-23T22:53:41.785 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:41.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:41.899 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:41 smithi151 ceph-mon[159210]: pgmap v849: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 10201/171867 objects degraded (5.935%); 1654/171867 objects misplaced (0.962%); 397 KiB/s, 23 objects/s recovering 2023-09-23T22:53:41.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:42.004 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:42.077 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:53:42.165 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:42.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:42.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:42.282 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:42.314 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:42.372 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:42.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:42.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:42.481 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:42.517 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:42.566 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:53:42.573 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:42.576 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:53:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:42.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:42.682 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:42.731 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:42.812 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:42.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:42.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:42.922 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:43.044 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:43.348 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:53:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:53:43] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-09-23T22:53:43.677 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[187743]: pgmap v850: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 695 op/s; 10085/165957 objects degraded (6.077%); 1654/165957 objects misplaced (0.997%); 0 B/s, 23 objects/s recovering 2023-09-23T22:53:43.677 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[187743]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:43.677 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:43.677 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:43 smithi151 ceph-mon[159210]: pgmap v850: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 695 op/s; 10085/165957 objects degraded (6.077%); 1654/165957 objects misplaced (0.997%); 0 B/s, 23 objects/s recovering 2023-09-23T22:53:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:43 smithi151 ceph-mon[159210]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:43.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[180979]: pgmap v850: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 695 op/s; 10085/165957 objects degraded (6.077%); 1654/165957 objects misplaced (0.997%); 0 B/s, 23 objects/s recovering 2023-09-23T22:53:43.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[180979]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:53:43.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:43.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:44.530 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:44.557 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:44.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:44.565 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":850,"stamp":"2023-09-23T22:53:42.401680+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3624343480,"num_objects":55319,"num_object_clones":0,"num_object_copies":165957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10085,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":55319,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":124243,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5095,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111577,"ondisk_log_size":111577,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12932692,"kb_used_data":10785636,"kb_used_omap":121,"kb_used_meta":2146502,"kb_avail":549538220,"statfs":{"total":575970213888,"available":562727137280,"internally_reserved":0,"allocated":11044491264,"data_stored":11039607761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2198018165},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,8,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-456196096,"num_objects":-6961,"num_object_clones":0,"num_object_copies":-20883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-232,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6961,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005315"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":292,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583758+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:53:26.583758+0000","last_peered":"2023-09-23T22:53:26.583758+0000","last_clean":"2023-09-23T22:53:26.583758+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:53:26.583758+0000","last_undegraded":"2023-09-23T22:53:26.583758+0000","last_fullsized":"2023-09-23T22:53:26.583758+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584962+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:53:26.584962+0000","last_peered":"2023-09-23T22:53:26.584962+0000","last_clean":"2023-09-23T22:53:26.584962+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:53:26.584962+0000","last_undegraded":"2023-09-23T22:53:26.584962+0000","last_fullsized":"2023-09-23T22:53:26.584962+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583951+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:53:26.583951+0000","last_peered":"2023-09-23T22:53:26.583951+0000","last_clean":"2023-09-23T22:53:26.583951+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:53:26.583951+0000","last_undegraded":"2023-09-23T22:53:26.583951+0000","last_fullsized":"2023-09-23T22:53:26.583951+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":273,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.586180+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:53:26.586180+0000","last_peered":"2023-09-23T22:53:26.586180+0000","last_clean":"2023-09-23T22:53:26.586180+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:53:26.586180+0000","last_undegraded":"2023-09-23T22:53:26.586180+0000","last_fullsized":"2023-09-23T22:53:26.586180+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583936+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:53:26.583936+0000","last_peered":"2023-09-23T22:53:26.583936+0000","last_clean":"2023-09-23T22:53:26.583936+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:53:26.583936+0000","last_undegraded":"2023-09-23T22:53:26.583936+0000","last_fullsized":"2023-09-23T22:53:26.583936+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.578638+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:53:26.578638+0000","last_peered":"2023-09-23T22:53:26.578638+0000","last_clean":"2023-09-23T22:53:26.578638+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:53:26.578638+0000","last_undegraded":"2023-09-23T22:53:26.578638+0000","last_fullsized":"2023-09-23T22:53:26.578638+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":205,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:28.190321+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:53:28.190321+0000","last_peered":"2023-09-23T22:53:28.190321+0000","last_clean":"2023-09-23T22:53:28.190321+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:53:28.190321+0000","last_undegraded":"2023-09-23T22:53:28.190321+0000","last_fullsized":"2023-09-23T22:53:28.190321+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":156,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.588185+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:53:26.588185+0000","last_peered":"2023-09-23T22:53:26.588185+0000","last_clean":"2023-09-23T22:53:26.588185+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:53:26.588185+0000","last_undegraded":"2023-09-23T22:53:26.588185+0000","last_fullsized":"2023-09-23T22:53:26.588185+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.587545+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:53:26.587545+0000","last_peered":"2023-09-23T22:53:26.587545+0000","last_clean":"2023-09-23T22:53:26.587545+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:53:26.587545+0000","last_undegraded":"2023-09-23T22:53:26.587545+0000","last_fullsized":"2023-09-23T22:53:26.587545+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":179,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.578544+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:53:26.578544+0000","last_peered":"2023-09-23T22:53:26.578544+0000","last_clean":"2023-09-23T22:53:26.578544+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:53:26.578544+0000","last_undegraded":"2023-09-23T22:53:26.578544+0000","last_fullsized":"2023-09-23T22:53:26.578544+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":53,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584843+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:53:26.584843+0000","last_peered":"2023-09-23T22:53:26.584843+0000","last_clean":"2023-09-23T22:53:26.584843+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:53:26.584843+0000","last_undegraded":"2023-09-23T22:53:26.584843+0000","last_fullsized":"2023-09-23T22:53:26.584843+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":169,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577417+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:53:26.577417+0000","last_peered":"2023-09-23T22:53:26.577417+0000","last_clean":"2023-09-23T22:53:26.577417+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:53:26.577417+0000","last_undegraded":"2023-09-23T22:53:26.577417+0000","last_fullsized":"2023-09-23T22:53:26.577417+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":314,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.589186+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:53:26.589186+0000","last_peered":"2023-09-23T22:53:26.589186+0000","last_clean":"2023-09-23T22:53:26.589186+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:53:26.589186+0000","last_undegraded":"2023-09-23T22:53:26.589186+0000","last_fullsized":"2023-09-23T22:53:26.589186+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583803+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:53:26.583803+0000","last_peered":"2023-09-23T22:53:26.583803+0000","last_clean":"2023-09-23T22:53:26.583803+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:53:26.583803+0000","last_undegraded":"2023-09-23T22:53:26.583803+0000","last_fullsized":"2023-09-23T22:53:26.583803+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":153,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584186+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:53:26.584186+0000","last_peered":"2023-09-23T22:53:26.584186+0000","last_clean":"2023-09-23T22:53:26.584186+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:53:26.584186+0000","last_undegraded":"2023-09-23T22:53:26.584186+0000","last_fullsized":"2023-09-23T22:53:26.584186+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.587075+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:53:26.587075+0000","last_peered":"2023-09-23T22:53:26.587075+0000","last_clean":"2023-09-23T22:53:26.587075+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:53:26.587075+0000","last_undegraded":"2023-09-23T22:53:26.587075+0000","last_fullsized":"2023-09-23T22:53:26.587075+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":261,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584044+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:53:26.584044+0000","last_peered":"2023-09-23T22:53:26.584044+0000","last_clean":"2023-09-23T22:53:26.584044+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:53:26.584044+0000","last_undegraded":"2023-09-23T22:53:26.584044+0000","last_fullsized":"2023-09-23T22:53:26.584044+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":218,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577502+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:53:26.577502+0000","last_peered":"2023-09-23T22:53:26.577502+0000","last_clean":"2023-09-23T22:53:26.577502+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:53:26.577502+0000","last_undegraded":"2023-09-23T22:53:26.577502+0000","last_fullsized":"2023-09-23T22:53:26.577502+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":145,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584093+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:53:26.584093+0000","last_peered":"2023-09-23T22:53:26.584093+0000","last_clean":"2023-09-23T22:53:26.584093+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:53:26.584093+0000","last_undegraded":"2023-09-23T22:53:26.584093+0000","last_fullsized":"2023-09-23T22:53:26.584093+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":262,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:28.190292+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:53:28.190292+0000","last_peered":"2023-09-23T22:53:28.190292+0000","last_clean":"2023-09-23T22:53:28.190292+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:53:28.190292+0000","last_undegraded":"2023-09-23T22:53:28.190292+0000","last_fullsized":"2023-09-23T22:53:28.190292+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":339,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.576843+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:53:26.576843+0000","last_peered":"2023-09-23T22:53:26.576843+0000","last_clean":"2023-09-23T22:53:26.576843+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:53:26.576843+0000","last_undegraded":"2023-09-23T22:53:26.576843+0000","last_fullsized":"2023-09-23T22:53:26.576843+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":221,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.588372+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:53:26.588372+0000","last_peered":"2023-09-23T22:53:26.588372+0000","last_clean":"2023-09-23T22:53:26.588372+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:53:26.588372+0000","last_undegraded":"2023-09-23T22:53:26.588372+0000","last_fullsized":"2023-09-23T22:53:26.588372+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"295'7623","reported_seq":7664,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:38.155108+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:53:38.155108+0000","last_peered":"2023-09-23T22:53:38.155108+0000","last_clean":"2023-09-23T22:53:38.155108+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:53:38.155108+0000","last_undegraded":"2023-09-23T22:53:38.155108+0000","last_fullsized":"2023-09-23T22:53:38.155108+0000","mapping_epoch":264,"log_start":"268'100","ondisk_log_start":"268'100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229965824,"num_objects":3509,"num_object_clones":0,"num_object_copies":10527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7623,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":164,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584420+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:53:26.584420+0000","last_peered":"2023-09-23T22:53:26.584420+0000","last_clean":"2023-09-23T22:53:26.584420+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:53:26.584420+0000","last_undegraded":"2023-09-23T22:53:26.584420+0000","last_fullsized":"2023-09-23T22:53:26.584420+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":259,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.576955+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:53:26.576955+0000","last_peered":"2023-09-23T22:53:26.576955+0000","last_clean":"2023-09-23T22:53:26.576955+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:53:26.576955+0000","last_undegraded":"2023-09-23T22:53:26.576955+0000","last_fullsized":"2023-09-23T22:53:26.576955+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"295'7941","reported_seq":7987,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:41.504111+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:53:41.504111+0000","last_peered":"2023-09-23T22:53:41.504111+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:53:41.504111+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6741,"log_dups_size":0,"ondisk_log_size":6741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232718336,"num_objects":3551,"num_object_clones":0,"num_object_copies":10653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7941,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":1890}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":174,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.588270+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:53:26.588270+0000","last_peered":"2023-09-23T22:53:26.588270+0000","last_clean":"2023-09-23T22:53:26.588270+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:53:26.588270+0000","last_undegraded":"2023-09-23T22:53:26.588270+0000","last_fullsized":"2023-09-23T22:53:26.588270+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":284,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584019+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:53:26.584019+0000","last_peered":"2023-09-23T22:53:26.584019+0000","last_clean":"2023-09-23T22:53:26.584019+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:53:26.584019+0000","last_undegraded":"2023-09-23T22:53:26.584019+0000","last_fullsized":"2023-09-23T22:53:26.584019+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"295'7869","reported_seq":7915,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:39.940859+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:53:39.940859+0000","last_peered":"2023-09-23T22:53:39.940859+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:53:39.940859+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7869,"log_dups_size":0,"ondisk_log_size":7869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224067584,"num_objects":3419,"num_object_clones":0,"num_object_copies":10257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7869,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":1648}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":118,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577411+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:53:26.577411+0000","last_peered":"2023-09-23T22:53:26.577411+0000","last_clean":"2023-09-23T22:53:26.577411+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:53:26.577411+0000","last_undegraded":"2023-09-23T22:53:26.577411+0000","last_fullsized":"2023-09-23T22:53:26.577411+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":287,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577260+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:53:26.577260+0000","last_peered":"2023-09-23T22:53:26.577260+0000","last_clean":"2023-09-23T22:53:26.577260+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:53:26.577260+0000","last_undegraded":"2023-09-23T22:53:26.577260+0000","last_fullsized":"2023-09-23T22:53:26.577260+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"295'7805","reported_seq":7846,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:39.958472+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:53:39.958472+0000","last_peered":"2023-09-23T22:53:39.958472+0000","last_clean":"2023-09-23T22:53:39.958472+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:53:39.958472+0000","last_undegraded":"2023-09-23T22:53:39.958472+0000","last_fullsized":"2023-09-23T22:53:39.958472+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7805,"log_dups_size":0,"ondisk_log_size":7805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230096896,"num_objects":3511,"num_object_clones":0,"num_object_copies":10533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7805,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":90,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577624+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:53:26.577624+0000","last_peered":"2023-09-23T22:53:26.577624+0000","last_clean":"2023-09-23T22:53:26.577624+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:53:26.577624+0000","last_undegraded":"2023-09-23T22:53:26.577624+0000","last_fullsized":"2023-09-23T22:53:26.577624+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":189,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583854+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:53:26.583854+0000","last_peered":"2023-09-23T22:53:26.583854+0000","last_clean":"2023-09-23T22:53:26.583854+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:53:26.583854+0000","last_undegraded":"2023-09-23T22:53:26.583854+0000","last_fullsized":"2023-09-23T22:53:26.583854+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"295'7681","reported_seq":11086,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:40.984710+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:53:40.984710+0000","last_peered":"2023-09-23T22:53:40.984710+0000","last_clean":"2023-09-23T22:53:40.984710+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:53:40.984710+0000","last_undegraded":"2023-09-23T22:53:40.984710+0000","last_fullsized":"2023-09-23T22:53:40.984710+0000","mapping_epoch":289,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218038272,"num_objects":3327,"num_object_clones":0,"num_object_copies":9981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7681,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583948+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:53:26.583948+0000","last_peered":"2023-09-23T22:53:26.583948+0000","last_clean":"2023-09-23T22:53:26.583948+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:53:26.583948+0000","last_undegraded":"2023-09-23T22:53:26.583948+0000","last_fullsized":"2023-09-23T22:53:26.583948+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":268,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.589128+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:53:26.589128+0000","last_peered":"2023-09-23T22:53:26.589128+0000","last_clean":"2023-09-23T22:53:26.589128+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:53:26.589128+0000","last_undegraded":"2023-09-23T22:53:26.589128+0000","last_fullsized":"2023-09-23T22:53:26.589128+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"295'7661","reported_seq":7700,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:40.960701+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:53:40.960701+0000","last_peered":"2023-09-23T22:53:40.960701+0000","last_clean":"2023-09-23T22:53:40.960701+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:53:40.960701+0000","last_undegraded":"2023-09-23T22:53:40.960701+0000","last_fullsized":"2023-09-23T22:53:40.960701+0000","mapping_epoch":264,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224198656,"num_objects":3421,"num_object_clones":0,"num_object_copies":10263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7661,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":279,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577108+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:53:26.577108+0000","last_peered":"2023-09-23T22:53:26.577108+0000","last_clean":"2023-09-23T22:53:26.577108+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:53:26.577108+0000","last_undegraded":"2023-09-23T22:53:26.577108+0000","last_fullsized":"2023-09-23T22:53:26.577108+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":290,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584680+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:53:26.584680+0000","last_peered":"2023-09-23T22:53:26.584680+0000","last_clean":"2023-09-23T22:53:26.584680+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:53:26.584680+0000","last_undegraded":"2023-09-23T22:53:26.584680+0000","last_fullsized":"2023-09-23T22:53:26.584680+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"295'7802","reported_seq":7847,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:42.098077+0000","last_change":"2023-09-23T22:51:35.518569+0000","last_active":"2023-09-23T22:53:42.098077+0000","last_peered":"2023-09-23T22:53:42.098077+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:53:42.098077+0000","last_undegraded":"2023-09-23T22:51:35.445687+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":271,"log_start":"275'2700","ondisk_log_start":"275'2700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5102,"log_dups_size":0,"ondisk_log_size":5102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:39.333773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221380608,"num_objects":3378,"num_object_clones":0,"num_object_copies":10134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7802,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,5","objects":1713}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":303,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583798+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:53:26.583798+0000","last_peered":"2023-09-23T22:53:26.583798+0000","last_clean":"2023-09-23T22:53:26.583798+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:53:26.583798+0000","last_undegraded":"2023-09-23T22:53:26.583798+0000","last_fullsized":"2023-09-23T22:53:26.583798+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":278,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.588033+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:53:26.588033+0000","last_peered":"2023-09-23T22:53:26.588033+0000","last_clean":"2023-09-23T22:53:26.588033+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:53:26.588033+0000","last_undegraded":"2023-09-23T22:53:26.588033+0000","last_fullsized":"2023-09-23T22:53:26.588033+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"295'7594","reported_seq":7639,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:39.977304+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:53:39.977304+0000","last_peered":"2023-09-23T22:53:39.977304+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:53:39.977304+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227803136,"num_objects":3476,"num_object_clones":0,"num_object_copies":10428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7594,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":1861}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":223,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583490+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:53:26.583490+0000","last_peered":"2023-09-23T22:53:26.583490+0000","last_clean":"2023-09-23T22:53:26.583490+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:53:26.583490+0000","last_undegraded":"2023-09-23T22:53:26.583490+0000","last_fullsized":"2023-09-23T22:53:26.583490+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":232,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.588497+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:53:26.588497+0000","last_peered":"2023-09-23T22:53:26.588497+0000","last_clean":"2023-09-23T22:53:26.588497+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:53:26.588497+0000","last_undegraded":"2023-09-23T22:53:26.588497+0000","last_fullsized":"2023-09-23T22:53:26.588497+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"295'7882","reported_seq":7927,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:41.950794+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:53:41.950794+0000","last_peered":"2023-09-23T22:53:41.950794+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:53:41.950794+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"275'2700","ondisk_log_start":"275'2700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,5","objects":1840}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":333,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.586279+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:53:26.586279+0000","last_peered":"2023-09-23T22:53:26.586279+0000","last_clean":"2023-09-23T22:53:26.586279+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:53:26.586279+0000","last_undegraded":"2023-09-23T22:53:26.586279+0000","last_fullsized":"2023-09-23T22:53:26.586279+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":158,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:37.574009+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:53:37.574009+0000","last_peered":"2023-09-23T22:53:37.574009+0000","last_clean":"2023-09-23T22:53:37.574009+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:53:37.574009+0000","last_undegraded":"2023-09-23T22:53:37.574009+0000","last_fullsized":"2023-09-23T22:53:37.574009+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583654+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:53:26.583654+0000","last_peered":"2023-09-23T22:53:26.583654+0000","last_clean":"2023-09-23T22:53:26.583654+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:53:26.583654+0000","last_undegraded":"2023-09-23T22:53:26.583654+0000","last_fullsized":"2023-09-23T22:53:26.583654+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"295'7647","reported_seq":11090,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:38.170818+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:53:38.170818+0000","last_peered":"2023-09-23T22:53:38.170818+0000","last_clean":"2023-09-23T22:53:38.170818+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:53:38.170818+0000","last_undegraded":"2023-09-23T22:53:38.170818+0000","last_fullsized":"2023-09-23T22:53:38.170818+0000","mapping_epoch":282,"log_start":"268'100","ondisk_log_start":"268'100","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231407616,"num_objects":3531,"num_object_clones":0,"num_object_copies":10593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7647,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":129,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577185+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:53:26.577185+0000","last_peered":"2023-09-23T22:53:26.577185+0000","last_clean":"2023-09-23T22:53:26.577185+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:53:26.577185+0000","last_undegraded":"2023-09-23T22:53:26.577185+0000","last_fullsized":"2023-09-23T22:53:26.577185+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":291,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583956+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:53:26.583956+0000","last_peered":"2023-09-23T22:53:26.583956+0000","last_clean":"2023-09-23T22:53:26.583956+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:53:26.583956+0000","last_undegraded":"2023-09-23T22:53:26.583956+0000","last_fullsized":"2023-09-23T22:53:26.583956+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":221,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.587162+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:53:26.587162+0000","last_peered":"2023-09-23T22:53:26.587162+0000","last_clean":"2023-09-23T22:53:26.587162+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:53:26.587162+0000","last_undegraded":"2023-09-23T22:53:26.587162+0000","last_fullsized":"2023-09-23T22:53:26.587162+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"295'7791","reported_seq":7841,"reported_epoch":295,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:53:41.151471+0000","last_change":"2023-09-23T22:51:35.528654+0000","last_active":"2023-09-23T22:53:41.151471+0000","last_peered":"2023-09-23T22:53:41.151471+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:53:41.151471+0000","last_undegraded":"2023-09-23T22:53:41.151471+0000","last_fullsized":"2023-09-23T22:53:41.151471+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228392960,"num_objects":3485,"num_object_clones":0,"num_object_copies":10455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":3485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7791,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":1831}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":297,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:37.574656+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:53:37.574656+0000","last_peered":"2023-09-23T22:53:37.574656+0000","last_clean":"2023-09-23T22:53:37.574656+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:53:37.574656+0000","last_undegraded":"2023-09-23T22:53:37.574656+0000","last_fullsized":"2023-09-23T22:53:37.574656+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":176,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.588060+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:53:26.588060+0000","last_peered":"2023-09-23T22:53:26.588060+0000","last_clean":"2023-09-23T22:53:26.588060+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:53:26.588060+0000","last_undegraded":"2023-09-23T22:53:26.588060+0000","last_fullsized":"2023-09-23T22:53:26.588060+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":302,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584192+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:53:26.584192+0000","last_peered":"2023-09-23T22:53:26.584192+0000","last_clean":"2023-09-23T22:53:26.584192+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:53:26.584192+0000","last_undegraded":"2023-09-23T22:53:26.584192+0000","last_fullsized":"2023-09-23T22:53:26.584192+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"295'7587","reported_seq":7634,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:38.177616+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:53:38.177616+0000","last_peered":"2023-09-23T22:53:38.177616+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:53:38.177616+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7587,"log_dups_size":0,"ondisk_log_size":7587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226951168,"num_objects":3463,"num_object_clones":0,"num_object_copies":10389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7587,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":1751}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":306,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.587007+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:53:26.587007+0000","last_peered":"2023-09-23T22:53:26.587007+0000","last_clean":"2023-09-23T22:53:26.587007+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:53:26.587007+0000","last_undegraded":"2023-09-23T22:53:26.587007+0000","last_fullsized":"2023-09-23T22:53:26.587007+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":383,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583855+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:53:26.583855+0000","last_peered":"2023-09-23T22:53:26.583855+0000","last_clean":"2023-09-23T22:53:26.583855+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:53:26.583855+0000","last_undegraded":"2023-09-23T22:53:26.583855+0000","last_fullsized":"2023-09-23T22:53:26.583855+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":49,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577344+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:53:26.577344+0000","last_peered":"2023-09-23T22:53:26.577344+0000","last_clean":"2023-09-23T22:53:26.577344+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:53:26.577344+0000","last_undegraded":"2023-09-23T22:53:26.577344+0000","last_fullsized":"2023-09-23T22:53:26.577344+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":287,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577246+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:53:26.577246+0000","last_peered":"2023-09-23T22:53:26.577246+0000","last_clean":"2023-09-23T22:53:26.577246+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:53:26.577246+0000","last_undegraded":"2023-09-23T22:53:26.577246+0000","last_fullsized":"2023-09-23T22:53:26.577246+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"295'7636","reported_seq":7677,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:38.125065+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:53:38.125065+0000","last_peered":"2023-09-23T22:53:38.125065+0000","last_clean":"2023-09-23T22:53:38.125065+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:53:38.125065+0000","last_undegraded":"2023-09-23T22:53:38.125065+0000","last_fullsized":"2023-09-23T22:53:38.125065+0000","mapping_epoch":264,"log_start":"268'100","ondisk_log_start":"268'100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235274240,"num_objects":3590,"num_object_clones":0,"num_object_copies":10770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7636,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":271,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583986+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:53:26.583986+0000","last_peered":"2023-09-23T22:53:26.583986+0000","last_clean":"2023-09-23T22:53:26.583986+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:53:26.583986+0000","last_undegraded":"2023-09-23T22:53:26.583986+0000","last_fullsized":"2023-09-23T22:53:26.583986+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":242,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.586019+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:53:26.586019+0000","last_peered":"2023-09-23T22:53:26.586019+0000","last_clean":"2023-09-23T22:53:26.586019+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:53:26.586019+0000","last_undegraded":"2023-09-23T22:53:26.586019+0000","last_fullsized":"2023-09-23T22:53:26.586019+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":220,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.585986+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:53:26.585986+0000","last_peered":"2023-09-23T22:53:26.585986+0000","last_clean":"2023-09-23T22:53:26.585986+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:53:26.585986+0000","last_undegraded":"2023-09-23T22:53:26.585986+0000","last_fullsized":"2023-09-23T22:53:26.585986+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"295'7664","reported_seq":7703,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:41.599973+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:53:41.599973+0000","last_peered":"2023-09-23T22:53:41.599973+0000","last_clean":"2023-09-23T22:53:41.599973+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:53:41.599973+0000","last_undegraded":"2023-09-23T22:53:41.599973+0000","last_fullsized":"2023-09-23T22:53:41.599973+0000","mapping_epoch":264,"log_start":"269'900","ondisk_log_start":"269'900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216924160,"num_objects":3310,"num_object_clones":0,"num_object_copies":9930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7664,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":225,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583819+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:53:26.583819+0000","last_peered":"2023-09-23T22:53:26.583819+0000","last_clean":"2023-09-23T22:53:26.583819+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:53:26.583819+0000","last_undegraded":"2023-09-23T22:53:26.583819+0000","last_fullsized":"2023-09-23T22:53:26.583819+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":175,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.587323+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:53:26.587323+0000","last_peered":"2023-09-23T22:53:26.587323+0000","last_clean":"2023-09-23T22:53:26.587323+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:53:26.587323+0000","last_undegraded":"2023-09-23T22:53:26.587323+0000","last_fullsized":"2023-09-23T22:53:26.587323+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":225,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584186+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:53:26.584186+0000","last_peered":"2023-09-23T22:53:26.584186+0000","last_clean":"2023-09-23T22:53:26.584186+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:53:26.584186+0000","last_undegraded":"2023-09-23T22:53:26.584186+0000","last_fullsized":"2023-09-23T22:53:26.584186+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"295'7885","reported_seq":7925,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:41.640638+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:53:41.640638+0000","last_peered":"2023-09-23T22:53:41.640638+0000","last_clean":"2023-09-23T22:53:41.640638+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:53:41.640638+0000","last_undegraded":"2023-09-23T22:53:41.640638+0000","last_fullsized":"2023-09-23T22:53:41.640638+0000","mapping_epoch":264,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227803160,"num_objects":3477,"num_object_clones":0,"num_object_copies":10431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7885,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":51,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584461+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:53:26.584461+0000","last_peered":"2023-09-23T22:53:26.584461+0000","last_clean":"2023-09-23T22:53:26.584461+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:53:26.584461+0000","last_undegraded":"2023-09-23T22:53:26.584461+0000","last_fullsized":"2023-09-23T22:53:26.584461+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":93,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577462+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:53:26.577462+0000","last_peered":"2023-09-23T22:53:26.577462+0000","last_clean":"2023-09-23T22:53:26.577462+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:53:26.577462+0000","last_undegraded":"2023-09-23T22:53:26.577462+0000","last_fullsized":"2023-09-23T22:53:26.577462+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T22:53:37.572691+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:53:37.572691+0000","last_peered":"2023-09-23T22:53:37.572691+0000","last_clean":"2023-09-23T22:53:37.572691+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:53:37.572691+0000","last_undegraded":"2023-09-23T22:53:37.572691+0000","last_fullsized":"2023-09-23T22:53:37.572691+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"295'7915","reported_seq":11359,"reported_epoch":295,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:41.637757+0000","last_change":"2023-09-23T22:53:13.718777+0000","last_active":"2023-09-23T22:53:41.637757+0000","last_peered":"2023-09-23T22:53:41.637757+0000","last_clean":"2023-09-23T22:51:33.377854+0000","last_became_active":"2023-09-23T22:51:37.033330+0000","last_became_peered":"2023-09-23T22:51:37.033330+0000","last_unstale":"2023-09-23T22:53:41.637757+0000","last_undegraded":"2023-09-23T22:53:13.714740+0000","last_fullsized":"2023-09-23T22:51:35.425861+0000","mapping_epoch":271,"log_start":"269'1200","ondisk_log_start":"269'1200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221315072,"num_objects":3377,"num_object_clones":0,"num_object_copies":10131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7915,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1698,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,4","objects":3346}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":336,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.578501+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:53:26.578501+0000","last_peered":"2023-09-23T22:53:26.578501+0000","last_clean":"2023-09-23T22:53:26.578501+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:53:26.578501+0000","last_undegraded":"2023-09-23T22:53:26.578501+0000","last_fullsized":"2023-09-23T22:53:26.578501+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":192,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.586093+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:53:26.586093+0000","last_peered":"2023-09-23T22:53:26.586093+0000","last_clean":"2023-09-23T22:53:26.586093+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:53:26.586093+0000","last_undegraded":"2023-09-23T22:53:26.586093+0000","last_fullsized":"2023-09-23T22:53:26.586093+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583875+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:53:26.583875+0000","last_peered":"2023-09-23T22:53:26.583875+0000","last_clean":"2023-09-23T22:53:26.583875+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:53:26.583875+0000","last_undegraded":"2023-09-23T22:53:26.583875+0000","last_fullsized":"2023-09-23T22:53:26.583875+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":227,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584025+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:53:26.584025+0000","last_peered":"2023-09-23T22:53:26.584025+0000","last_clean":"2023-09-23T22:53:26.584025+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:53:26.584025+0000","last_undegraded":"2023-09-23T22:53:26.584025+0000","last_fullsized":"2023-09-23T22:53:26.584025+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":78,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.587241+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:53:26.587241+0000","last_peered":"2023-09-23T22:53:26.587241+0000","last_clean":"2023-09-23T22:53:26.587241+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:53:26.587241+0000","last_undegraded":"2023-09-23T22:53:26.587241+0000","last_fullsized":"2023-09-23T22:53:26.587241+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":104,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.589137+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:53:26.589137+0000","last_peered":"2023-09-23T22:53:26.589137+0000","last_clean":"2023-09-23T22:53:26.589137+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:53:26.589137+0000","last_undegraded":"2023-09-23T22:53:26.589137+0000","last_fullsized":"2023-09-23T22:53:26.589137+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":303,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584031+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:53:26.584031+0000","last_peered":"2023-09-23T22:53:26.584031+0000","last_clean":"2023-09-23T22:53:26.584031+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:53:26.584031+0000","last_undegraded":"2023-09-23T22:53:26.584031+0000","last_fullsized":"2023-09-23T22:53:26.584031+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":105,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584757+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:53:26.584757+0000","last_peered":"2023-09-23T22:53:26.584757+0000","last_clean":"2023-09-23T22:53:26.584757+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:53:26.584757+0000","last_undegraded":"2023-09-23T22:53:26.584757+0000","last_fullsized":"2023-09-23T22:53:26.584757+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583845+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:53:26.583845+0000","last_peered":"2023-09-23T22:53:26.583845+0000","last_clean":"2023-09-23T22:53:26.583845+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:53:26.583845+0000","last_undegraded":"2023-09-23T22:53:26.583845+0000","last_fullsized":"2023-09-23T22:53:26.583845+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":45,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.577320+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:53:26.577320+0000","last_peered":"2023-09-23T22:53:26.577320+0000","last_clean":"2023-09-23T22:53:26.577320+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:53:26.577320+0000","last_undegraded":"2023-09-23T22:53:26.577320+0000","last_fullsized":"2023-09-23T22:53:26.577320+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":277,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.586859+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:53:26.586859+0000","last_peered":"2023-09-23T22:53:26.586859+0000","last_clean":"2023-09-23T22:53:26.586859+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:53:26.586859+0000","last_undegraded":"2023-09-23T22:53:26.586859+0000","last_fullsized":"2023-09-23T22:53:26.586859+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":241,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.584343+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:53:26.584343+0000","last_peered":"2023-09-23T22:53:26.584343+0000","last_clean":"2023-09-23T22:53:26.584343+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:53:26.584343+0000","last_undegraded":"2023-09-23T22:53:26.584343+0000","last_fullsized":"2023-09-23T22:53:26.584343+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T22:53:26.583571+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:53:26.583571+0000","last_peered":"2023-09-23T22:53:26.583571+0000","last_clean":"2023-09-23T22:53:26.583571+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:53:26.583571+0000","last_undegraded":"2023-09-23T22:53:26.583571+0000","last_fullsized":"2023-09-23T22:53:26.583571+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3623747608,"num_objects":55295,"num_object_clones":0,"num_object_copies":165885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10085,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":55295,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123983,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5065,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12041138176,"data_stored":12041125960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111383,"ondisk_log_size":111383,"up":48,"acting":41,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430528,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626756,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2081660,"kb_used_data":1772084,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91663492,"statfs":{"total":95995035648,"available":93863415808,"internally_reserved":0,"allocated":1814614016,"data_stored":1814057963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":6.9699999999999998,"5min":9.5869999999999997,"15min":4.2800000000000002},"min":{"1min":0.24199999999999999,"5min":0.218,"15min":0.218},"max":{"1min":112.51000000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":7.6239999999999997,"5min":9.9469999999999992,"15min":4.2679999999999998},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":112.154,"5min":243.41499999999999,"15min":243.41499999999999},"last":2.8759999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":14.752000000000001,"5min":5.0700000000000003,"15min":3.6389999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":254.62100000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":1.248},{"interface":"front","average":{"1min":20.399999999999999,"5min":7.9690000000000003,"15min":4.6319999999999997},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.247},"max":{"1min":255.20099999999999,"5min":255.20099999999999,"15min":255.20099999999999},"last":2.835}]},{"osd":2,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":3.867,"15min":3.4199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":38.527999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":3.468},{"interface":"front","average":{"1min":4.3499999999999996,"5min":3.4969999999999999,"15min":3.786},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":38.776000000000003,"5min":101.13,"15min":185.21799999999999},"last":3.617}]},{"osd":3,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":5.6150000000000002,"15min":3.706},"min":{"1min":0.36199999999999999,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":7.734,"5min":185.19399999999999,"15min":185.19399999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.919,"5min":3.919,"15min":3.3170000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.7770000000000001,"5min":128.19800000000001,"15min":128.19800000000001},"last":2.9239999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.875,"15min":0.753},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.7290000000000001,"5min":6.5309999999999997,"15min":18.085999999999999},"last":2.9990000000000001},{"interface":"front","average":{"1min":1.3320000000000001,"5min":0.78900000000000003,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4800000000000004,"5min":7.4800000000000004,"15min":18.597000000000001},"last":0.36099999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:52:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.0529999999999999,"15min":0.85299999999999998},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":4.7560000000000002,"5min":10.148999999999999,"15min":18.972000000000001},"last":2.9009999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.83799999999999997,"15min":0.80900000000000005},"min":{"1min":0.316,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":2.782,"5min":4.9089999999999998,"15min":18.625},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":0.86499999999999999,"15min":0.82999999999999996},"min":{"1min":0.35799999999999998,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":4.2869999999999999,"5min":4.3049999999999997,"15min":12.481999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.03,"15min":0.86199999999999999},"min":{"1min":0.38200000000000001,"5min":0.30499999999999999,"15min":0.27100000000000002},"max":{"1min":6.6269999999999998,"5min":6.6269999999999998,"15min":16.661999999999999},"last":3.161}]}]},{"osd":1,"up_from":84,"seq":360777253066,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3129004,"kb_used_data":2627556,"kb_used_omap":9,"kb_used_meta":501302,"kb_avail":90616148,"statfs":{"total":95995035648,"available":92790935552,"internally_reserved":0,"allocated":2690617344,"data_stored":2689554222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":513334249},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.6829999999999998,"5min":6.5149999999999997,"15min":5.3140000000000001},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.691000000000001,"5min":82.903999999999996,"15min":101.605},"last":4.4859999999999998},{"interface":"front","average":{"1min":3.944,"5min":5.3680000000000003,"15min":5.1929999999999996},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":13.601000000000001,"5min":82.748999999999995,"15min":101.892},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.6749999999999998,"5min":4.1029999999999998,"15min":3.0419999999999998},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":10.952999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":4.6950000000000003},{"interface":"front","average":{"1min":4.4930000000000003,"5min":4.1340000000000003,"15min":3.1190000000000002},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.429,"5min":37.957000000000001,"15min":37.957000000000001},"last":4.7359999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":3.3079999999999998,"15min":2.528},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.1660000000000004,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.20999999999999999},{"interface":"front","average":{"1min":2.9540000000000002,"5min":3.8650000000000002,"15min":3.0059999999999998},"min":{"1min":0.29299999999999998,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":10.845000000000001,"5min":89.861999999999995,"15min":89.861999999999995},"last":0.746}]},{"osd":4,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.9710000000000001,"15min":2.25},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":8.7469999999999999,"5min":11.432,"15min":78.948999999999998},"last":11.388},{"interface":"front","average":{"1min":3.544,"5min":2.649,"15min":2.2850000000000001},"min":{"1min":0.315,"5min":0.313,"15min":0.313},"max":{"1min":10.894,"5min":14.058,"15min":78.963999999999999},"last":9.2579999999999991}]},{"osd":5,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.597,"15min":1.792},"min":{"1min":0.42799999999999999,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":9.141,"5min":9.141,"15min":84.254999999999995},"last":4.601},{"interface":"front","average":{"1min":2.903,"5min":1.958,"15min":2.125},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":7.1150000000000002,"5min":11.458,"15min":87.515000000000001},"last":3.746}]},{"osd":6,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":4.0899999999999999,"5min":2.7519999999999998,"15min":2.1549999999999998},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":10.853999999999999,"5min":13.991,"15min":79.311000000000007},"last":4.6239999999999997},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.7609999999999999,"15min":1.7809999999999999},"min":{"1min":0.40899999999999997,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":79.328999999999994},"last":11.521000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.1339999999999999,"15min":2.5049999999999999},"min":{"1min":0.438,"5min":0.40799999999999997,"15min":0.32800000000000001},"max":{"1min":8.702,"5min":11.481999999999999,"15min":159.018},"last":0.78500000000000003},{"interface":"front","average":{"1min":4.008,"5min":2.911,"15min":2.6669999999999998},"min":{"1min":0.49199999999999999,"5min":0.371,"15min":0.37},"max":{"1min":10.923,"5min":14.032999999999999,"15min":158.96000000000001},"last":4.7560000000000002}]}]},{"osd":0,"up_from":219,"seq":940597837914,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1908000,"kb_used_data":1598744,"kb_used_omap":31,"kb_used_meta":309024,"kb_avail":91837152,"statfs":{"total":95995035648,"available":94041243648,"internally_reserved":0,"allocated":1637113856,"data_stored":1636003470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":316441522},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":5.3840000000000003,"5min":4.0069999999999997,"15min":3.758},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":40.180999999999997,"5min":40.180999999999997,"15min":40.180999999999997},"last":8.6340000000000003},{"interface":"front","average":{"1min":4.9249999999999998,"5min":3.6819999999999999,"15min":3.605},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":40.137999999999998,"5min":40.137999999999998,"15min":40.137999999999998},"last":7.9379999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":4.4560000000000004,"5min":5.0910000000000002,"15min":3.2189999999999999},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":46.387999999999998,"5min":111.508,"15min":111.508},"last":0.89600000000000002},{"interface":"front","average":{"1min":5.7619999999999996,"5min":4.7220000000000004,"15min":2.8460000000000001},"min":{"1min":0.41599999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":46.249000000000002,"5min":111.532,"15min":111.532},"last":4.0789999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":4.9660000000000002,"15min":3.0289999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.2759999999999998,"5min":126.73,"15min":126.73},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.2829999999999999,"5min":3.581,"15min":2.044},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.7539999999999996,"5min":121.483,"15min":121.483},"last":4.3360000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":2.2709999999999999,"15min":2.1179999999999999},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":8.8239999999999998,"5min":18.670999999999999,"15min":18.670999999999999},"last":8.5820000000000007},{"interface":"front","average":{"1min":2.218,"5min":2.6080000000000001,"15min":2.036},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.8620000000000001,"5min":20.009,"15min":20.009},"last":0.86499999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":2.766,"15min":1.901},"min":{"1min":0.39000000000000001,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":13.851000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":4.1870000000000003},{"interface":"front","average":{"1min":3.4079999999999999,"5min":3.4380000000000002,"15min":2.2839999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.436,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.72899999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.6760000000000002,"15min":1.9059999999999999},"min":{"1min":0.44500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.683,"5min":59.146999999999998,"15min":59.146999999999998},"last":4.1459999999999999},{"interface":"front","average":{"1min":4.0019999999999998,"5min":3.3780000000000001,"15min":2.4009999999999998},"min":{"1min":0.41199999999999998,"5min":0.379,"15min":0.379},"max":{"1min":12.470000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":2.468,"15min":2.0110000000000001},"min":{"1min":0.45500000000000002,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":12.385999999999999,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.0550000000000002,"15min":1.5489999999999999},"min":{"1min":0.40400000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.5750000000000002,"5min":21.954999999999998,"15min":21.954999999999998},"last":4.2199999999999998}]}]},{"osd":2,"up_from":131,"seq":562640715927,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1756264,"kb_used_data":1352608,"kb_used_omap":81,"kb_used_meta":403566,"kb_avail":91988888,"statfs":{"total":95995035648,"available":94196621312,"internally_reserved":0,"allocated":1385070592,"data_stored":1384014228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":413251802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:32 2023","interfaces":[{"interface":"back","average":{"1min":6.1900000000000004,"5min":6.9480000000000004,"15min":5.8170000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":73.388000000000005,"5min":132.79499999999999,"15min":132.79499999999999},"last":1.6120000000000001},{"interface":"front","average":{"1min":6.2869999999999999,"5min":7.4640000000000004,"15min":6.2400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":73.405000000000001,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.95399999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.5310000000000001,"15min":3.0230000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":12.035,"5min":15.622999999999999,"15min":88.316000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":3.5329999999999999,"5min":3.4220000000000002,"15min":3.1589999999999998},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":14.159000000000001,"5min":28.596,"15min":88.436999999999998},"last":0.80200000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.3250000000000002,"15min":4.1890000000000001},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":6.5419999999999998,"5min":112.895,"15min":112.895},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.524,"5min":4.617,"15min":3.4169999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.251},"max":{"1min":9.3840000000000003,"5min":113.995,"15min":113.995},"last":6.2670000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":3.597,"15min":2.4169999999999998},"min":{"1min":0.37,"5min":0.253,"15min":0.253},"max":{"1min":7.1520000000000001,"5min":84.778999999999996,"15min":108.878},"last":1.6699999999999999},{"interface":"front","average":{"1min":1.901,"5min":3.641,"15min":3.056},"min":{"1min":0.40500000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.8460000000000001,"5min":84.753,"15min":108.93000000000001},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":3.0750000000000002,"15min":2.9009999999999998},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.5970000000000004,"5min":90.082999999999998,"15min":97.322999999999993},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.0049999999999999,"5min":3.3399999999999999,"15min":2.718},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.5380000000000003,"5min":89.986000000000004,"15min":97.301000000000002},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":4.8330000000000002,"15min":2.9390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":4.5789999999999997,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.607,"5min":4.3360000000000003,"15min":2.3180000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.4089999999999998,"5min":98.509,"15min":98.509},"last":1.7130000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:52:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.0459999999999998,"15min":1.5740000000000001},"min":{"1min":0.42799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.7440000000000002,"5min":10.92,"15min":10.92},"last":1.748},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.1240000000000001,"15min":2.323},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":4.2489999999999997,"5min":11.311,"15min":88.584000000000003},"last":0.90600000000000003}]}]},{"osd":3,"up_from":0,"seq":403726926013,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019212,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1523648,"kb_used_data":1214136,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92221504,"statfs":{"total":95995035648,"available":94434820096,"internally_reserved":0,"allocated":1243275264,"data_stored":1242714617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":8.0700000000000003,"5min":4.423,"15min":6.367},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":82.376000000000005,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.251},{"interface":"front","average":{"1min":8.1430000000000007,"5min":4.9420000000000002,"15min":6.9160000000000004},"min":{"1min":0.34100000000000003,"5min":0.217,"15min":0.217},"max":{"1min":79.403000000000006,"5min":113.417,"15min":113.417},"last":0.41199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":11.753,"5min":3.7679999999999998,"15min":2.4649999999999999},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":132.304,"5min":132.304,"15min":132.304},"last":4.5199999999999996},{"interface":"front","average":{"1min":11.4,"5min":3.6560000000000001,"15min":2.3180000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":132.453,"5min":132.453,"15min":132.453},"last":4.1399999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.3860000000000001,"15min":3.1400000000000001},"min":{"1min":0.35999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.8070000000000004,"5min":20.308,"15min":84.909000000000006},"last":1.5349999999999999},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.4790000000000001,"15min":3.1469999999999998},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.673,"5min":21.704000000000001,"15min":83.840000000000003},"last":4.1079999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":5.3339999999999996,"5min":5.3849999999999998,"15min":4.3410000000000002},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":74.138999999999996,"5min":164.453,"15min":164.453},"last":4.0350000000000001},{"interface":"front","average":{"1min":4.9329999999999998,"5min":5.3520000000000003,"15min":4.4249999999999998},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":75.099999999999994,"5min":167.05199999999999,"15min":167.05199999999999},"last":3.169}]},{"osd":5,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.097,"15min":0.93799999999999994},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":10.321999999999999,"5min":12.238,"15min":43.189999999999998},"last":3.0800000000000001},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.079,"15min":0.93999999999999995},"min":{"1min":0.34200000000000003,"5min":0.248,"15min":0.248},"max":{"1min":6.4349999999999996,"5min":7.5460000000000003,"15min":42.450000000000003},"last":4.0179999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.95199999999999996,"15min":0.84399999999999997},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.758,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.88600000000000001,"15min":0.84299999999999997},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":1.978,"5min":12.269,"15min":26.940999999999999},"last":3.113}]},{"osd":7,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.0629999999999999,"15min":0.82199999999999995},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":10.278,"5min":12.311,"15min":12.311},"last":3.137},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.98099999999999998,"15min":0.76100000000000001},"min":{"1min":0.38800000000000001,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":3.7909999999999999,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":4.077}]}]},{"osd":5,"up_from":36,"seq":154618822984,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2534116,"kb_used_data":2220508,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91211036,"statfs":{"total":95995035648,"available":93400100864,"internally_reserved":0,"allocated":2273800192,"data_stored":2273263261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":12.433,"5min":6.899,"15min":4.6070000000000002},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":185.62,"5min":185.62,"15min":185.62},"last":0.39200000000000002},{"interface":"front","average":{"1min":12.449999999999999,"5min":7.0140000000000002,"15min":4.569},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":185.499,"5min":185.499,"15min":185.499},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0069999999999997,"5min":2.2759999999999998,"15min":2.2850000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.22500000000000001},"max":{"1min":17.658999999999999,"5min":41.463999999999999,"15min":149.203},"last":2.863},{"interface":"front","average":{"1min":4.5030000000000001,"5min":2.6880000000000002,"15min":2.5899999999999999},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":17.84,"5min":41.954000000000001,"15min":149.554},"last":2.1120000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":12.965,"5min":5.3209999999999997,"15min":5.8570000000000002},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":189.126,"5min":189.126,"15min":189.126},"last":0.435},{"interface":"front","average":{"1min":12.991,"5min":4.931,"15min":5.7809999999999997},"min":{"1min":0.498,"5min":0.318,"15min":0.28999999999999998},"max":{"1min":188.74799999999999,"5min":188.74799999999999,"15min":188.74799999999999},"last":2.9609999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":3.3929999999999998,"5min":5.4790000000000001,"15min":5.5510000000000002},"min":{"1min":0.316,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.99,"5min":98.641000000000005,"15min":257.05000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":2.6779999999999999,"5min":5.5949999999999998,"15min":5.4180000000000001},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.27800000000000002},"max":{"1min":10.362,"5min":98.906000000000006,"15min":257.35000000000002},"last":0.51800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":0.84499999999999997,"15min":0.72199999999999998},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":6.3949999999999996,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.956,"5min":0.83799999999999997,"15min":0.72299999999999998},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":6.3550000000000004,"5min":6.3550000000000004,"15min":10.535},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.054,"15min":1.5649999999999999},"min":{"1min":0.34399999999999997,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.649,"5min":8.0830000000000002,"15min":140.07499999999999},"last":2.0209999999999999},{"interface":"front","average":{"1min":2.1699999999999999,"5min":1.0089999999999999,"15min":1.5880000000000001},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":10.314,"5min":10.314,"15min":139.739},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:53:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":0.97899999999999998,"15min":0.82799999999999996},"min":{"1min":0.40100000000000002,"5min":0.33400000000000002,"15min":0.29499999999999998},"max":{"1min":10.367000000000001,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.353,"5min":0.89400000000000002,"15min":0.76100000000000001},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":6.1120000000000001,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.55700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1624965120,"data_stored":1624965120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2670661632,"data_stored":2670657560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1363279872,"data_stored":1363279872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1220935680,"data_stored":1220935680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1114771456,"data_stored":1114767384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2254049280,"data_stored":2254045208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1792475136,"data_stored":1792475136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:53:44.572 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:53:44.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:44.693 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:44.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:44.776 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[187743]: osdmap e296: 8 total, 8 up, 6 in 2023-09-23T22:53:44.776 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[187743]: failed to encode map e296 with expected crc 2023-09-23T22:53:44.776 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[187743]: failed to encode map e296 with expected crc 2023-09-23T22:53:44.776 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:44.776 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:44.776 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:44.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[180979]: osdmap e296: 8 total, 8 up, 6 in 2023-09-23T22:53:44.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[180979]: failed to encode map e296 with expected crc 2023-09-23T22:53:44.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[180979]: failed to encode map e296 with expected crc 2023-09-23T22:53:44.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:44.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:44.777 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:44.849 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:44.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:44 smithi151 ceph-mon[159210]: osdmap e296: 8 total, 8 up, 6 in 2023-09-23T22:53:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:44 smithi151 ceph-mon[159210]: failed to encode map e296 with expected crc 2023-09-23T22:53:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:44 smithi151 ceph-mon[159210]: failed to encode map e296 with expected crc 2023-09-23T22:53:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:53:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:53:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:44.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:45.108 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:45.351 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:53:45.351 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:53:45.351 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:126618 9.8 9:1d2301f2:::benchmark_data_smithi052_402718_object36977:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:45.331629+0000", 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.016380938000000001, 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016431458999999999, 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:45.352 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 126618 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331629+0000", 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331629+0000", 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:45.354 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331627+0000", 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331638+0000", 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1287999999999999e-05 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.355 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331662+0000", 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4244999999999999e-05 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331673+0000", 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0068000000000001e-05 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.356 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.347131+0000", 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01545877 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.347302+0000", 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017037899999999999 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.357 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.347450+0000", 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148197 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.358 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:126619 9.5 9:ae9c7cfd:::benchmark_data_smithi052_402718_object36978:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:45.331692+0000", 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.016317759000000001, 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016466650999999999, 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 126619 2023-09-23T22:53:45.359 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331692+0000", 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331692+0000", 2023-09-23T22:53:45.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331692+0000", 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:45.361 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331695+0000", 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3119999999999998e-06 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331706+0000", 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1110999999999999e-05 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331713+0000", 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5270000000000004e-06 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331801+0000", 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8165999999999996e-05 2023-09-23T22:53:45.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331950+0000", 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014935799999999999 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332100+0000", 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015007 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.364 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:126621 9.5 9:a3949c88:::benchmark_data_smithi052_402718_object36980:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:45.331727+0000", 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.016283215, 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016519953, 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:45.365 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 126621 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331727+0000", 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331727+0000", 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331726+0000", 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331729+0000", 2023-09-23T22:53:45.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1540000000000002e-06 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331742+0000", 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2639999999999999e-05 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331750+0000", 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4420000000000001e-06 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.335996+0000", 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042469070000000003 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.336131+0000", 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013458 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:45.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.336245+0000", 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011375900000000001 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:126617 9.c e297/296 9:3d939e0f:::benchmark_data_smithi052_402718_object36976:head v 297'8021, mlcod=297'8019)", 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:45.331823+0000", 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.016187041999999999, 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016511719000000001, 2023-09-23T22:53:45.370 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331823+0000", 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331823+0000", 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331822+0000", 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331831+0000", 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9500000000000007e-06 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331845+0000", 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4151999999999999e-05 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331851+0000", 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2040000000000002e-06 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331917+0000", 2023-09-23T22:53:45.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5161999999999997e-05 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.331938+0000", 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1562000000000001e-05 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.374 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:126620 9.4 e297/271 9:21c2553e:::benchmark_data_smithi052_402718_object36979:head v 297'8048, mlcod=0'0)", 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:45.332455+0000", 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.015554790000000001, 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015945035, 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332455+0000", 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332455+0000", 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332454+0000", 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:45.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332462+0000", 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4900000000000003e-06 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332478+0000", 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6067e-05 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.332488+0000", 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.043e-05 2023-09-23T22:53:45.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.335570+0000", 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030811760000000001 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.335598+0000", 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8249999999999999e-05 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.378 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:126629 9.5 9:af608d68:::benchmark_data_smithi052_402718_object36988:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:45.344909+0000", 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.003100542, 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035540849999999998, 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:45.379 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 126629 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.344909+0000", 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.344909+0000", 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.344907+0000", 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:45.381 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.344916+0000", 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1370000000000008e-06 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.344933+0000", 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7059999999999999e-05 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.344944+0000", 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1185e-05 2023-09-23T22:53:45.382 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.345015+0000", 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0592000000000004e-05 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.345199+0000", 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018373200000000001 2023-09-23T22:53:45.383 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:45.345379+0000", 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000180149 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 6 2023-09-23T22:53:45.384 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:53:45.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[180979]: pgmap v852: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 10085/163449 objects degraded (6.170%); 1654/163449 objects misplaced (1.012%); 0 B/s, 28 objects/s recovering 2023-09-23T22:53:45.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[180979]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[180979]: osdmap e297: 8 total, 8 up, 6 in 2023-09-23T22:53:45.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[180979]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[180979]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 10085/163449 objects degraded (6.170%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:45 smithi151 ceph-mon[159210]: pgmap v852: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 10085/163449 objects degraded (6.170%); 1654/163449 objects misplaced (1.012%); 0 B/s, 28 objects/s recovering 2023-09-23T22:53:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:45 smithi151 ceph-mon[159210]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:45 smithi151 ceph-mon[159210]: osdmap e297: 8 total, 8 up, 6 in 2023-09-23T22:53:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:45 smithi151 ceph-mon[159210]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:45 smithi151 ceph-mon[159210]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 10085/163449 objects degraded (6.170%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[187743]: pgmap v852: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 10085/163449 objects degraded (6.170%); 1654/163449 objects misplaced (1.012%); 0 B/s, 28 objects/s recovering 2023-09-23T22:53:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[187743]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[187743]: osdmap e297: 8 total, 8 up, 6 in 2023-09-23T22:53:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[187743]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[187743]: failed to encode map e297 with expected crc 2023-09-23T22:53:45.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 10085/163449 objects degraded (6.170%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:47.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:53:47.712 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:47 smithi052 ceph-mon[180979]: pgmap v854: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 10085/160356 objects degraded (6.289%); 1654/160356 objects misplaced (1.031%); 0 B/s, 14 objects/s recovering 2023-09-23T22:53:47.757 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:47.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:47.867 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:47.902 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:47.971 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:47.974 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:47 smithi151 ceph-mon[159210]: pgmap v854: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 10085/160356 objects degraded (6.289%); 1654/160356 objects misplaced (1.031%); 0 B/s, 14 objects/s recovering 2023-09-23T22:53:47.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:47.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:48.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:47 smithi052 ceph-mon[187743]: pgmap v854: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 10085/160356 objects degraded (6.289%); 1654/160356 objects misplaced (1.031%); 0 B/s, 14 objects/s recovering 2023-09-23T22:53:48.076 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:48.119 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:48.200 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:48.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:48.305 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:48.398 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:48.512 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:48.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:48.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:48.618 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:48.653 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:48.740 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:48.751 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:48.856 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:48.908 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:48.926 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:48.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:48.942 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:48.983 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:49.012 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:49.042 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:49.059 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:49.119 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:49.133 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:49.158 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:53:49.169 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.170 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:53:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[180979]: pgmap v855: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 9597/155247 objects degraded (6.182%); 1654/155247 objects misplaced (1.065%); 0 B/s, 75 objects/s recovering 2023-09-23T22:53:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2432789378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:49.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2154055594' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:49.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:49.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[187743]: pgmap v855: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 9597/155247 objects degraded (6.182%); 1654/155247 objects misplaced (1.065%); 0 B/s, 75 objects/s recovering 2023-09-23T22:53:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2432789378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2154055594' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:49.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:49 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:50.034 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:50.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:50.141 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:49 smithi151 ceph-mon[159210]: pgmap v855: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 9597/155247 objects degraded (6.182%); 1654/155247 objects misplaced (1.065%); 0 B/s, 75 objects/s recovering 2023-09-23T22:53:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2432789378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:53:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:53:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:53:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2154055594' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:49 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:53:50.167 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:50.209 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:50.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:50.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:50.313 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:50.344 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:50.441 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:50.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:50.536 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:53:50.542 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:50.580 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:50.652 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:50.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:50.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 9597/155247 objects degraded (6.182%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:50.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[187743]: pgmap v856: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 9597/153522 objects degraded (6.251%); 1654/153522 objects misplaced (1.077%); 0 B/s, 60 objects/s recovering 2023-09-23T22:53:50.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:50.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:50.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[187743]: osdmap e298: 8 total, 8 up, 6 in 2023-09-23T22:53:50.763 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:53:50.812 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:50.887 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:53:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:50.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:53:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 9597/155247 objects degraded (6.182%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[180979]: pgmap v856: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 9597/153522 objects degraded (6.251%); 1654/153522 objects misplaced (1.077%); 0 B/s, 60 objects/s recovering 2023-09-23T22:53:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:50 smithi052 ceph-mon[180979]: osdmap e298: 8 total, 8 up, 6 in 2023-09-23T22:53:51.000 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:51.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 9597/155247 objects degraded (6.182%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:51.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:50 smithi151 ceph-mon[159210]: pgmap v856: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 9597/153522 objects degraded (6.251%); 1654/153522 objects misplaced (1.077%); 0 B/s, 60 objects/s recovering 2023-09-23T22:53:51.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:50 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:53:51.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:50 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:53:51.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:50 smithi151 ceph-mon[159210]: osdmap e298: 8 total, 8 up, 6 in 2023-09-23T22:53:51.045 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:51.125 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:51.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:51.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:51.232 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:51.289 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:51.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:51 smithi052 ceph-mon[180979]: failed to encode map e298 with expected crc 2023-09-23T22:53:51.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:51 smithi052 ceph-mon[180979]: failed to encode map e298 with expected crc 2023-09-23T22:53:51.907 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:51 smithi052 ceph-mon[180979]: failed to encode map e298 with expected crc 2023-09-23T22:53:51.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:51 smithi052 ceph-mon[187743]: failed to encode map e298 with expected crc 2023-09-23T22:53:51.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:51 smithi052 ceph-mon[187743]: failed to encode map e298 with expected crc 2023-09-23T22:53:51.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:51 smithi052 ceph-mon[187743]: failed to encode map e298 with expected crc 2023-09-23T22:53:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:51 smithi151 ceph-mon[159210]: failed to encode map e298 with expected crc 2023-09-23T22:53:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:51 smithi151 ceph-mon[159210]: failed to encode map e298 with expected crc 2023-09-23T22:53:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:51 smithi151 ceph-mon[159210]: failed to encode map e298 with expected crc 2023-09-23T22:53:52.486 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:52.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:52.592 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:52.624 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:52.689 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:52.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:52.800 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:52.847 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:52.925 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:52.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:52.935 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:52 smithi151 ceph-mon[159210]: pgmap v858: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 641 op/s; 9193/148047 objects degraded (6.210%); 1654/148047 objects misplaced (1.117%); 0 B/s, 111 objects/s recovering 2023-09-23T22:53:52.948 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:53:53.022 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:52 smithi052 ceph-mon[180979]: pgmap v858: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 641 op/s; 9193/148047 objects degraded (6.210%); 1654/148047 objects misplaced (1.117%); 0 B/s, 111 objects/s recovering 2023-09-23T22:53:53.023 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:52 smithi052 ceph-mon[187743]: pgmap v858: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 641 op/s; 9193/148047 objects degraded (6.210%); 1654/148047 objects misplaced (1.117%); 0 B/s, 111 objects/s recovering 2023-09-23T22:53:53.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:53.175 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:53.218 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:53:53.219 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:53:53.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:53.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:53.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:53.535 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:53.577 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:53.640 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:53.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:53.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:53:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:53:53] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-09-23T22:53:53.747 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:53:53.790 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:53.842 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:53:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:53.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:53:53.948 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:54.023 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:54.340 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:53:54.341 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9193/148047 objects degraded (6.210%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1167,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":298,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":6,"osd_in_since":1695509493,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":82},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":49349,"data_bytes":3233093560,"bytes_used":12049666048,"bytes_avail":563920547840,"bytes_total":575970213888,"degraded_objects":9193,"degraded_total":148047,"degraded_ratio":0.062095145460563196,"misplaced_objects":1654,"misplaced_total":148047,"misplaced_ratio":0.011172127770235128,"recovering_objects_per_sec":111,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":892,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":641},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T22:53:02.381180+0000","services":{}},"progress_events":{"6b27ada5-699f-4815-a812-68a0dbe12316":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 11s)","progress":0.92134833335876465,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:53:54.393 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:54.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:54.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:54.500 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:54.511 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:53:54.541 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:54.610 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:54.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:54.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:54.634 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:53:54.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1896308035' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:53:54.659 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1896308035' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:53:54.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1896308035' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:53:54.719 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:53:54.762 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:54.827 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:53:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:54.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:53:54.937 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:54.986 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:55.046 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:55.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:53:55.157 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:55.218 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:55.419 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:53:55.420 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:53:55.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:55 smithi151 ceph-mon[159210]: pgmap v859: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 9193/145614 objects degraded (6.313%); 1654/145614 objects misplaced (1.136%); 0 B/s, 91 objects/s recovering 2023-09-23T22:53:55.432 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 9193/148047 objects degraded (6.210%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:55.461 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:55.476 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:55 smithi052 ceph-mon[187743]: pgmap v859: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 9193/145614 objects degraded (6.313%); 1654/145614 objects misplaced (1.136%); 0 B/s, 91 objects/s recovering 2023-09-23T22:53:55.476 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 9193/148047 objects degraded (6.210%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:55.477 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:55 smithi052 ceph-mon[180979]: pgmap v859: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 9193/145614 objects degraded (6.313%); 1654/145614 objects misplaced (1.136%); 0 B/s, 91 objects/s recovering 2023-09-23T22:53:55.477 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 9193/148047 objects degraded (6.210%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:53:55.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:55.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:55.579 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:55.644 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:55.687 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:55.688 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:55.822 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:55.850 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:55.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:55.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:55.963 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:53:56.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:57.130 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:53:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:57.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:53:57.249 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:53:57.278 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:57.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:57 smithi151 ceph-mon[159210]: pgmap v860: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 600 op/s; 9193/142323 objects degraded (6.459%); 1654/142323 objects misplaced (1.162%); 0 B/s, 89 objects/s recovering 2023-09-23T22:53:57.706 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:57 smithi052 ceph-mon[180979]: pgmap v860: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 600 op/s; 9193/142323 objects degraded (6.459%); 1654/142323 objects misplaced (1.162%); 0 B/s, 89 objects/s recovering 2023-09-23T22:53:57.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:57 smithi052 ceph-mon[187743]: pgmap v860: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 600 op/s; 9193/142323 objects degraded (6.459%); 1654/142323 objects misplaced (1.162%); 0 B/s, 89 objects/s recovering 2023-09-23T22:53:58.695 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:53:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:58.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:53:58.808 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:53:58.959 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:59.272 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:53:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:59.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:53:59.376 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:53:59.447 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:59.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:53:59 smithi052 ceph-mon[187743]: pgmap v861: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 8676/136449 objects degraded (6.358%); 1654/136449 objects misplaced (1.212%); 0 B/s, 92 objects/s recovering 2023-09-23T22:53:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:53:59 smithi052 ceph-mon[180979]: pgmap v861: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 8676/136449 objects degraded (6.358%); 1654/136449 objects misplaced (1.212%); 0 B/s, 92 objects/s recovering 2023-09-23T22:53:59.637 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:53:59.649 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:53:59.656 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:53:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:59.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:53:59.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:53:59 smithi151 ceph-mon[159210]: pgmap v861: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 8676/136449 objects degraded (6.358%); 1654/136449 objects misplaced (1.212%); 0 B/s, 92 objects/s recovering 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48261 8.1 8:87871911:::benchmark_data_smithi052_337357_object48260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:59.716 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.77824887200001, 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27316745399999998, 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48261 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.717 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927090+0000", 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927089+0000", 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927116+0000", 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7282999999999999e-05 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927128+0000", 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1987000000000001e-05 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927135+0000", 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4120000000000004e-06 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927186+0000", 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0982000000000001e-05 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927313+0000", 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012680600000000001 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:53:59.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.927452+0000", 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013846099999999999 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.930719+0000", 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032677600000000002 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934336+0000", 2023-09-23T22:53:59.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036163720000000001 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200180+0000", 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26584448100000002 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200227+0000", 2023-09-23T22:53:59.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7108999999999999e-05 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200257+0000", 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0210999999999999e-05 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48276 8.c 8:3458bc4f:::benchmark_data_smithi052_337357_object48275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.77088393100001, 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26596831100000001, 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48276 2023-09-23T22:53:59.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934455+0000", 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934454+0000", 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934485+0000", 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1665999999999998e-05 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934497+0000", 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1136e-05 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934505+0000", 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3399999999999998e-06 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934560+0000", 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4852000000000003e-05 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934713+0000", 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000153129 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.934856+0000", 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143199 2023-09-23T22:53:59.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.936652+0000", 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001796157 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.936814+0000", 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016158000000000001 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200327+0000", 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26351350400000001 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200382+0000", 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4231000000000003e-05 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200423+0000", 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1641999999999999e-05 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48281 8.c 8:3ea84d87:::benchmark_data_smithi052_337357_object48280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:53:59.732 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.76797122800002, 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26513863300000001, 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48281 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937368+0000", 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937366+0000", 2023-09-23T22:53:59.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937390+0000", 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3839e-05 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937401+0000", 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1238e-05 2023-09-23T22:53:59.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937408+0000", 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.148e-06 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937510+0000", 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101549 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937622+0000", 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111784 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.937748+0000", 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012662900000000001 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941281+0000", 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003532355 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941456+0000", 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017523800000000001 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202425+0000", 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26096917400000003 2023-09-23T22:53:59.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202471+0000", 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6286999999999999e-05 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.202506+0000", 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4978999999999998e-05 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.739 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48284 8.e 8:76fd02f7:::benchmark_data_smithi052_337357_object48283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.76582882299999, 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.264529664, 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:59.740 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48284 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939510+0000", 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939509+0000", 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939533+0000", 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4029e-05 2023-09-23T22:53:59.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939543+0000", 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0251999999999999e-05 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939550+0000", 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3560000000000004e-06 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939596+0000", 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5862000000000002e-05 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939720+0000", 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124368 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.939922+0000", 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020159799999999999 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201651+0000", 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.261728827 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201691+0000", 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9750999999999999e-05 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203971+0000", 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022803329999999998 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204008+0000", 2023-09-23T22:53:59.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7373000000000003e-05 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204040+0000", 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1350999999999998e-05 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48287 8.e 8:77bdb954:::benchmark_data_smithi052_337357_object48286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:59.747 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.76386357400003, 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26634659700000002, 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48287 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941475+0000", 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941474+0000", 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941498+0000", 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3611000000000001e-05 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941507+0000", 2023-09-23T22:53:59.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8169999999999994e-06 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941519+0000", 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1724e-05 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941642+0000", 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012283800000000001 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941737+0000", 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4828999999999996e-05 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941823+0000", 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6247000000000003e-05 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201726+0000", 2023-09-23T22:53:59.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25990260700000001 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.204073+0000", 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023471569999999999 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207731+0000", 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036582989999999998 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207785+0000", 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3621e-05 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.207822+0000", 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7115e-05 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.754 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48288 8.6 8:6358135e:::benchmark_data_smithi052_337357_object48287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.76380630099999, 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.262402097, 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:53:59.755 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48288 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941533+0000", 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941532+0000", 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941551+0000", 2023-09-23T22:53:59.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9185999999999999e-05 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941558+0000", 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6479999999999997e-06 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941563+0000", 2023-09-23T22:53:59.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3430000000000003e-06 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941634+0000", 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0407000000000005e-05 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941713+0000", 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9330000000000001e-05 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941808+0000", 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5241000000000005e-05 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201641+0000", 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25983307300000003 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201657+0000", 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5184000000000001e-05 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203850+0000", 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021936019999999998 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203898+0000", 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8186999999999998e-05 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.203935+0000", 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6375000000000003e-05 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48276 8.c e228/219 ondisk, result = 0)", 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.763420607, 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25851532799999999, 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941918+0000", 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.941922+0000", 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3780000000000003e-06 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200261+0000", 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25833932399999998 2023-09-23T22:53:59.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200269+0000", 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.255e-06 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200312+0000", 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3016999999999998e-05 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200321+0000", 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2790000000000004e-06 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200434+0000", 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000112623 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48261 8.1 e228/226 ondisk, result = 0)", 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.76318555099999, 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25811125899999998, 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942153+0000", 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.942157+0000", 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5770000000000004e-06 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200098+0000", 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25794090400000003 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200110+0000", 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2119999999999999e-05 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200163+0000", 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2859999999999999e-05 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200175+0000", 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1765e-05 2023-09-23T22:53:59.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200265+0000", 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9689000000000001e-05 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48284 8.e e228/219 ondisk, result = 0)", 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:53:59.772 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 428.760272869, 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25662737099999999, 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945066+0000", 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945065+0000", 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.945071+0000", 2023-09-23T22:53:59.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.088e-06 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200085+0000", 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25501373599999999 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.200109+0000", 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3581e-05 2023-09-23T22:53:59.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201679+0000", 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015704989999999999 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201685+0000", 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.404e-06 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.201693+0000", 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9060000000000002e-06 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13476 9.9 9:90c01bcf:::benchmark_data_smithi052_402718_object13475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.332745+0000", 2023-09-23T22:53:59.777 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 154.37259396900001, 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32173884899999999, 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13476 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332745+0000", 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332745+0000", 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332744+0000", 2023-09-23T22:53:59.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332798+0000", 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3698e-05 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332809+0000", 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1049e-05 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332824+0000", 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5863e-05 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.332879+0000", 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4376999999999998e-05 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.333001+0000", 2023-09-23T22:53:59.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012225699999999999 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.333144+0000", 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014329499999999999 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.333932+0000", 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00078775000000000004 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335087+0000", 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011546239999999999 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654409+0000", 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31932254799999998 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654452+0000", 2023-09-23T22:53:59.784 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2259999999999999e-05 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654484+0000", 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2169999999999999e-05 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.785 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13478 9.9 9:92c51427:::benchmark_data_smithi052_402718_object13477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.334045+0000", 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 154.37129411699999, 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.320549849, 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.786 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13478 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334045+0000", 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334045+0000", 2023-09-23T22:53:59.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334044+0000", 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334078+0000", 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4103000000000002e-05 2023-09-23T22:53:59.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334090+0000", 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1783000000000001e-05 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334097+0000", 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6340000000000001e-06 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334145+0000", 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7917000000000002e-05 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334264+0000", 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119301 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:53:59.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334415+0000", 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000150025 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335300+0000", 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088519900000000003 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654326+0000", 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31902618199999999 2023-09-23T22:53:59.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654524+0000", 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019771400000000001 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654564+0000", 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0296000000000002e-05 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654595+0000", 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0700000000000001e-05 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13479 9.5 9:ad6f3e65:::benchmark_data_smithi052_402718_object13478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.334600+0000", 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 154.37073936799999, 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31995974199999999, 2023-09-23T22:53:59.793 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13479 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334600+0000", 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334600+0000", 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334598+0000", 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:59.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334622+0000", 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4139e-05 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334634+0000", 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2109999999999999e-05 2023-09-23T22:53:59.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334642+0000", 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0020000000000006e-06 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334695+0000", 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2142999999999999e-05 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334823+0000", 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012874199999999999 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.334954+0000", 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013098499999999999 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654328+0000", 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31937344499999998 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654413+0000", 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4905000000000001e-05 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654477+0000", 2023-09-23T22:53:59.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4732000000000003e-05 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654521+0000", 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4029999999999997e-05 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654559+0000", 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7799999999999997e-05 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.800 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:13479 9.5 e269/264 ondisk, result = 0)", 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.335840+0000", 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 154.369499082, 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31857624899999998, 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335840+0000", 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335840+0000", 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335839+0000", 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.335843+0000", 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0799999999999999e-06 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654332+0000", 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31848875700000001 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654347+0000", 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4958e-05 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654393+0000", 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5671999999999999e-05 2023-09-23T22:53:59.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654403+0000", 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0395e-05 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654416+0000", 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2937e-05 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.805 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:13476 9.9 e269/264 ondisk, result = 0)", 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.336205+0000", 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 154.36913401199999, 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31828632699999998, 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336205+0000", 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336205+0000", 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336204+0000", 2023-09-23T22:53:59.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336209+0000", 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2070000000000003e-06 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654332+0000", 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31812279700000001 2023-09-23T22:53:59.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654347+0000", 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5012e-05 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654392+0000", 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5139000000000003e-05 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.809 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654403+0000", 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0797e-05 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.654491+0000", 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8190999999999996e-05 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.810 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:101509 9.9 9:9de5aa29:::benchmark_data_smithi052_402718_object11868:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:07.545697+0000", 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.159642206000001, 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30923869700000001, 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.811 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101509 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545697+0000", 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.812 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545697+0000", 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545695+0000", 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545702+0000", 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7240000000000003e-06 2023-09-23T22:53:59.813 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545714+0000", 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1916000000000001e-05 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545721+0000", 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5039999999999996e-06 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.814 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545770+0000", 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8511999999999999e-05 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.545894+0000", 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012400099999999999 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:53:59.815 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.546000+0000", 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106913 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.557116+0000", 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011115715 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.557636+0000", 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00051984000000000004 2023-09-23T22:53:59.816 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.854841+0000", 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29720492900000001 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.854889+0000", 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8340000000000001e-05 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.817 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.854935+0000", 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6093000000000001e-05 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:101511 9.9 9:987020cb:::benchmark_data_smithi052_402718_object11870:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:07.549085+0000", 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.156253595999999, 2023-09-23T22:53:59.818 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30599520499999999, 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101511 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.819 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.549085+0000", 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.549085+0000", 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.820 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.549084+0000", 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.549090+0000", 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0329999999999997e-06 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.549103+0000", 2023-09-23T22:53:59.821 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.252e-05 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.549109+0000", 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2859999999999996e-06 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.550304+0000", 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011954310000000001 2023-09-23T22:53:59.822 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.550441+0000", 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013697700000000001 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.550561+0000", 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011934099999999999 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.823 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.557628+0000", 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070669679999999999 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.557643+0000", 2023-09-23T22:53:59.824 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5311999999999998e-05 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.854990+0000", 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29734670299999999 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855027+0000", 2023-09-23T22:53:59.825 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7508000000000001e-05 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855080+0000", 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3251999999999998e-05 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.826 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:101526 9.9 9:974fb29c:::benchmark_data_smithi052_402718_object11885:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:07.578083+0000", 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.127256008000003, 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27710831400000002, 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.827 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101526 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578083+0000", 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.828 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578083+0000", 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578080+0000", 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:53:59.829 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578090+0000", 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9189999999999994e-06 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578111+0000", 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0599999999999999e-05 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.830 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578121+0000", 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0613e-05 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578175+0000", 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3545000000000003e-05 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.831 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578307+0000", 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013169200000000001 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.578425+0000", 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000117945 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.832 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.581815+0000", 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033905060000000002 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583258+0000", 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014429060000000001 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855123+0000", 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27186546 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855160+0000", 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6955000000000001e-05 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855191+0000", 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0771000000000001e-05 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:101530 9.9 9:988b84f7:::benchmark_data_smithi052_402718_object11889:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:07.583610+0000", 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 52.121729023, 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.27168004099999998, 2023-09-23T22:53:59.835 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 101530 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583610+0000", 2023-09-23T22:53:59.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583610+0000", 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583608+0000", 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583616+0000", 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5270000000000001e-06 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583633+0000", 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6840000000000001e-05 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.838 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583644+0000", 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.099e-05 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583701+0000", 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7599999999999997e-05 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583865+0000", 2023-09-23T22:53:59.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016380799999999999 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.583996+0000", 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013055 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.587236+0000", 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032404339999999999 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.587653+0000", 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041646700000000001 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855228+0000", 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26757584099999998 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855261+0000", 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2539000000000001e-05 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:07.855290+0000", 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8975e-05 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110901 9.8 9:16285fac:::benchmark_data_smithi052_402718_object21260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063260+0000", 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 38.642078890999997, 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.2472589, 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110901 2023-09-23T22:53:59.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063260+0000", 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063260+0000", 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063258+0000", 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063266+0000", 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5480000000000002e-06 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.845 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063283+0000", 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6778000000000001e-05 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063292+0000", 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9220000000000003e-06 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063362+0000", 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0130999999999997e-05 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063528+0000", 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016615900000000001 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:59.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063674+0000", 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014544800000000001 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072836+0000", 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0091623099999999999 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.074703+0000", 2023-09-23T22:53:59.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018666990000000001 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.309421+0000", 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23471850399999999 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.309480+0000", 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8730000000000002e-05 2023-09-23T22:53:59.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.310519+0000", 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010388400000000001 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110906 9.5 9:aa70d84b:::benchmark_data_smithi052_402718_object21265:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:53:59.850 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.069967+0000", 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 38.635371415999998, 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24639351900000001, 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110906 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.851 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069967+0000", 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069967+0000", 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069966+0000", 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069972+0000", 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9800000000000003e-06 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069981+0000", 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.867e-06 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069987+0000", 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5419999999999996e-06 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.070035+0000", 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8350999999999998e-05 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.070165+0000", 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000130238 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.070274+0000", 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000108676 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.073936+0000", 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003661999 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.081294+0000", 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073583199999999998 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.316281+0000", 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23498682200000001 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.316319+0000", 2023-09-23T22:53:59.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8330000000000001e-05 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.316361+0000", 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1489000000000002e-05 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:53:59.858 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:53:59.859 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:53:59.859 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:53:59.859 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:53:59.860 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:53:59.895 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:53:59.899 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:53:59.945 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:53:59.962 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:53:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:53:59.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:00.000 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:54:00.024 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.025 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:54:00.068 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:00.117 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:00.174 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:00.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:00.282 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:00.370 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2129582744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8676/136449 objects degraded (6.358%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[187743]: osdmap e299: 8 total, 8 up, 6 in 2023-09-23T22:54:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3765184729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2129582744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8676/136449 objects degraded (6.358%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[180979]: osdmap e299: 8 total, 8 up, 6 in 2023-09-23T22:54:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3765184729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:00.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2129582744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8676/136449 objects degraded (6.358%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:00 smithi151 ceph-mon[159210]: osdmap e299: 8 total, 8 up, 6 in 2023-09-23T22:54:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3765184729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:01.130 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:54:01.678 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:54:01.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: failed to encode map e299 with expected crc 2023-09-23T22:54:01.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: pgmap v863: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 632 op/s; 8676/134727 objects degraded (6.440%); 1654/134727 objects misplaced (1.228%); 0 B/s, 92 objects/s recovering 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: failed to encode map e300 with expected crc 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3765184729' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: osdmap e300: 8 total, 8 up, 6 in 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[187743]: failed to encode map e300 with expected crc 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: failed to encode map e299 with expected crc 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: pgmap v863: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 632 op/s; 8676/134727 objects degraded (6.440%); 1654/134727 objects misplaced (1.228%); 0 B/s, 92 objects/s recovering 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:01.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: failed to encode map e300 with expected crc 2023-09-23T22:54:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3765184729' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:54:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: osdmap e300: 8 total, 8 up, 6 in 2023-09-23T22:54:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:01 smithi052 ceph-mon[180979]: failed to encode map e300 with expected crc 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: failed to encode map e299 with expected crc 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: pgmap v863: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 632 op/s; 8676/134727 objects degraded (6.440%); 1654/134727 objects misplaced (1.228%); 0 B/s, 92 objects/s recovering 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: failed to encode map e300 with expected crc 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3765184729' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: osdmap e300: 8 total, 8 up, 6 in 2023-09-23T22:54:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:01 smithi151 ceph-mon[159210]: failed to encode map e300 with expected crc 2023-09-23T22:54:01.958 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:01.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:01.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:02.075 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:02.177 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:02.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:02 smithi052 ceph-mon[180979]: failed to encode map e300 with expected crc 2023-09-23T22:54:02.468 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:02 smithi052 ceph-mon[180979]: failed to encode map e300 with expected crc 2023-09-23T22:54:02.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:02 smithi052 ceph-mon[187743]: failed to encode map e300 with expected crc 2023-09-23T22:54:02.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:02 smithi052 ceph-mon[187743]: failed to encode map e300 with expected crc 2023-09-23T22:54:02.774 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:02.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:02.883 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:02.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:02 smithi151 ceph-mon[159210]: failed to encode map e300 with expected crc 2023-09-23T22:54:02.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:02 smithi151 ceph-mon[159210]: failed to encode map e300 with expected crc 2023-09-23T22:54:02.919 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:03.015 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:03.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:03.125 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:03.192 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:03.466 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:54:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:54:03] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-09-23T22:54:03.466 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:03 smithi052 ceph-mon[187743]: pgmap v865: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 8389/130311 objects degraded (6.438%); 1478/130311 objects misplaced (1.134%); 0 B/s, 122 objects/s recovering 2023-09-23T22:54:03.467 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:03 smithi052 ceph-mon[180979]: pgmap v865: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 8389/130311 objects degraded (6.438%); 1478/130311 objects misplaced (1.134%); 0 B/s, 122 objects/s recovering 2023-09-23T22:54:03.846 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:03.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:03.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:03 smithi151 ceph-mon[159210]: pgmap v865: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 8389/130311 objects degraded (6.438%); 1478/130311 objects misplaced (1.134%); 0 B/s, 122 objects/s recovering 2023-09-23T22:54:03.951 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:04.013 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:04.035 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:04.165 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:04.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:04.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:04.282 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:04.324 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:04.407 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:04.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:04.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:04.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:04.576 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:54:04.610 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:04.738 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:04.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:04.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:04.844 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:04.887 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:54:04.992 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:05.033 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:54:05.043 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:54:05.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:05 smithi052 ceph-mon[180979]: pgmap v866: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 590 op/s; 8389/128136 objects degraded (6.547%); 1478/128136 objects misplaced (1.153%); 0 B/s, 122 objects/s recovering 2023-09-23T22:54:05.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:05.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8389/128136 objects degraded (6.547%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:05.498 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:05 smithi052 ceph-mon[187743]: pgmap v866: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 590 op/s; 8389/128136 objects degraded (6.547%); 1478/128136 objects misplaced (1.153%); 0 B/s, 122 objects/s recovering 2023-09-23T22:54:05.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:05.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8389/128136 objects degraded (6.547%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:05 smithi151 ceph-mon[159210]: pgmap v866: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 590 op/s; 8389/128136 objects degraded (6.547%); 1478/128136 objects misplaced (1.153%); 0 B/s, 122 objects/s recovering 2023-09-23T22:54:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8389/128136 objects degraded (6.547%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:06.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:06 smithi052 ceph-mon[180979]: from='client.45800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:06.621 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:06.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:06.725 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":866,"stamp":"2023-09-23T22:54:04.411465+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2798131128,"num_objects":42712,"num_object_clones":0,"num_object_copies":128136,"num_objects_missing_on_primary":0,"num_objects_missing":2,"num_objects_degraded":8389,"num_objects_misplaced":1478,"num_objects_unfound":0,"num_objects_dirty":42712,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":136850,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6967,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114384,"ondisk_log_size":114384,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10495348,"kb_used_data":8352772,"kb_used_omap":121,"kb_used_meta":2142406,"kb_avail":551975564,"statfs":{"total":575970213888,"available":565222977536,"internally_reserved":0,"allocated":8553238528,"data_stored":8548331009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2193823861},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,5,1,0,0,6,8],"upper_bound":1024},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-309919744,"num_objects":-4729,"num_object_clones":0,"num_object_copies":-14187,"num_objects_missing_on_primary":0,"num_objects_missing":2,"num_objects_degraded":-804,"num_objects_misplaced":-176,"num_objects_unfound":0,"num_objects_dirty":-4729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4729,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":980,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004872"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":297,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132199+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:54:00.132199+0000","last_peered":"2023-09-23T22:54:00.132199+0000","last_clean":"2023-09-23T22:54:00.132199+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:54:00.132199+0000","last_undegraded":"2023-09-23T22:54:00.132199+0000","last_fullsized":"2023-09-23T22:54:00.132199+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":96,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132146+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:54:00.132146+0000","last_peered":"2023-09-23T22:54:00.132146+0000","last_clean":"2023-09-23T22:54:00.132146+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:54:00.132146+0000","last_undegraded":"2023-09-23T22:54:00.132146+0000","last_fullsized":"2023-09-23T22:54:00.132146+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132511+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:54:00.132511+0000","last_peered":"2023-09-23T22:54:00.132511+0000","last_clean":"2023-09-23T22:54:00.132511+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:54:00.132511+0000","last_undegraded":"2023-09-23T22:54:00.132511+0000","last_fullsized":"2023-09-23T22:54:00.132511+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":278,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.153386+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:54:00.153386+0000","last_peered":"2023-09-23T22:54:00.153386+0000","last_clean":"2023-09-23T22:54:00.153386+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:54:00.153386+0000","last_undegraded":"2023-09-23T22:54:00.153386+0000","last_fullsized":"2023-09-23T22:54:00.153386+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":283,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194852+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:54:01.194852+0000","last_peered":"2023-09-23T22:54:01.194852+0000","last_clean":"2023-09-23T22:54:01.194852+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:54:01.194852+0000","last_undegraded":"2023-09-23T22:54:01.194852+0000","last_fullsized":"2023-09-23T22:54:01.194852+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":264,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519945+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:53:50.519945+0000","last_peered":"2023-09-23T22:53:50.519945+0000","last_clean":"2023-09-23T22:53:50.519945+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:53:50.519945+0000","last_undegraded":"2023-09-23T22:53:50.519945+0000","last_fullsized":"2023-09-23T22:53:50.519945+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":210,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152700+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:54:00.152700+0000","last_peered":"2023-09-23T22:54:00.152700+0000","last_clean":"2023-09-23T22:54:00.152700+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:54:00.152700+0000","last_undegraded":"2023-09-23T22:54:00.152700+0000","last_fullsized":"2023-09-23T22:54:00.152700+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":160,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.523804+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:53:50.523804+0000","last_peered":"2023-09-23T22:53:50.523804+0000","last_clean":"2023-09-23T22:53:50.523804+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:53:50.523804+0000","last_undegraded":"2023-09-23T22:53:50.523804+0000","last_fullsized":"2023-09-23T22:53:50.523804+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":39,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.523955+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:53:50.523955+0000","last_peered":"2023-09-23T22:53:50.523955+0000","last_clean":"2023-09-23T22:53:50.523955+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:53:50.523955+0000","last_undegraded":"2023-09-23T22:53:50.523955+0000","last_fullsized":"2023-09-23T22:53:50.523955+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":183,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519228+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:53:50.519228+0000","last_peered":"2023-09-23T22:53:50.519228+0000","last_clean":"2023-09-23T22:53:50.519228+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:53:50.519228+0000","last_undegraded":"2023-09-23T22:53:50.519228+0000","last_fullsized":"2023-09-23T22:53:50.519228+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":58,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132002+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:54:00.132002+0000","last_peered":"2023-09-23T22:54:00.132002+0000","last_clean":"2023-09-23T22:54:00.132002+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:54:00.132002+0000","last_undegraded":"2023-09-23T22:54:00.132002+0000","last_fullsized":"2023-09-23T22:54:00.132002+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":173,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519835+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:53:50.519835+0000","last_peered":"2023-09-23T22:53:50.519835+0000","last_clean":"2023-09-23T22:53:50.519835+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:53:50.519835+0000","last_undegraded":"2023-09-23T22:53:50.519835+0000","last_fullsized":"2023-09-23T22:53:50.519835+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":319,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131860+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:54:00.131860+0000","last_peered":"2023-09-23T22:54:00.131860+0000","last_clean":"2023-09-23T22:54:00.131860+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:54:00.131860+0000","last_undegraded":"2023-09-23T22:54:00.131860+0000","last_fullsized":"2023-09-23T22:54:00.131860+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":265,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131675+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:54:00.131675+0000","last_peered":"2023-09-23T22:54:00.131675+0000","last_clean":"2023-09-23T22:54:00.131675+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:54:00.131675+0000","last_undegraded":"2023-09-23T22:54:00.131675+0000","last_fullsized":"2023-09-23T22:54:00.131675+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":158,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132144+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:54:00.132144+0000","last_peered":"2023-09-23T22:54:00.132144+0000","last_clean":"2023-09-23T22:54:00.132144+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:54:00.132144+0000","last_undegraded":"2023-09-23T22:54:00.132144+0000","last_fullsized":"2023-09-23T22:54:00.132144+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.522726+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:53:50.522726+0000","last_peered":"2023-09-23T22:53:50.522726+0000","last_clean":"2023-09-23T22:53:50.522726+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:53:50.522726+0000","last_undegraded":"2023-09-23T22:53:50.522726+0000","last_fullsized":"2023-09-23T22:53:50.522726+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":266,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128822+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:54:00.128822+0000","last_peered":"2023-09-23T22:54:00.128822+0000","last_clean":"2023-09-23T22:54:00.128822+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:54:00.128822+0000","last_undegraded":"2023-09-23T22:54:00.128822+0000","last_fullsized":"2023-09-23T22:54:00.128822+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":222,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519833+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:53:50.519833+0000","last_peered":"2023-09-23T22:53:50.519833+0000","last_clean":"2023-09-23T22:53:50.519833+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:53:50.519833+0000","last_undegraded":"2023-09-23T22:53:50.519833+0000","last_fullsized":"2023-09-23T22:53:50.519833+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132692+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:54:00.132692+0000","last_peered":"2023-09-23T22:54:00.132692+0000","last_clean":"2023-09-23T22:54:00.132692+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:54:00.132692+0000","last_undegraded":"2023-09-23T22:54:00.132692+0000","last_fullsized":"2023-09-23T22:54:00.132692+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":267,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.153464+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:54:00.153464+0000","last_peered":"2023-09-23T22:54:00.153464+0000","last_clean":"2023-09-23T22:54:00.153464+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:54:00.153464+0000","last_undegraded":"2023-09-23T22:54:00.153464+0000","last_fullsized":"2023-09-23T22:54:00.153464+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":343,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519135+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:53:50.519135+0000","last_peered":"2023-09-23T22:53:50.519135+0000","last_clean":"2023-09-23T22:53:50.519135+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:53:50.519135+0000","last_undegraded":"2023-09-23T22:53:50.519135+0000","last_fullsized":"2023-09-23T22:53:50.519135+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":225,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.533362+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:53:50.533362+0000","last_peered":"2023-09-23T22:53:50.533362+0000","last_clean":"2023-09-23T22:53:50.533362+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:53:50.533362+0000","last_undegraded":"2023-09-23T22:53:50.533362+0000","last_fullsized":"2023-09-23T22:53:50.533362+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"300'8543","reported_seq":8589,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:03.241320+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:54:03.241320+0000","last_peered":"2023-09-23T22:54:03.241320+0000","last_clean":"2023-09-23T22:54:03.241320+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:54:03.241320+0000","last_undegraded":"2023-09-23T22:54:03.241320+0000","last_fullsized":"2023-09-23T22:54:03.241320+0000","mapping_epoch":264,"log_start":"269'1000","ondisk_log_start":"269'1000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7543,"log_dups_size":0,"ondisk_log_size":7543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169672704,"num_objects":2589,"num_object_clones":0,"num_object_copies":7767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8543,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":169,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131814+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:54:00.131814+0000","last_peered":"2023-09-23T22:54:00.131814+0000","last_clean":"2023-09-23T22:54:00.131814+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:54:00.131814+0000","last_undegraded":"2023-09-23T22:54:00.131814+0000","last_fullsized":"2023-09-23T22:54:00.131814+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":263,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519222+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:53:50.519222+0000","last_peered":"2023-09-23T22:53:50.519222+0000","last_clean":"2023-09-23T22:53:50.519222+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:53:50.519222+0000","last_undegraded":"2023-09-23T22:53:50.519222+0000","last_fullsized":"2023-09-23T22:53:50.519222+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"300'8722","reported_seq":8773,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:01.632571+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:54:01.632571+0000","last_peered":"2023-09-23T22:54:01.632571+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:54:01.632571+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"273'2000","ondisk_log_start":"273'2000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181534720,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8722,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":1109}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":179,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.133698+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:54:00.133698+0000","last_peered":"2023-09-23T22:54:00.133698+0000","last_clean":"2023-09-23T22:54:00.133698+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:54:00.133698+0000","last_undegraded":"2023-09-23T22:54:00.133698+0000","last_fullsized":"2023-09-23T22:54:00.133698+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":289,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132608+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:54:00.132608+0000","last_peered":"2023-09-23T22:54:00.132608+0000","last_clean":"2023-09-23T22:54:00.132608+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:54:00.132608+0000","last_undegraded":"2023-09-23T22:54:00.132608+0000","last_fullsized":"2023-09-23T22:54:00.132608+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"298'8621","reported_seq":8669,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:59.961776+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:53:59.961776+0000","last_peered":"2023-09-23T22:53:59.961776+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:53:59.961776+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8621,"log_dups_size":0,"ondisk_log_size":8621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8621,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":896}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":122,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519810+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:53:50.519810+0000","last_peered":"2023-09-23T22:53:50.519810+0000","last_clean":"2023-09-23T22:53:50.519810+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:53:50.519810+0000","last_undegraded":"2023-09-23T22:53:50.519810+0000","last_fullsized":"2023-09-23T22:53:50.519810+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":291,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519753+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:53:50.519753+0000","last_peered":"2023-09-23T22:53:50.519753+0000","last_clean":"2023-09-23T22:53:50.519753+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:53:50.519753+0000","last_undegraded":"2023-09-23T22:53:50.519753+0000","last_fullsized":"2023-09-23T22:53:50.519753+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"298'8523","reported_seq":8567,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:59.885831+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:53:59.885831+0000","last_peered":"2023-09-23T22:53:59.885831+0000","last_clean":"2023-09-23T22:53:59.885831+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:53:59.885831+0000","last_undegraded":"2023-09-23T22:53:59.885831+0000","last_fullsized":"2023-09-23T22:53:59.885831+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8523,"log_dups_size":0,"ondisk_log_size":8523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183042048,"num_objects":2793,"num_object_clones":0,"num_object_copies":8379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8523,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519532+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:53:50.519532+0000","last_peered":"2023-09-23T22:53:50.519532+0000","last_clean":"2023-09-23T22:53:50.519532+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:53:50.519532+0000","last_undegraded":"2023-09-23T22:53:50.519532+0000","last_fullsized":"2023-09-23T22:53:50.519532+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":194,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.153822+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:54:00.153822+0000","last_peered":"2023-09-23T22:54:00.153822+0000","last_clean":"2023-09-23T22:54:00.153822+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:54:00.153822+0000","last_undegraded":"2023-09-23T22:54:00.153822+0000","last_fullsized":"2023-09-23T22:54:00.153822+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"299'8390","reported_seq":11799,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.456569+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:54:00.456569+0000","last_peered":"2023-09-23T22:54:00.456569+0000","last_clean":"2023-09-23T22:54:00.456569+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:54:00.456569+0000","last_undegraded":"2023-09-23T22:54:00.456569+0000","last_fullsized":"2023-09-23T22:54:00.456569+0000","mapping_epoch":289,"log_start":"273'2000","ondisk_log_start":"273'2000","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171573248,"num_objects":2618,"num_object_clones":0,"num_object_copies":7854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8390,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":66,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152652+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:54:00.152652+0000","last_peered":"2023-09-23T22:54:00.152652+0000","last_clean":"2023-09-23T22:54:00.152652+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:54:00.152652+0000","last_undegraded":"2023-09-23T22:54:00.152652+0000","last_fullsized":"2023-09-23T22:54:00.152652+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":273,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131778+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:54:00.131778+0000","last_peered":"2023-09-23T22:54:00.131778+0000","last_clean":"2023-09-23T22:54:00.131778+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:54:00.131778+0000","last_undegraded":"2023-09-23T22:54:00.131778+0000","last_fullsized":"2023-09-23T22:54:00.131778+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"299'8369","reported_seq":8412,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.369635+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:54:00.369635+0000","last_peered":"2023-09-23T22:54:00.369635+0000","last_clean":"2023-09-23T22:54:00.369635+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:54:00.369635+0000","last_undegraded":"2023-09-23T22:54:00.369635+0000","last_fullsized":"2023-09-23T22:54:00.369635+0000","mapping_epoch":264,"log_start":"273'1900","ondisk_log_start":"273'1900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177799168,"num_objects":2713,"num_object_clones":0,"num_object_copies":8139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8369,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519659+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:53:50.519659+0000","last_peered":"2023-09-23T22:53:50.519659+0000","last_clean":"2023-09-23T22:53:50.519659+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:53:50.519659+0000","last_undegraded":"2023-09-23T22:53:50.519659+0000","last_fullsized":"2023-09-23T22:53:50.519659+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":295,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131811+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:54:00.131811+0000","last_peered":"2023-09-23T22:54:00.131811+0000","last_clean":"2023-09-23T22:54:00.131811+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:54:00.131811+0000","last_undegraded":"2023-09-23T22:54:00.131811+0000","last_fullsized":"2023-09-23T22:54:00.131811+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"298'8514","reported_seq":11897,"reported_epoch":299,"state":"creating+peering","last_fresh":"2023-09-23T22:54:00.096146+0000","last_change":"2023-09-23T22:54:00.178056+0000","last_active":"2023-09-23T22:54:00.178056+0000","last_peered":"2023-09-23T22:54:00.096146+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:51:35.514865+0000","last_became_peered":"2023-09-23T22:51:35.514865+0000","last_unstale":"2023-09-23T22:54:00.096146+0000","last_undegraded":"2023-09-23T22:54:00.096146+0000","last_fullsized":"2023-09-23T22:51:35.423604+0000","mapping_epoch":299,"log_start":"277'3400","ondisk_log_start":"277'3400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5114,"log_dups_size":0,"ondisk_log_size":5114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174718976,"num_objects":2666,"num_object_clones":0,"num_object_copies":7998,"num_objects_missing_on_primary":0,"num_objects_missing":2,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8514,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1665,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":308,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132107+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:54:00.132107+0000","last_peered":"2023-09-23T22:54:00.132107+0000","last_clean":"2023-09-23T22:54:00.132107+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:54:00.132107+0000","last_undegraded":"2023-09-23T22:54:00.132107+0000","last_fullsized":"2023-09-23T22:54:00.132107+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":283,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.133747+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:54:00.133747+0000","last_peered":"2023-09-23T22:54:00.133747+0000","last_clean":"2023-09-23T22:54:00.133747+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:54:00.133747+0000","last_undegraded":"2023-09-23T22:54:00.133747+0000","last_fullsized":"2023-09-23T22:54:00.133747+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"298'8325","reported_seq":8375,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:53:59.807341+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:53:59.807341+0000","last_peered":"2023-09-23T22:53:59.807341+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:53:59.807341+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8325,"log_dups_size":0,"ondisk_log_size":8325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179896320,"num_objects":2745,"num_object_clones":0,"num_object_copies":8235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8325,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":1130}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":228,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131404+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:54:00.131404+0000","last_peered":"2023-09-23T22:54:00.131404+0000","last_clean":"2023-09-23T22:54:00.131404+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:54:00.131404+0000","last_undegraded":"2023-09-23T22:54:00.131404+0000","last_fullsized":"2023-09-23T22:54:00.131404+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":235,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.774143+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:53:44.774143+0000","last_peered":"2023-09-23T22:53:44.774143+0000","last_clean":"2023-09-23T22:53:44.774143+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:53:44.774143+0000","last_undegraded":"2023-09-23T22:53:44.774143+0000","last_fullsized":"2023-09-23T22:53:44.774143+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"300'8620","reported_seq":8671,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:02.161918+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:54:02.161918+0000","last_peered":"2023-09-23T22:54:02.161918+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:54:02.161918+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"277'3500","ondisk_log_start":"277'3500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179044352,"num_objects":2732,"num_object_clones":0,"num_object_copies":8196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8620,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,5","objects":1102}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":338,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.151474+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:54:00.151474+0000","last_peered":"2023-09-23T22:54:00.151474+0000","last_clean":"2023-09-23T22:54:00.151474+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:54:00.151474+0000","last_undegraded":"2023-09-23T22:54:00.151474+0000","last_fullsized":"2023-09-23T22:54:00.151474+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.773781+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:53:44.773781+0000","last_peered":"2023-09-23T22:53:44.773781+0000","last_clean":"2023-09-23T22:53:44.773781+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:53:44.773781+0000","last_undegraded":"2023-09-23T22:53:44.773781+0000","last_fullsized":"2023-09-23T22:53:44.773781+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":131,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132102+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:54:00.132102+0000","last_peered":"2023-09-23T22:54:00.132102+0000","last_clean":"2023-09-23T22:54:00.132102+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:54:00.132102+0000","last_undegraded":"2023-09-23T22:54:00.132102+0000","last_fullsized":"2023-09-23T22:54:00.132102+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"300'8618","reported_seq":12066,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:03.244851+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:54:03.244851+0000","last_peered":"2023-09-23T22:54:03.244851+0000","last_clean":"2023-09-23T22:54:03.244851+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:54:03.244851+0000","last_undegraded":"2023-09-23T22:54:03.244851+0000","last_fullsized":"2023-09-23T22:54:03.244851+0000","mapping_epoch":282,"log_start":"269'1100","ondisk_log_start":"269'1100","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167772160,"num_objects":2560,"num_object_clones":0,"num_object_copies":7680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8618,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519710+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:53:50.519710+0000","last_peered":"2023-09-23T22:53:50.519710+0000","last_clean":"2023-09-23T22:53:50.519710+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:53:50.519710+0000","last_undegraded":"2023-09-23T22:53:50.519710+0000","last_fullsized":"2023-09-23T22:53:50.519710+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":296,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131839+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:54:00.131839+0000","last_peered":"2023-09-23T22:54:00.131839+0000","last_clean":"2023-09-23T22:54:00.131839+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:54:00.131839+0000","last_undegraded":"2023-09-23T22:54:00.131839+0000","last_fullsized":"2023-09-23T22:54:00.131839+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":224,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.773367+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:53:44.773367+0000","last_peered":"2023-09-23T22:53:44.773367+0000","last_clean":"2023-09-23T22:53:44.773367+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:53:44.773367+0000","last_undegraded":"2023-09-23T22:53:44.773367+0000","last_fullsized":"2023-09-23T22:53:44.773367+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"299'8543","reported_seq":8951,"reported_epoch":300,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:54:01.180424+0000","last_change":"2023-09-23T22:53:59.962748+0000","last_active":"2023-09-23T22:54:01.180424+0000","last_peered":"2023-09-23T22:54:01.180424+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:54:01.180424+0000","last_undegraded":"2023-09-23T22:54:01.180424+0000","last_fullsized":"2023-09-23T22:54:01.180424+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8543,"log_dups_size":0,"ondisk_log_size":8543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179109888,"num_objects":2733,"num_object_clones":0,"num_object_copies":8199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1478,"num_objects_unfound":0,"num_objects_dirty":2733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8543,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":177,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":1255}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":300,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.772981+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:53:44.772981+0000","last_peered":"2023-09-23T22:53:44.772981+0000","last_clean":"2023-09-23T22:53:44.772981+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:53:44.772981+0000","last_undegraded":"2023-09-23T22:53:44.772981+0000","last_fullsized":"2023-09-23T22:53:44.772981+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":179,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.772590+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:53:44.772590+0000","last_peered":"2023-09-23T22:53:44.772590+0000","last_clean":"2023-09-23T22:53:44.772590+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:53:44.772590+0000","last_undegraded":"2023-09-23T22:53:44.772590+0000","last_fullsized":"2023-09-23T22:53:44.772590+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":307,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152596+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:54:00.152596+0000","last_peered":"2023-09-23T22:54:00.152596+0000","last_clean":"2023-09-23T22:54:00.152596+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:54:00.152596+0000","last_undegraded":"2023-09-23T22:54:00.152596+0000","last_fullsized":"2023-09-23T22:54:00.152596+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"300'8524","reported_seq":8576,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:03.233298+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:54:03.233298+0000","last_peered":"2023-09-23T22:54:03.233298+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:54:03.233298+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"269'1000","ondisk_log_start":"269'1000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8524,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":814}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":309,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.773175+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:53:44.773175+0000","last_peered":"2023-09-23T22:53:44.773175+0000","last_clean":"2023-09-23T22:53:44.773175+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:53:44.773175+0000","last_undegraded":"2023-09-23T22:53:44.773175+0000","last_fullsized":"2023-09-23T22:53:44.773175+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":388,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128782+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:54:00.128782+0000","last_peered":"2023-09-23T22:54:00.128782+0000","last_clean":"2023-09-23T22:54:00.128782+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:54:00.128782+0000","last_undegraded":"2023-09-23T22:54:00.128782+0000","last_fullsized":"2023-09-23T22:54:00.128782+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":53,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519801+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:53:50.519801+0000","last_peered":"2023-09-23T22:53:50.519801+0000","last_clean":"2023-09-23T22:53:50.519801+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:53:50.519801+0000","last_undegraded":"2023-09-23T22:53:50.519801+0000","last_fullsized":"2023-09-23T22:53:50.519801+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":291,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519702+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:53:50.519702+0000","last_peered":"2023-09-23T22:53:50.519702+0000","last_clean":"2023-09-23T22:53:50.519702+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:53:50.519702+0000","last_undegraded":"2023-09-23T22:53:50.519702+0000","last_fullsized":"2023-09-23T22:53:50.519702+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"300'8630","reported_seq":8676,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:03.231585+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:54:03.231585+0000","last_peered":"2023-09-23T22:54:03.231585+0000","last_clean":"2023-09-23T22:54:03.231585+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:54:03.231585+0000","last_undegraded":"2023-09-23T22:54:03.231585+0000","last_fullsized":"2023-09-23T22:54:03.231585+0000","mapping_epoch":264,"log_start":"269'1100","ondisk_log_start":"269'1100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7530,"log_dups_size":0,"ondisk_log_size":7530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170131456,"num_objects":2596,"num_object_clones":0,"num_object_copies":7788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8630,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":276,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.151558+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:54:00.151558+0000","last_peered":"2023-09-23T22:54:00.151558+0000","last_clean":"2023-09-23T22:54:00.151558+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:54:00.151558+0000","last_undegraded":"2023-09-23T22:54:00.151558+0000","last_fullsized":"2023-09-23T22:54:00.151558+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":247,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128802+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:54:00.128802+0000","last_peered":"2023-09-23T22:54:00.128802+0000","last_clean":"2023-09-23T22:54:00.128802+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:54:00.128802+0000","last_undegraded":"2023-09-23T22:54:00.128802+0000","last_fullsized":"2023-09-23T22:54:00.128802+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":225,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.153207+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:54:00.153207+0000","last_peered":"2023-09-23T22:54:00.153207+0000","last_clean":"2023-09-23T22:54:00.153207+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:54:00.153207+0000","last_undegraded":"2023-09-23T22:54:00.153207+0000","last_fullsized":"2023-09-23T22:54:00.153207+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"300'8353","reported_seq":8396,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.639183+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:54:01.639183+0000","last_peered":"2023-09-23T22:54:01.639183+0000","last_clean":"2023-09-23T22:54:01.639183+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:54:01.639183+0000","last_undegraded":"2023-09-23T22:54:01.639183+0000","last_fullsized":"2023-09-23T22:54:01.639183+0000","mapping_epoch":264,"log_start":"269'1600","ondisk_log_start":"269'1600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171769856,"num_objects":2621,"num_object_clones":0,"num_object_copies":7863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8353,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":230,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.154189+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:54:00.154189+0000","last_peered":"2023-09-23T22:54:00.154189+0000","last_clean":"2023-09-23T22:54:00.154189+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:54:00.154189+0000","last_undegraded":"2023-09-23T22:54:00.154189+0000","last_fullsized":"2023-09-23T22:54:00.154189+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":178,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.773518+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:53:44.773518+0000","last_peered":"2023-09-23T22:53:44.773518+0000","last_clean":"2023-09-23T22:53:44.773518+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:53:44.773518+0000","last_undegraded":"2023-09-23T22:53:44.773518+0000","last_fullsized":"2023-09-23T22:53:44.773518+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152901+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:54:00.152901+0000","last_peered":"2023-09-23T22:54:00.152901+0000","last_clean":"2023-09-23T22:54:00.152901+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:54:00.152901+0000","last_undegraded":"2023-09-23T22:54:00.152901+0000","last_fullsized":"2023-09-23T22:54:00.152901+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"300'8631","reported_seq":8676,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.585214+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:54:01.585214+0000","last_peered":"2023-09-23T22:54:01.585214+0000","last_clean":"2023-09-23T22:54:01.585214+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:54:01.585214+0000","last_undegraded":"2023-09-23T22:54:01.585214+0000","last_fullsized":"2023-09-23T22:54:01.585214+0000","mapping_epoch":264,"log_start":"273'1900","ondisk_log_start":"273'1900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178913304,"num_objects":2731,"num_object_clones":0,"num_object_copies":8193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2731,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8631,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":57,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152992+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:54:00.152992+0000","last_peered":"2023-09-23T22:54:00.152992+0000","last_clean":"2023-09-23T22:54:00.152992+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:54:00.152992+0000","last_undegraded":"2023-09-23T22:54:00.152992+0000","last_fullsized":"2023-09-23T22:54:00.152992+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":97,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519818+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:53:50.519818+0000","last_peered":"2023-09-23T22:53:50.519818+0000","last_clean":"2023-09-23T22:53:50.519818+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:53:50.519818+0000","last_undegraded":"2023-09-23T22:53:50.519818+0000","last_fullsized":"2023-09-23T22:53:50.519818+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":277,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128666+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:54:00.128666+0000","last_peered":"2023-09-23T22:54:00.128666+0000","last_clean":"2023-09-23T22:54:00.128666+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:54:00.128666+0000","last_undegraded":"2023-09-23T22:54:00.128666+0000","last_fullsized":"2023-09-23T22:54:00.128666+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"300'8664","reported_seq":12182,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.584605+0000","last_change":"2023-09-23T22:53:44.765920+0000","last_active":"2023-09-23T22:54:01.584605+0000","last_peered":"2023-09-23T22:54:01.584605+0000","last_clean":"2023-09-23T22:54:01.584605+0000","last_became_active":"2023-09-23T22:53:44.707193+0000","last_became_peered":"2023-09-23T22:53:44.707193+0000","last_unstale":"2023-09-23T22:54:01.584605+0000","last_undegraded":"2023-09-23T22:54:01.584605+0000","last_fullsized":"2023-09-23T22:54:01.584605+0000","mapping_epoch":296,"log_start":"273'1900","ondisk_log_start":"273'1900","created":264,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172228608,"num_objects":2628,"num_object_clones":0,"num_object_copies":7884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8664,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":340,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519861+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:53:50.519861+0000","last_peered":"2023-09-23T22:53:50.519861+0000","last_clean":"2023-09-23T22:53:50.519861+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:53:50.519861+0000","last_undegraded":"2023-09-23T22:53:50.519861+0000","last_fullsized":"2023-09-23T22:53:50.519861+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":197,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.153301+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:54:00.153301+0000","last_peered":"2023-09-23T22:54:00.153301+0000","last_clean":"2023-09-23T22:54:00.153301+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:54:00.153301+0000","last_undegraded":"2023-09-23T22:54:00.153301+0000","last_fullsized":"2023-09-23T22:54:00.153301+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":250,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132206+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:54:00.132206+0000","last_peered":"2023-09-23T22:54:00.132206+0000","last_clean":"2023-09-23T22:54:00.132206+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:54:00.132206+0000","last_undegraded":"2023-09-23T22:54:00.132206+0000","last_fullsized":"2023-09-23T22:54:00.132206+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":233,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152752+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:54:00.152752+0000","last_peered":"2023-09-23T22:54:00.152752+0000","last_clean":"2023-09-23T22:54:00.152752+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:54:00.152752+0000","last_undegraded":"2023-09-23T22:54:00.152752+0000","last_fullsized":"2023-09-23T22:54:00.152752+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":81,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T22:53:44.773426+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:53:44.773426+0000","last_peered":"2023-09-23T22:53:44.773426+0000","last_clean":"2023-09-23T22:53:44.773426+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:53:44.773426+0000","last_undegraded":"2023-09-23T22:53:44.773426+0000","last_fullsized":"2023-09-23T22:53:44.773426+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":109,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131677+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:54:00.131677+0000","last_peered":"2023-09-23T22:54:00.131677+0000","last_clean":"2023-09-23T22:54:00.131677+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:54:00.131677+0000","last_undegraded":"2023-09-23T22:54:00.131677+0000","last_fullsized":"2023-09-23T22:54:00.131677+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":308,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131915+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:54:00.131915+0000","last_peered":"2023-09-23T22:54:00.131915+0000","last_clean":"2023-09-23T22:54:00.131915+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:54:00.131915+0000","last_undegraded":"2023-09-23T22:54:00.131915+0000","last_fullsized":"2023-09-23T22:54:00.131915+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":110,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131904+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:54:00.131904+0000","last_peered":"2023-09-23T22:54:00.131904+0000","last_clean":"2023-09-23T22:54:00.131904+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:54:00.131904+0000","last_undegraded":"2023-09-23T22:54:00.131904+0000","last_fullsized":"2023-09-23T22:54:00.131904+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132252+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:54:00.132252+0000","last_peered":"2023-09-23T22:54:00.132252+0000","last_clean":"2023-09-23T22:54:00.132252+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:54:00.132252+0000","last_undegraded":"2023-09-23T22:54:00.132252+0000","last_fullsized":"2023-09-23T22:54:00.132252+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":49,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.519712+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:53:50.519712+0000","last_peered":"2023-09-23T22:53:50.519712+0000","last_clean":"2023-09-23T22:53:50.519712+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:53:50.519712+0000","last_undegraded":"2023-09-23T22:53:50.519712+0000","last_fullsized":"2023-09-23T22:53:50.519712+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":281,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T22:53:50.523316+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:53:50.523316+0000","last_peered":"2023-09-23T22:53:50.523316+0000","last_clean":"2023-09-23T22:53:50.523316+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:53:50.523316+0000","last_undegraded":"2023-09-23T22:53:50.523316+0000","last_fullsized":"2023-09-23T22:53:50.523316+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":246,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152810+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:54:00.152810+0000","last_peered":"2023-09-23T22:54:00.152810+0000","last_clean":"2023-09-23T22:54:00.152810+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:54:00.152810+0000","last_undegraded":"2023-09-23T22:54:00.152810+0000","last_fullsized":"2023-09-23T22:54:00.152810+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":265,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131446+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:54:00.131446+0000","last_peered":"2023-09-23T22:54:00.131446+0000","last_clean":"2023-09-23T22:54:00.131446+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:54:00.131446+0000","last_undegraded":"2023-09-23T22:54:00.131446+0000","last_fullsized":"2023-09-23T22:54:00.131446+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2797535256,"num_objects":42688,"num_object_clones":0,"num_object_copies":128064,"num_objects_missing_on_primary":0,"num_objects_missing":2,"num_objects_degraded":8389,"num_objects_misplaced":1478,"num_objects_unfound":0,"num_objects_dirty":42688,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136590,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6937,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9571414016,"data_stored":9571401800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114190,"ondisk_log_size":114190,"up":48,"acting":43,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430532,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626760,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1693140,"kb_used_data":1383628,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92052012,"statfs":{"total":95995035648,"available":94261260288,"internally_reserved":0,"allocated":1416835072,"data_stored":1416278839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,6,8],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":6.9699999999999998,"5min":9.5869999999999997,"15min":4.2800000000000002},"min":{"1min":0.24199999999999999,"5min":0.218,"15min":0.218},"max":{"1min":112.51000000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.25},{"interface":"front","average":{"1min":7.6239999999999997,"5min":9.9469999999999992,"15min":4.2679999999999998},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":112.154,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":5.2249999999999996,"15min":3.8079999999999998},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":10.98,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":3.21,"5min":6.5389999999999997,"15min":4.8079999999999998},"min":{"1min":0.45000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":12.069000000000001,"5min":255.20099999999999,"15min":255.20099999999999},"last":0.24199999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":3.867,"15min":3.4199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":38.527999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":4.3499999999999996,"5min":3.4969999999999999,"15min":3.786},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":38.776000000000003,"5min":101.13,"15min":185.21799999999999},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.1520000000000001,"15min":3.8029999999999999},"min":{"1min":0.309,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":6.5090000000000003,"5min":17.742999999999999,"15min":185.19399999999999},"last":2.3929999999999998},{"interface":"front","average":{"1min":3.2850000000000001,"5min":2.2799999999999998,"15min":3.4950000000000001},"min":{"1min":0.42599999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":10.378,"5min":10.773999999999999,"15min":128.19800000000001},"last":3.3620000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":1.1359999999999999,"15min":0.871},"min":{"1min":0.309,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.317,"5min":10.317,"15min":18.085999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.107,"15min":0.88500000000000001},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4610000000000003,"5min":7.4800000000000004,"15min":18.597000000000001},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.339,"15min":0.99099999999999999},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.353999999999999,"5min":10.353999999999999,"15min":18.972000000000001},"last":0.374},{"interface":"front","average":{"1min":2.5459999999999998,"5min":1.1779999999999999,"15min":0.93999999999999995},"min":{"1min":0.41599999999999998,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":7.1459999999999999,"5min":7.1459999999999999,"15min":18.625},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.123,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":7.8650000000000002,"5min":7.8650000000000002,"15min":12.481999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.694,"5min":1.21,"15min":0.93600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.774,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.48399999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253070,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2560096,"kb_used_data":2066968,"kb_used_omap":9,"kb_used_meta":493110,"kb_avail":91185056,"statfs":{"total":95995035648,"available":93373497344,"internally_reserved":0,"allocated":2116575232,"data_stored":2115500602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":504945641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.6829999999999998,"5min":6.5149999999999997,"15min":5.3140000000000001},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.691000000000001,"5min":82.903999999999996,"15min":101.605},"last":2.3380000000000001},{"interface":"front","average":{"1min":3.944,"5min":5.3680000000000003,"15min":5.1929999999999996},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":13.601000000000001,"5min":82.748999999999995,"15min":101.892},"last":0.20200000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.6749999999999998,"5min":4.1029999999999998,"15min":3.0419999999999998},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":10.952999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":2.3050000000000002},{"interface":"front","average":{"1min":4.4930000000000003,"5min":4.1340000000000003,"15min":3.1190000000000002},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.429,"5min":37.957000000000001,"15min":37.957000000000001},"last":2.3730000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":3.3079999999999998,"15min":2.528},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.1660000000000004,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.9540000000000002,"5min":3.8650000000000002,"15min":3.0059999999999998},"min":{"1min":0.29299999999999998,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":10.845000000000001,"5min":89.861999999999995,"15min":89.861999999999995},"last":2.3889999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.9710000000000001,"15min":2.25},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":8.7469999999999999,"5min":11.432,"15min":78.948999999999998},"last":1.4199999999999999},{"interface":"front","average":{"1min":3.544,"5min":2.649,"15min":2.2850000000000001},"min":{"1min":0.315,"5min":0.313,"15min":0.313},"max":{"1min":10.894,"5min":14.058,"15min":78.963999999999999},"last":2.4359999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.597,"15min":1.792},"min":{"1min":0.42799999999999999,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":9.141,"5min":9.141,"15min":84.254999999999995},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.903,"5min":1.958,"15min":2.125},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":7.1150000000000002,"5min":11.458,"15min":87.515000000000001},"last":1.353}]},{"osd":6,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":4.0899999999999999,"5min":2.7519999999999998,"15min":2.1549999999999998},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":10.853999999999999,"5min":13.991,"15min":79.311000000000007},"last":2.4660000000000002},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.7609999999999999,"15min":1.7809999999999999},"min":{"1min":0.40899999999999997,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":79.328999999999994},"last":0.91900000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.1339999999999999,"15min":2.5049999999999999},"min":{"1min":0.438,"5min":0.40799999999999997,"15min":0.32800000000000001},"max":{"1min":8.702,"5min":11.481999999999999,"15min":159.018},"last":1.3899999999999999},{"interface":"front","average":{"1min":4.008,"5min":2.911,"15min":2.6669999999999998},"min":{"1min":0.49199999999999999,"5min":0.371,"15min":0.37},"max":{"1min":10.923,"5min":14.032999999999999,"15min":158.96000000000001},"last":2.4100000000000001}]}]},{"osd":0,"up_from":219,"seq":940597837919,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1486476,"kb_used_data":1169220,"kb_used_omap":31,"kb_used_meta":317216,"kb_avail":92258676,"statfs":{"total":95995035648,"available":94472884224,"internally_reserved":0,"allocated":1197281280,"data_stored":1196166808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":324830130},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":5.3840000000000003,"5min":4.0069999999999997,"15min":3.758},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":40.180999999999997,"5min":40.180999999999997,"15min":40.180999999999997},"last":10.326000000000001},{"interface":"front","average":{"1min":4.9249999999999998,"5min":3.6819999999999999,"15min":3.605},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":40.137999999999998,"5min":40.137999999999998,"15min":40.137999999999998},"last":4.5119999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":4.4560000000000004,"5min":5.0910000000000002,"15min":3.2189999999999999},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":46.387999999999998,"5min":111.508,"15min":111.508},"last":4.585},{"interface":"front","average":{"1min":5.7619999999999996,"5min":4.7220000000000004,"15min":2.8460000000000001},"min":{"1min":0.41599999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":46.249000000000002,"5min":111.532,"15min":111.532},"last":4.3159999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":4.9660000000000002,"15min":3.0289999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.2759999999999998,"5min":126.73,"15min":126.73},"last":4.6269999999999998},{"interface":"front","average":{"1min":1.2829999999999999,"5min":3.581,"15min":2.044},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.7539999999999996,"5min":121.483,"15min":121.483},"last":4.3520000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":2.2709999999999999,"15min":2.1179999999999999},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":8.8239999999999998,"5min":18.670999999999999,"15min":18.670999999999999},"last":5.0209999999999999},{"interface":"front","average":{"1min":2.218,"5min":2.6080000000000001,"15min":2.036},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.8620000000000001,"5min":20.009,"15min":20.009},"last":4.6669999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":2.766,"15min":1.901},"min":{"1min":0.39000000000000001,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":13.851000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":4.4059999999999997},{"interface":"front","average":{"1min":3.4079999999999999,"5min":3.4380000000000002,"15min":2.2839999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.436,"5min":61.777000000000001,"15min":61.777000000000001},"last":5.1120000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.6760000000000002,"15min":1.9059999999999999},"min":{"1min":0.44500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.683,"5min":59.146999999999998,"15min":59.146999999999998},"last":4.4409999999999998},{"interface":"front","average":{"1min":4.0019999999999998,"5min":3.3780000000000001,"15min":2.4009999999999998},"min":{"1min":0.41199999999999998,"5min":0.379,"15min":0.379},"max":{"1min":12.470000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":5.0579999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":2.468,"15min":2.0110000000000001},"min":{"1min":0.45500000000000002,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":12.385999999999999,"5min":18.640000000000001,"15min":18.640000000000001},"last":5.0869999999999997},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.0550000000000002,"15min":1.5489999999999999},"min":{"1min":0.40400000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.5750000000000002,"5min":21.954999999999998,"15min":21.954999999999998},"last":4.468}]}]},{"osd":2,"up_from":131,"seq":562640715931,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1466632,"kb_used_data":1062976,"kb_used_omap":81,"kb_used_meta":403566,"kb_avail":92278520,"statfs":{"total":95995035648,"available":94493204480,"internally_reserved":0,"allocated":1088487424,"data_stored":1087423445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":413251802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:32 2023","interfaces":[{"interface":"back","average":{"1min":6.1900000000000004,"5min":6.9480000000000004,"15min":5.8170000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":73.388000000000005,"5min":132.79499999999999,"15min":132.79499999999999},"last":9.3819999999999997},{"interface":"front","average":{"1min":6.2869999999999999,"5min":7.4640000000000004,"15min":6.2400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":73.405000000000001,"5min":132.58099999999999,"15min":132.58099999999999},"last":3.1920000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":5.9770000000000003,"5min":3.052,"15min":3.2160000000000002},"min":{"1min":0.27700000000000002,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":77.004999999999995,"5min":77.004999999999995,"15min":88.316000000000003},"last":3.3029999999999999},{"interface":"front","average":{"1min":6.3490000000000002,"5min":3.8029999999999999,"15min":3.4430000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":76.956000000000003,"5min":76.956000000000003,"15min":88.436999999999998},"last":1.3069999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":4.8250000000000002,"15min":4.306},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.999000000000001,"5min":112.895,"15min":112.895},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.3969999999999998,"5min":4.3090000000000002,"15min":3.5059999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":6.2670000000000003,"5min":113.995,"15min":113.995},"last":4.4500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":5.5330000000000004,"5min":2.6659999999999999,"15min":2.7250000000000001},"min":{"1min":0.44700000000000001,"5min":0.253,"15min":0.253},"max":{"1min":69.113,"5min":69.113,"15min":108.878},"last":3.4809999999999999},{"interface":"front","average":{"1min":5.8780000000000001,"5min":2.5960000000000001,"15min":3.3599999999999999},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":69.028999999999996,"5min":69.028999999999996,"15min":108.93000000000001},"last":3.335}]},{"osd":5,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":2.0830000000000002,"15min":2.9910000000000001},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":22.646000000000001,"5min":22.646000000000001,"15min":97.322999999999993},"last":3.2610000000000001},{"interface":"front","average":{"1min":4.1840000000000002,"5min":2.589,"15min":2.9460000000000002},"min":{"1min":0.39300000000000002,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":22.501000000000001,"5min":22.501000000000001,"15min":97.301000000000002},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":6.1219999999999999,"5min":3.911,"15min":3.2949999999999999},"min":{"1min":0.36099999999999999,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":59.655000000000001,"5min":98.597999999999999,"15min":98.597999999999999},"last":1.351},{"interface":"front","average":{"1min":5.125,"5min":3.375,"15min":2.6000000000000001},"min":{"1min":0.46999999999999997,"5min":0.313,"15min":0.313},"max":{"1min":59.612000000000002,"5min":98.509,"15min":98.509},"last":3.512}]},{"osd":7,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.1549999999999998,"15min":1.6599999999999999},"min":{"1min":0.40300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.9450000000000003,"5min":10.92,"15min":10.92},"last":3.5379999999999998},{"interface":"front","average":{"1min":2.149,"5min":1.9670000000000001,"15min":2.3759999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":6.9109999999999996,"5min":11.311,"15min":88.584000000000003},"last":3.4100000000000001}]}]},{"osd":3,"up_from":0,"seq":403726926017,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019216,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1249960,"kb_used_data":940448,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92495192,"statfs":{"total":95995035648,"available":94715076608,"internally_reserved":0,"allocated":963018752,"data_stored":962457747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":8.0700000000000003,"5min":4.423,"15min":6.367},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":82.376000000000005,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.878},{"interface":"front","average":{"1min":8.1430000000000007,"5min":4.9420000000000002,"15min":6.9160000000000004},"min":{"1min":0.34100000000000003,"5min":0.217,"15min":0.217},"max":{"1min":79.403000000000006,"5min":113.417,"15min":113.417},"last":4.0069999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":7.5090000000000003,"5min":4.9939999999999998,"15min":2.9249999999999998},"min":{"1min":0.504,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":70.418000000000006,"5min":132.304,"15min":132.304},"last":1.681},{"interface":"front","average":{"1min":6.6749999999999998,"5min":4.7859999999999996,"15min":2.726},"min":{"1min":0.29899999999999999,"5min":0.253,"15min":0.253},"max":{"1min":70.432000000000002,"5min":132.453,"15min":132.453},"last":1.7869999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.3860000000000001,"15min":3.1400000000000001},"min":{"1min":0.35999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.8070000000000004,"5min":20.308,"15min":84.909000000000006},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.4790000000000001,"15min":3.1469999999999998},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.673,"5min":21.704000000000001,"15min":83.840000000000003},"last":2.4289999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6429999999999998,"5min":5.4409999999999998,"15min":4.4729999999999999},"min":{"1min":0.39500000000000002,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":12.622999999999999,"5min":164.453,"15min":164.453},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.6230000000000002,"5min":5.5229999999999997,"15min":4.5609999999999999},"min":{"1min":0.40500000000000003,"5min":0.33300000000000002,"15min":0.27200000000000002},"max":{"1min":10.347,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.36399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.097,"15min":0.93799999999999994},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":10.321999999999999,"5min":12.238,"15min":43.189999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.079,"15min":0.93999999999999995},"min":{"1min":0.34200000000000003,"5min":0.248,"15min":0.248},"max":{"1min":6.4349999999999996,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.34300000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.0920000000000001,"15min":0.90400000000000003},"min":{"1min":0.378,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4039999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.379},{"interface":"front","average":{"1min":1.8109999999999999,"5min":1.091,"15min":0.92400000000000004},"min":{"1min":0.44400000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":10.423999999999999,"5min":12.269,"15min":26.940999999999999},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:53:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.0629999999999999,"15min":0.82199999999999995},"min":{"1min":0.435,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":10.278,"5min":12.311,"15min":12.311},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.98099999999999998,"15min":0.76100000000000001},"min":{"1min":0.38800000000000001,"5min":0.35799999999999998,"15min":0.29399999999999998},"max":{"1min":3.7909999999999999,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.77700000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822988,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2039044,"kb_used_data":1729532,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91706108,"statfs":{"total":95995035648,"available":93907054592,"internally_reserved":0,"allocated":1771040768,"data_stored":1770503568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":12.433,"5min":6.899,"15min":4.6070000000000002},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":185.62,"5min":185.62,"15min":185.62},"last":0.37},{"interface":"front","average":{"1min":12.449999999999999,"5min":7.0140000000000002,"15min":4.569},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":185.499,"5min":185.499,"15min":185.499},"last":0.248}]},{"osd":1,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0069999999999997,"5min":2.2759999999999998,"15min":2.2850000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.22500000000000001},"max":{"1min":17.658999999999999,"5min":41.463999999999999,"15min":149.203},"last":0.27900000000000003},{"interface":"front","average":{"1min":4.5030000000000001,"5min":2.6880000000000002,"15min":2.5899999999999999},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":17.84,"5min":41.954000000000001,"15min":149.554},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":12.965,"5min":5.3209999999999997,"15min":5.8570000000000002},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":189.126,"5min":189.126,"15min":189.126},"last":0.35999999999999999},{"interface":"front","average":{"1min":12.991,"5min":4.931,"15min":5.7809999999999997},"min":{"1min":0.498,"5min":0.318,"15min":0.28999999999999998},"max":{"1min":188.74799999999999,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":3.3929999999999998,"5min":5.4790000000000001,"15min":5.5510000000000002},"min":{"1min":0.316,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.99,"5min":98.641000000000005,"15min":257.05000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.6779999999999999,"5min":5.5949999999999998,"15min":5.4180000000000001},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.27800000000000002},"max":{"1min":10.362,"5min":98.906000000000006,"15min":257.35000000000002},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":0.84499999999999997,"15min":0.72199999999999998},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":6.3949999999999996,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.956,"5min":0.83799999999999997,"15min":0.72299999999999998},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":6.3550000000000004,"5min":6.3550000000000004,"15min":10.535},"last":0.316}]},{"osd":6,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.054,"15min":1.5649999999999999},"min":{"1min":0.34399999999999997,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.649,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.1699999999999999,"5min":1.0089999999999999,"15min":1.5880000000000001},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":10.314,"5min":10.314,"15min":139.739},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:53:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":0.97899999999999998,"15min":0.82799999999999996},"min":{"1min":0.40100000000000002,"5min":0.33400000000000002,"15min":0.29499999999999998},"max":{"1min":10.367000000000001,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.435},{"interface":"front","average":{"1min":1.353,"5min":0.89400000000000002,"15min":0.76100000000000001},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":6.1120000000000001,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.39300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1185087488,"data_stored":1185087488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2094796800,"data_stored":2094792728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1072365568,"data_stored":1072365568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1171849216,"data_stored":1171849216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":874582016,"data_stored":874577944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1767510016,"data_stored":1767505944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1405222912,"data_stored":1405222912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:54:06.732 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:06 smithi052 ceph-mon[187743]: from='client.45800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:06.858 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:06.930 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:06 smithi151 ceph-mon[159210]: from='client.45800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:07.192 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:07.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:07.300 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:07.349 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:07.405 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:07.492 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:54:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:54:07.492 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:140399 9.f 9:fb7600f6:::benchmark_data_smithi052_402718_object50758:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.444214+0000", 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.036113566, 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036147947999999999, 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:07.493 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140399 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444214+0000", 2023-09-23T22:54:07.494 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444214+0000", 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444212+0000", 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:07.495 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444218+0000", 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.276e-06 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.496 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444226+0000", 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5490000000000006e-06 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444232+0000", 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0680000000000002e-06 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.449776+0000", 2023-09-23T22:54:07.497 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055435299999999996 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.449919+0000", 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014324100000000001 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.450068+0000", 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014862499999999999 2023-09-23T22:54:07.498 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.455879+0000", 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058110979999999998 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.467836+0000", 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011957168000000001 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.499 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.476567+0000", 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0087306390000000001 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.476631+0000", 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4219000000000003e-05 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.500 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:140406 9.f 9:f1da3979:::benchmark_data_smithi052_402718_object50765:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.444247+0000", 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.036080815000000002, 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036223186999999997, 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:07.501 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140406 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444247+0000", 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.502 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444247+0000", 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444246+0000", 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:07.503 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444249+0000", 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4109999999999999e-06 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.504 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444254+0000", 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.339e-06 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.444259+0000", 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5909999999999998e-06 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.505 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.452783+0000", 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085245670000000003 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.452934+0000", 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015122599999999999 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:54:07.506 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.453087+0000", 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015304500000000001 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.456171+0000", 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030833420000000002 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:07.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.467884+0000", 2023-09-23T22:54:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01171313 2023-09-23T22:54:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:140398 9.2 e300/271 9:4bf0da3a:::benchmark_data_smithi052_402718_object50757:head v 300'8641, mlcod=0'0)", 2023-09-23T22:54:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.457683+0000", 2023-09-23T22:54:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.022644282000000002, 2023-09-23T22:54:07.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022871452, 2023-09-23T22:54:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.510 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457683+0000", 2023-09-23T22:54:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457683+0000", 2023-09-23T22:54:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457683+0000", 2023-09-23T22:54:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457690+0000", 2023-09-23T22:54:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9689999999999997e-06 2023-09-23T22:54:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457697+0000", 2023-09-23T22:54:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9e-06 2023-09-23T22:54:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457701+0000", 2023-09-23T22:54:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4989999999999996e-06 2023-09-23T22:54:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.468644+0000", 2023-09-23T22:54:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010942754000000001 2023-09-23T22:54:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.468675+0000", 2023-09-23T22:54:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0883000000000002e-05 2023-09-23T22:54:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.518 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:140407 9.f 9:ff255116:::benchmark_data_smithi052_402718_object50766:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-09-23T22:54:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.457851+0000", 2023-09-23T22:54:07.519 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.022476172999999999, 2023-09-23T22:54:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022766812000000001, 2023-09-23T22:54:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:54:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:07.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140407 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457851+0000", 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457851+0000", 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457850+0000", 2023-09-23T22:54:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:54:07.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457856+0000", 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.849e-06 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457868+0000", 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1775e-05 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.457876+0000", 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3690000000000001e-06 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.464113+0000", 2023-09-23T22:54:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062370079999999996 2023-09-23T22:54:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.464296+0000", 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000183878 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.464468+0000", 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017149900000000001 2023-09-23T22:54:07.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.467924+0000", 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034558710000000001 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469543+0000", 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001619182 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.528 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:140410 9.8 e300/271 9:1d7aa28f:::benchmark_data_smithi052_402718_object50769:head v 300'8707, mlcod=0'0)", 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.469909+0000", 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.010418393999999999, 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0107934, 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469909+0000", 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469909+0000", 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469909+0000", 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469915+0000", 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2260000000000002e-06 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469925+0000", 2023-09-23T22:54:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0443e-05 2023-09-23T22:54:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469932+0000", 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7299999999999999e-06 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473389+0000", 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034573070000000002 2023-09-23T22:54:07.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473412+0000", 2023-09-23T22:54:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3071e-05 2023-09-23T22:54:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.534 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:140411 9.5 e300/264 9:aa0adb53:::benchmark_data_smithi052_402718_object50770:head v 300'8721, mlcod=300'8719)", 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.469938+0000", 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.010389749, 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010824474000000001, 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.535 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469938+0000", 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469938+0000", 2023-09-23T22:54:07.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469937+0000", 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469942+0000", 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0849999999999999e-06 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469950+0000", 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5909999999999996e-06 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.469955+0000", 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.724e-06 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.470040+0000", 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5031999999999995e-05 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.470112+0000", 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2142000000000001e-05 2023-09-23T22:54:07.539 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:140416 9.3 e300/299 9:c3a7b118:::benchmark_data_smithi052_402718_object50775:head v 300'8758, mlcod=300'8756)", 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.472609+0000", 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0077183240000000004, 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0082126620000000008, 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472609+0000", 2023-09-23T22:54:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472609+0000", 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472608+0000", 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472617+0000", 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2059999999999996e-06 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472631+0000", 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3419e-05 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472639+0000", 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2949999999999994e-06 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472700+0000", 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0985e-05 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.472724+0000", 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4000999999999999e-05 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:140413 9.d e300/264 9:bbdb26ea:::benchmark_data_smithi052_402718_object50772:head v 300'8889, mlcod=300'8887)", 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.473900+0000", 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0064278130000000001, 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070301790000000001, 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473900+0000", 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473900+0000", 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473899+0000", 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473906+0000", 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8019999999999992e-06 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473917+0000", 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0200000000000001e-05 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.473924+0000", 2023-09-23T22:54:07.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9659999999999999e-06 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475743+0000", 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018189269999999999 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475762+0000", 2023-09-23T22:54:07.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9627999999999999e-05 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:140399 9.f e300/271 ondisk, result = 0)", 2023-09-23T22:54:07.551 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.475556+0000", 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0047714289999999998, 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054320460000000003, 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475556+0000", 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475556+0000", 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475555+0000", 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475560+0000", 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9779999999999996e-06 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475577+0000", 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7107000000000001e-05 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475586+0000", 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0490000000000004e-06 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.476548+0000", 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00096275600000000005 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.476559+0000", 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0423e-05 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:140406 9.f e300/271)", 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.475598+0000", 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0047296209999999998, 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054457350000000002, 2023-09-23T22:54:07.557 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475598+0000", 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475598+0000", 2023-09-23T22:54:07.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475598+0000", 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475600+0000", 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1859999999999999e-06 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475605+0000", 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3000000000000001e-06 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475609+0000", 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9920000000000001e-06 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.560 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45593.0:140407 9.f e300/271)", 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.475612+0000", 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0047149180000000002, 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054830499999999997, 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "queued for pg", 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475612+0000", 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475612+0000", 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475612+0000", 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475614+0000", 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7689999999999999e-06 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475619+0000", 2023-09-23T22:54:07.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5340000000000001e-06 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475623+0000", 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8689999999999997e-06 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45593.0:140409 9.6 e300/264 9:63859d9a:::benchmark_data_smithi052_402718_object50768:head v 300'8804, mlcod=300'8802)", 2023-09-23T22:54:07.564 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:07.475627+0000", 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0047007009999999998, 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055183640000000004, 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475627+0000", 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475627+0000", 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475627+0000", 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T22:54:07.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475632+0000", 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.22e-06 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475637+0000", 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1549999999999998e-06 2023-09-23T22:54:07.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475641+0000", 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9600000000000002e-06 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475719+0000", 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8589000000000003e-05 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:07.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:07.475746+0000", 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6176000000000001e-05 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 12 2023-09-23T22:54:07.569 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:54:07.598 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:07.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:07.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:07 smithi052 ceph-mon[180979]: pgmap v867: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 8389/124161 objects degraded (6.757%); 1136/124161 objects misplaced (0.915%); 0 B/s, 100 objects/s recovering 2023-09-23T22:54:07.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:07 smithi052 ceph-mon[187743]: pgmap v867: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 8389/124161 objects degraded (6.757%); 1136/124161 objects misplaced (0.915%); 0 B/s, 100 objects/s recovering 2023-09-23T22:54:07.703 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:07.820 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:07 smithi151 ceph-mon[159210]: pgmap v867: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 8389/124161 objects degraded (6.757%); 1136/124161 objects misplaced (0.915%); 0 B/s, 100 objects/s recovering 2023-09-23T22:54:08.865 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:08.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:08.972 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:08.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:54:09.085 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:09.732 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:54:09.788 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:09 smithi052 ceph-mon[187743]: pgmap v868: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 8389/118332 objects degraded (7.089%); 1136/118332 objects misplaced (0.960%); 0 B/s, 97 objects/s recovering 2023-09-23T22:54:09.789 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:09 smithi052 ceph-mon[180979]: pgmap v868: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 8389/118332 objects degraded (7.089%); 1136/118332 objects misplaced (0.960%); 0 B/s, 97 objects/s recovering 2023-09-23T22:54:09.825 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:09.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:09.936 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:09.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:09 smithi151 ceph-mon[159210]: pgmap v868: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 8389/118332 objects degraded (7.089%); 1136/118332 objects misplaced (0.960%); 0 B/s, 97 objects/s recovering 2023-09-23T22:54:09.971 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:10.434 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:10.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:10.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:10.539 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:10.628 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:10.648 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:10.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8389/118332 objects degraded (7.089%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:10.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8389/118332 objects degraded (7.089%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:10.836 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:10.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:10.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:10.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8389/118332 objects degraded (7.089%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:11.009 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:11.053 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:11.155 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:11.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:11.262 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:11.368 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:11.604 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:54:11.653 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:54:11.690 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:54:11.732 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:54:11.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:11 smithi052 ceph-mon[180979]: pgmap v869: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 8389/116358 objects degraded (7.210%); 1136/116358 objects misplaced (0.976%); 0 B/s, 80 objects/s recovering 2023-09-23T22:54:11.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:11 smithi052 ceph-mon[187743]: pgmap v869: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 8389/116358 objects degraded (7.210%); 1136/116358 objects misplaced (0.976%); 0 B/s, 80 objects/s recovering 2023-09-23T22:54:11.772 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:11.772 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:54:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:11.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:11.840 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:54:11.869 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:54:11.879 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:11.910 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:54:11.929 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:54:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:11 smithi151 ceph-mon[159210]: pgmap v869: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 612 op/s; 8389/116358 objects degraded (7.210%); 1136/116358 objects misplaced (0.976%); 0 B/s, 80 objects/s recovering 2023-09-23T22:54:11.983 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:12.579 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:54:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3743259589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1622971648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:12 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:12.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3743259589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:12.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1622971648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:12.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:12 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:12.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3743259589' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1622971648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:12 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:13.409 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:54:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:54:13] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.33.4" 2023-09-23T22:54:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[187743]: pgmap v870: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 8389/110991 objects degraded (7.558%); 754/110991 objects misplaced (0.679%); 0 B/s, 79 objects/s recovering 2023-09-23T22:54:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:54:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[187743]: failed to encode map e301 with expected crc 2023-09-23T22:54:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[187743]: osdmap e301: 8 total, 8 up, 6 in 2023-09-23T22:54:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[187743]: failed to encode map e301 with expected crc 2023-09-23T22:54:13.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[180979]: pgmap v870: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 8389/110991 objects degraded (7.558%); 754/110991 objects misplaced (0.679%); 0 B/s, 79 objects/s recovering 2023-09-23T22:54:13.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:54:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[180979]: failed to encode map e301 with expected crc 2023-09-23T22:54:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[180979]: osdmap e301: 8 total, 8 up, 6 in 2023-09-23T22:54:13.748 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:13 smithi052 ceph-mon[180979]: failed to encode map e301 with expected crc 2023-09-23T22:54:13.819 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:13.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:13.928 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:13.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:13 smithi151 ceph-mon[159210]: pgmap v870: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 8389/110991 objects degraded (7.558%); 754/110991 objects misplaced (0.679%); 0 B/s, 79 objects/s recovering 2023-09-23T22:54:13.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:13 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:54:13.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:13 smithi151 ceph-mon[159210]: failed to encode map e301 with expected crc 2023-09-23T22:54:13.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:13.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:13 smithi151 ceph-mon[159210]: osdmap e301: 8 total, 8 up, 6 in 2023-09-23T22:54:13.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:13 smithi151 ceph-mon[159210]: failed to encode map e301 with expected crc 2023-09-23T22:54:14.019 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:14.585 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:14.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:14.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:14.693 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:14.791 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:14.915 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:14.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:15.027 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:15.087 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:15.162 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:15.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:15.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:15.270 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:15.315 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:15.389 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:15.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:15.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:15.503 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:15.575 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:15 smithi052 ceph-mon[180979]: pgmap v872: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 8389/108714 objects degraded (7.717%); 754/108714 objects misplaced (0.694%); 0 B/s, 72 objects/s recovering 2023-09-23T22:54:15.575 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8389/108714 objects degraded (7.717%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:15.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:15 smithi052 ceph-mon[187743]: pgmap v872: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 8389/108714 objects degraded (7.717%); 754/108714 objects misplaced (0.694%); 0 B/s, 72 objects/s recovering 2023-09-23T22:54:15.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8389/108714 objects degraded (7.717%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:15.618 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:15.683 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:54:15.683 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:54:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:15 smithi151 ceph-mon[159210]: pgmap v872: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 8389/108714 objects degraded (7.717%); 754/108714 objects misplaced (0.694%); 0 B/s, 72 objects/s recovering 2023-09-23T22:54:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8389/108714 objects degraded (7.717%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:15.726 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:15.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:15.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:15.849 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:15.940 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:54:15.946 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:15.955 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:54:16.052 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:16.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:16.160 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:16.195 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:16.270 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:16.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:16.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:16.373 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:54:16.380 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:16.484 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:16.784 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:54:17.113 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:17.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:17.227 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:17.299 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:17.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:17.433 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:54:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:17.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:17.532 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":872,"stamp":"2023-09-23T22:54:14.414680+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2373851064,"num_objects":36238,"num_object_clones":0,"num_object_copies":108714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8389,"num_objects_misplaced":754,"num_objects_unfound":0,"num_objects_dirty":36238,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":143324,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7694,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115658,"ondisk_log_size":115658,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9513128,"kb_used_data":7089976,"kb_used_omap":121,"kb_used_meta":2422598,"kb_avail":552957784,"statfs":{"total":575970213888,"available":566228770816,"internally_reserved":0,"allocated":7260135424,"data_stored":7255223643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2480740469},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,3,1,6,4,0,9],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-424280064,"num_objects":-6474,"num_object_clones":0,"num_object_copies":-19422,"num_objects_missing_on_primary":0,"num_objects_missing":-2,"num_objects_degraded":0,"num_objects_misplaced":-724,"num_objects_unfound":0,"num_objects_dirty":-6474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6474,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":727,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003215"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":297,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132199+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:54:00.132199+0000","last_peered":"2023-09-23T22:54:00.132199+0000","last_clean":"2023-09-23T22:54:00.132199+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:54:00.132199+0000","last_undegraded":"2023-09-23T22:54:00.132199+0000","last_fullsized":"2023-09-23T22:54:00.132199+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":96,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132146+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:54:00.132146+0000","last_peered":"2023-09-23T22:54:00.132146+0000","last_clean":"2023-09-23T22:54:00.132146+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:54:00.132146+0000","last_undegraded":"2023-09-23T22:54:00.132146+0000","last_fullsized":"2023-09-23T22:54:00.132146+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132511+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:54:00.132511+0000","last_peered":"2023-09-23T22:54:00.132511+0000","last_clean":"2023-09-23T22:54:00.132511+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:54:00.132511+0000","last_undegraded":"2023-09-23T22:54:00.132511+0000","last_fullsized":"2023-09-23T22:54:00.132511+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":280,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T22:54:12.565884+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:54:12.565884+0000","last_peered":"2023-09-23T22:54:12.565884+0000","last_clean":"2023-09-23T22:54:12.565884+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:54:12.565884+0000","last_undegraded":"2023-09-23T22:54:12.565884+0000","last_fullsized":"2023-09-23T22:54:12.565884+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":284,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:03.254976+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:54:03.254976+0000","last_peered":"2023-09-23T22:54:03.254976+0000","last_clean":"2023-09-23T22:54:03.254976+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:54:03.254976+0000","last_undegraded":"2023-09-23T22:54:03.254976+0000","last_fullsized":"2023-09-23T22:54:03.254976+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":265,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118883+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:54:00.118883+0000","last_peered":"2023-09-23T22:54:00.118883+0000","last_clean":"2023-09-23T22:54:00.118883+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:54:00.118883+0000","last_undegraded":"2023-09-23T22:54:00.118883+0000","last_fullsized":"2023-09-23T22:54:00.118883+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":211,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194604+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:54:01.194604+0000","last_peered":"2023-09-23T22:54:01.194604+0000","last_clean":"2023-09-23T22:54:01.194604+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:54:01.194604+0000","last_undegraded":"2023-09-23T22:54:01.194604+0000","last_fullsized":"2023-09-23T22:54:01.194604+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":162,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.127153+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:54:00.127153+0000","last_peered":"2023-09-23T22:54:00.127153+0000","last_clean":"2023-09-23T22:54:00.127153+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:54:00.127153+0000","last_undegraded":"2023-09-23T22:54:00.127153+0000","last_fullsized":"2023-09-23T22:54:00.127153+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":41,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.153734+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:54:00.153734+0000","last_peered":"2023-09-23T22:54:00.153734+0000","last_clean":"2023-09-23T22:54:00.153734+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:54:00.153734+0000","last_undegraded":"2023-09-23T22:54:00.153734+0000","last_fullsized":"2023-09-23T22:54:00.153734+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.117410+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:54:00.117410+0000","last_peered":"2023-09-23T22:54:00.117410+0000","last_clean":"2023-09-23T22:54:00.117410+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:54:00.117410+0000","last_undegraded":"2023-09-23T22:54:00.117410+0000","last_fullsized":"2023-09-23T22:54:00.117410+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":58,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132002+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:54:00.132002+0000","last_peered":"2023-09-23T22:54:00.132002+0000","last_clean":"2023-09-23T22:54:00.132002+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:54:00.132002+0000","last_undegraded":"2023-09-23T22:54:00.132002+0000","last_fullsized":"2023-09-23T22:54:00.132002+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":174,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118296+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:54:00.118296+0000","last_peered":"2023-09-23T22:54:00.118296+0000","last_clean":"2023-09-23T22:54:00.118296+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:54:00.118296+0000","last_undegraded":"2023-09-23T22:54:00.118296+0000","last_fullsized":"2023-09-23T22:54:00.118296+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":319,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131860+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:54:00.131860+0000","last_peered":"2023-09-23T22:54:00.131860+0000","last_clean":"2023-09-23T22:54:00.131860+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:54:00.131860+0000","last_undegraded":"2023-09-23T22:54:00.131860+0000","last_fullsized":"2023-09-23T22:54:00.131860+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":265,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131675+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:54:00.131675+0000","last_peered":"2023-09-23T22:54:00.131675+0000","last_clean":"2023-09-23T22:54:00.131675+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:54:00.131675+0000","last_undegraded":"2023-09-23T22:54:00.131675+0000","last_fullsized":"2023-09-23T22:54:00.131675+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":158,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132144+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:54:00.132144+0000","last_peered":"2023-09-23T22:54:00.132144+0000","last_clean":"2023-09-23T22:54:00.132144+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:54:00.132144+0000","last_undegraded":"2023-09-23T22:54:00.132144+0000","last_fullsized":"2023-09-23T22:54:00.132144+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":247,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128188+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:54:00.128188+0000","last_peered":"2023-09-23T22:54:00.128188+0000","last_clean":"2023-09-23T22:54:00.128188+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:54:00.128188+0000","last_undegraded":"2023-09-23T22:54:00.128188+0000","last_fullsized":"2023-09-23T22:54:00.128188+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":267,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.198535+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:54:01.198535+0000","last_peered":"2023-09-23T22:54:01.198535+0000","last_clean":"2023-09-23T22:54:01.198535+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:54:01.198535+0000","last_undegraded":"2023-09-23T22:54:01.198535+0000","last_fullsized":"2023-09-23T22:54:01.198535+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":223,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.117792+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:54:00.117792+0000","last_peered":"2023-09-23T22:54:00.117792+0000","last_clean":"2023-09-23T22:54:00.117792+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:54:00.117792+0000","last_undegraded":"2023-09-23T22:54:00.117792+0000","last_fullsized":"2023-09-23T22:54:00.117792+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132692+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:54:00.132692+0000","last_peered":"2023-09-23T22:54:00.132692+0000","last_clean":"2023-09-23T22:54:00.132692+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:54:00.132692+0000","last_undegraded":"2023-09-23T22:54:00.132692+0000","last_fullsized":"2023-09-23T22:54:00.132692+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":268,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.195265+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:54:01.195265+0000","last_peered":"2023-09-23T22:54:01.195265+0000","last_clean":"2023-09-23T22:54:01.195265+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:54:01.195265+0000","last_undegraded":"2023-09-23T22:54:01.195265+0000","last_fullsized":"2023-09-23T22:54:01.195265+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":344,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.119350+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:54:00.119350+0000","last_peered":"2023-09-23T22:54:00.119350+0000","last_clean":"2023-09-23T22:54:00.119350+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:54:00.119350+0000","last_undegraded":"2023-09-23T22:54:00.119350+0000","last_fullsized":"2023-09-23T22:54:00.119350+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":227,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152314+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:54:00.152314+0000","last_peered":"2023-09-23T22:54:00.152314+0000","last_clean":"2023-09-23T22:54:00.152314+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:54:00.152314+0000","last_undegraded":"2023-09-23T22:54:00.152314+0000","last_fullsized":"2023-09-23T22:54:00.152314+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"301'8926","reported_seq":8972,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T22:54:13.134506+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:54:13.134506+0000","last_peered":"2023-09-23T22:54:13.134506+0000","last_clean":"2023-09-23T22:54:13.134506+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:54:13.134506+0000","last_undegraded":"2023-09-23T22:54:13.134506+0000","last_fullsized":"2023-09-23T22:54:13.134506+0000","mapping_epoch":264,"log_start":"269'1400","ondisk_log_start":"269'1400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7526,"log_dups_size":0,"ondisk_log_size":7526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144572416,"num_objects":2206,"num_object_clones":0,"num_object_copies":6618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8926,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":169,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131814+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:54:00.131814+0000","last_peered":"2023-09-23T22:54:00.131814+0000","last_clean":"2023-09-23T22:54:00.131814+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:54:00.131814+0000","last_undegraded":"2023-09-23T22:54:00.131814+0000","last_fullsized":"2023-09-23T22:54:00.131814+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":264,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.119439+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:54:00.119439+0000","last_peered":"2023-09-23T22:54:00.119439+0000","last_clean":"2023-09-23T22:54:00.119439+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:54:00.119439+0000","last_undegraded":"2023-09-23T22:54:00.119439+0000","last_fullsized":"2023-09-23T22:54:00.119439+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"300'9140","reported_seq":9191,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:11.618019+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:54:11.618019+0000","last_peered":"2023-09-23T22:54:11.618019+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:54:11.618019+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"273'2400","ondisk_log_start":"273'2400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154140672,"num_objects":2352,"num_object_clones":0,"num_object_copies":7056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9140,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":691}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":179,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.133698+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:54:00.133698+0000","last_peered":"2023-09-23T22:54:00.133698+0000","last_clean":"2023-09-23T22:54:00.133698+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:54:00.133698+0000","last_undegraded":"2023-09-23T22:54:00.133698+0000","last_fullsized":"2023-09-23T22:54:00.133698+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":289,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132608+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:54:00.132608+0000","last_peered":"2023-09-23T22:54:00.132608+0000","last_clean":"2023-09-23T22:54:00.132608+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:54:00.132608+0000","last_undegraded":"2023-09-23T22:54:00.132608+0000","last_fullsized":"2023-09-23T22:54:00.132608+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"300'8986","reported_seq":9038,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:09.930698+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:54:09.930698+0000","last_peered":"2023-09-23T22:54:09.930698+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:54:09.930698+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"268'400","ondisk_log_start":"268'400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8586,"log_dups_size":0,"ondisk_log_size":8586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150863872,"num_objects":2302,"num_object_clones":0,"num_object_copies":6906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8986,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":531}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":123,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.119617+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:54:00.119617+0000","last_peered":"2023-09-23T22:54:00.119617+0000","last_clean":"2023-09-23T22:54:00.119617+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:54:00.119617+0000","last_undegraded":"2023-09-23T22:54:00.119617+0000","last_fullsized":"2023-09-23T22:54:00.119617+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":292,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118188+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:54:00.118188+0000","last_peered":"2023-09-23T22:54:00.118188+0000","last_clean":"2023-09-23T22:54:00.118188+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:54:00.118188+0000","last_undegraded":"2023-09-23T22:54:00.118188+0000","last_fullsized":"2023-09-23T22:54:00.118188+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"300'8916","reported_seq":8961,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:09.947574+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:54:09.947574+0000","last_peered":"2023-09-23T22:54:09.947574+0000","last_clean":"2023-09-23T22:54:09.947574+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:54:09.947574+0000","last_undegraded":"2023-09-23T22:54:09.947574+0000","last_fullsized":"2023-09-23T22:54:09.947574+0000","mapping_epoch":264,"log_start":"268'300","ondisk_log_start":"268'300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8916,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.117805+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:54:00.117805+0000","last_peered":"2023-09-23T22:54:00.117805+0000","last_clean":"2023-09-23T22:54:00.117805+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:54:00.117805+0000","last_undegraded":"2023-09-23T22:54:00.117805+0000","last_fullsized":"2023-09-23T22:54:00.117805+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":195,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.198419+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:54:01.198419+0000","last_peered":"2023-09-23T22:54:01.198419+0000","last_clean":"2023-09-23T22:54:01.198419+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:54:01.198419+0000","last_undegraded":"2023-09-23T22:54:01.198419+0000","last_fullsized":"2023-09-23T22:54:01.198419+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"300'8817","reported_seq":12227,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:10.995596+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:54:10.995596+0000","last_peered":"2023-09-23T22:54:10.995596+0000","last_clean":"2023-09-23T22:54:10.995596+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:54:10.995596+0000","last_undegraded":"2023-09-23T22:54:10.995596+0000","last_fullsized":"2023-09-23T22:54:10.995596+0000","mapping_epoch":289,"log_start":"273'2400","ondisk_log_start":"273'2400","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6417,"log_dups_size":0,"ondisk_log_size":6417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143589376,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8817,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":67,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.198435+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:54:01.198435+0000","last_peered":"2023-09-23T22:54:01.198435+0000","last_clean":"2023-09-23T22:54:01.198435+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:54:01.198435+0000","last_undegraded":"2023-09-23T22:54:01.198435+0000","last_fullsized":"2023-09-23T22:54:01.198435+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":273,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131778+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:54:00.131778+0000","last_peered":"2023-09-23T22:54:00.131778+0000","last_clean":"2023-09-23T22:54:00.131778+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:54:00.131778+0000","last_undegraded":"2023-09-23T22:54:00.131778+0000","last_fullsized":"2023-09-23T22:54:00.131778+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"300'8764","reported_seq":8808,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:10.992690+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:54:10.992690+0000","last_peered":"2023-09-23T22:54:10.992690+0000","last_clean":"2023-09-23T22:54:10.992690+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:54:10.992690+0000","last_undegraded":"2023-09-23T22:54:10.992690+0000","last_fullsized":"2023-09-23T22:54:10.992690+0000","mapping_epoch":264,"log_start":"273'2300","ondisk_log_start":"273'2300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151912448,"num_objects":2318,"num_object_clones":0,"num_object_copies":6954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8764,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":284,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118037+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:54:00.118037+0000","last_peered":"2023-09-23T22:54:00.118037+0000","last_clean":"2023-09-23T22:54:00.118037+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:54:00.118037+0000","last_undegraded":"2023-09-23T22:54:00.118037+0000","last_fullsized":"2023-09-23T22:54:00.118037+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":295,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131811+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:54:00.131811+0000","last_peered":"2023-09-23T22:54:00.131811+0000","last_clean":"2023-09-23T22:54:00.131811+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:54:00.131811+0000","last_undegraded":"2023-09-23T22:54:00.131811+0000","last_fullsized":"2023-09-23T22:54:00.131811+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"300'8900","reported_seq":12298,"reported_epoch":300,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:54:10.989660+0000","last_change":"2023-09-23T22:54:01.194508+0000","last_active":"2023-09-23T22:54:10.989660+0000","last_peered":"2023-09-23T22:54:10.989660+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:54:01.140849+0000","last_became_peered":"2023-09-23T22:54:01.140849+0000","last_unstale":"2023-09-23T22:54:10.989660+0000","last_undegraded":"2023-09-23T22:54:10.989660+0000","last_fullsized":"2023-09-23T22:54:10.989660+0000","mapping_epoch":299,"log_start":"277'3400","ondisk_log_start":"277'3400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149422080,"num_objects":2280,"num_object_clones":0,"num_object_copies":6840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8900,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1668,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":["6","1","3","5"],"object_location_counts":[{"shards":"1,5,6","objects":2280}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":308,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132107+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:54:00.132107+0000","last_peered":"2023-09-23T22:54:00.132107+0000","last_clean":"2023-09-23T22:54:00.132107+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:54:00.132107+0000","last_undegraded":"2023-09-23T22:54:00.132107+0000","last_fullsized":"2023-09-23T22:54:00.132107+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":283,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.133747+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:54:00.133747+0000","last_peered":"2023-09-23T22:54:00.133747+0000","last_clean":"2023-09-23T22:54:00.133747+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:54:00.133747+0000","last_undegraded":"2023-09-23T22:54:00.133747+0000","last_fullsized":"2023-09-23T22:54:00.133747+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"300'8754","reported_seq":8806,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:09.909717+0000","last_change":"2023-09-23T22:51:35.515100+0000","last_active":"2023-09-23T22:54:09.909717+0000","last_peered":"2023-09-23T22:54:09.909717+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:54:09.909717+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"268'100","ondisk_log_start":"268'100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8654,"log_dups_size":0,"ondisk_log_size":8654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151781376,"num_objects":2316,"num_object_clones":0,"num_object_copies":6948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8754,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":701}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":228,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131404+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:54:00.131404+0000","last_peered":"2023-09-23T22:54:00.131404+0000","last_clean":"2023-09-23T22:54:00.131404+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:54:00.131404+0000","last_undegraded":"2023-09-23T22:54:00.131404+0000","last_fullsized":"2023-09-23T22:54:00.131404+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":237,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.152443+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:54:00.152443+0000","last_peered":"2023-09-23T22:54:00.152443+0000","last_clean":"2023-09-23T22:54:00.152443+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:54:00.152443+0000","last_undegraded":"2023-09-23T22:54:00.152443+0000","last_fullsized":"2023-09-23T22:54:00.152443+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"300'9077","reported_seq":9128,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:12.149444+0000","last_change":"2023-09-23T22:51:35.526543+0000","last_active":"2023-09-23T22:54:12.149444+0000","last_peered":"2023-09-23T22:54:12.149444+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:54:12.149444+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"278'3900","ondisk_log_start":"278'3900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149094400,"num_objects":2275,"num_object_clones":0,"num_object_copies":6825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9077,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,5","objects":645}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":339,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.196731+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:54:01.196731+0000","last_peered":"2023-09-23T22:54:01.196731+0000","last_clean":"2023-09-23T22:54:01.196731+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:54:01.196731+0000","last_undegraded":"2023-09-23T22:54:01.196731+0000","last_fullsized":"2023-09-23T22:54:01.196731+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":163,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128441+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:54:00.128441+0000","last_peered":"2023-09-23T22:54:00.128441+0000","last_clean":"2023-09-23T22:54:00.128441+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:54:00.128441+0000","last_undegraded":"2023-09-23T22:54:00.128441+0000","last_fullsized":"2023-09-23T22:54:00.128441+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":131,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132102+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:54:00.132102+0000","last_peered":"2023-09-23T22:54:00.132102+0000","last_clean":"2023-09-23T22:54:00.132102+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:54:00.132102+0000","last_undegraded":"2023-09-23T22:54:00.132102+0000","last_fullsized":"2023-09-23T22:54:00.132102+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"301'9022","reported_seq":12471,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T22:54:13.071211+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:54:13.071211+0000","last_peered":"2023-09-23T22:54:13.071211+0000","last_clean":"2023-09-23T22:54:13.071211+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:54:13.071211+0000","last_undegraded":"2023-09-23T22:54:13.071211+0000","last_fullsized":"2023-09-23T22:54:13.071211+0000","mapping_epoch":282,"log_start":"269'1500","ondisk_log_start":"269'1500","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141295616,"num_objects":2156,"num_object_clones":0,"num_object_copies":6468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9022,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":134,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118115+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:54:00.118115+0000","last_peered":"2023-09-23T22:54:00.118115+0000","last_clean":"2023-09-23T22:54:00.118115+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:54:00.118115+0000","last_undegraded":"2023-09-23T22:54:00.118115+0000","last_fullsized":"2023-09-23T22:54:00.118115+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":296,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131839+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:54:00.131839+0000","last_peered":"2023-09-23T22:54:00.131839+0000","last_clean":"2023-09-23T22:54:00.131839+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:54:00.131839+0000","last_undegraded":"2023-09-23T22:54:00.131839+0000","last_fullsized":"2023-09-23T22:54:00.131839+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128237+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:54:00.128237+0000","last_peered":"2023-09-23T22:54:00.128237+0000","last_clean":"2023-09-23T22:54:00.128237+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:54:00.128237+0000","last_undegraded":"2023-09-23T22:54:00.128237+0000","last_fullsized":"2023-09-23T22:54:00.128237+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"300'8952","reported_seq":10808,"reported_epoch":300,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:54:11.185402+0000","last_change":"2023-09-23T22:53:59.962748+0000","last_active":"2023-09-23T22:54:11.185402+0000","last_peered":"2023-09-23T22:54:11.185402+0000","last_clean":"2023-09-23T22:51:33.378550+0000","last_became_active":"2023-09-23T22:51:35.503481+0000","last_became_peered":"2023-09-23T22:51:35.503481+0000","last_unstale":"2023-09-23T22:54:11.185402+0000","last_undegraded":"2023-09-23T22:54:11.185402+0000","last_fullsized":"2023-09-23T22:54:11.185402+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8952,"log_dups_size":0,"ondisk_log_size":8952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:33.902250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152305664,"num_objects":2324,"num_object_clones":0,"num_object_copies":6972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":754,"num_objects_unfound":0,"num_objects_dirty":2324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8952,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":901,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":1570}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":302,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.127230+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:54:00.127230+0000","last_peered":"2023-09-23T22:54:00.127230+0000","last_clean":"2023-09-23T22:54:00.127230+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:54:00.127230+0000","last_undegraded":"2023-09-23T22:54:00.127230+0000","last_fullsized":"2023-09-23T22:54:00.127230+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":181,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.127072+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:54:00.127072+0000","last_peered":"2023-09-23T22:54:00.127072+0000","last_clean":"2023-09-23T22:54:00.127072+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:54:00.127072+0000","last_undegraded":"2023-09-23T22:54:00.127072+0000","last_fullsized":"2023-09-23T22:54:00.127072+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":308,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194510+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:54:01.194510+0000","last_peered":"2023-09-23T22:54:01.194510+0000","last_clean":"2023-09-23T22:54:01.194510+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:54:01.194510+0000","last_undegraded":"2023-09-23T22:54:01.194510+0000","last_fullsized":"2023-09-23T22:54:01.194510+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"301'8958","reported_seq":9012,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:13.126086+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:54:13.126086+0000","last_peered":"2023-09-23T22:54:13.126086+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:54:13.126086+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"269'1400","ondisk_log_start":"269'1400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8958,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":380}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":311,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128163+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:54:00.128163+0000","last_peered":"2023-09-23T22:54:00.128163+0000","last_clean":"2023-09-23T22:54:00.128163+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:54:00.128163+0000","last_undegraded":"2023-09-23T22:54:00.128163+0000","last_fullsized":"2023-09-23T22:54:00.128163+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":390,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T22:54:12.566226+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:54:12.566226+0000","last_peered":"2023-09-23T22:54:12.566226+0000","last_clean":"2023-09-23T22:54:12.566226+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:54:12.566226+0000","last_undegraded":"2023-09-23T22:54:12.566226+0000","last_fullsized":"2023-09-23T22:54:12.566226+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":54,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118245+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:54:00.118245+0000","last_peered":"2023-09-23T22:54:00.118245+0000","last_clean":"2023-09-23T22:54:00.118245+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:54:00.118245+0000","last_undegraded":"2023-09-23T22:54:00.118245+0000","last_fullsized":"2023-09-23T22:54:00.118245+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":292,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.117633+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:54:00.117633+0000","last_peered":"2023-09-23T22:54:00.117633+0000","last_clean":"2023-09-23T22:54:00.117633+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:54:00.117633+0000","last_undegraded":"2023-09-23T22:54:00.117633+0000","last_fullsized":"2023-09-23T22:54:00.117633+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"301'9018","reported_seq":9065,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T22:54:13.041327+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:54:13.041327+0000","last_peered":"2023-09-23T22:54:13.041327+0000","last_clean":"2023-09-23T22:54:13.041327+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:54:13.041327+0000","last_undegraded":"2023-09-23T22:54:13.041327+0000","last_fullsized":"2023-09-23T22:54:13.041327+0000","mapping_epoch":264,"log_start":"269'1500","ondisk_log_start":"269'1500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144703488,"num_objects":2208,"num_object_clones":0,"num_object_copies":6624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9018,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194926+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:54:01.194926+0000","last_peered":"2023-09-23T22:54:01.194926+0000","last_clean":"2023-09-23T22:54:01.194926+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:54:01.194926+0000","last_undegraded":"2023-09-23T22:54:01.194926+0000","last_fullsized":"2023-09-23T22:54:01.194926+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":248,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.203233+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:54:01.203233+0000","last_peered":"2023-09-23T22:54:01.203233+0000","last_clean":"2023-09-23T22:54:01.203233+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:54:01.203233+0000","last_undegraded":"2023-09-23T22:54:01.203233+0000","last_fullsized":"2023-09-23T22:54:01.203233+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":226,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.195038+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:54:01.195038+0000","last_peered":"2023-09-23T22:54:01.195038+0000","last_clean":"2023-09-23T22:54:01.195038+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:54:01.195038+0000","last_undegraded":"2023-09-23T22:54:01.195038+0000","last_fullsized":"2023-09-23T22:54:01.195038+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"300'8739","reported_seq":8783,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:11.572987+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:54:11.572987+0000","last_peered":"2023-09-23T22:54:11.572987+0000","last_clean":"2023-09-23T22:54:11.572987+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:54:11.572987+0000","last_undegraded":"2023-09-23T22:54:11.572987+0000","last_fullsized":"2023-09-23T22:54:11.572987+0000","mapping_epoch":264,"log_start":"273'2000","ondisk_log_start":"273'2000","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146472960,"num_objects":2235,"num_object_clones":0,"num_object_copies":6705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8739,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":231,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.196656+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:54:01.196656+0000","last_peered":"2023-09-23T22:54:01.196656+0000","last_clean":"2023-09-23T22:54:01.196656+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:54:01.196656+0000","last_undegraded":"2023-09-23T22:54:01.196656+0000","last_fullsized":"2023-09-23T22:54:01.196656+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":180,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128304+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:54:00.128304+0000","last_peered":"2023-09-23T22:54:00.128304+0000","last_clean":"2023-09-23T22:54:00.128304+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:54:00.128304+0000","last_undegraded":"2023-09-23T22:54:00.128304+0000","last_fullsized":"2023-09-23T22:54:00.128304+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":231,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.198582+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:54:01.198582+0000","last_peered":"2023-09-23T22:54:01.198582+0000","last_clean":"2023-09-23T22:54:01.198582+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:54:01.198582+0000","last_undegraded":"2023-09-23T22:54:01.198582+0000","last_fullsized":"2023-09-23T22:54:01.198582+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"300'9044","reported_seq":9089,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:11.568531+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:54:11.568531+0000","last_peered":"2023-09-23T22:54:11.568531+0000","last_clean":"2023-09-23T22:54:11.568531+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:54:11.568531+0000","last_undegraded":"2023-09-23T22:54:11.568531+0000","last_fullsized":"2023-09-23T22:54:11.568531+0000","mapping_epoch":264,"log_start":"273'2300","ondisk_log_start":"273'2300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846936,"num_objects":2318,"num_object_clones":0,"num_object_copies":6954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2318,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9044,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":58,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194873+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:54:01.194873+0000","last_peered":"2023-09-23T22:54:01.194873+0000","last_clean":"2023-09-23T22:54:01.194873+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:54:01.194873+0000","last_undegraded":"2023-09-23T22:54:01.194873+0000","last_fullsized":"2023-09-23T22:54:01.194873+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118722+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:54:00.118722+0000","last_peered":"2023-09-23T22:54:00.118722+0000","last_clean":"2023-09-23T22:54:00.118722+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:54:00.118722+0000","last_undegraded":"2023-09-23T22:54:00.118722+0000","last_fullsized":"2023-09-23T22:54:00.118722+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194313+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:54:01.194313+0000","last_peered":"2023-09-23T22:54:01.194313+0000","last_clean":"2023-09-23T22:54:01.194313+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:54:01.194313+0000","last_undegraded":"2023-09-23T22:54:01.194313+0000","last_fullsized":"2023-09-23T22:54:01.194313+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"300'9051","reported_seq":12572,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:11.650067+0000","last_change":"2023-09-23T22:53:44.765920+0000","last_active":"2023-09-23T22:54:11.650067+0000","last_peered":"2023-09-23T22:54:11.650067+0000","last_clean":"2023-09-23T22:54:11.650067+0000","last_became_active":"2023-09-23T22:53:44.707193+0000","last_became_peered":"2023-09-23T22:53:44.707193+0000","last_unstale":"2023-09-23T22:54:11.650067+0000","last_undegraded":"2023-09-23T22:54:11.650067+0000","last_fullsized":"2023-09-23T22:54:11.650067+0000","mapping_epoch":296,"log_start":"273'2300","ondisk_log_start":"273'2300","created":264,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146866176,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9051,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":341,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.118807+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:54:00.118807+0000","last_peered":"2023-09-23T22:54:00.118807+0000","last_clean":"2023-09-23T22:54:00.118807+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:54:00.118807+0000","last_undegraded":"2023-09-23T22:54:00.118807+0000","last_fullsized":"2023-09-23T22:54:00.118807+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":198,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.195138+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:54:01.195138+0000","last_peered":"2023-09-23T22:54:01.195138+0000","last_clean":"2023-09-23T22:54:01.195138+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:54:01.195138+0000","last_undegraded":"2023-09-23T22:54:01.195138+0000","last_fullsized":"2023-09-23T22:54:01.195138+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":250,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132206+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:54:00.132206+0000","last_peered":"2023-09-23T22:54:00.132206+0000","last_clean":"2023-09-23T22:54:00.132206+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:54:00.132206+0000","last_undegraded":"2023-09-23T22:54:00.132206+0000","last_fullsized":"2023-09-23T22:54:00.132206+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":234,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.198669+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:54:01.198669+0000","last_peered":"2023-09-23T22:54:01.198669+0000","last_clean":"2023-09-23T22:54:01.198669+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:54:01.198669+0000","last_undegraded":"2023-09-23T22:54:01.198669+0000","last_fullsized":"2023-09-23T22:54:01.198669+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":83,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.128266+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:54:00.128266+0000","last_peered":"2023-09-23T22:54:00.128266+0000","last_clean":"2023-09-23T22:54:00.128266+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:54:00.128266+0000","last_undegraded":"2023-09-23T22:54:00.128266+0000","last_fullsized":"2023-09-23T22:54:00.128266+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":109,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131677+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:54:00.131677+0000","last_peered":"2023-09-23T22:54:00.131677+0000","last_clean":"2023-09-23T22:54:00.131677+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:54:00.131677+0000","last_undegraded":"2023-09-23T22:54:00.131677+0000","last_fullsized":"2023-09-23T22:54:00.131677+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":308,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131915+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:54:00.131915+0000","last_peered":"2023-09-23T22:54:00.131915+0000","last_clean":"2023-09-23T22:54:00.131915+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:54:00.131915+0000","last_undegraded":"2023-09-23T22:54:00.131915+0000","last_fullsized":"2023-09-23T22:54:00.131915+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":110,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131904+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:54:00.131904+0000","last_peered":"2023-09-23T22:54:00.131904+0000","last_clean":"2023-09-23T22:54:00.131904+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:54:00.131904+0000","last_undegraded":"2023-09-23T22:54:00.131904+0000","last_fullsized":"2023-09-23T22:54:00.131904+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.132252+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:54:00.132252+0000","last_peered":"2023-09-23T22:54:00.132252+0000","last_clean":"2023-09-23T22:54:00.132252+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:54:00.132252+0000","last_undegraded":"2023-09-23T22:54:00.132252+0000","last_fullsized":"2023-09-23T22:54:00.132252+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":50,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.117519+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:54:00.117519+0000","last_peered":"2023-09-23T22:54:00.117519+0000","last_clean":"2023-09-23T22:54:00.117519+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:54:00.117519+0000","last_undegraded":"2023-09-23T22:54:00.117519+0000","last_fullsized":"2023-09-23T22:54:00.117519+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":283,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.127109+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:54:00.127109+0000","last_peered":"2023-09-23T22:54:00.127109+0000","last_clean":"2023-09-23T22:54:00.127109+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:54:00.127109+0000","last_undegraded":"2023-09-23T22:54:00.127109+0000","last_fullsized":"2023-09-23T22:54:00.127109+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":247,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T22:54:01.194709+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:54:01.194709+0000","last_peered":"2023-09-23T22:54:01.194709+0000","last_clean":"2023-09-23T22:54:01.194709+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:54:01.194709+0000","last_undegraded":"2023-09-23T22:54:01.194709+0000","last_fullsized":"2023-09-23T22:54:01.194709+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":265,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T22:54:00.131446+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:54:00.131446+0000","last_peered":"2023-09-23T22:54:00.131446+0000","last_clean":"2023-09-23T22:54:00.131446+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:54:00.131446+0000","last_undegraded":"2023-09-23T22:54:00.131446+0000","last_fullsized":"2023-09-23T22:54:00.131446+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2373255192,"num_objects":36214,"num_object_clones":0,"num_object_copies":108642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8389,"num_objects_misplaced":754,"num_objects_unfound":0,"num_objects_dirty":36214,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143064,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7664,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8266919936,"data_stored":8266907720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115464,"ondisk_log_size":115464,"up":48,"acting":43,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430534,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626762,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1498532,"kb_used_data":1184924,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92246620,"statfs":{"total":95995035648,"available":94460538880,"internally_reserved":0,"allocated":1213362176,"data_stored":1212805765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":6.9699999999999998,"5min":9.5869999999999997,"15min":4.2800000000000002},"min":{"1min":0.24199999999999999,"5min":0.218,"15min":0.218},"max":{"1min":112.51000000000001,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":7.6239999999999997,"5min":9.9469999999999992,"15min":4.2679999999999998},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":112.154,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":5.2249999999999996,"15min":3.8079999999999998},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":10.98,"5min":254.62100000000001,"15min":254.62100000000001},"last":1.2889999999999999},{"interface":"front","average":{"1min":3.21,"5min":6.5389999999999997,"15min":4.8079999999999998},"min":{"1min":0.45000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":12.069000000000001,"5min":255.20099999999999,"15min":255.20099999999999},"last":1.018}]},{"osd":2,"last update":"Sat Sep 23 22:53:23 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":3.867,"15min":3.4199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":38.527999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":4.3499999999999996,"5min":3.4969999999999999,"15min":3.786},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.27100000000000002},"max":{"1min":38.776000000000003,"5min":101.13,"15min":185.21799999999999},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.1520000000000001,"15min":3.8029999999999999},"min":{"1min":0.309,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":6.5090000000000003,"5min":17.742999999999999,"15min":185.19399999999999},"last":1.238},{"interface":"front","average":{"1min":3.2850000000000001,"5min":2.2799999999999998,"15min":3.4950000000000001},"min":{"1min":0.42599999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":10.378,"5min":10.773999999999999,"15min":128.19800000000001},"last":6.3109999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":1.1359999999999999,"15min":0.871},"min":{"1min":0.309,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.317,"5min":10.317,"15min":18.085999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.107,"15min":0.88500000000000001},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4610000000000003,"5min":7.4800000000000004,"15min":18.597000000000001},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.339,"15min":0.99099999999999999},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.353999999999999,"5min":10.353999999999999,"15min":18.972000000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":2.5459999999999998,"5min":1.1779999999999999,"15min":0.93999999999999995},"min":{"1min":0.41599999999999998,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":7.1459999999999999,"5min":7.1459999999999999,"15min":18.625},"last":1.9430000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.123,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":7.8650000000000002,"5min":7.8650000000000002,"15min":12.481999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.694,"5min":1.21,"15min":0.93600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.774,"5min":6.6269999999999998,"15min":16.661999999999999},"last":1.262}]}]},{"osd":1,"up_from":84,"seq":360777253072,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2248352,"kb_used_data":1754968,"kb_used_omap":9,"kb_used_meta":493110,"kb_avail":91496800,"statfs":{"total":95995035648,"available":93692723200,"internally_reserved":0,"allocated":1797087232,"data_stored":1796012602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":504945641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,9],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.6829999999999998,"5min":6.5149999999999997,"15min":5.3140000000000001},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.691000000000001,"5min":82.903999999999996,"15min":101.605},"last":14.766},{"interface":"front","average":{"1min":3.944,"5min":5.3680000000000003,"15min":5.1929999999999996},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":13.601000000000001,"5min":82.748999999999995,"15min":101.892},"last":20.414000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.6749999999999998,"5min":4.1029999999999998,"15min":3.0419999999999998},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":10.952999999999999,"5min":37.869999999999997,"15min":37.889000000000003},"last":3.8690000000000002},{"interface":"front","average":{"1min":4.4930000000000003,"5min":4.1340000000000003,"15min":3.1190000000000002},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.429,"5min":37.957000000000001,"15min":37.957000000000001},"last":4.4260000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":3.3079999999999998,"15min":2.528},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.1660000000000004,"5min":87.787999999999997,"15min":87.787999999999997},"last":2.9910000000000001},{"interface":"front","average":{"1min":2.9540000000000002,"5min":3.8650000000000002,"15min":3.0059999999999998},"min":{"1min":0.29299999999999998,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":10.845000000000001,"5min":89.861999999999995,"15min":89.861999999999995},"last":3.0939999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.9710000000000001,"15min":2.25},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":8.7469999999999999,"5min":11.432,"15min":78.948999999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":3.544,"5min":2.649,"15min":2.2850000000000001},"min":{"1min":0.315,"5min":0.313,"15min":0.313},"max":{"1min":10.894,"5min":14.058,"15min":78.963999999999999},"last":3.5640000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.597,"15min":1.792},"min":{"1min":0.42799999999999999,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":9.141,"5min":9.141,"15min":84.254999999999995},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.903,"5min":1.958,"15min":2.125},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":7.1150000000000002,"5min":11.458,"15min":87.515000000000001},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":4.0899999999999999,"5min":2.7519999999999998,"15min":2.1549999999999998},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":10.853999999999999,"5min":13.991,"15min":79.311000000000007},"last":3.6360000000000001},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.7609999999999999,"15min":1.7809999999999999},"min":{"1min":0.40899999999999997,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":79.328999999999994},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.1339999999999999,"15min":2.5049999999999999},"min":{"1min":0.438,"5min":0.40799999999999997,"15min":0.32800000000000001},"max":{"1min":8.702,"5min":11.481999999999999,"15min":159.018},"last":0.48099999999999998},{"interface":"front","average":{"1min":4.008,"5min":2.911,"15min":2.6669999999999998},"min":{"1min":0.49199999999999999,"5min":0.371,"15min":0.37},"max":{"1min":10.923,"5min":14.032999999999999,"15min":158.96000000000001},"last":3.601}]}]},{"osd":0,"up_from":219,"seq":940597837921,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292564,"kb_used_data":983372,"kb_used_omap":31,"kb_used_meta":309024,"kb_avail":92452588,"statfs":{"total":95995035648,"available":94671450112,"internally_reserved":0,"allocated":1006972928,"data_stored":1005858367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":316441522},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":5.3840000000000003,"5min":4.0069999999999997,"15min":3.758},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":40.180999999999997,"5min":40.180999999999997,"15min":40.180999999999997},"last":0.85099999999999998},{"interface":"front","average":{"1min":4.9249999999999998,"5min":3.6819999999999999,"15min":3.605},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":40.137999999999998,"5min":40.137999999999998,"15min":40.137999999999998},"last":0.91600000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":4.4560000000000004,"5min":5.0910000000000002,"15min":3.2189999999999999},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":46.387999999999998,"5min":111.508,"15min":111.508},"last":0.86599999999999999},{"interface":"front","average":{"1min":5.7619999999999996,"5min":4.7220000000000004,"15min":2.8460000000000001},"min":{"1min":0.41599999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":46.249000000000002,"5min":111.532,"15min":111.532},"last":0.55300000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":4.9660000000000002,"15min":3.0289999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.2759999999999998,"5min":126.73,"15min":126.73},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":3.581,"15min":2.044},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.7539999999999996,"5min":121.483,"15min":121.483},"last":0.68700000000000006}]},{"osd":4,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":2.2709999999999999,"15min":2.1179999999999999},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":8.8239999999999998,"5min":18.670999999999999,"15min":18.670999999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":2.218,"5min":2.6080000000000001,"15min":2.036},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.8620000000000001,"5min":20.009,"15min":20.009},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":2.766,"15min":1.901},"min":{"1min":0.39000000000000001,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":13.851000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":3.4079999999999999,"5min":3.4380000000000002,"15min":2.2839999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.436,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.80900000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.6760000000000002,"15min":1.9059999999999999},"min":{"1min":0.44500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.683,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":4.0019999999999998,"5min":3.3780000000000001,"15min":2.4009999999999998},"min":{"1min":0.41199999999999998,"5min":0.379,"15min":0.379},"max":{"1min":12.470000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.751}]},{"osd":7,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":2.468,"15min":2.0110000000000001},"min":{"1min":0.45500000000000002,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":12.385999999999999,"5min":18.640000000000001,"15min":18.640000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.0550000000000002,"15min":1.5489999999999999},"min":{"1min":0.40400000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.5750000000000002,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.48299999999999998}]}]},{"osd":2,"up_from":131,"seq":562640715933,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309468,"kb_used_data":897620,"kb_used_omap":81,"kb_used_meta":411758,"kb_avail":92435684,"statfs":{"total":95995035648,"available":94654140416,"internally_reserved":0,"allocated":919162880,"data_stored":918094995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":421640410},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:32 2023","interfaces":[{"interface":"back","average":{"1min":6.1900000000000004,"5min":6.9480000000000004,"15min":5.8170000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":73.388000000000005,"5min":132.79499999999999,"15min":132.79499999999999},"last":7.0449999999999999},{"interface":"front","average":{"1min":6.2869999999999999,"5min":7.4640000000000004,"15min":6.2400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":73.405000000000001,"5min":132.58099999999999,"15min":132.58099999999999},"last":7.1580000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":5.9770000000000003,"5min":3.052,"15min":3.2160000000000002},"min":{"1min":0.27700000000000002,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":77.004999999999995,"5min":77.004999999999995,"15min":88.316000000000003},"last":2.6890000000000001},{"interface":"front","average":{"1min":6.3490000000000002,"5min":3.8029999999999999,"15min":3.4430000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":76.956000000000003,"5min":76.956000000000003,"15min":88.436999999999998},"last":2.8660000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":4.8250000000000002,"15min":4.306},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.999000000000001,"5min":112.895,"15min":112.895},"last":2.6509999999999998},{"interface":"front","average":{"1min":2.3969999999999998,"5min":4.3090000000000002,"15min":3.5059999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":6.2670000000000003,"5min":113.995,"15min":113.995},"last":3.9900000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":5.5330000000000004,"5min":2.6659999999999999,"15min":2.7250000000000001},"min":{"1min":0.44700000000000001,"5min":0.253,"15min":0.253},"max":{"1min":69.113,"5min":69.113,"15min":108.878},"last":4.0259999999999998},{"interface":"front","average":{"1min":5.8780000000000001,"5min":2.5960000000000001,"15min":3.3599999999999999},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":69.028999999999996,"5min":69.028999999999996,"15min":108.93000000000001},"last":0.29399999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":2.0830000000000002,"15min":2.9910000000000001},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":22.646000000000001,"5min":22.646000000000001,"15min":97.322999999999993},"last":0.32700000000000001},{"interface":"front","average":{"1min":4.1840000000000002,"5min":2.589,"15min":2.9460000000000002},"min":{"1min":0.39300000000000002,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":22.501000000000001,"5min":22.501000000000001,"15min":97.301000000000002},"last":2.7010000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":6.1219999999999999,"5min":3.911,"15min":3.2949999999999999},"min":{"1min":0.36099999999999999,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":59.655000000000001,"5min":98.597999999999999,"15min":98.597999999999999},"last":2.7440000000000002},{"interface":"front","average":{"1min":5.125,"5min":3.375,"15min":2.6000000000000001},"min":{"1min":0.46999999999999997,"5min":0.313,"15min":0.313},"max":{"1min":59.612000000000002,"5min":98.509,"15min":98.509},"last":4.0519999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.1549999999999998,"15min":1.6599999999999999},"min":{"1min":0.40300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.9450000000000003,"5min":10.92,"15min":10.92},"last":4.0780000000000003},{"interface":"front","average":{"1min":2.149,"5min":1.9670000000000001,"15min":2.3759999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":6.9109999999999996,"5min":11.311,"15min":88.584000000000003},"last":0.38800000000000001}]}]},{"osd":3,"up_from":0,"seq":403726926019,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1091496,"kb_used_data":790176,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92653656,"statfs":{"total":95995035648,"available":94877343744,"internally_reserved":0,"allocated":809140224,"data_stored":808579219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":8.0700000000000003,"5min":4.423,"15min":6.367},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":82.376000000000005,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":8.1430000000000007,"5min":4.9420000000000002,"15min":6.9160000000000004},"min":{"1min":0.34100000000000003,"5min":0.217,"15min":0.217},"max":{"1min":79.403000000000006,"5min":113.417,"15min":113.417},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":7.5090000000000003,"5min":4.9939999999999998,"15min":2.9249999999999998},"min":{"1min":0.504,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":70.418000000000006,"5min":132.304,"15min":132.304},"last":3.2210000000000001},{"interface":"front","average":{"1min":6.6749999999999998,"5min":4.7859999999999996,"15min":2.726},"min":{"1min":0.29899999999999999,"5min":0.253,"15min":0.253},"max":{"1min":70.432000000000002,"5min":132.453,"15min":132.453},"last":2.9860000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.3860000000000001,"15min":3.1400000000000001},"min":{"1min":0.35999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.8070000000000004,"5min":20.308,"15min":84.909000000000006},"last":0.376},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.4790000000000001,"15min":3.1469999999999998},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.673,"5min":21.704000000000001,"15min":83.840000000000003},"last":3.6219999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6429999999999998,"5min":5.4409999999999998,"15min":4.4729999999999999},"min":{"1min":0.39500000000000002,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":12.622999999999999,"5min":164.453,"15min":164.453},"last":3.3929999999999998},{"interface":"front","average":{"1min":2.6230000000000002,"5min":5.5229999999999997,"15min":4.5609999999999999},"min":{"1min":0.40500000000000003,"5min":0.33300000000000002,"15min":0.27200000000000002},"max":{"1min":10.347,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.70499999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:53:21 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.097,"15min":0.93799999999999994},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":10.321999999999999,"5min":12.238,"15min":43.189999999999998},"last":0.315},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.079,"15min":0.93999999999999995},"min":{"1min":0.34200000000000003,"5min":0.248,"15min":0.248},"max":{"1min":6.4349999999999996,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.0920000000000001,"15min":0.90400000000000003},"min":{"1min":0.378,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4039999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.8109999999999999,"5min":1.091,"15min":0.92400000000000004},"min":{"1min":0.44400000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":10.423999999999999,"5min":12.269,"15min":26.940999999999999},"last":0.34899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.2250000000000001,"15min":0.876},"min":{"1min":0.36699999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":5.1449999999999996,"5min":12.311,"15min":12.311},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.171,"5min":1.0569999999999999,"15min":0.79900000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":4.077,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":3.0169999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822990,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2072716,"kb_used_data":1478916,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91672436,"statfs":{"total":95995035648,"available":93872574464,"internally_reserved":0,"allocated":1514409984,"data_stored":1513872695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":12.433,"5min":6.899,"15min":4.6070000000000002},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":185.62,"5min":185.62,"15min":185.62},"last":2.1099999999999999},{"interface":"front","average":{"1min":12.449999999999999,"5min":7.0140000000000002,"15min":4.569},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":185.499,"5min":185.499,"15min":185.499},"last":1.4590000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0069999999999997,"5min":2.2759999999999998,"15min":2.2850000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.22500000000000001},"max":{"1min":17.658999999999999,"5min":41.463999999999999,"15min":149.203},"last":4.2510000000000003},{"interface":"front","average":{"1min":4.5030000000000001,"5min":2.6880000000000002,"15min":2.5899999999999999},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":17.84,"5min":41.954000000000001,"15min":149.554},"last":4.1580000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":12.965,"5min":5.3209999999999997,"15min":5.8570000000000002},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":189.126,"5min":189.126,"15min":189.126},"last":6.492},{"interface":"front","average":{"1min":12.991,"5min":4.931,"15min":5.7809999999999997},"min":{"1min":0.498,"5min":0.318,"15min":0.28999999999999998},"max":{"1min":188.74799999999999,"5min":188.74799999999999,"15min":188.74799999999999},"last":1.4079999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":3.3929999999999998,"5min":5.4790000000000001,"15min":5.5510000000000002},"min":{"1min":0.316,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.99,"5min":98.641000000000005,"15min":257.05000000000001},"last":1.222},{"interface":"front","average":{"1min":2.6779999999999999,"5min":5.5949999999999998,"15min":5.4180000000000001},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.27800000000000002},"max":{"1min":10.362,"5min":98.906000000000006,"15min":257.35000000000002},"last":1.3109999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:53:22 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":0.84499999999999997,"15min":0.72199999999999998},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":6.3949999999999996,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.956,"5min":0.83799999999999997,"15min":0.72299999999999998},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":6.3550000000000004,"5min":6.3550000000000004,"15min":10.535},"last":0.34100000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:53:27 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.054,"15min":1.5649999999999999},"min":{"1min":0.34399999999999997,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.649,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.1699999999999999,"5min":1.0089999999999999,"15min":1.5880000000000001},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.254},"max":{"1min":10.314,"5min":10.314,"15min":139.739},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.133,"15min":0.88700000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.9509999999999996,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.437},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.99299999999999999,"15min":0.80300000000000005},"min":{"1min":0.378,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.39600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1001914368,"data_stored":1001914368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1771966464,"data_stored":1771962392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":915210240,"data_stored":915210240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1145044992,"data_stored":1145044992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":740954112,"data_stored":740950040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1499533312,"data_stored":1499529240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1192296448,"data_stored":1192296448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:54:17.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:54:17.551 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:17.642 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:17.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:17 smithi052 ceph-mon[187743]: from='client.35718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:17.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:17 smithi052 ceph-mon[187743]: pgmap v873: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 8389/104478 objects degraded (8.029%); 324/104478 objects misplaced (0.310%); 0 B/s, 81 objects/s recovering 2023-09-23T22:54:17.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:17 smithi052 ceph-mon[180979]: from='client.35718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:17.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:17 smithi052 ceph-mon[180979]: pgmap v873: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 8389/104478 objects degraded (8.029%); 324/104478 objects misplaced (0.310%); 0 B/s, 81 objects/s recovering 2023-09-23T22:54:17.696 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:17.833 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:17.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:17.947 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:17 smithi151 ceph-mon[159210]: from='client.35718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:17 smithi151 ceph-mon[159210]: pgmap v873: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 8389/104478 objects degraded (8.029%); 324/104478 objects misplaced (0.310%); 0 B/s, 81 objects/s recovering 2023-09-23T22:54:17.977 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:18.042 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:18.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:18.152 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:18.193 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:18.262 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:18.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:18.320 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:18.372 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:18.471 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:19.473 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:19.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:19.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:19.582 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:19.618 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:19.623 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:19.686 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:19.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:19.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:19 smithi052 ceph-mon[180979]: pgmap v874: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 642 op/s; 8389/99048 objects degraded (8.470%); 324/99048 objects misplaced (0.327%); 0 B/s, 81 objects/s recovering 2023-09-23T22:54:19.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:19.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:19.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:19 smithi052 ceph-mon[187743]: pgmap v874: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 642 op/s; 8389/99048 objects degraded (8.470%); 324/99048 objects misplaced (0.327%); 0 B/s, 81 objects/s recovering 2023-09-23T22:54:19.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:19.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:19.796 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:19.873 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:19 smithi151 ceph-mon[159210]: pgmap v874: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 642 op/s; 8389/99048 objects degraded (8.470%); 324/99048 objects misplaced (0.327%); 0 B/s, 81 objects/s recovering 2023-09-23T22:54:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:19.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:20.209 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:20.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:20.320 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:20.377 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8389/99048 objects degraded (8.470%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:20 smithi151 ceph-mon[159210]: osdmap e302: 8 total, 8 up, 6 in 2023-09-23T22:54:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:20 smithi151 ceph-mon[159210]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:20 smithi151 ceph-mon[159210]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:20 smithi151 ceph-mon[159210]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8389/99048 objects degraded (8.470%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:20.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[180979]: osdmap e302: 8 total, 8 up, 6 in 2023-09-23T22:54:20.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[180979]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[180979]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[180979]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8389/99048 objects degraded (8.470%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[187743]: osdmap e302: 8 total, 8 up, 6 in 2023-09-23T22:54:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[187743]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[187743]: failed to encode map e302 with expected crc 2023-09-23T22:54:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:20 smithi052 ceph-mon[187743]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.150 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:21.194 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:21.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:21.295 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:21.333 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:21.395 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:21.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:21.480 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:54:21.503 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:21.514 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:54:21.543 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:21.572 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:54:21.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: pgmap v876: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 575 op/s; 8389/97167 objects degraded (8.634%); 324/97167 objects misplaced (0.333%); 0 B/s, 53 objects/s recovering 2023-09-23T22:54:21.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: osdmap e303: 8 total, 8 up, 6 in 2023-09-23T22:54:21.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.615 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:54:21.622 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:21.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:21.661 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:54:21.709 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:54:21.731 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:21.754 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:54:21.799 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:21.800 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.826 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:54:21.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.861 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: pgmap v876: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 575 op/s; 8389/97167 objects degraded (8.634%); 324/97167 objects misplaced (0.333%); 0 B/s, 53 objects/s recovering 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: osdmap e303: 8 total, 8 up, 6 in 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/895736398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: failed to encode map e302 with expected crc 2023-09-23T22:54:21.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: pgmap v876: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 575 op/s; 8389/97167 objects degraded (8.634%); 324/97167 objects misplaced (0.333%); 0 B/s, 53 objects/s recovering 2023-09-23T22:54:21.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: osdmap e303: 8 total, 8 up, 6 in 2023-09-23T22:54:21.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: failed to encode map e303 with expected crc 2023-09-23T22:54:21.863 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/895736398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/895736398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:22.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:22 smithi151 ceph-mon[159210]: failed to encode map e303 with expected crc 2023-09-23T22:54:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:22 smithi052 ceph-mon[187743]: failed to encode map e303 with expected crc 2023-09-23T22:54:22.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:22 smithi052 ceph-mon[180979]: failed to encode map e303 with expected crc 2023-09-23T22:54:23.153 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:23.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:23.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:23.321 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:23.397 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:23.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:23.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:23.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:54:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:54:23] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-09-23T22:54:23.513 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:23.556 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:23.633 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:23.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:23.748 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:23.791 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:23.871 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:23.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:23.882 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:23 smithi151 ceph-mon[159210]: pgmap v878: 89 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 8189/92301 objects degraded (8.872%); 0 B/s, 119 objects/s recovering 2023-09-23T22:54:23.882 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:23 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:54:23.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:23 smithi052 ceph-mon[187743]: pgmap v878: 89 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 8189/92301 objects degraded (8.872%); 0 B/s, 119 objects/s recovering 2023-09-23T22:54:23.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:23 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:54:23.944 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:23 smithi052 ceph-mon[180979]: pgmap v878: 89 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 8189/92301 objects degraded (8.872%); 0 B/s, 119 objects/s recovering 2023-09-23T22:54:23.945 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:23 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:54:23.983 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:24.027 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:24.089 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:24.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:24.198 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:24.229 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:24.288 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:24.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:24.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:24.395 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:24.433 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:24.502 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:24.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:24.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:24.637 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184397 7.0 7:0017d177:::benchmark_data_smithi052_277165_object91905:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:54:24.655 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.14515262600003, 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0558009070000001, 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184397 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.656 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495201+0000", 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495200+0000", 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:24.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495205+0000", 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9799999999999998e-06 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495218+0000", 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3602e-05 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.658 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495227+0000", 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1780000000000003e-06 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495336+0000", 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000109238 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op must wait for map", 2023-09-23T22:54:24.659 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495345+0000", 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5910000000000002e-06 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500507+0000", 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051612130000000004 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500517+0000", 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0794e-05 2023-09-23T22:54:24.660 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.550859+0000", 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0503421209999999 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551002+0000", 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014262000000000001 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.661 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184406 7.0 7:01b010b6:::benchmark_data_smithi052_277165_object91914:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.145109553, 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0749326939999999, 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.662 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184406 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.663 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495244+0000", 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495247+0000", 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.853e-06 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.664 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495261+0000", 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4344e-05 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495266+0000", 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3090000000000003e-06 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.665 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495397+0000", 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000130827 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495403+0000", 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.682e-06 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500537+0000", 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051338649999999996 2023-09-23T22:54:24.666 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500541+0000", 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3510000000000004e-06 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551486+0000", 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.050945196 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:54:24.667 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551618+0000", 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013216700000000001 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570084+0000", 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01846515 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570177+0000", 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3393999999999994e-05 2023-09-23T22:54:24.668 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184408 7.0 7:04875276:::benchmark_data_smithi052_277165_object91916:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.14502646400001, 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0894791740000001, 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.669 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184408 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.670 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495327+0000", 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495331+0000", 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6959999999999996e-06 2023-09-23T22:54:24.671 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495343+0000", 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.201e-05 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495351+0000", 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0190000000000006e-06 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.672 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495428+0000", 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6434999999999996e-05 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495433+0000", 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3870000000000003e-06 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500552+0000", 2023-09-23T22:54:24.673 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051191750000000001 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500556+0000", 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9040000000000002e-06 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551667+0000", 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.051111047 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.674 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for missing object", 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551765+0000", 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8115999999999994e-05 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584729+0000", 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.032963237999999999 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584807+0000", 2023-09-23T22:54:24.675 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7855999999999995e-05 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184411 7.0 7:04cf78da:::benchmark_data_smithi052_277165_object91919:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e212)", 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.14499707100003, 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0680791730000001, 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.676 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184411 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.677 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495357+0000", 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495356+0000", 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495359+0000", 2023-09-23T22:54:24.678 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4090000000000001e-06 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495365+0000", 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0310000000000004e-06 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.495370+0000", 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3120000000000001e-06 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.679 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.496599+0000", 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012286339999999999 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting_for_map not empty", 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.496605+0000", 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1870000000000002e-06 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500568+0000", 2023-09-23T22:54:24.680 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003962559 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.500571+0000", 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.698e-06 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551817+0000", 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0512461209999999 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.681 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551945+0000", 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012785800000000001 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.552407+0000", 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000461896 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.682 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.561382+0000", 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0089752110000000003 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563179+0000", 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017967440000000001 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563267+0000", 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7570000000000004e-05 2023-09-23T22:54:24.683 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563330+0000", 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2973999999999997e-05 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563436+0000", 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106332 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.684 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184468 7.10 7:092ecb73:::benchmark_data_smithi052_277165_object91976:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.09217246000003, 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0031411560000001, 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184468 2023-09-23T22:54:24.685 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.686 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548181+0000", 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548184+0000", 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3390000000000001e-06 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.687 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548194+0000", 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6420000000000002e-06 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548199+0000", 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6160000000000001e-06 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548264+0000", 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4523999999999994e-05 2023-09-23T22:54:24.688 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.548276+0000", 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1939e-05 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534263+0000", 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98598719300000004 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.689 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534418+0000", 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015466899999999999 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.534800+0000", 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00038215800000000001 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536718+0000", 2023-09-23T22:54:24.690 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019180110000000001 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551075+0000", 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014356733 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551223+0000", 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014790699999999999 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.691 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551276+0000", 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3312000000000001e-05 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551323+0000", 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6589999999999999e-05 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.692 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184490 7.10 7:0fb33c7c:::benchmark_data_smithi052_277165_object91998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.07004626900004, 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98115489099999997, 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184490 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.693 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570308+0000", 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570305+0000", 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570316+0000", 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0579e-05 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570338+0000", 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2246999999999999e-05 2023-09-23T22:54:24.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570347+0000", 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8189999999999998e-06 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570426+0000", 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9136999999999998e-05 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.696 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.570441+0000", 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4269e-05 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536742+0000", 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.96630105200000005 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.536853+0000", 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111598 2023-09-23T22:54:24.697 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.537044+0000", 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019121900000000001 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551089+0000", 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014044137 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.698 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551122+0000", 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3930000000000002e-05 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551393+0000", 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027049300000000001 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551428+0000", 2023-09-23T22:54:24.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4764999999999997e-05 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.551462+0000", 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4730000000000001e-05 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184514 7.10 7:08068522:::benchmark_data_smithi052_277165_object92022:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.700 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.04828947500005, 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.99227452999999999, 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184514 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.701 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592064+0000", 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.702 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592062+0000", 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592072+0000", 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7999999999999993e-06 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592091+0000", 2023-09-23T22:54:24.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8834e-05 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592102+0000", 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.079e-05 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592169+0000", 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7201999999999998e-05 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.704 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.592178+0000", 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.312e-06 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570201+0000", 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.97802318300000002 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570317+0000", 2023-09-23T22:54:24.705 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115149 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.570714+0000", 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039780000000000002 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579033+0000", 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0083185269999999992 2023-09-23T22:54:24.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579064+0000", 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1152999999999998e-05 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584277+0000", 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052129189999999999 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584310+0000", 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3500000000000001e-05 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584339+0000", 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8464999999999999e-05 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.708 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184527 7.10 7:0f7ed45d:::benchmark_data_smithi052_277165_object92035:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.03693500700001, 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98101475999999999, 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184527 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.709 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603419+0000", 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.710 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603417+0000", 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603425+0000", 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.294e-06 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603441+0000", 2023-09-23T22:54:24.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6146000000000001e-05 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603450+0000", 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1949999999999992e-06 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603514+0000", 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4456999999999996e-05 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.603524+0000", 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0066e-05 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.573937+0000", 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.97041304399999995 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574065+0000", 2023-09-23T22:54:24.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000127327 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574459+0000", 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039406800000000002 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579109+0000", 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046502430000000001 2023-09-23T22:54:24.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584224+0000", 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051145239999999996 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584362+0000", 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138436 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.715 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584392+0000", 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9927999999999999e-05 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584434+0000", 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1569999999999997e-05 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.716 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184549 7.10 7:0d0c97e5:::benchmark_data_smithi052_277165_object92057:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.00614392099999, 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.95037141400000003, 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184549 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634210+0000", 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634208+0000", 2023-09-23T22:54:24.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634216+0000", 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.123e-06 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634229+0000", 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2289999999999999e-05 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634235+0000", 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6270000000000002e-06 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634296+0000", 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1091000000000002e-05 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634308+0000", 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1647999999999999e-05 2023-09-23T22:54:24.720 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576583+0000", 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.94227499800000003 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576682+0000", 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9207000000000001e-05 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576998+0000", 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031594400000000002 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579138+0000", 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021399700000000002 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584232+0000", 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050934769999999999 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584502+0000", 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027061099999999997 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584546+0000", 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4187999999999998e-05 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584581+0000", 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4848999999999998e-05 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184550 7.0 7:012735c8:::benchmark_data_smithi052_277165_object92058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 514.006101532, 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.93521286599999998, 2023-09-23T22:54:24.724 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184550 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634252+0000", 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634256+0000", 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9559999999999999e-06 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634264+0000", 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9179999999999994e-06 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634269+0000", 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5249999999999996e-06 2023-09-23T22:54:24.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634322+0000", 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3062000000000001e-05 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.634332+0000", 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8339999999999994e-06 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.554914+0000", 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.92058223699999997 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.555025+0000", 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110911 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.555443+0000", 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041786699999999999 2023-09-23T22:54:24.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563188+0000", 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0077454409999999996 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563504+0000", 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00031558900000000002 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569298+0000", 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057944709999999998 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569357+0000", 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8242000000000002e-05 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.731 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569465+0000", 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010841299999999999 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184557 7.10 7:0c22b33f:::benchmark_data_smithi052_277165_object92065:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 513.995998773, 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.94057564299999996, 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.732 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184557 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644355+0000", 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644354+0000", 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644361+0000", 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3660000000000003e-06 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644376+0000", 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4778e-05 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644384+0000", 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7519999999999996e-06 2023-09-23T22:54:24.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644433+0000", 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9512999999999998e-05 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.644441+0000", 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2390000000000002e-06 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579160+0000", 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.93471871500000003 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579275+0000", 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011521799999999999 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579546+0000", 2023-09-23T22:54:24.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027091600000000001 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584451+0000", 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049044199999999996 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584842+0000", 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00039113700000000001 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584870+0000", 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8340999999999999e-05 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584897+0000", 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6838999999999998e-05 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584931+0000", 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3798999999999999e-05 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184560 7.0 7:00bb5b7f:::benchmark_data_smithi052_277165_object92068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 513.92719466400001, 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85650294199999999, 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.740 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184560 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713159+0000", 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713158+0000", 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713165+0000", 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.165e-06 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713180+0000", 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5169e-05 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713187+0000", 2023-09-23T22:54:24.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5560000000000002e-06 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713588+0000", 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040024400000000001 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.713601+0000", 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3078000000000001e-05 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557158+0000", 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84355768200000003 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557274+0000", 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115092 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.557528+0000", 2023-09-23T22:54:24.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025477200000000001 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563196+0000", 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056677539999999997 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563652+0000", 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00045571700000000001 2023-09-23T22:54:24.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569510+0000", 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058584919999999999 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569559+0000", 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8903000000000003e-05 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569662+0000", 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000102871 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184562 7.0 7:02f4adfb:::benchmark_data_smithi052_277165_object92070:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 513.92498615800002, 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85882703400000004, 2023-09-23T22:54:24.748 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184562 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715368+0000", 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715366+0000", 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715373+0000", 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0190000000000001e-06 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715389+0000", 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6200000000000001e-05 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715398+0000", 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8440000000000004e-06 2023-09-23T22:54:24.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715448+0000", 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9747000000000003e-05 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.715458+0000", 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0378e-05 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563719+0000", 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84826098699999997 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.563811+0000", 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1298999999999994e-05 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.564148+0000", 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000337757 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.569476+0000", 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0053272930000000003 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.573915+0000", 2023-09-23T22:54:24.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004439386 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574045+0000", 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012963800000000001 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574102+0000", 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6929999999999999e-05 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574195+0000", 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3134000000000006e-05 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184566 7.0 7:034423e1:::benchmark_data_smithi052_277165_object92074:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 513.92118080700004, 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.86009980500000005, 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.756 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184566 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719173+0000", 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719172+0000", 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719176+0000", 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5370000000000002e-06 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719184+0000", 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8159999999999994e-06 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719190+0000", 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4980000000000003e-06 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719260+0000", 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0012000000000007e-05 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.719271+0000", 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0862999999999999e-05 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.760 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.566596+0000", 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84732440099999995 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.566722+0000", 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012632999999999999 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.567133+0000", 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041073900000000003 2023-09-23T22:54:24.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.574211+0000", 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070784969999999996 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.576559+0000", 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023475700000000002 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579125+0000", 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025667670000000002 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579184+0000", 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8736999999999997e-05 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579273+0000", 2023-09-23T22:54:24.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8580999999999998e-05 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184606 7.0 7:0419327f:::benchmark_data_smithi052_277165_object92114:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 513.81243259799999, 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.75649038599999996, 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.764 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184606 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827921+0000", 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827920+0000", 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.766 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827925+0000", 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0579999999999997e-06 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827937+0000", 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2271e-05 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.827943+0000", 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5840000000000002e-06 2023-09-23T22:54:24.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.828005+0000", 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2479999999999998e-05 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.828019+0000", 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3534e-05 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571101+0000", 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.74308152100000002 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571233+0000", 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013277300000000001 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.571648+0000", 2023-09-23T22:54:24.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00041445300000000001 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579015+0000", 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073670360000000004 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.579081+0000", 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5988000000000006e-05 2023-09-23T22:54:24.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584275+0000", 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051946539999999999 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584318+0000", 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3046000000000001e-05 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584412+0000", 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3121999999999994e-05 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:184620 7.10 7:0f84e4b0:::benchmark_data_smithi052_277165_object92128:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 513.74864282600004, 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.69549317099999997, 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.772 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 184620 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891711+0000", 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891709+0000", 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891720+0000", 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0986e-05 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891747+0000", 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7750999999999999e-05 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891758+0000", 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1043000000000001e-05 2023-09-23T22:54:24.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891835+0000", 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6794999999999999e-05 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:50.891850+0000", 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4929999999999999e-05 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.581707+0000", 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.68985722999999999 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.581829+0000", 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012162700000000001 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.582150+0000", 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032150899999999999 2023-09-23T22:54:24.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584591+0000", 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024402400000000002 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.584964+0000", 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00037371099999999999 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587111+0000", 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021470249999999999 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587159+0000", 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7295e-05 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:51.587204+0000", 2023-09-23T22:54:24.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5411000000000003e-05 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110891 9.3 9:c27b0003:::benchmark_data_smithi052_402718_object21250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 63.584720075, 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25767365199999998, 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.780 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110891 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055633+0000", 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055637+0000", 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.126e-06 2023-09-23T22:54:24.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055645+0000", 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9850000000000006e-06 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055650+0000", 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4990000000000002e-06 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055758+0000", 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010737 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055921+0000", 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163437 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:54:24.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.056098+0000", 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000176563 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.059728+0000", 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036304570000000001 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.062797+0000", 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00306851 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312384+0000", 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.249587273 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312438+0000", 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4354999999999997e-05 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313307+0000", 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00086921899999999996 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110899 9.3 9:c9f9c7b0:::benchmark_data_smithi052_402718_object21258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 63.577056765000002, 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29968218299999999, 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.787 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110899 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063299+0000", 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5469999999999999e-06 2023-09-23T22:54:24.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063306+0000", 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6889999999999998e-06 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063312+0000", 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6489999999999996e-06 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066300+0000", 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029886909999999999 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066461+0000", 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016098600000000001 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:54:24.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066620+0000", 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015845099999999999 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312307+0000", 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24568741599999999 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312349+0000", 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1686e-05 2023-09-23T22:54:24.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362127+0000", 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049778356000000003 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362181+0000", 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3421999999999998e-05 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362979+0000", 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079871 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110900 9.3 9:cfcfdec5:::benchmark_data_smithi052_402718_object21259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 63.577036688, 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.300308513, 2023-09-23T22:54:24.794 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110900 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063319+0000", 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.525e-06 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063325+0000", 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5999999999999997e-06 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063329+0000", 2023-09-23T22:54:24.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5170000000000001e-06 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069300+0000", 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059708189999999996 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069454+0000", 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015409899999999999 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069610+0000", 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155332 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312314+0000", 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24270393900000001 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313359+0000", 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010457520000000001 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363082+0000", 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049722865999999998 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363133+0000", 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0757999999999998e-05 2023-09-23T22:54:24.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363626+0000", 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000492659 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110905 9.3 9:cdd24a36:::benchmark_data_smithi052_402718_object21264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 63.576152049000001, 2023-09-23T22:54:24.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29997646300000003, 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110905 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:54:24.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064201+0000", 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064206+0000", 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7419999999999997e-06 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064217+0000", 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.094e-05 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:24.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064224+0000", 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2529999999999998e-06 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072314+0000", 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0080899820000000008 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072465+0000", 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015051600000000001 2023-09-23T22:54:24.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.309128+0000", 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23666314399999999 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.314036+0000", 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049078869999999997 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.806 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.320379+0000", 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00634324 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363720+0000", 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.043340956999999999 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363771+0000", 2023-09-23T22:54:24.807 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1734999999999998e-05 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.364178+0000", 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040673700000000001 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:24.808 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:54:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:24.809 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:24.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:24.910 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:24.958 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:25.034 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:25.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:25.141 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:25.168 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:25.248 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:25.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:25.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:25.462 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:25.626 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:25 smithi151 ceph-mon[159210]: pgmap v879: 89 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 8189/90162 objects degraded (9.083%); 0 B/s, 65 objects/s recovering 2023-09-23T22:54:25.626 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2858427932' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:54:25.627 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 8189/90162 objects degraded (9.083%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:25.690 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:54:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:25 smithi052 ceph-mon[180979]: pgmap v879: 89 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 8189/90162 objects degraded (9.083%); 0 B/s, 65 objects/s recovering 2023-09-23T22:54:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2858427932' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:54:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 8189/90162 objects degraded (9.083%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:25 smithi052 ceph-mon[187743]: pgmap v879: 89 pgs: 1 creating+peering, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 8189/90162 objects degraded (9.083%); 0 B/s, 65 objects/s recovering 2023-09-23T22:54:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2858427932' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:54:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 8189/90162 objects degraded (9.083%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:25.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:54:26.344 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:26.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:26.457 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:26.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:26.565 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:26.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:26.632 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:26.632 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2858427932' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:54:26.632 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[180979]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T22:54:26.633 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[180979]: failed to encode map e304 with expected crc 2023-09-23T22:54:26.670 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:26.763 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:54:26.778 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:26 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2858427932' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:54:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:26 smithi151 ceph-mon[159210]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T22:54:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:26 smithi151 ceph-mon[159210]: failed to encode map e304 with expected crc 2023-09-23T22:54:26.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:26.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2858427932' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:54:26.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[187743]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T22:54:26.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:26 smithi052 ceph-mon[187743]: failed to encode map e304 with expected crc 2023-09-23T22:54:27.069 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:27.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:27.177 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:27.285 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:27.476 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:27.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:27.549 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:27.589 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:27.618 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:27.624 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:27.682 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:27.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:27.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:27 smithi052 ceph-mon[180979]: pgmap v881: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 600 op/s; 7748/85815 objects degraded (9.029%); 0 B/s, 153 objects/s recovering 2023-09-23T22:54:27.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:27 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:54:27.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:27 smithi052 ceph-mon[187743]: pgmap v881: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 600 op/s; 7748/85815 objects degraded (9.029%); 0 B/s, 153 objects/s recovering 2023-09-23T22:54:27.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:27 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:54:27.787 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:27.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:27.921 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:27.924 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:27 smithi151 ceph-mon[159210]: pgmap v881: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 600 op/s; 7748/85815 objects degraded (9.029%); 0 B/s, 153 objects/s recovering 2023-09-23T22:54:27.924 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:27 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:54:27.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:27.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:28.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:28.063 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:54:28.064 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7748/85815 objects degraded (9.029%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1200,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":304,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":6,"osd_in_since":1695509493,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":28605,"data_bytes":1873614776,"bytes_used":8193118208,"bytes_avail":567777095680,"bytes_total":575970213888,"degraded_objects":7748,"degraded_total":85815,"degraded_ratio":0.090287245819495432,"recovering_objects_per_sec":153,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":965,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":600},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T22:53:02.381180+0000","services":{}},"progress_events":{"6b27ada5-699f-4815-a812-68a0dbe12316":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 14s)","progress":0.92134833335876465,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:54:28.135 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:28.860 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2576753657' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:54:28.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2576753657' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:54:28.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2576753657' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:54:29.125 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:54:29.862 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:29.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:29.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:29 smithi151 ceph-mon[159210]: pgmap v882: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 710 op/s; 7519/80103 objects degraded (9.387%); 0 B/s, 149 objects/s recovering 2023-09-23T22:54:29.970 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:29.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:29 smithi052 ceph-mon[187743]: pgmap v882: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 710 op/s; 7519/80103 objects degraded (9.387%); 0 B/s, 149 objects/s recovering 2023-09-23T22:54:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:29 smithi052 ceph-mon[180979]: pgmap v882: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 710 op/s; 7519/80103 objects degraded (9.387%); 0 B/s, 149 objects/s recovering 2023-09-23T22:54:30.052 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:30.256 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:30.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:30.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:30.361 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:30.394 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:30.470 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:30.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:30.563 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:54:30.563 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:54:30.576 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:30.671 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:30.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 7519/80103 objects degraded (9.387%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:30.912 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 7519/80103 objects degraded (9.387%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:30.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 7519/80103 objects degraded (9.387%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:31.209 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:31.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:31.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:31.329 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:31.372 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:31.414 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:31.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:31 smithi052 ceph-mon[187743]: pgmap v883: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 7137/78087 objects degraded (9.140%); 0 B/s, 134 objects/s recovering 2023-09-23T22:54:31.753 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:31 smithi052 ceph-mon[180979]: pgmap v883: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 7137/78087 objects degraded (9.140%); 0 B/s, 134 objects/s recovering 2023-09-23T22:54:31.782 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:31.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:31.866 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:54:31.892 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:31 smithi151 ceph-mon[159210]: pgmap v883: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 7137/78087 objects degraded (9.140%); 0 B/s, 134 objects/s recovering 2023-09-23T22:54:31.952 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:54:31.997 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:32.007 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:54:32.051 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:54:32.093 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:54:32.128 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:54:32.128 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:54:32.136 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:54:32.202 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:54:32.217 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:32.235 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:54:32.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:54:32.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1231675697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:32.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1231675697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1231675697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:33.518 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:33.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:33.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:33.626 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:33.673 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:33.711 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:33 smithi052 ceph-mon[180979]: pgmap v884: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 656 op/s; 6872/72582 objects degraded (9.468%); 0 B/s, 131 objects/s recovering 2023-09-23T22:54:33.711 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:54:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:54:33] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-09-23T22:54:33.711 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:33 smithi052 ceph-mon[187743]: pgmap v884: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 656 op/s; 6872/72582 objects degraded (9.468%); 0 B/s, 131 objects/s recovering 2023-09-23T22:54:33.764 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:33.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:33.875 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:33.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:33.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:33 smithi151 ceph-mon[159210]: pgmap v884: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 656 op/s; 6872/72582 objects degraded (9.468%); 0 B/s, 131 objects/s recovering 2023-09-23T22:54:34.557 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:34.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:34.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:34.701 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:34.703 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:54:34.773 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:34.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:34.882 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:34.922 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:34.994 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:35.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:35.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:35.103 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:35.216 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:35 smithi151 ceph-mon[159210]: pgmap v885: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 6872/70197 objects degraded (9.790%); 0 B/s, 131 objects/s recovering 2023-09-23T22:54:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6872/70197 objects degraded (9.790%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:35.967 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:35.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[180979]: pgmap v885: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 6872/70197 objects degraded (9.790%); 0 B/s, 131 objects/s recovering 2023-09-23T22:54:35.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:35.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:35.976 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6872/70197 objects degraded (9.790%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:35.976 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[187743]: pgmap v885: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 6872/70197 objects degraded (9.790%); 0 B/s, 131 objects/s recovering 2023-09-23T22:54:35.977 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:35.977 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:35.977 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6872/70197 objects degraded (9.790%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:35.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:35.995 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:54:36.078 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:36.157 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:36.322 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:36.411 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:36.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:36.521 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:36.582 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:54:36.622 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:36.692 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2335543337' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:36.693 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2335543337' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2335543337' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:37.379 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:54:37.384 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:54:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[180979]: pgmap v886: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 701 op/s; 6440/65787 objects degraded (9.789%); 0 B/s, 121 objects/s recovering 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[180979]: failed to encode map e305 with expected crc 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2335543337' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[180979]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[187743]: pgmap v886: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 701 op/s; 6440/65787 objects degraded (9.789%); 0 B/s, 121 objects/s recovering 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[187743]: failed to encode map e305 with expected crc 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2335543337' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:37 smithi052 ceph-mon[187743]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T22:54:37.878 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:37.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:37 smithi151 ceph-mon[159210]: pgmap v886: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 701 op/s; 6440/65787 objects degraded (9.789%); 0 B/s, 121 objects/s recovering 2023-09-23T22:54:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:37 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:37 smithi151 ceph-mon[159210]: failed to encode map e305 with expected crc 2023-09-23T22:54:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2335543337' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:37 smithi151 ceph-mon[159210]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T22:54:37.990 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:38.064 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:38.727 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:38.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:38.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:38.833 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:38.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:38 smithi052 ceph-mon[187743]: failed to encode map e305 with expected crc 2023-09-23T22:54:38.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:38 smithi052 ceph-mon[187743]: from='client.45836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:38.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:38 smithi052 ceph-mon[180979]: failed to encode map e305 with expected crc 2023-09-23T22:54:38.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:38 smithi052 ceph-mon[180979]: from='client.45836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:38.861 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:38.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:38 smithi151 ceph-mon[159210]: failed to encode map e305 with expected crc 2023-09-23T22:54:38.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:38 smithi151 ceph-mon[159210]: from='client.45836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:39.196 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:39.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:39.304 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:39.325 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":887,"stamp":"2023-09-23T22:54:36.541514+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1436096440,"num_objects":21929,"num_object_clones":0,"num_object_copies":65787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21929,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":157633,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10397,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114117,"ondisk_log_size":114117,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6738544,"kb_used_data":4273472,"kb_used_omap":121,"kb_used_meta":2464454,"kb_avail":555732368,"statfs":{"total":575970213888,"available":569069944832,"internally_reserved":0,"allocated":4376035328,"data_stored":4371106049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2523601013},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,6,8],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-437518336,"num_objects":-6676,"num_object_clones":0,"num_object_copies":-20028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1308,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6676,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1308,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.115436"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":301,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214349+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:54:21.214349+0000","last_peered":"2023-09-23T22:54:21.214349+0000","last_clean":"2023-09-23T22:54:21.214349+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:54:21.214349+0000","last_undegraded":"2023-09-23T22:54:21.214349+0000","last_fullsized":"2023-09-23T22:54:21.214349+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214865+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:54:21.214865+0000","last_peered":"2023-09-23T22:54:21.214865+0000","last_clean":"2023-09-23T22:54:21.214865+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:54:21.214865+0000","last_undegraded":"2023-09-23T22:54:21.214865+0000","last_fullsized":"2023-09-23T22:54:21.214865+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214430+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:54:21.214430+0000","last_peered":"2023-09-23T22:54:21.214430+0000","last_clean":"2023-09-23T22:54:21.214430+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:54:21.214430+0000","last_undegraded":"2023-09-23T22:54:21.214430+0000","last_fullsized":"2023-09-23T22:54:21.214430+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":283,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.224970+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:54:21.224970+0000","last_peered":"2023-09-23T22:54:21.224970+0000","last_clean":"2023-09-23T22:54:21.224970+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:54:21.224970+0000","last_undegraded":"2023-09-23T22:54:21.224970+0000","last_fullsized":"2023-09-23T22:54:21.224970+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":287,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.222329+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:54:21.222329+0000","last_peered":"2023-09-23T22:54:21.222329+0000","last_clean":"2023-09-23T22:54:21.222329+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:54:21.222329+0000","last_undegraded":"2023-09-23T22:54:21.222329+0000","last_fullsized":"2023-09-23T22:54:21.222329+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":269,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.127154+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:54:21.127154+0000","last_peered":"2023-09-23T22:54:21.127154+0000","last_clean":"2023-09-23T22:54:21.127154+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:54:21.127154+0000","last_undegraded":"2023-09-23T22:54:21.127154+0000","last_fullsized":"2023-09-23T22:54:21.127154+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":214,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.224722+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:54:21.224722+0000","last_peered":"2023-09-23T22:54:21.224722+0000","last_clean":"2023-09-23T22:54:21.224722+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:54:21.224722+0000","last_undegraded":"2023-09-23T22:54:21.224722+0000","last_fullsized":"2023-09-23T22:54:21.224722+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":166,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128124+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:54:21.128124+0000","last_peered":"2023-09-23T22:54:21.128124+0000","last_clean":"2023-09-23T22:54:21.128124+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:54:21.128124+0000","last_undegraded":"2023-09-23T22:54:21.128124+0000","last_fullsized":"2023-09-23T22:54:21.128124+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":45,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128798+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:54:21.128798+0000","last_peered":"2023-09-23T22:54:21.128798+0000","last_clean":"2023-09-23T22:54:21.128798+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:54:21.128798+0000","last_undegraded":"2023-09-23T22:54:21.128798+0000","last_fullsized":"2023-09-23T22:54:21.128798+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126418+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:54:21.126418+0000","last_peered":"2023-09-23T22:54:21.126418+0000","last_clean":"2023-09-23T22:54:21.126418+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:54:21.126418+0000","last_undegraded":"2023-09-23T22:54:21.126418+0000","last_fullsized":"2023-09-23T22:54:21.126418+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":62,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214739+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:54:21.214739+0000","last_peered":"2023-09-23T22:54:21.214739+0000","last_clean":"2023-09-23T22:54:21.214739+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:54:21.214739+0000","last_undegraded":"2023-09-23T22:54:21.214739+0000","last_fullsized":"2023-09-23T22:54:21.214739+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":178,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.127033+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:54:21.127033+0000","last_peered":"2023-09-23T22:54:21.127033+0000","last_clean":"2023-09-23T22:54:21.127033+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:54:21.127033+0000","last_undegraded":"2023-09-23T22:54:21.127033+0000","last_fullsized":"2023-09-23T22:54:21.127033+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":323,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.215175+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:54:21.215175+0000","last_peered":"2023-09-23T22:54:21.215175+0000","last_clean":"2023-09-23T22:54:21.215175+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:54:21.215175+0000","last_undegraded":"2023-09-23T22:54:21.215175+0000","last_fullsized":"2023-09-23T22:54:21.215175+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":269,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214572+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:54:21.214572+0000","last_peered":"2023-09-23T22:54:21.214572+0000","last_clean":"2023-09-23T22:54:21.214572+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:54:21.214572+0000","last_undegraded":"2023-09-23T22:54:21.214572+0000","last_fullsized":"2023-09-23T22:54:21.214572+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":162,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214681+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:54:21.214681+0000","last_peered":"2023-09-23T22:54:21.214681+0000","last_clean":"2023-09-23T22:54:21.214681+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:54:21.214681+0000","last_undegraded":"2023-09-23T22:54:21.214681+0000","last_fullsized":"2023-09-23T22:54:21.214681+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128733+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:54:21.128733+0000","last_peered":"2023-09-23T22:54:21.128733+0000","last_clean":"2023-09-23T22:54:21.128733+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:54:21.128733+0000","last_undegraded":"2023-09-23T22:54:21.128733+0000","last_fullsized":"2023-09-23T22:54:21.128733+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":270,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.221381+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:54:21.221381+0000","last_peered":"2023-09-23T22:54:21.221381+0000","last_clean":"2023-09-23T22:54:21.221381+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:54:21.221381+0000","last_undegraded":"2023-09-23T22:54:21.221381+0000","last_fullsized":"2023-09-23T22:54:21.221381+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":227,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126931+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:54:21.126931+0000","last_peered":"2023-09-23T22:54:21.126931+0000","last_clean":"2023-09-23T22:54:21.126931+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:54:21.126931+0000","last_undegraded":"2023-09-23T22:54:21.126931+0000","last_fullsized":"2023-09-23T22:54:21.126931+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214758+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:54:21.214758+0000","last_peered":"2023-09-23T22:54:21.214758+0000","last_clean":"2023-09-23T22:54:21.214758+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:54:21.214758+0000","last_undegraded":"2023-09-23T22:54:21.214758+0000","last_fullsized":"2023-09-23T22:54:21.214758+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.225017+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:54:21.225017+0000","last_peered":"2023-09-23T22:54:21.225017+0000","last_clean":"2023-09-23T22:54:21.225017+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:54:21.225017+0000","last_undegraded":"2023-09-23T22:54:21.225017+0000","last_fullsized":"2023-09-23T22:54:21.225017+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":348,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126224+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:54:21.126224+0000","last_peered":"2023-09-23T22:54:21.126224+0000","last_clean":"2023-09-23T22:54:21.126224+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:54:21.126224+0000","last_undegraded":"2023-09-23T22:54:21.126224+0000","last_fullsized":"2023-09-23T22:54:21.126224+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":231,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.129178+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:54:21.129178+0000","last_peered":"2023-09-23T22:54:21.129178+0000","last_clean":"2023-09-23T22:54:21.129178+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:54:21.129178+0000","last_undegraded":"2023-09-23T22:54:21.129178+0000","last_fullsized":"2023-09-23T22:54:21.129178+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"304'9718","reported_seq":9768,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:33.261147+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:54:33.261147+0000","last_peered":"2023-09-23T22:54:33.261147+0000","last_clean":"2023-09-23T22:54:33.261147+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:54:33.261147+0000","last_undegraded":"2023-09-23T22:54:33.261147+0000","last_fullsized":"2023-09-23T22:54:33.261147+0000","mapping_epoch":264,"log_start":"273'2200","ondisk_log_start":"273'2200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92667904,"num_objects":1414,"num_object_clones":0,"num_object_copies":4242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9718,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":173,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.215774+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:54:21.215774+0000","last_peered":"2023-09-23T22:54:21.215774+0000","last_clean":"2023-09-23T22:54:21.215774+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:54:21.215774+0000","last_undegraded":"2023-09-23T22:54:21.215774+0000","last_fullsized":"2023-09-23T22:54:21.215774+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126254+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:54:21.126254+0000","last_peered":"2023-09-23T22:54:21.126254+0000","last_clean":"2023-09-23T22:54:21.126254+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:54:21.126254+0000","last_undegraded":"2023-09-23T22:54:21.126254+0000","last_fullsized":"2023-09-23T22:54:21.126254+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"304'9962","reported_seq":10017,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:31.651074+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:54:31.651074+0000","last_peered":"2023-09-23T22:54:31.651074+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:54:31.651074+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"276'3200","ondisk_log_start":"276'3200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6762,"log_dups_size":0,"ondisk_log_size":6762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9962,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":-131}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":183,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.215796+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:54:21.215796+0000","last_peered":"2023-09-23T22:54:21.215796+0000","last_clean":"2023-09-23T22:54:21.215796+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:54:21.215796+0000","last_undegraded":"2023-09-23T22:54:21.215796+0000","last_fullsized":"2023-09-23T22:54:21.215796+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":293,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214514+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:54:21.214514+0000","last_peered":"2023-09-23T22:54:21.214514+0000","last_clean":"2023-09-23T22:54:21.214514+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:54:21.214514+0000","last_undegraded":"2023-09-23T22:54:21.214514+0000","last_fullsized":"2023-09-23T22:54:21.214514+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"304'10053","reported_seq":10106,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:35.188480+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:54:35.188480+0000","last_peered":"2023-09-23T22:54:35.188480+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:54:35.188480+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"269'1400","ondisk_log_start":"269'1400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8653,"log_dups_size":0,"ondisk_log_size":8653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80936960,"num_objects":1235,"num_object_clones":0,"num_object_copies":3705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10053,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-536}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":127,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126827+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:54:21.126827+0000","last_peered":"2023-09-23T22:54:21.126827+0000","last_clean":"2023-09-23T22:54:21.126827+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:54:21.126827+0000","last_undegraded":"2023-09-23T22:54:21.126827+0000","last_fullsized":"2023-09-23T22:54:21.126827+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":296,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126918+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:54:21.126918+0000","last_peered":"2023-09-23T22:54:21.126918+0000","last_clean":"2023-09-23T22:54:21.126918+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:54:21.126918+0000","last_undegraded":"2023-09-23T22:54:21.126918+0000","last_fullsized":"2023-09-23T22:54:21.126918+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"304'9958","reported_seq":10008,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:35.148909+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:54:35.148909+0000","last_peered":"2023-09-23T22:54:35.148909+0000","last_clean":"2023-09-23T22:54:35.148909+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:54:35.148909+0000","last_undegraded":"2023-09-23T22:54:35.148909+0000","last_fullsized":"2023-09-23T22:54:35.148909+0000","mapping_epoch":264,"log_start":"269'1300","ondisk_log_start":"269'1300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8658,"log_dups_size":0,"ondisk_log_size":8658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9958,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126696+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:54:21.126696+0000","last_peered":"2023-09-23T22:54:21.126696+0000","last_clean":"2023-09-23T22:54:21.126696+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:54:21.126696+0000","last_undegraded":"2023-09-23T22:54:21.126696+0000","last_fullsized":"2023-09-23T22:54:21.126696+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":198,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.221663+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:54:21.221663+0000","last_peered":"2023-09-23T22:54:21.221663+0000","last_clean":"2023-09-23T22:54:21.221663+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:54:21.221663+0000","last_undegraded":"2023-09-23T22:54:21.221663+0000","last_fullsized":"2023-09-23T22:54:21.221663+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"304'9789","reported_seq":13203,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:35.863621+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:54:35.863621+0000","last_peered":"2023-09-23T22:54:35.863621+0000","last_clean":"2023-09-23T22:54:35.863621+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:54:35.863621+0000","last_undegraded":"2023-09-23T22:54:35.863621+0000","last_fullsized":"2023-09-23T22:54:35.863621+0000","mapping_epoch":289,"log_start":"277'3400","ondisk_log_start":"277'3400","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6389,"log_dups_size":0,"ondisk_log_size":6389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79888384,"num_objects":1219,"num_object_clones":0,"num_object_copies":3657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9789,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":70,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.221757+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:54:21.221757+0000","last_peered":"2023-09-23T22:54:21.221757+0000","last_clean":"2023-09-23T22:54:21.221757+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:54:21.221757+0000","last_undegraded":"2023-09-23T22:54:21.221757+0000","last_fullsized":"2023-09-23T22:54:21.221757+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":277,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.215098+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:54:21.215098+0000","last_peered":"2023-09-23T22:54:21.215098+0000","last_clean":"2023-09-23T22:54:21.215098+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:54:21.215098+0000","last_undegraded":"2023-09-23T22:54:21.215098+0000","last_fullsized":"2023-09-23T22:54:21.215098+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"304'9830","reported_seq":9877,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:36.002923+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:54:36.002923+0000","last_peered":"2023-09-23T22:54:36.002923+0000","last_clean":"2023-09-23T22:54:36.002923+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:54:36.002923+0000","last_undegraded":"2023-09-23T22:54:36.002923+0000","last_fullsized":"2023-09-23T22:54:36.002923+0000","mapping_epoch":264,"log_start":"277'3400","ondisk_log_start":"277'3400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9830,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":288,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126852+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:54:21.126852+0000","last_peered":"2023-09-23T22:54:21.126852+0000","last_clean":"2023-09-23T22:54:21.126852+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:54:21.126852+0000","last_undegraded":"2023-09-23T22:54:21.126852+0000","last_fullsized":"2023-09-23T22:54:21.126852+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":299,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214532+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:54:21.214532+0000","last_peered":"2023-09-23T22:54:21.214532+0000","last_clean":"2023-09-23T22:54:21.214532+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:54:21.214532+0000","last_undegraded":"2023-09-23T22:54:21.214532+0000","last_fullsized":"2023-09-23T22:54:21.214532+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"304'9939","reported_seq":13341,"reported_epoch":304,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:54:35.990905+0000","last_change":"2023-09-23T22:54:01.194508+0000","last_active":"2023-09-23T22:54:35.990905+0000","last_peered":"2023-09-23T22:54:35.990905+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:54:01.140849+0000","last_became_peered":"2023-09-23T22:54:01.140849+0000","last_unstale":"2023-09-23T22:54:35.990905+0000","last_undegraded":"2023-09-23T22:54:35.990905+0000","last_fullsized":"2023-09-23T22:54:35.990905+0000","mapping_epoch":299,"log_start":"277'3500","ondisk_log_start":"277'3500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6439,"log_dups_size":0,"ondisk_log_size":6439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81330176,"num_objects":1241,"num_object_clones":0,"num_object_copies":3723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9939,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1668,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":["6","1","3","5"],"object_location_counts":[{"shards":"1,5,6","objects":1241}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":312,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214260+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:54:21.214260+0000","last_peered":"2023-09-23T22:54:21.214260+0000","last_clean":"2023-09-23T22:54:21.214260+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:54:21.214260+0000","last_undegraded":"2023-09-23T22:54:21.214260+0000","last_fullsized":"2023-09-23T22:54:21.214260+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":287,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.215874+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:54:21.215874+0000","last_peered":"2023-09-23T22:54:21.215874+0000","last_clean":"2023-09-23T22:54:21.215874+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:54:21.215874+0000","last_undegraded":"2023-09-23T22:54:21.215874+0000","last_fullsized":"2023-09-23T22:54:21.215874+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"304'9724","reported_seq":12291,"reported_epoch":304,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:35.190321+0000","last_change":"2023-09-23T22:54:20.023846+0000","last_active":"2023-09-23T22:54:35.190321+0000","last_peered":"2023-09-23T22:54:35.190321+0000","last_clean":"2023-09-23T22:51:33.358390+0000","last_became_active":"2023-09-23T22:51:35.512259+0000","last_became_peered":"2023-09-23T22:51:35.512259+0000","last_unstale":"2023-09-23T22:54:35.190321+0000","last_undegraded":"2023-09-23T22:51:35.423892+0000","last_fullsized":"2023-09-23T22:51:35.414090+0000","mapping_epoch":271,"log_start":"269'1100","ondisk_log_start":"269'1100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8624,"log_dups_size":0,"ondisk_log_size":8624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:04.342867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":360,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9724,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1255,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":986}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":232,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.213995+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:54:21.213995+0000","last_peered":"2023-09-23T22:54:21.213995+0000","last_clean":"2023-09-23T22:54:21.213995+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:54:21.213995+0000","last_undegraded":"2023-09-23T22:54:21.213995+0000","last_fullsized":"2023-09-23T22:54:21.213995+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":241,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.129271+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:54:21.129271+0000","last_peered":"2023-09-23T22:54:21.129271+0000","last_clean":"2023-09-23T22:54:21.129271+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:54:21.129271+0000","last_undegraded":"2023-09-23T22:54:21.129271+0000","last_fullsized":"2023-09-23T22:54:21.129271+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"304'9880","reported_seq":11322,"reported_epoch":304,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:32.248444+0000","last_change":"2023-09-23T22:54:20.000403+0000","last_active":"2023-09-23T22:54:32.248444+0000","last_peered":"2023-09-23T22:54:32.248444+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:54:32.248444+0000","last_undegraded":"2023-09-23T22:51:35.440939+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"284'4700","ondisk_log_start":"284'4700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5180,"log_dups_size":0,"ondisk_log_size":5180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96468992,"num_objects":1472,"num_object_clones":0,"num_object_copies":4416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":936,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9880,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":694,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,5","objects":536}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":342,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.225236+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:54:21.225236+0000","last_peered":"2023-09-23T22:54:21.225236+0000","last_clean":"2023-09-23T22:54:21.225236+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:54:21.225236+0000","last_undegraded":"2023-09-23T22:54:21.225236+0000","last_fullsized":"2023-09-23T22:54:21.225236+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":167,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128717+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:54:21.128717+0000","last_peered":"2023-09-23T22:54:21.128717+0000","last_clean":"2023-09-23T22:54:21.128717+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:54:21.128717+0000","last_undegraded":"2023-09-23T22:54:21.128717+0000","last_fullsized":"2023-09-23T22:54:21.128717+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214262+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:54:21.214262+0000","last_peered":"2023-09-23T22:54:21.214262+0000","last_clean":"2023-09-23T22:54:21.214262+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:54:21.214262+0000","last_undegraded":"2023-09-23T22:54:21.214262+0000","last_fullsized":"2023-09-23T22:54:21.214262+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"304'9825","reported_seq":13277,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:33.286340+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:54:33.286340+0000","last_peered":"2023-09-23T22:54:33.286340+0000","last_clean":"2023-09-23T22:54:33.286340+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:54:33.286340+0000","last_undegraded":"2023-09-23T22:54:33.286340+0000","last_fullsized":"2023-09-23T22:54:33.286340+0000","mapping_epoch":282,"log_start":"273'2300","ondisk_log_start":"273'2300","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7525,"log_dups_size":0,"ondisk_log_size":7525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9825,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":138,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126834+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:54:21.126834+0000","last_peered":"2023-09-23T22:54:21.126834+0000","last_clean":"2023-09-23T22:54:21.126834+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:54:21.126834+0000","last_undegraded":"2023-09-23T22:54:21.126834+0000","last_fullsized":"2023-09-23T22:54:21.126834+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":300,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214663+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:54:21.214663+0000","last_peered":"2023-09-23T22:54:21.214663+0000","last_clean":"2023-09-23T22:54:21.214663+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:54:21.214663+0000","last_undegraded":"2023-09-23T22:54:21.214663+0000","last_fullsized":"2023-09-23T22:54:21.214663+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":231,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128816+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:54:21.128816+0000","last_peered":"2023-09-23T22:54:21.128816+0000","last_clean":"2023-09-23T22:54:21.128816+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:54:21.128816+0000","last_undegraded":"2023-09-23T22:54:21.128816+0000","last_fullsized":"2023-09-23T22:54:21.128816+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"304'9980","reported_seq":13359,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:35.985244+0000","last_change":"2023-09-23T22:54:21.209191+0000","last_active":"2023-09-23T22:54:35.985244+0000","last_peered":"2023-09-23T22:54:35.985244+0000","last_clean":"2023-09-23T22:54:35.985244+0000","last_became_active":"2023-09-23T22:54:21.131570+0000","last_became_peered":"2023-09-23T22:54:21.131570+0000","last_unstale":"2023-09-23T22:54:35.985244+0000","last_undegraded":"2023-09-23T22:54:35.985244+0000","last_fullsized":"2023-09-23T22:54:35.985244+0000","mapping_epoch":302,"log_start":"277'3550","ondisk_log_start":"277'3550","created":264,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84934656,"num_objects":1296,"num_object_clones":0,"num_object_copies":3888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9980,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1655,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":306,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128411+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:54:21.128411+0000","last_peered":"2023-09-23T22:54:21.128411+0000","last_clean":"2023-09-23T22:54:21.128411+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:54:21.128411+0000","last_undegraded":"2023-09-23T22:54:21.128411+0000","last_fullsized":"2023-09-23T22:54:21.128411+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":185,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128021+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:54:21.128021+0000","last_peered":"2023-09-23T22:54:21.128021+0000","last_clean":"2023-09-23T22:54:21.128021+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:54:21.128021+0000","last_undegraded":"2023-09-23T22:54:21.128021+0000","last_fullsized":"2023-09-23T22:54:21.128021+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":311,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.224634+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:54:21.224634+0000","last_peered":"2023-09-23T22:54:21.224634+0000","last_clean":"2023-09-23T22:54:21.224634+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:54:21.224634+0000","last_undegraded":"2023-09-23T22:54:21.224634+0000","last_fullsized":"2023-09-23T22:54:21.224634+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"304'9648","reported_seq":9705,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:33.261001+0000","last_change":"2023-09-23T22:51:35.532533+0000","last_active":"2023-09-23T22:54:33.261001+0000","last_peered":"2023-09-23T22:54:33.261001+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:54:33.261001+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"273'2100","ondisk_log_start":"273'2100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9648,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":-310}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":316,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128541+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:54:21.128541+0000","last_peered":"2023-09-23T22:54:21.128541+0000","last_clean":"2023-09-23T22:54:21.128541+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:54:21.128541+0000","last_undegraded":"2023-09-23T22:54:21.128541+0000","last_fullsized":"2023-09-23T22:54:21.128541+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":393,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.222189+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:54:21.222189+0000","last_peered":"2023-09-23T22:54:21.222189+0000","last_clean":"2023-09-23T22:54:21.222189+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:54:21.222189+0000","last_undegraded":"2023-09-23T22:54:21.222189+0000","last_fullsized":"2023-09-23T22:54:21.222189+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":58,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126932+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:54:21.126932+0000","last_peered":"2023-09-23T22:54:21.126932+0000","last_clean":"2023-09-23T22:54:21.126932+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:54:21.126932+0000","last_undegraded":"2023-09-23T22:54:21.126932+0000","last_fullsized":"2023-09-23T22:54:21.126932+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.126725+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:54:21.126725+0000","last_peered":"2023-09-23T22:54:21.126725+0000","last_clean":"2023-09-23T22:54:21.126725+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:54:21.126725+0000","last_undegraded":"2023-09-23T22:54:21.126725+0000","last_fullsized":"2023-09-23T22:54:21.126725+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"304'9830","reported_seq":9880,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:33.261840+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:54:33.261840+0000","last_peered":"2023-09-23T22:54:33.261840+0000","last_clean":"2023-09-23T22:54:33.261840+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:54:33.261840+0000","last_undegraded":"2023-09-23T22:54:33.261840+0000","last_fullsized":"2023-09-23T22:54:33.261840+0000","mapping_epoch":264,"log_start":"273'2300","ondisk_log_start":"273'2300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7530,"log_dups_size":0,"ondisk_log_size":7530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91488256,"num_objects":1396,"num_object_clones":0,"num_object_copies":4188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9830,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":280,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.222413+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:54:21.222413+0000","last_peered":"2023-09-23T22:54:21.222413+0000","last_clean":"2023-09-23T22:54:21.222413+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:54:21.222413+0000","last_undegraded":"2023-09-23T22:54:21.222413+0000","last_fullsized":"2023-09-23T22:54:21.222413+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":252,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:25.657079+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:54:25.657079+0000","last_peered":"2023-09-23T22:54:25.657079+0000","last_clean":"2023-09-23T22:54:25.657079+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:54:25.657079+0000","last_undegraded":"2023-09-23T22:54:25.657079+0000","last_fullsized":"2023-09-23T22:54:25.657079+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":229,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.224846+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:54:21.224846+0000","last_peered":"2023-09-23T22:54:21.224846+0000","last_clean":"2023-09-23T22:54:21.224846+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:54:21.224846+0000","last_undegraded":"2023-09-23T22:54:21.224846+0000","last_fullsized":"2023-09-23T22:54:21.224846+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"304'9559","reported_seq":9607,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:31.634981+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:54:31.634981+0000","last_peered":"2023-09-23T22:54:31.634981+0000","last_clean":"2023-09-23T22:54:31.634981+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:54:31.634981+0000","last_undegraded":"2023-09-23T22:54:31.634981+0000","last_fullsized":"2023-09-23T22:54:31.634981+0000","mapping_epoch":264,"log_start":"275'2800","ondisk_log_start":"275'2800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6759,"log_dups_size":0,"ondisk_log_size":6759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92733440,"num_objects":1415,"num_object_clones":0,"num_object_copies":4245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9559,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":235,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:25.657945+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:54:25.657945+0000","last_peered":"2023-09-23T22:54:25.657945+0000","last_clean":"2023-09-23T22:54:25.657945+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:54:25.657945+0000","last_undegraded":"2023-09-23T22:54:25.657945+0000","last_fullsized":"2023-09-23T22:54:25.657945+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":184,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128981+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:54:21.128981+0000","last_peered":"2023-09-23T22:54:21.128981+0000","last_clean":"2023-09-23T22:54:21.128981+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:54:21.128981+0000","last_undegraded":"2023-09-23T22:54:21.128981+0000","last_fullsized":"2023-09-23T22:54:21.128981+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":235,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:25.657491+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:54:25.657491+0000","last_peered":"2023-09-23T22:54:25.657491+0000","last_clean":"2023-09-23T22:54:25.657491+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:54:25.657491+0000","last_undegraded":"2023-09-23T22:54:25.657491+0000","last_fullsized":"2023-09-23T22:54:25.657491+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"304'9835","reported_seq":9884,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:31.616849+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:54:31.616849+0000","last_peered":"2023-09-23T22:54:31.616849+0000","last_clean":"2023-09-23T22:54:31.616849+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:54:31.616849+0000","last_undegraded":"2023-09-23T22:54:31.616849+0000","last_fullsized":"2023-09-23T22:54:31.616849+0000","mapping_epoch":264,"log_start":"275'3100","ondisk_log_start":"275'3100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100007960,"num_objects":1527,"num_object_clones":0,"num_object_copies":4581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1527,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9835,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":61,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.224893+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:54:21.224893+0000","last_peered":"2023-09-23T22:54:21.224893+0000","last_clean":"2023-09-23T22:54:21.224893+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:54:21.224893+0000","last_undegraded":"2023-09-23T22:54:21.224893+0000","last_fullsized":"2023-09-23T22:54:21.224893+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.127096+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:54:21.127096+0000","last_peered":"2023-09-23T22:54:21.127096+0000","last_clean":"2023-09-23T22:54:21.127096+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:54:21.127096+0000","last_undegraded":"2023-09-23T22:54:21.127096+0000","last_fullsized":"2023-09-23T22:54:21.127096+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":281,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.221104+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:54:21.221104+0000","last_peered":"2023-09-23T22:54:21.221104+0000","last_clean":"2023-09-23T22:54:21.221104+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:54:21.221104+0000","last_undegraded":"2023-09-23T22:54:21.221104+0000","last_fullsized":"2023-09-23T22:54:21.221104+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"304'9843","reported_seq":13368,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:31.650607+0000","last_change":"2023-09-23T22:53:44.765920+0000","last_active":"2023-09-23T22:54:31.650607+0000","last_peered":"2023-09-23T22:54:31.650607+0000","last_clean":"2023-09-23T22:54:31.650607+0000","last_became_active":"2023-09-23T22:53:44.707193+0000","last_became_peered":"2023-09-23T22:53:44.707193+0000","last_unstale":"2023-09-23T22:54:31.650607+0000","last_undegraded":"2023-09-23T22:54:31.650607+0000","last_fullsized":"2023-09-23T22:54:31.650607+0000","mapping_epoch":296,"log_start":"275'3100","ondisk_log_start":"275'3100","created":264,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94961664,"num_objects":1449,"num_object_clones":0,"num_object_copies":4347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9843,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":345,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.127081+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:54:21.127081+0000","last_peered":"2023-09-23T22:54:21.127081+0000","last_clean":"2023-09-23T22:54:21.127081+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:54:21.127081+0000","last_undegraded":"2023-09-23T22:54:21.127081+0000","last_fullsized":"2023-09-23T22:54:21.127081+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":201,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.224935+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:54:21.224935+0000","last_peered":"2023-09-23T22:54:21.224935+0000","last_clean":"2023-09-23T22:54:21.224935+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:54:21.224935+0000","last_undegraded":"2023-09-23T22:54:21.224935+0000","last_fullsized":"2023-09-23T22:54:21.224935+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214343+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:54:21.214343+0000","last_peered":"2023-09-23T22:54:21.214343+0000","last_clean":"2023-09-23T22:54:21.214343+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:54:21.214343+0000","last_undegraded":"2023-09-23T22:54:21.214343+0000","last_fullsized":"2023-09-23T22:54:21.214343+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":237,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.221850+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:54:21.221850+0000","last_peered":"2023-09-23T22:54:21.221850+0000","last_clean":"2023-09-23T22:54:21.221850+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:54:21.221850+0000","last_undegraded":"2023-09-23T22:54:21.221850+0000","last_fullsized":"2023-09-23T22:54:21.221850+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":87,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128896+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:54:21.128896+0000","last_peered":"2023-09-23T22:54:21.128896+0000","last_clean":"2023-09-23T22:54:21.128896+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:54:21.128896+0000","last_undegraded":"2023-09-23T22:54:21.128896+0000","last_fullsized":"2023-09-23T22:54:21.128896+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":113,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.215031+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:54:21.215031+0000","last_peered":"2023-09-23T22:54:21.215031+0000","last_clean":"2023-09-23T22:54:21.215031+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:54:21.215031+0000","last_undegraded":"2023-09-23T22:54:21.215031+0000","last_fullsized":"2023-09-23T22:54:21.215031+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":312,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214752+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:54:21.214752+0000","last_peered":"2023-09-23T22:54:21.214752+0000","last_clean":"2023-09-23T22:54:21.214752+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:54:21.214752+0000","last_undegraded":"2023-09-23T22:54:21.214752+0000","last_fullsized":"2023-09-23T22:54:21.214752+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":114,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214634+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:54:21.214634+0000","last_peered":"2023-09-23T22:54:21.214634+0000","last_clean":"2023-09-23T22:54:21.214634+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:54:21.214634+0000","last_undegraded":"2023-09-23T22:54:21.214634+0000","last_fullsized":"2023-09-23T22:54:21.214634+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214426+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:54:21.214426+0000","last_peered":"2023-09-23T22:54:21.214426+0000","last_clean":"2023-09-23T22:54:21.214426+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:54:21.214426+0000","last_undegraded":"2023-09-23T22:54:21.214426+0000","last_fullsized":"2023-09-23T22:54:21.214426+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":54,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.127023+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:54:21.127023+0000","last_peered":"2023-09-23T22:54:21.127023+0000","last_clean":"2023-09-23T22:54:21.127023+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:54:21.127023+0000","last_undegraded":"2023-09-23T22:54:21.127023+0000","last_fullsized":"2023-09-23T22:54:21.127023+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":287,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.128292+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:54:21.128292+0000","last_peered":"2023-09-23T22:54:21.128292+0000","last_clean":"2023-09-23T22:54:21.128292+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:54:21.128292+0000","last_undegraded":"2023-09-23T22:54:21.128292+0000","last_fullsized":"2023-09-23T22:54:21.128292+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":251,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T22:54:25.657375+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:54:25.657375+0000","last_peered":"2023-09-23T22:54:25.657375+0000","last_clean":"2023-09-23T22:54:25.657375+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:54:25.657375+0000","last_undegraded":"2023-09-23T22:54:25.657375+0000","last_fullsized":"2023-09-23T22:54:25.657375+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":269,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T22:54:21.214085+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:54:21.214085+0000","last_peered":"2023-09-23T22:54:21.214085+0000","last_clean":"2023-09-23T22:54:21.214085+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:54:21.214085+0000","last_undegraded":"2023-09-23T22:54:21.214085+0000","last_fullsized":"2023-09-23T22:54:21.214085+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1435500568,"num_objects":21905,"num_object_clones":0,"num_object_copies":65715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6440,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21905,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157373,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10367,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5272907776,"data_stored":5272895560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113923,"ondisk_log_size":113923,"up":48,"acting":43,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430539,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626767,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":964100,"kb_used_data":654588,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92781052,"statfs":{"total":95995035648,"available":95007797248,"internally_reserved":0,"allocated":670298112,"data_stored":669741246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":8.0800000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.087,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.3559999999999999,"5min":8.8829999999999991,"15min":4.4269999999999996},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":12.081,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":5.2249999999999996,"15min":3.8079999999999998},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":10.98,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":3.21,"5min":6.5389999999999997,"15min":4.8079999999999998},"min":{"1min":0.45000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":12.069000000000001,"5min":255.20099999999999,"15min":255.20099999999999},"last":5.0570000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":7.5069999999999997,"5min":4.4580000000000002,"15min":3.738},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":81.719999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.1970000000000001,"5min":3.1539999999999999,"15min":3.7509999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":9.3529999999999998,"5min":101.13,"15min":185.21799999999999},"last":2.7490000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.1520000000000001,"15min":3.8029999999999999},"min":{"1min":0.309,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":6.5090000000000003,"5min":17.742999999999999,"15min":185.19399999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":3.2850000000000001,"5min":2.2799999999999998,"15min":3.4950000000000001},"min":{"1min":0.42599999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":10.378,"5min":10.773999999999999,"15min":128.19800000000001},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":1.1359999999999999,"15min":0.871},"min":{"1min":0.309,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.317,"5min":10.317,"15min":18.085999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.107,"15min":0.88500000000000001},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4610000000000003,"5min":7.4800000000000004,"15min":18.597000000000001},"last":0.32000000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.339,"15min":0.99099999999999999},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.353999999999999,"5min":10.353999999999999,"15min":18.972000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.5459999999999998,"5min":1.1779999999999999,"15min":0.93999999999999995},"min":{"1min":0.41599999999999998,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":7.1459999999999999,"5min":7.1459999999999999,"15min":18.625},"last":2.0059999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.123,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":7.8650000000000002,"5min":7.8650000000000002,"15min":12.481999999999999},"last":1.9530000000000001},{"interface":"front","average":{"1min":1.694,"5min":1.21,"15min":0.93600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.774,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.40500000000000003}]}]},{"osd":1,"up_from":84,"seq":360777253076,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1638148,"kb_used_data":1130620,"kb_used_omap":9,"kb_used_meta":507062,"kb_avail":92107004,"statfs":{"total":95995035648,"available":94317572096,"internally_reserved":0,"allocated":1157754880,"data_stored":1156676088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":519232489},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":4.5519999999999996,"15min":5.4329999999999998},"min":{"1min":0.27700000000000002,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":37.991,"5min":37.991,"15min":101.605},"last":1.5449999999999999},{"interface":"front","average":{"1min":6.1550000000000002,"5min":3.819,"15min":5.3019999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":42.191000000000003,"5min":42.191000000000003,"15min":101.892},"last":4.6260000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":3.8900000000000001,"15min":3.2549999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":21.215,"5min":26.129000000000001,"15min":37.889000000000003},"last":0.628},{"interface":"front","average":{"1min":3.9689999999999999,"5min":3.633,"15min":3.3239999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":21.254000000000001,"5min":26.001999999999999,"15min":37.957000000000001},"last":1.974}]},{"osd":3,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":3.3079999999999998,"15min":2.528},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":9.1660000000000004,"5min":87.787999999999997,"15min":87.787999999999997},"last":0.92300000000000004},{"interface":"front","average":{"1min":2.9540000000000002,"5min":3.8650000000000002,"15min":3.0059999999999998},"min":{"1min":0.29299999999999998,"5min":0.28899999999999998,"15min":0.20999999999999999},"max":{"1min":10.845000000000001,"5min":89.861999999999995,"15min":89.861999999999995},"last":1.3129999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.9710000000000001,"15min":2.25},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":8.7469999999999999,"5min":11.432,"15min":78.948999999999998},"last":1.343},{"interface":"front","average":{"1min":3.544,"5min":2.649,"15min":2.2850000000000001},"min":{"1min":0.315,"5min":0.313,"15min":0.313},"max":{"1min":10.894,"5min":14.058,"15min":78.963999999999999},"last":0.66100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.597,"15min":1.792},"min":{"1min":0.42799999999999999,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":9.141,"5min":9.141,"15min":84.254999999999995},"last":0.86499999999999999},{"interface":"front","average":{"1min":2.903,"5min":1.958,"15min":2.125},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":7.1150000000000002,"5min":11.458,"15min":87.515000000000001},"last":1.3660000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":4.0899999999999999,"5min":2.7519999999999998,"15min":2.1549999999999998},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":10.853999999999999,"5min":13.991,"15min":79.311000000000007},"last":0.70799999999999996},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.7609999999999999,"15min":1.7809999999999999},"min":{"1min":0.40899999999999997,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":79.328999999999994},"last":0.89100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.1339999999999999,"15min":2.5049999999999999},"min":{"1min":0.438,"5min":0.40799999999999997,"15min":0.32800000000000001},"max":{"1min":8.702,"5min":11.481999999999999,"15min":159.018},"last":1.3939999999999999},{"interface":"front","average":{"1min":4.008,"5min":2.911,"15min":2.6669999999999998},"min":{"1min":0.49199999999999999,"5min":0.371,"15min":0.37},"max":{"1min":10.923,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.68500000000000005}]}]},{"osd":0,"up_from":219,"seq":940597837925,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":957744,"kb_used_data":630632,"kb_used_omap":31,"kb_used_meta":327072,"kb_avail":92787408,"statfs":{"total":95995035648,"available":95014305792,"internally_reserved":0,"allocated":645767168,"data_stored":644648534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":334922674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":5.3840000000000003,"5min":4.0069999999999997,"15min":3.758},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":40.180999999999997,"5min":40.180999999999997,"15min":40.180999999999997},"last":1.798},{"interface":"front","average":{"1min":4.9249999999999998,"5min":3.6819999999999999,"15min":3.605},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":40.137999999999998,"5min":40.137999999999998,"15min":40.137999999999998},"last":8.6240000000000006}]},{"osd":2,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":4.4560000000000004,"5min":5.0910000000000002,"15min":3.2189999999999999},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":46.387999999999998,"5min":111.508,"15min":111.508},"last":8.7129999999999992},{"interface":"front","average":{"1min":5.7619999999999996,"5min":4.7220000000000004,"15min":2.8460000000000001},"min":{"1min":0.41599999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":46.249000000000002,"5min":111.532,"15min":111.532},"last":1.5920000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":4.9660000000000002,"15min":3.0289999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.2759999999999998,"5min":126.73,"15min":126.73},"last":8.5380000000000003},{"interface":"front","average":{"1min":1.2829999999999999,"5min":3.581,"15min":2.044},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.7539999999999996,"5min":121.483,"15min":121.483},"last":0.40699999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":2.2709999999999999,"15min":2.1179999999999999},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":8.8239999999999998,"5min":18.670999999999999,"15min":18.670999999999999},"last":1.718},{"interface":"front","average":{"1min":2.218,"5min":2.6080000000000001,"15min":2.036},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.8620000000000001,"5min":20.009,"15min":20.009},"last":8.5899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":2.766,"15min":1.901},"min":{"1min":0.39000000000000001,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":13.851000000000001,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.4079999999999999,"5min":3.4380000000000002,"15min":2.2839999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":12.436,"5min":61.777000000000001,"15min":61.777000000000001},"last":1.7689999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.6760000000000002,"15min":1.9059999999999999},"min":{"1min":0.44500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.683,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.0019999999999998,"5min":3.3780000000000001,"15min":2.4009999999999998},"min":{"1min":0.41199999999999998,"5min":0.379,"15min":0.379},"max":{"1min":12.470000000000001,"5min":59.164000000000001,"15min":59.164000000000001},"last":1.6779999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:53:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":2.468,"15min":2.0110000000000001},"min":{"1min":0.45500000000000002,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":12.385999999999999,"5min":18.640000000000001,"15min":18.640000000000001},"last":1.742},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.0550000000000002,"15min":1.5489999999999999},"min":{"1min":0.40400000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.5750000000000002,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.49299999999999999}]}]},{"osd":2,"up_from":131,"seq":562640715938,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":945988,"kb_used_data":520188,"kb_used_omap":81,"kb_used_meta":425710,"kb_avail":92799164,"statfs":{"total":95995035648,"available":95026343936,"internally_reserved":0,"allocated":532672512,"data_stored":531596729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":435927258},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:53:32 2023","interfaces":[{"interface":"back","average":{"1min":6.1900000000000004,"5min":6.9480000000000004,"15min":5.8170000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":73.388000000000005,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":6.2869999999999999,"5min":7.4640000000000004,"15min":6.2400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":73.405000000000001,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":5.9770000000000003,"5min":3.052,"15min":3.2160000000000002},"min":{"1min":0.27700000000000002,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":77.004999999999995,"5min":77.004999999999995,"15min":88.316000000000003},"last":0.498},{"interface":"front","average":{"1min":6.3490000000000002,"5min":3.8029999999999999,"15min":3.4430000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":76.956000000000003,"5min":76.956000000000003,"15min":88.436999999999998},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":4.8250000000000002,"15min":4.306},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.999000000000001,"5min":112.895,"15min":112.895},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.3969999999999998,"5min":4.3090000000000002,"15min":3.5059999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":6.2670000000000003,"5min":113.995,"15min":113.995},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":5.5330000000000004,"5min":2.6659999999999999,"15min":2.7250000000000001},"min":{"1min":0.44700000000000001,"5min":0.253,"15min":0.253},"max":{"1min":69.113,"5min":69.113,"15min":108.878},"last":0.70099999999999996},{"interface":"front","average":{"1min":5.8780000000000001,"5min":2.5960000000000001,"15min":3.3599999999999999},"min":{"1min":0.36899999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":69.028999999999996,"5min":69.028999999999996,"15min":108.93000000000001},"last":0.68200000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":2.0830000000000002,"15min":2.9910000000000001},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":22.646000000000001,"5min":22.646000000000001,"15min":97.322999999999993},"last":0.57699999999999996},{"interface":"front","average":{"1min":4.1840000000000002,"5min":2.589,"15min":2.9460000000000002},"min":{"1min":0.39300000000000002,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":22.501000000000001,"5min":22.501000000000001,"15min":97.301000000000002},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":6.1219999999999999,"5min":3.911,"15min":3.2949999999999999},"min":{"1min":0.36099999999999999,"5min":0.34000000000000002,"15min":0.32700000000000001},"max":{"1min":59.655000000000001,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":5.125,"5min":3.375,"15min":2.6000000000000001},"min":{"1min":0.46999999999999997,"5min":0.313,"15min":0.313},"max":{"1min":59.612000000000002,"5min":98.509,"15min":98.509},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.1549999999999998,"15min":1.6599999999999999},"min":{"1min":0.40300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.9450000000000003,"5min":10.92,"15min":10.92},"last":0.72999999999999998},{"interface":"front","average":{"1min":2.149,"5min":1.9670000000000001,"15min":2.3759999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.37},"max":{"1min":6.9109999999999996,"5min":11.311,"15min":88.584000000000003},"last":0.625}]}]},{"osd":3,"up_from":0,"seq":403726926024,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019222,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":795464,"kb_used_data":498240,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92949688,"statfs":{"total":95995035648,"available":95180480512,"internally_reserved":0,"allocated":510197760,"data_stored":509636300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.3140000000000001,"15min":5.9390000000000001},"min":{"1min":0.251,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":5.9509999999999996,"5min":113.35599999999999,"15min":113.35599999999999},"last":1.2},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.7229999999999999,"15min":6.5099999999999998},"min":{"1min":0.41199999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.6760000000000002,"5min":113.417,"15min":113.417},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":7.5090000000000003,"5min":4.9939999999999998,"15min":2.9249999999999998},"min":{"1min":0.504,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":70.418000000000006,"5min":132.304,"15min":132.304},"last":1.1040000000000001},{"interface":"front","average":{"1min":6.6749999999999998,"5min":4.7859999999999996,"15min":2.726},"min":{"1min":0.29899999999999999,"5min":0.253,"15min":0.253},"max":{"1min":70.432000000000002,"5min":132.453,"15min":132.453},"last":0.41199999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.3860000000000001,"15min":3.1400000000000001},"min":{"1min":0.35999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.8070000000000004,"5min":20.308,"15min":84.909000000000006},"last":0.377},{"interface":"front","average":{"1min":2.6779999999999999,"5min":2.4790000000000001,"15min":3.1469999999999998},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.673,"5min":21.704000000000001,"15min":83.840000000000003},"last":3.1440000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6429999999999998,"5min":5.4409999999999998,"15min":4.4729999999999999},"min":{"1min":0.39500000000000002,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":12.622999999999999,"5min":164.453,"15min":164.453},"last":0.79700000000000004},{"interface":"front","average":{"1min":2.6230000000000002,"5min":5.5229999999999997,"15min":4.5609999999999999},"min":{"1min":0.40500000000000003,"5min":0.33300000000000002,"15min":0.27200000000000002},"max":{"1min":10.347,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.85499999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.1779999999999999,"15min":1},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.25900000000000001},"max":{"1min":5.0880000000000001,"5min":12.238,"15min":43.189999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1060000000000001,"15min":1},"min":{"1min":0.32900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.3079999999999998,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.0920000000000001,"15min":0.90400000000000003},"min":{"1min":0.378,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4039999999999999,"5min":10.952999999999999,"15min":27.056000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.8109999999999999,"5min":1.091,"15min":0.92400000000000004},"min":{"1min":0.44400000000000001,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":10.423999999999999,"5min":12.269,"15min":26.940999999999999},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.2250000000000001,"15min":0.876},"min":{"1min":0.36699999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":5.1449999999999996,"5min":12.311,"15min":12.311},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.171,"5min":1.0569999999999999,"15min":0.79900000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":4.077,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.61699999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822995,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1437100,"kb_used_data":839204,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92308052,"statfs":{"total":95995035648,"available":94523445248,"internally_reserved":0,"allocated":859344896,"data_stored":858807152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":9.6929999999999996,"5min":8.1189999999999998,"15min":5.133},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":80.450999999999993,"5min":185.62,"15min":185.62},"last":3.6440000000000001},{"interface":"front","average":{"1min":9.734,"5min":8.1829999999999998,"15min":5.1079999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":80.171999999999997,"5min":185.499,"15min":185.499},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":9.452,"5min":3.9129999999999998,"15min":2.8799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":110.36499999999999,"5min":110.36499999999999,"15min":149.203},"last":3.5600000000000001},{"interface":"front","average":{"1min":9.5340000000000007,"5min":4.3019999999999996,"15min":3.1909999999999998},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":110.89,"5min":110.89,"15min":149.554},"last":6.2140000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":5.54,"15min":5.5140000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.997,"5min":189.126,"15min":189.126},"last":7.5309999999999997},{"interface":"front","average":{"1min":3.5379999999999998,"5min":5.2309999999999999,"15min":5.4059999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":41.070999999999998,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.843,"15min":5.6660000000000004},"min":{"1min":0.34899999999999998,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.298999999999999,"5min":98.641000000000005,"15min":257.05000000000001},"last":4.6580000000000004},{"interface":"front","average":{"1min":2.085,"5min":4.9130000000000003,"15min":5.516},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27800000000000002},"max":{"1min":11.097,"5min":98.906000000000006,"15min":257.35000000000002},"last":3.8660000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.86899999999999999,"15min":0.73399999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":3.669,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.434},{"interface":"front","average":{"1min":0.996,"5min":0.92100000000000004,"15min":0.75700000000000001},"min":{"1min":0.29399999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":5.8019999999999996,"5min":6.3550000000000004,"15min":10.535},"last":0.37}]},{"osd":6,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1759999999999999,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.3369999999999997,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.139,"15min":1.629},"min":{"1min":0.374,"5min":0.315,"15min":0.254},"max":{"1min":7.9909999999999997,"5min":10.314,"15min":139.739},"last":0.38200000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.133,"15min":0.88700000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.9509999999999996,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.99299999999999999,"15min":0.80300000000000005},"min":{"1min":0.378,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":639893504,"data_stored":639893504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1140592640,"data_stored":1140588568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":512294912,"data_stored":512294912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":990642176,"data_stored":990642176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":478482432,"data_stored":478478360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":845484032,"data_stored":845479960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":665518080,"data_stored":665518080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:54:39.336 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:39.405 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:39.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:39.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:39.555 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:39.610 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:39.635 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:39.648 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:39 smithi151 ceph-mon[159210]: pgmap v888: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 659 op/s; 6231/60318 objects degraded (10.330%); 0 B/s, 128 objects/s recovering 2023-09-23T22:54:39.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:39.737 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:54:39.737 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:54:39.749 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:39.782 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:39.854 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:39.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:39.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:39.962 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:39.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:39 smithi052 ceph-mon[187743]: pgmap v888: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 659 op/s; 6231/60318 objects degraded (10.330%); 0 B/s, 128 objects/s recovering 2023-09-23T22:54:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:39 smithi052 ceph-mon[180979]: pgmap v888: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 659 op/s; 6231/60318 objects degraded (10.330%); 0 B/s, 128 objects/s recovering 2023-09-23T22:54:39.995 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:40.091 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:40.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:40.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:40.201 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:40.271 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:40.553 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 6231/60318 objects degraded (10.330%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:40.553 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[187743]: failed to encode map e306 with expected crc 2023-09-23T22:54:40.553 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[187743]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T22:54:40.553 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[187743]: failed to encode map e306 with expected crc 2023-09-23T22:54:40.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 6231/60318 objects degraded (10.330%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:40.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[180979]: failed to encode map e306 with expected crc 2023-09-23T22:54:40.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[180979]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T22:54:40.554 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:40 smithi052 ceph-mon[180979]: failed to encode map e306 with expected crc 2023-09-23T22:54:40.609 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:54:40.852 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:40.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:40.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:40.860 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:54:40.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 6231/60318 objects degraded (10.330%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:40 smithi151 ceph-mon[159210]: failed to encode map e306 with expected crc 2023-09-23T22:54:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:40 smithi151 ceph-mon[159210]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T22:54:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:40 smithi151 ceph-mon[159210]: failed to encode map e306 with expected crc 2023-09-23T22:54:40.956 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:40.989 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:41.157 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:41.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:41.262 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:41.298 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:41.688 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:41.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:41.793 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:41.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[187743]: failed to encode map e306 with expected crc 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[187743]: pgmap v890: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 5871/58419 objects degraded (10.050%); 0 B/s, 125 objects/s recovering 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[187743]: failed to encode map e307 with expected crc 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[187743]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[187743]: failed to encode map e307 with expected crc 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[180979]: failed to encode map e306 with expected crc 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[180979]: pgmap v890: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 5871/58419 objects degraded (10.050%); 0 B/s, 125 objects/s recovering 2023-09-23T22:54:41.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[180979]: failed to encode map e307 with expected crc 2023-09-23T22:54:41.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[180979]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T22:54:41.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:41 smithi052 ceph-mon[180979]: failed to encode map e307 with expected crc 2023-09-23T22:54:41.933 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:41.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:41 smithi151 ceph-mon[159210]: failed to encode map e306 with expected crc 2023-09-23T22:54:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:41 smithi151 ceph-mon[159210]: pgmap v890: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 5871/58419 objects degraded (10.050%); 0 B/s, 125 objects/s recovering 2023-09-23T22:54:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:41 smithi151 ceph-mon[159210]: failed to encode map e307 with expected crc 2023-09-23T22:54:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:41 smithi151 ceph-mon[159210]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T22:54:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:41 smithi151 ceph-mon[159210]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.036 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:42.037 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:42.142 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:42.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:42.248 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:42.331 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:54:42.356 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:42.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:54:42.640 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:54:42.683 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:42 smithi052 ceph-mon[187743]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.683 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:42 smithi052 ceph-mon[187743]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.683 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4252593332' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:42.684 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:42 smithi052 ceph-mon[180979]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.684 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:42 smithi052 ceph-mon[180979]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.684 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4252593332' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:42.706 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:42.708 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:54:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:42.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:42.747 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:54:42.789 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:54:42.822 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:42.827 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:54:42.860 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:54:42.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:42 smithi151 ceph-mon[159210]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:42 smithi151 ceph-mon[159210]: failed to encode map e307 with expected crc 2023-09-23T22:54:42.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4252593332' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:42.863 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:42.898 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:54:42.916 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.916 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.917 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.917 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.917 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.917 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.917 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.917 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:54:42.921 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:42.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:43.028 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:43.114 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:43.617 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:54:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:54:43] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.33.4" 2023-09-23T22:54:43.617 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:43 smithi052 ceph-mon[180979]: pgmap v892: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 5388/53280 objects degraded (10.113%); 0 B/s, 175 objects/s recovering 2023-09-23T22:54:43.928 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:43 smithi052 ceph-mon[187743]: pgmap v892: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 5388/53280 objects degraded (10.113%); 0 B/s, 175 objects/s recovering 2023-09-23T22:54:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:43 smithi151 ceph-mon[159210]: pgmap v892: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 5388/53280 objects degraded (10.113%); 0 B/s, 175 objects/s recovering 2023-09-23T22:54:44.985 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:45.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:45.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:45.139 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:45.203 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:45.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:45.310 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:45.335 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:45.402 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:45.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:45.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:45.514 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:45.559 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:45.560 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:45.630 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:45.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: pgmap v893: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 5388/51000 objects degraded (10.565%); 0 B/s, 133 objects/s recovering 2023-09-23T22:54:45.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:45.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5388/51000 objects degraded (10.565%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:45.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:45 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.734 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: pgmap v893: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 5388/51000 objects degraded (10.565%); 0 B/s, 133 objects/s recovering 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5388/51000 objects degraded (10.565%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: pgmap v893: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 5388/51000 objects degraded (10.565%); 0 B/s, 133 objects/s recovering 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5388/51000 objects degraded (10.565%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:45 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:45.776 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:45.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:45.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:45.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:45.848 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:54:45.848 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:54:45.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164683 7.f 7:f08671e8:::benchmark_data_smithi052_277165_object72191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.376581+0000", 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.46781040999997, 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31209462100000002, 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.849 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164683 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376581+0000", 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376581+0000", 2023-09-23T22:54:45.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376580+0000", 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376585+0000", 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6990000000000002e-06 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376593+0000", 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7700000000000007e-06 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.376599+0000", 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5210000000000002e-06 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.379215+0000", 2023-09-23T22:54:45.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026156759999999999 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for degraded object", 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.382234+0000", 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030192750000000001 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.393352+0000", 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011118213 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.853 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.393458+0000", 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106211 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.393611+0000", 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015228099999999999 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683900+0000", 2023-09-23T22:54:45.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29028897799999998 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683910+0000", 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0356999999999999e-05 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.687485+0000", 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035752599999999998 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.687540+0000", 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4867999999999997e-05 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.688676+0000", 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011357870000000001 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.856 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164698 7.3 7:ca17f318:::benchmark_data_smithi052_277165_object72206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396189+0000", 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44820244899995, 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29793002099999999, 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164698 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396189+0000", 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396189+0000", 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396188+0000", 2023-09-23T22:54:45.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396194+0000", 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3960000000000003e-06 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396206+0000", 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1249e-05 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396214+0000", 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.055e-06 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396255+0000", 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1656999999999998e-05 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396379+0000", 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012328799999999999 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396492+0000", 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113832 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690191+0000", 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29369885699999998 2023-09-23T22:54:45.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690228+0000", 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6956000000000003e-05 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694017+0000", 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037884310000000001 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694080+0000", 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3461999999999996e-05 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694119+0000", 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9007000000000001e-05 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.863 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164699 7.3 7:c3dd2f2c:::benchmark_data_smithi052_277165_object72207:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396243+0000", 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44814850099999, 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.298988266, 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164699 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396243+0000", 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396243+0000", 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396242+0000", 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396247+0000", 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7140000000000001e-06 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396260+0000", 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2383e-05 2023-09-23T22:54:45.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396266+0000", 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5810000000000002e-06 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686458+0000", 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.290191692 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686599+0000", 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000141361 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686735+0000", 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013546200000000001 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694006+0000", 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072714729999999997 2023-09-23T22:54:45.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694126+0000", 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120029 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.695156+0000", 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001029877 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.695195+0000", 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9167e-05 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.695231+0000", 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6157e-05 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164700 7.6 7:61d26359:::benchmark_data_smithi052_277165_object72208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396276+0000", 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44811558200001, 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.302929104, 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164700 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396276+0000", 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396276+0000", 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396275+0000", 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396279+0000", 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6339999999999999e-06 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396285+0000", 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4080000000000003e-06 2023-09-23T22:54:45.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396289+0000", 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2329999999999998e-06 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683153+0000", 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.286864073 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683301+0000", 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014825400000000001 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683436+0000", 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013492199999999999 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.692875+0000", 2023-09-23T22:54:45.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0094391779999999995 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.692912+0000", 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6439999999999997e-05 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.698838+0000", 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059267909999999998 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.698866+0000", 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7628999999999999e-05 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699205+0000", 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033919400000000002 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164703 7.6 7:65560176:::benchmark_data_smithi052_277165_object72211:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44809934, 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30325594700000003, 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164703 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.878 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396292+0000", 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396294+0000", 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.688e-06 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396297+0000", 2023-09-23T22:54:45.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3079999999999999e-06 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396300+0000", 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8940000000000002e-06 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.688726+0000", 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29242600200000002 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.688873+0000", 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014661899999999999 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.689025+0000", 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015209500000000001 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699257+0000", 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010232194999999999 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699265+0000", 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2779999999999993e-06 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699302+0000", 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6424999999999998e-05 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699367+0000", 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5044999999999998e-05 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.699548+0000", 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018163700000000001 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164706 7.3 7:c61dc2a5:::benchmark_data_smithi052_277165_object72214:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44808914400005, 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30120987500000002, 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164706 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.885 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396302+0000", 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396304+0000", 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4750000000000001e-06 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396307+0000", 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1080000000000001e-06 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396310+0000", 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4809999999999998e-06 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690272+0000", 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29396191900000002 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690388+0000", 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011600899999999999 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.690498+0000", 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000109812 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.694320+0000", 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0038220609999999999 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697402+0000", 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030819300000000001 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697423+0000", 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.101e-05 2023-09-23T22:54:45.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697470+0000", 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7324999999999998e-05 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.697512+0000", 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1996000000000002e-05 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.35001.0:164708 7.2 7:4a1cbce6:::benchmark_data_smithi052_277165_object72216:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396315+0000", 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44807694799999, 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29540316500000002, 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.35001", 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 164708 2023-09-23T22:54:45.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396315+0000", 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396315+0000", 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396314+0000", 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396316+0000", 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0509999999999999e-06 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396322+0000", 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4940000000000003e-06 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396326+0000", 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1779999999999996e-06 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396369+0000", 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2608000000000003e-05 2023-09-23T22:54:45.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396441+0000", 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2803000000000001e-05 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396505+0000", 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3466000000000005e-05 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686583+0000", 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29007853 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.686615+0000", 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1557999999999998e-05 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.691629+0000", 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050141949999999999 2023-09-23T22:54:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.691677+0000", 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7290000000000003e-05 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.691718+0000", 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1288999999999997e-05 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:164704 7.8 e205/196 7:160987c2:::benchmark_data_smithi052_277165_object72212:head v 205'10330, mlcod=167'4547)", 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44769415899998, 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31435047700000002, 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:54:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396697+0000", 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.396702+0000", 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1159999999999998e-06 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683041+0000", 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28633941800000001 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683049+0000", 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7819999999999994e-06 2023-09-23T22:54:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683120+0000", 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0648000000000005e-05 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683150+0000", 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0161000000000001e-05 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711014+0000", 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.027863675000000001 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.711048+0000", 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4217000000000001e-05 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.35001.0:164705 7.0 e205/146 7:0e09ba28:::benchmark_data_smithi052_277165_object72213:head v 205'10244, mlcod=205'10242)", 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 564.44383519799999, 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.30802488500000003, 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400556+0000", 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.400563+0000", 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8009999999999998e-06 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.682993+0000", 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.282430454 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683007+0000", 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3760999999999999e-05 2023-09-23T22:54:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683134+0000", 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012747100000000001 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.683157+0000", 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2535e-05 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.708543+0000", 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.025385650999999999 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:45:21.708581+0000", 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8730999999999999e-05 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:140973 8.f 8:f33d72b0:::benchmark_data_smithi052_337357_object38522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 347.04928896299998, 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.329060238, 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140973 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795101+0000", 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795109+0000", 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6429999999999995e-06 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795122+0000", 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3022e-05 2023-09-23T22:54:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795130+0000", 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7400000000000004e-06 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810865+0000", 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015735702000000001 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for rw locks", 2023-09-23T22:54:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810975+0000", 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010935200000000001 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067039+0000", 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.256064085 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067161+0000", 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121952 2023-09-23T22:54:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067334+0000", 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017327 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.083778+0000", 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016443607999999998 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.084031+0000", 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025363099999999998 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120168+0000", 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036136433000000003 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120220+0000", 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2271000000000001e-05 2023-09-23T22:54:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.124163+0000", 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039430070000000001 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5823 9.c 9:36ec13c5:::benchmark_data_smithi052_402718_object5822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:54:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 204.22661178600001, 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22852289200000001, 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5823 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617779+0000", 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617800+0000", 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0721999999999999e-05 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617807+0000", 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.878e-06 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617814+0000", 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8519999999999998e-06 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618047+0000", 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023353299999999999 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618201+0000", 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015408199999999999 2023-09-23T22:54:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618341+0000", 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139765 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619519+0000", 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011783220000000001 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619626+0000", 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107017 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846195+0000", 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22656818400000001 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846251+0000", 2023-09-23T22:54:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6786000000000001e-05 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846303+0000", 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1248999999999997e-05 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5845 9.c 9:32588725:::benchmark_data_smithi052_402718_object5844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:54:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 204.219648363, 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.221823086, 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5845 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624742+0000", 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624765+0000", 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2909e-05 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624775+0000", 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.023e-05 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624783+0000", 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624865+0000", 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1631999999999994e-05 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625069+0000", 2023-09-23T22:54:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000203685 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625237+0000", 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168426 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625970+0000", 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00073263699999999996 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.626858+0000", 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088813800000000001 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846472+0000", 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21961366099999999 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846523+0000", 2023-09-23T22:54:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0986000000000003e-05 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846566+0000", 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3708000000000002e-05 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5878 9.c 9:34f7d49b:::benchmark_data_smithi052_402718_object5877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:54:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 204.205480223, 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.208484483, 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5878 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638910+0000", 2023-09-23T22:54:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638928+0000", 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7866000000000001e-05 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638941+0000", 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2618e-05 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638947+0000", 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5639999999999996e-06 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638986+0000", 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9031e-05 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639111+0000", 2023-09-23T22:54:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124955 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639216+0000", 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105696 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639724+0000", 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00050770100000000005 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.640023+0000", 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029943599999999998 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847306+0000", 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20728270800000001 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847359+0000", 2023-09-23T22:54:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3319999999999998e-05 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847396+0000", 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6473999999999997e-05 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13481 9.8 9:1f01482c:::benchmark_data_smithi052_402718_object13480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:54:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 200.50838843400001, 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.322004134, 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13481 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336002+0000", 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336029+0000", 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6545000000000001e-05 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336044+0000", 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5444999999999999e-05 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336063+0000", 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8729e-05 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336119+0000", 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6112000000000001e-05 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336242+0000", 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012232500000000001 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336383+0000", 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014170900000000001 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337199+0000", 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00081607499999999998 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337730+0000", 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00053117300000000002 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657923+0000", 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32019289400000001 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657967+0000", 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3399999999999998e-05 2023-09-23T22:54:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.658007+0000", 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0534999999999997e-05 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13489 9.c 9:33ad037a:::benchmark_data_smithi052_402718_object13488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:54:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 200.50731374899999, 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32053898800000002, 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13489 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337077+0000", 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337098+0000", 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1191000000000001e-05 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337107+0000", 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2179999999999999e-06 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337112+0000", 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337159+0000", 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6615e-05 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337308+0000", 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014894100000000001 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337456+0000", 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148066 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338621+0000", 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011655420000000001 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.339454+0000", 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083201599999999998 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657516+0000", 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.318062489 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657570+0000", 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4024000000000003e-05 2023-09-23T22:54:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657617+0000", 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6826000000000002e-05 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59534 9.a 9:56ae8077:::benchmark_data_smithi052_402718_object59533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 156.75008935, 2023-09-23T22:54:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20214791800000001, 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59534 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:54:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094300+0000", 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094331+0000", 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0930999999999999e-05 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094353+0000", 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2340999999999998e-05 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094362+0000", 2023-09-23T22:54:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2519999999999999e-06 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094415+0000", 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2997000000000001e-05 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094637+0000", 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000222208 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094844+0000", 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000207631 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097382+0000", 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025371360000000002 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097400+0000", 2023-09-23T22:54:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8714000000000001e-05 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.098411+0000", 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001011131 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294679+0000", 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19626732399999999 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294738+0000", 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8801000000000002e-05 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296450+0000", 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001712618 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59542 9.a 9:51ad1323:::benchmark_data_smithi052_402718_object59541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 156.74513515999999, 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19894724899999999, 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59542 2023-09-23T22:54:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099254+0000", 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099282+0000", 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7971999999999999e-05 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099301+0000", 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8593e-05 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099311+0000", 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0074999999999999e-05 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099389+0000", 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7689999999999996e-05 2023-09-23T22:54:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099595+0000", 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020635099999999999 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099809+0000", 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021421 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.102469+0000", 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002660314 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104681+0000", 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022113889999999998 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104831+0000", 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015015699999999999 2023-09-23T22:54:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296512+0000", 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19168089399999999 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296580+0000", 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8103999999999997e-05 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298204+0000", 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016236810000000001 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98500 9.a 9:5e200aba:::benchmark_data_smithi052_402718_object8859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 102.876895709, 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22489546499999999, 2023-09-23T22:54:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98500 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967494+0000", 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967502+0000", 2023-09-23T22:54:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6190000000000002e-06 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967516+0000", 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4651000000000001e-05 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967525+0000", 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6279999999999994e-06 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968657+0000", 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011323209999999999 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968810+0000", 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015326000000000001 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:54:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968949+0000", 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138429 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.974764+0000", 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058149839999999996 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.975363+0000", 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059912599999999996 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977959+0000", 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025958399999999999 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190744+0000", 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.212785264 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190803+0000", 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8777000000000003e-05 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.192391+0000", 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015884969999999999 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110895 9.a 9:5225ae0d:::benchmark_data_smithi052_402718_object21254:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 84.780812745000006, 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25354543899999998, 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110895 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063577+0000", 2023-09-23T22:54:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063584+0000", 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1860000000000003e-06 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063611+0000", 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6279e-05 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063621+0000", 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8179999999999999e-06 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063692+0000", 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1106000000000007e-05 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063841+0000", 2023-09-23T22:54:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014895899999999999 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063967+0000", 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012629000000000001 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.071444+0000", 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074768090000000001 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072687+0000", 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00124325 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072764+0000", 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7496999999999993e-05 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315667+0000", 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24290224499999999 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315734+0000", 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.317124+0000", 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001390575 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110898 9.a 9:538cf70b:::benchmark_data_smithi052_402718_object21257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 84.780745648999996, 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25922483400000002, 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110898 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:54:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063645+0000", 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063650+0000", 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6210000000000003e-06 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063660+0000", 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8379999999999997e-06 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063667+0000", 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9879999999999998e-06 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:54:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067524+0000", 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003857624 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067678+0000", 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015363900000000001 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067821+0000", 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143096 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072775+0000", 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049535079999999997 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.073473+0000", 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069833799999999995 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.074443+0000", 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097013000000000004 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321598+0000", 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24715498 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321672+0000", 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4229999999999999e-05 2023-09-23T22:54:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.322871+0000", 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001198483 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:54:45.988 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:54:45.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:46.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:46.378 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:46.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:46.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:46.531 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:46.654 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:46.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:46.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3808014407' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:46 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:46.717 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:54:46.748 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:54:46.762 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:46.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3808014407' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:46.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:46 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:46.815 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3808014407' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:46.816 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:46 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:54:46.858 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:47.269 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:47.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:47.375 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:47.436 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:47.530 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:47.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:47.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:47.644 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: pgmap v894: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 980 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5388/47097 objects degraded (11.440%); 0 B/s, 105 objects/s recovering 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:47 smithi151 ceph-mon[159210]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: pgmap v894: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 980 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5388/47097 objects degraded (11.440%); 0 B/s, 105 objects/s recovering 2023-09-23T22:54:47.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:47.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:54:47.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T22:54:47.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.711 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:47.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[180979]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: pgmap v894: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 980 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5388/47097 objects degraded (11.440%); 0 B/s, 105 objects/s recovering 2023-09-23T22:54:47.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:54:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:54:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T22:54:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: failed to encode map e308 with expected crc 2023-09-23T22:54:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:47 smithi052 ceph-mon[187743]: failed to encode map e308 with expected crc 2023-09-23T22:54:48.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:54:48.558 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:48.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:48.662 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:48.693 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:48.800 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:48.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:48.902 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:54:48.905 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:48 smithi151 ceph-mon[159210]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T22:54:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:48 smithi151 ceph-mon[159210]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:48 smithi151 ceph-mon[159210]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:48 smithi151 ceph-mon[159210]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[187743]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T22:54:48.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[187743]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[187743]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[187743]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[180979]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T22:54:48.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[180979]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[180979]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:48 smithi052 ceph-mon[180979]: failed to encode map e309 with expected crc 2023-09-23T22:54:48.976 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:49.201 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:49.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:49.277 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:49.308 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:49.362 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:49.563 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:49.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:49.674 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:49.773 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:49.798 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:54:49.820 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 869 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 657 op/s; 5007/41766 objects degraded (11.988%); 0 B/s, 118 objects/s recovering 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: failed to encode map e310 with expected crc 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: failed to encode map e310 with expected crc 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: osdmap e310: 8 total, 8 up, 6 in 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 869 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 657 op/s; 5007/41766 objects degraded (11.988%); 0 B/s, 118 objects/s recovering 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: failed to encode map e310 with expected crc 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: failed to encode map e310 with expected crc 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: osdmap e310: 8 total, 8 up, 6 in 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:49.823 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 869 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 657 op/s; 5007/41766 objects degraded (11.988%); 0 B/s, 118 objects/s recovering 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: failed to encode map e310 with expected crc 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: failed to encode map e310 with expected crc 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: osdmap e310: 8 total, 8 up, 6 in 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:54:50.101 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:50.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:50.208 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:54:50.254 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:50.505 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:54:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:50.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:54:50.611 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:50.664 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:50.665 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:54:50.770 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:50 smithi151 ceph-mon[159210]: from='client.45851 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:50.770 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 5007/41766 objects degraded (11.988%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:50.792 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:50.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:50.896 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:50.935 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:50 smithi052 ceph-mon[187743]: from='client.45851 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 5007/41766 objects degraded (11.988%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:50 smithi052 ceph-mon[180979]: from='client.45851 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:54:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 5007/41766 objects degraded (11.988%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:50.998 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:51.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:167759 9.4 9:2ab3c930:::benchmark_data_smithi052_402718_object78118:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:54:50.970554+0000", 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.038718608000000002, 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.038745966, 2023-09-23T22:54:51.010 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 167759 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.970554+0000", 2023-09-23T22:54:51.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.970554+0000", 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.970552+0000", 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1810000000000004e-06 2023-09-23T22:54:51.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.970560+0000", 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1790999999999999e-05 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.970571+0000", 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3263e-05 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.970585+0000", 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001092081 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.971677+0000", 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016496300000000001 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:54:51.014 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.971842+0000", 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019581700000000001 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.972037+0000", 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039621860000000004 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.976000+0000", 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041800379999999996 2023-09-23T22:54:51.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.980180+0000", 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0096257670000000004 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:167763 9.e 9:7969931d:::benchmark_data_smithi052_402718_object78122:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:54:50.974919+0000", 2023-09-23T22:54:51.016 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.034353308999999999, 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034436498000000003, 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 167763 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:54:51.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974919+0000", 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974919+0000", 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974917+0000", 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2741e-05 2023-09-23T22:54:51.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974930+0000", 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7200000000000001e-05 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974947+0000", 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3300000000000001e-06 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974955+0000", 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3514e-05 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.975038+0000", 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010868 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.975147+0000", 2023-09-23T22:54:51.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000127179 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.975274+0000", 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077770770000000003 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983051+0000", 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00058703200000000003 2023-09-23T22:54:51.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983638+0000", 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0087188779999999994 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:167768 9.e 9:7cd693c6:::benchmark_data_smithi052_402718_object78127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:54:50.974972+0000", 2023-09-23T22:54:51.022 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.034300465000000002, 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034431834000000001, 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 167768 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:54:51.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974972+0000", 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974972+0000", 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974971+0000", 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7570000000000002e-06 2023-09-23T22:54:51.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974975+0000", 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2439999999999996e-06 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974982+0000", 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5750000000000002e-06 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.974987+0000", 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038619380000000001 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.978849+0000", 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014201099999999999 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:54:51.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.978991+0000", 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015362199999999999 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.979145+0000", 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040925850000000001 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983237+0000", 2023-09-23T22:54:51.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00042920700000000003 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983666+0000", 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0086943200000000002 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:167773 9.d 9:bcba9c6a:::benchmark_data_smithi052_402718_object78132:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-09-23T22:54:51.028 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:54:50.983938+0000", 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.025334624, 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.025514433999999999, 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 167773 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983938+0000", 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983938+0000", 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983937+0000", 2023-09-23T22:54:51.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1409999999999998e-06 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983941+0000", 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5859999999999999e-06 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983949+0000", 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8049999999999997e-06 2023-09-23T22:54:51.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983954+0000", 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4313000000000001e-05 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.983998+0000", 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010847 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.984107+0000", 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011916100000000001 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.984226+0000", 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073781539999999996 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:54:51.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.991604+0000", 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00043197 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:50.992036+0000", 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0080979539999999992 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:54:51.034 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 4 2023-09-23T22:54:51.035 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:54:51.102 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:51.214 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:51.705 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":898,"stamp":"2023-09-23T22:54:48.704985+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":911349688,"num_objects":13922,"num_object_clones":0,"num_object_copies":41766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5007,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13922,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":165640,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11830,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114424,"ondisk_log_size":114424,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5231844,"kb_used_data":2754676,"kb_used_omap":121,"kb_used_meta":2476742,"kb_avail":557239068,"statfs":{"total":575970213888,"available":570612805632,"internally_reserved":0,"allocated":2820788224,"data_stored":2815837493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2536183925},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,8,5,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-251527168,"num_objects":-3838,"num_object_clones":0,"num_object_copies":-11514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":381,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.272616"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":304,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138066+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:54:40.138066+0000","last_peered":"2023-09-23T22:54:40.138066+0000","last_clean":"2023-09-23T22:54:40.138066+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:54:40.138066+0000","last_undegraded":"2023-09-23T22:54:40.138066+0000","last_fullsized":"2023-09-23T22:54:40.138066+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138646+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:54:40.138646+0000","last_peered":"2023-09-23T22:54:40.138646+0000","last_clean":"2023-09-23T22:54:40.138646+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:54:40.138646+0000","last_undegraded":"2023-09-23T22:54:40.138646+0000","last_fullsized":"2023-09-23T22:54:40.138646+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":89,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138217+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:54:40.138217+0000","last_peered":"2023-09-23T22:54:40.138217+0000","last_clean":"2023-09-23T22:54:40.138217+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:54:40.138217+0000","last_undegraded":"2023-09-23T22:54:40.138217+0000","last_fullsized":"2023-09-23T22:54:40.138217+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":288,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688462+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:54:46.688462+0000","last_peered":"2023-09-23T22:54:46.688462+0000","last_clean":"2023-09-23T22:54:46.688462+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:54:46.688462+0000","last_undegraded":"2023-09-23T22:54:46.688462+0000","last_fullsized":"2023-09-23T22:54:46.688462+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":292,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.687848+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:54:46.687848+0000","last_peered":"2023-09-23T22:54:46.687848+0000","last_clean":"2023-09-23T22:54:46.687848+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:54:46.687848+0000","last_undegraded":"2023-09-23T22:54:46.687848+0000","last_fullsized":"2023-09-23T22:54:46.687848+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.154208+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:54:40.154208+0000","last_peered":"2023-09-23T22:54:40.154208+0000","last_clean":"2023-09-23T22:54:40.154208+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:54:40.154208+0000","last_undegraded":"2023-09-23T22:54:40.154208+0000","last_fullsized":"2023-09-23T22:54:40.154208+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":219,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688086+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:54:46.688086+0000","last_peered":"2023-09-23T22:54:46.688086+0000","last_clean":"2023-09-23T22:54:46.688086+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:54:46.688086+0000","last_undegraded":"2023-09-23T22:54:46.688086+0000","last_fullsized":"2023-09-23T22:54:46.688086+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":169,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134550+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:54:40.134550+0000","last_peered":"2023-09-23T22:54:40.134550+0000","last_clean":"2023-09-23T22:54:40.134550+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:54:40.134550+0000","last_undegraded":"2023-09-23T22:54:40.134550+0000","last_fullsized":"2023-09-23T22:54:40.134550+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.135751+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:54:40.135751+0000","last_peered":"2023-09-23T22:54:40.135751+0000","last_clean":"2023-09-23T22:54:40.135751+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:54:40.135751+0000","last_undegraded":"2023-09-23T22:54:40.135751+0000","last_fullsized":"2023-09-23T22:54:40.135751+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":191,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153381+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:54:40.153381+0000","last_peered":"2023-09-23T22:54:40.153381+0000","last_clean":"2023-09-23T22:54:40.153381+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:54:40.153381+0000","last_undegraded":"2023-09-23T22:54:40.153381+0000","last_fullsized":"2023-09-23T22:54:40.153381+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":65,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138498+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:54:40.138498+0000","last_peered":"2023-09-23T22:54:40.138498+0000","last_clean":"2023-09-23T22:54:40.138498+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:54:40.138498+0000","last_undegraded":"2023-09-23T22:54:40.138498+0000","last_fullsized":"2023-09-23T22:54:40.138498+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":181,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153558+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:54:40.153558+0000","last_peered":"2023-09-23T22:54:40.153558+0000","last_clean":"2023-09-23T22:54:40.153558+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:54:40.153558+0000","last_undegraded":"2023-09-23T22:54:40.153558+0000","last_fullsized":"2023-09-23T22:54:40.153558+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":326,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.145442+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:54:40.145442+0000","last_peered":"2023-09-23T22:54:40.145442+0000","last_clean":"2023-09-23T22:54:40.145442+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:54:40.145442+0000","last_undegraded":"2023-09-23T22:54:40.145442+0000","last_fullsized":"2023-09-23T22:54:40.145442+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":272,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138019+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:54:40.138019+0000","last_peered":"2023-09-23T22:54:40.138019+0000","last_clean":"2023-09-23T22:54:40.138019+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:54:40.138019+0000","last_undegraded":"2023-09-23T22:54:40.138019+0000","last_fullsized":"2023-09-23T22:54:40.138019+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":165,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138124+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:54:40.138124+0000","last_peered":"2023-09-23T22:54:40.138124+0000","last_clean":"2023-09-23T22:54:40.138124+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:54:40.138124+0000","last_undegraded":"2023-09-23T22:54:40.138124+0000","last_fullsized":"2023-09-23T22:54:40.138124+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134552+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:54:40.134552+0000","last_peered":"2023-09-23T22:54:40.134552+0000","last_clean":"2023-09-23T22:54:40.134552+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:54:40.134552+0000","last_undegraded":"2023-09-23T22:54:40.134552+0000","last_fullsized":"2023-09-23T22:54:40.134552+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":275,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.687906+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:54:46.687906+0000","last_peered":"2023-09-23T22:54:46.687906+0000","last_clean":"2023-09-23T22:54:46.687906+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:54:46.687906+0000","last_undegraded":"2023-09-23T22:54:46.687906+0000","last_fullsized":"2023-09-23T22:54:46.687906+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.154025+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:54:40.154025+0000","last_peered":"2023-09-23T22:54:40.154025+0000","last_clean":"2023-09-23T22:54:40.154025+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:54:40.154025+0000","last_undegraded":"2023-09-23T22:54:40.154025+0000","last_fullsized":"2023-09-23T22:54:40.154025+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138116+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:54:40.138116+0000","last_peered":"2023-09-23T22:54:40.138116+0000","last_clean":"2023-09-23T22:54:40.138116+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:54:40.138116+0000","last_undegraded":"2023-09-23T22:54:40.138116+0000","last_fullsized":"2023-09-23T22:54:40.138116+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":276,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688534+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:54:46.688534+0000","last_peered":"2023-09-23T22:54:46.688534+0000","last_clean":"2023-09-23T22:54:46.688534+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:54:46.688534+0000","last_undegraded":"2023-09-23T22:54:46.688534+0000","last_fullsized":"2023-09-23T22:54:46.688534+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":351,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.149561+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:54:40.149561+0000","last_peered":"2023-09-23T22:54:40.149561+0000","last_clean":"2023-09-23T22:54:40.149561+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:54:40.149561+0000","last_undegraded":"2023-09-23T22:54:40.149561+0000","last_fullsized":"2023-09-23T22:54:40.149561+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":234,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.137394+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:54:40.137394+0000","last_peered":"2023-09-23T22:54:40.137394+0000","last_clean":"2023-09-23T22:54:40.137394+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:54:40.137394+0000","last_undegraded":"2023-09-23T22:54:40.137394+0000","last_fullsized":"2023-09-23T22:54:40.137394+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"309'10302","reported_seq":10357,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T22:54:47.962922+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:54:47.962922+0000","last_peered":"2023-09-23T22:54:47.962922+0000","last_clean":"2023-09-23T22:54:47.962922+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:54:47.962922+0000","last_undegraded":"2023-09-23T22:54:47.962922+0000","last_fullsized":"2023-09-23T22:54:47.962922+0000","mapping_epoch":264,"log_start":"275'2800","ondisk_log_start":"275'2800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7502,"log_dups_size":0,"ondisk_log_size":7502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54394880,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10302,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":176,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138230+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:54:40.138230+0000","last_peered":"2023-09-23T22:54:40.138230+0000","last_clean":"2023-09-23T22:54:40.138230+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:54:40.138230+0000","last_undegraded":"2023-09-23T22:54:40.138230+0000","last_fullsized":"2023-09-23T22:54:40.138230+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.149658+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:54:40.149658+0000","last_peered":"2023-09-23T22:54:40.149658+0000","last_clean":"2023-09-23T22:54:40.149658+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:54:40.149658+0000","last_undegraded":"2023-09-23T22:54:40.149658+0000","last_fullsized":"2023-09-23T22:54:40.149658+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"307'10590","reported_seq":10648,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:46.645588+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:54:46.645588+0000","last_peered":"2023-09-23T22:54:46.645588+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:54:46.645588+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"278'3800","ondisk_log_start":"278'3800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6790,"log_dups_size":0,"ondisk_log_size":6790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59113472,"num_objects":902,"num_object_clones":0,"num_object_copies":2706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":-759}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":186,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138418+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:54:40.138418+0000","last_peered":"2023-09-23T22:54:40.138418+0000","last_clean":"2023-09-23T22:54:40.138418+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:54:40.138418+0000","last_undegraded":"2023-09-23T22:54:40.138418+0000","last_fullsized":"2023-09-23T22:54:40.138418+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":296,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138029+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:54:40.138029+0000","last_peered":"2023-09-23T22:54:40.138029+0000","last_clean":"2023-09-23T22:54:40.138029+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:54:40.138029+0000","last_undegraded":"2023-09-23T22:54:40.138029+0000","last_fullsized":"2023-09-23T22:54:40.138029+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"307'10428","reported_seq":10487,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:45.128475+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:54:45.128475+0000","last_peered":"2023-09-23T22:54:45.128475+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:54:45.128475+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"273'1800","ondisk_log_start":"273'1800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8628,"log_dups_size":0,"ondisk_log_size":8628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56360960,"num_objects":860,"num_object_clones":0,"num_object_copies":2580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10428,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-911}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":130,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153991+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:54:40.153991+0000","last_peered":"2023-09-23T22:54:40.153991+0000","last_clean":"2023-09-23T22:54:40.153991+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:54:40.153991+0000","last_undegraded":"2023-09-23T22:54:40.153991+0000","last_fullsized":"2023-09-23T22:54:40.153991+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":299,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153408+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:54:40.153408+0000","last_peered":"2023-09-23T22:54:40.153408+0000","last_clean":"2023-09-23T22:54:40.153408+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:54:40.153408+0000","last_undegraded":"2023-09-23T22:54:40.153408+0000","last_fullsized":"2023-09-23T22:54:40.153408+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"307'10345","reported_seq":10397,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:45.219376+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:54:45.219376+0000","last_peered":"2023-09-23T22:54:45.219376+0000","last_clean":"2023-09-23T22:54:45.219376+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:54:45.219376+0000","last_undegraded":"2023-09-23T22:54:45.219376+0000","last_fullsized":"2023-09-23T22:54:45.219376+0000","mapping_epoch":264,"log_start":"273'1700","ondisk_log_start":"273'1700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8645,"log_dups_size":0,"ondisk_log_size":8645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63635456,"num_objects":971,"num_object_clones":0,"num_object_copies":2913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10345,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153852+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:54:40.153852+0000","last_peered":"2023-09-23T22:54:40.153852+0000","last_clean":"2023-09-23T22:54:40.153852+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:54:40.153852+0000","last_undegraded":"2023-09-23T22:54:40.153852+0000","last_fullsized":"2023-09-23T22:54:40.153852+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":203,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688737+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:54:46.688737+0000","last_peered":"2023-09-23T22:54:46.688737+0000","last_clean":"2023-09-23T22:54:46.688737+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:54:46.688737+0000","last_undegraded":"2023-09-23T22:54:46.688737+0000","last_fullsized":"2023-09-23T22:54:46.688737+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"307'10164","reported_seq":13581,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:45.929483+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:54:45.929483+0000","last_peered":"2023-09-23T22:54:45.929483+0000","last_clean":"2023-09-23T22:54:45.929483+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:54:45.929483+0000","last_undegraded":"2023-09-23T22:54:45.929483+0000","last_fullsized":"2023-09-23T22:54:45.929483+0000","mapping_epoch":289,"log_start":"278'3700","ondisk_log_start":"278'3700","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55312384,"num_objects":844,"num_object_clones":0,"num_object_copies":2532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10164,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":75,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688823+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:54:46.688823+0000","last_peered":"2023-09-23T22:54:46.688823+0000","last_clean":"2023-09-23T22:54:46.688823+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:54:46.688823+0000","last_undegraded":"2023-09-23T22:54:46.688823+0000","last_fullsized":"2023-09-23T22:54:46.688823+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":280,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.145362+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:54:40.145362+0000","last_peered":"2023-09-23T22:54:40.145362+0000","last_clean":"2023-09-23T22:54:40.145362+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:54:40.145362+0000","last_undegraded":"2023-09-23T22:54:40.145362+0000","last_fullsized":"2023-09-23T22:54:40.145362+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"307'10221","reported_seq":10272,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.003016+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:54:46.003016+0000","last_peered":"2023-09-23T22:54:46.003016+0000","last_clean":"2023-09-23T22:54:46.003016+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:54:46.003016+0000","last_undegraded":"2023-09-23T22:54:46.003016+0000","last_fullsized":"2023-09-23T22:54:46.003016+0000","mapping_epoch":264,"log_start":"279'3800","ondisk_log_start":"279'3800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6421,"log_dups_size":0,"ondisk_log_size":6421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56426496,"num_objects":861,"num_object_clones":0,"num_object_copies":2583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10221,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":291,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153261+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:54:40.153261+0000","last_peered":"2023-09-23T22:54:40.153261+0000","last_clean":"2023-09-23T22:54:40.153261+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:54:40.153261+0000","last_undegraded":"2023-09-23T22:54:40.153261+0000","last_fullsized":"2023-09-23T22:54:40.153261+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":302,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138296+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:54:40.138296+0000","last_peered":"2023-09-23T22:54:40.138296+0000","last_clean":"2023-09-23T22:54:40.138296+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:54:40.138296+0000","last_undegraded":"2023-09-23T22:54:40.138296+0000","last_fullsized":"2023-09-23T22:54:40.138296+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"307'10323","reported_seq":13728,"reported_epoch":307,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:54:45.927205+0000","last_change":"2023-09-23T22:54:01.194508+0000","last_active":"2023-09-23T22:54:45.927205+0000","last_peered":"2023-09-23T22:54:45.927205+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:54:01.140849+0000","last_became_peered":"2023-09-23T22:54:01.140849+0000","last_unstale":"2023-09-23T22:54:45.927205+0000","last_undegraded":"2023-09-23T22:54:45.927205+0000","last_fullsized":"2023-09-23T22:54:45.927205+0000","mapping_epoch":299,"log_start":"278'3900","ondisk_log_start":"278'3900","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56164352,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10323,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1668,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":["6","1","3","5"],"object_location_counts":[{"shards":"1,5,6","objects":857}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":315,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.137930+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:54:40.137930+0000","last_peered":"2023-09-23T22:54:40.137930+0000","last_clean":"2023-09-23T22:54:40.137930+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:54:40.137930+0000","last_undegraded":"2023-09-23T22:54:40.137930+0000","last_fullsized":"2023-09-23T22:54:40.137930+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":290,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138452+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:54:40.138452+0000","last_peered":"2023-09-23T22:54:40.138452+0000","last_clean":"2023-09-23T22:54:40.138452+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:54:40.138452+0000","last_undegraded":"2023-09-23T22:54:40.138452+0000","last_fullsized":"2023-09-23T22:54:40.138452+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"307'10123","reported_seq":13422,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:45.159137+0000","last_change":"2023-09-23T22:54:41.191574+0000","last_active":"2023-09-23T22:54:45.159137+0000","last_peered":"2023-09-23T22:54:45.159137+0000","last_clean":"2023-09-23T22:54:45.159137+0000","last_became_active":"2023-09-23T22:54:41.158218+0000","last_became_peered":"2023-09-23T22:54:41.158218+0000","last_unstale":"2023-09-23T22:54:45.159137+0000","last_undegraded":"2023-09-23T22:54:45.159137+0000","last_fullsized":"2023-09-23T22:54:45.159137+0000","mapping_epoch":306,"log_start":"269'1500","ondisk_log_start":"269'1500","created":264,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8623,"log_dups_size":0,"ondisk_log_size":8623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:01.903022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62062592,"num_objects":947,"num_object_clones":0,"num_object_copies":2841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10123,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1615,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":235,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.137808+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:54:40.137808+0000","last_peered":"2023-09-23T22:54:40.137808+0000","last_clean":"2023-09-23T22:54:40.137808+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:54:40.137808+0000","last_undegraded":"2023-09-23T22:54:40.137808+0000","last_fullsized":"2023-09-23T22:54:40.137808+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":245,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.214190+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:54:40.214190+0000","last_peered":"2023-09-23T22:54:40.214190+0000","last_clean":"2023-09-23T22:54:40.214190+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:54:40.214190+0000","last_undegraded":"2023-09-23T22:54:40.214190+0000","last_fullsized":"2023-09-23T22:54:40.214190+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"308'10482","reported_seq":13803,"reported_epoch":308,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:54:47.298108+0000","last_change":"2023-09-23T22:54:46.995007+0000","last_active":"2023-09-23T22:54:47.298108+0000","last_peered":"2023-09-23T22:54:47.298108+0000","last_clean":"2023-09-23T22:51:33.387072+0000","last_became_active":"2023-09-23T22:51:35.521270+0000","last_became_peered":"2023-09-23T22:51:35.521270+0000","last_unstale":"2023-09-23T22:54:47.298108+0000","last_undegraded":"2023-09-23T22:54:47.298108+0000","last_fullsized":"2023-09-23T22:51:35.432209+0000","mapping_epoch":271,"log_start":"286'5300","ondisk_log_start":"286'5300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:48.363532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57016320,"num_objects":870,"num_object_clones":0,"num_object_copies":2610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10482,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1630,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[1,5],"avail_no_missing":["1","2","3","5"],"object_location_counts":[{"shards":"1,5","objects":870}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":347,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.700354+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:54:46.700354+0000","last_peered":"2023-09-23T22:54:46.700354+0000","last_clean":"2023-09-23T22:54:46.700354+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:54:46.700354+0000","last_undegraded":"2023-09-23T22:54:46.700354+0000","last_fullsized":"2023-09-23T22:54:46.700354+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":170,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.135495+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:54:40.135495+0000","last_peered":"2023-09-23T22:54:40.135495+0000","last_clean":"2023-09-23T22:54:40.135495+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:54:40.135495+0000","last_undegraded":"2023-09-23T22:54:40.135495+0000","last_fullsized":"2023-09-23T22:54:40.135495+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":138,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.137975+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:54:40.137975+0000","last_peered":"2023-09-23T22:54:40.137975+0000","last_clean":"2023-09-23T22:54:40.137975+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:54:40.137975+0000","last_undegraded":"2023-09-23T22:54:40.137975+0000","last_fullsized":"2023-09-23T22:54:40.137975+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"309'10403","reported_seq":13860,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T22:54:47.974067+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:54:47.974067+0000","last_peered":"2023-09-23T22:54:47.974067+0000","last_clean":"2023-09-23T22:54:47.974067+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:54:47.974067+0000","last_undegraded":"2023-09-23T22:54:47.974067+0000","last_fullsized":"2023-09-23T22:54:47.974067+0000","mapping_epoch":282,"log_start":"275'2900","ondisk_log_start":"275'2900","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50790400,"num_objects":775,"num_object_clones":0,"num_object_copies":2325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10403,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":141,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153335+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:54:40.153335+0000","last_peered":"2023-09-23T22:54:40.153335+0000","last_clean":"2023-09-23T22:54:40.153335+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:54:40.153335+0000","last_undegraded":"2023-09-23T22:54:40.153335+0000","last_fullsized":"2023-09-23T22:54:40.153335+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":303,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138243+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:54:40.138243+0000","last_peered":"2023-09-23T22:54:40.138243+0000","last_clean":"2023-09-23T22:54:40.138243+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:54:40.138243+0000","last_undegraded":"2023-09-23T22:54:40.138243+0000","last_fullsized":"2023-09-23T22:54:40.138243+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":234,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134639+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:54:40.134639+0000","last_peered":"2023-09-23T22:54:40.134639+0000","last_clean":"2023-09-23T22:54:40.134639+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:54:40.134639+0000","last_undegraded":"2023-09-23T22:54:40.134639+0000","last_fullsized":"2023-09-23T22:54:40.134639+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"307'10358","reported_seq":13740,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:45.956125+0000","last_change":"2023-09-23T22:54:21.209191+0000","last_active":"2023-09-23T22:54:45.956125+0000","last_peered":"2023-09-23T22:54:45.956125+0000","last_clean":"2023-09-23T22:54:45.956125+0000","last_became_active":"2023-09-23T22:54:21.131570+0000","last_became_peered":"2023-09-23T22:54:21.131570+0000","last_unstale":"2023-09-23T22:54:45.956125+0000","last_undegraded":"2023-09-23T22:54:45.956125+0000","last_fullsized":"2023-09-23T22:54:45.956125+0000","mapping_epoch":302,"log_start":"279'3950","ondisk_log_start":"279'3950","created":264,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60162048,"num_objects":918,"num_object_clones":0,"num_object_copies":2754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10358,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1655,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":309,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134831+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:54:40.134831+0000","last_peered":"2023-09-23T22:54:40.134831+0000","last_clean":"2023-09-23T22:54:40.134831+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:54:40.134831+0000","last_undegraded":"2023-09-23T22:54:40.134831+0000","last_fullsized":"2023-09-23T22:54:40.134831+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":189,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:41.152325+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:54:41.152325+0000","last_peered":"2023-09-23T22:54:41.152325+0000","last_clean":"2023-09-23T22:54:41.152325+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:54:41.152325+0000","last_undegraded":"2023-09-23T22:54:41.152325+0000","last_fullsized":"2023-09-23T22:54:41.152325+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":316,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688038+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:54:46.688038+0000","last_peered":"2023-09-23T22:54:46.688038+0000","last_clean":"2023-09-23T22:54:46.688038+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:54:46.688038+0000","last_undegraded":"2023-09-23T22:54:46.688038+0000","last_fullsized":"2023-09-23T22:54:46.688038+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"309'10257","reported_seq":10594,"reported_epoch":309,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:54:48.301684+0000","last_change":"2023-09-23T22:54:46.998497+0000","last_active":"2023-09-23T22:54:48.301684+0000","last_peered":"2023-09-23T22:54:48.301684+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:54:48.301684+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"275'2700","ondisk_log_start":"275'2700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51970048,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10257,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":-782}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":319,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.135226+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:54:40.135226+0000","last_peered":"2023-09-23T22:54:40.135226+0000","last_clean":"2023-09-23T22:54:40.135226+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:54:40.135226+0000","last_undegraded":"2023-09-23T22:54:40.135226+0000","last_fullsized":"2023-09-23T22:54:40.135226+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":398,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.687764+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:54:46.687764+0000","last_peered":"2023-09-23T22:54:46.687764+0000","last_clean":"2023-09-23T22:54:46.687764+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:54:46.687764+0000","last_undegraded":"2023-09-23T22:54:46.687764+0000","last_fullsized":"2023-09-23T22:54:46.687764+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":61,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153486+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:54:40.153486+0000","last_peered":"2023-09-23T22:54:40.153486+0000","last_clean":"2023-09-23T22:54:40.153486+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:54:40.153486+0000","last_undegraded":"2023-09-23T22:54:40.153486+0000","last_fullsized":"2023-09-23T22:54:40.153486+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":299,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.153946+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:54:40.153946+0000","last_peered":"2023-09-23T22:54:40.153946+0000","last_clean":"2023-09-23T22:54:40.153946+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:54:40.153946+0000","last_undegraded":"2023-09-23T22:54:40.153946+0000","last_fullsized":"2023-09-23T22:54:40.153946+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"309'10384","reported_seq":10439,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T22:54:47.957167+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:54:47.957167+0000","last_peered":"2023-09-23T22:54:47.957167+0000","last_clean":"2023-09-23T22:54:47.957167+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:54:47.957167+0000","last_undegraded":"2023-09-23T22:54:47.957167+0000","last_fullsized":"2023-09-23T22:54:47.957167+0000","mapping_epoch":264,"log_start":"275'2800","ondisk_log_start":"275'2800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55181312,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10384,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":286,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T22:54:47.713549+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:54:47.713549+0000","last_peered":"2023-09-23T22:54:47.713549+0000","last_clean":"2023-09-23T22:54:47.713549+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:54:47.713549+0000","last_undegraded":"2023-09-23T22:54:47.713549+0000","last_fullsized":"2023-09-23T22:54:47.713549+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":257,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.691319+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:54:46.691319+0000","last_peered":"2023-09-23T22:54:46.691319+0000","last_clean":"2023-09-23T22:54:46.691319+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:54:46.691319+0000","last_undegraded":"2023-09-23T22:54:46.691319+0000","last_fullsized":"2023-09-23T22:54:46.691319+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":234,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688311+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:54:46.688311+0000","last_peered":"2023-09-23T22:54:46.688311+0000","last_clean":"2023-09-23T22:54:46.688311+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:54:46.688311+0000","last_undegraded":"2023-09-23T22:54:46.688311+0000","last_fullsized":"2023-09-23T22:54:46.688311+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"307'10147","reported_seq":10198,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.625480+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:54:46.625480+0000","last_peered":"2023-09-23T22:54:46.625480+0000","last_clean":"2023-09-23T22:54:46.625480+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:54:46.625480+0000","last_undegraded":"2023-09-23T22:54:46.625480+0000","last_fullsized":"2023-09-23T22:54:46.625480+0000","mapping_epoch":264,"log_start":"277'3300","ondisk_log_start":"277'3300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6847,"log_dups_size":0,"ondisk_log_size":6847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54198272,"num_objects":827,"num_object_clones":0,"num_object_copies":2481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10147,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":240,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688161+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:54:46.688161+0000","last_peered":"2023-09-23T22:54:46.688161+0000","last_clean":"2023-09-23T22:54:46.688161+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:54:46.688161+0000","last_undegraded":"2023-09-23T22:54:46.688161+0000","last_fullsized":"2023-09-23T22:54:46.688161+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":187,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134798+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:54:40.134798+0000","last_peered":"2023-09-23T22:54:40.134798+0000","last_clean":"2023-09-23T22:54:40.134798+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:54:40.134798+0000","last_undegraded":"2023-09-23T22:54:40.134798+0000","last_fullsized":"2023-09-23T22:54:40.134798+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":240,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.689035+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:54:46.689035+0000","last_peered":"2023-09-23T22:54:46.689035+0000","last_clean":"2023-09-23T22:54:46.689035+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:54:46.689035+0000","last_undegraded":"2023-09-23T22:54:46.689035+0000","last_fullsized":"2023-09-23T22:54:46.689035+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"307'10446","reported_seq":10498,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.641752+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:54:46.641752+0000","last_peered":"2023-09-23T22:54:46.641752+0000","last_clean":"2023-09-23T22:54:46.641752+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:54:46.641752+0000","last_undegraded":"2023-09-23T22:54:46.641752+0000","last_fullsized":"2023-09-23T22:54:46.641752+0000","mapping_epoch":264,"log_start":"277'3600","ondisk_log_start":"277'3600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59965464,"num_objects":916,"num_object_clones":0,"num_object_copies":2748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10446,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":66,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688257+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:54:46.688257+0000","last_peered":"2023-09-23T22:54:46.688257+0000","last_clean":"2023-09-23T22:54:46.688257+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:54:46.688257+0000","last_undegraded":"2023-09-23T22:54:46.688257+0000","last_fullsized":"2023-09-23T22:54:46.688257+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.154087+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:54:40.154087+0000","last_peered":"2023-09-23T22:54:40.154087+0000","last_clean":"2023-09-23T22:54:40.154087+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:54:40.154087+0000","last_undegraded":"2023-09-23T22:54:40.154087+0000","last_fullsized":"2023-09-23T22:54:40.154087+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":286,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.687668+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:54:46.687668+0000","last_peered":"2023-09-23T22:54:46.687668+0000","last_clean":"2023-09-23T22:54:46.687668+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:54:46.687668+0000","last_undegraded":"2023-09-23T22:54:46.687668+0000","last_fullsized":"2023-09-23T22:54:46.687668+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"307'10407","reported_seq":13935,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.654469+0000","last_change":"2023-09-23T22:53:44.765920+0000","last_active":"2023-09-23T22:54:46.654469+0000","last_peered":"2023-09-23T22:54:46.654469+0000","last_clean":"2023-09-23T22:54:46.654469+0000","last_became_active":"2023-09-23T22:53:44.707193+0000","last_became_peered":"2023-09-23T22:53:44.707193+0000","last_unstale":"2023-09-23T22:54:46.654469+0000","last_undegraded":"2023-09-23T22:54:46.654469+0000","last_fullsized":"2023-09-23T22:54:46.654469+0000","mapping_epoch":296,"log_start":"277'3600","ondisk_log_start":"277'3600","created":264,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6807,"log_dups_size":0,"ondisk_log_size":6807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999360,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10407,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":348,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.154167+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:54:40.154167+0000","last_peered":"2023-09-23T22:54:40.154167+0000","last_clean":"2023-09-23T22:54:40.154167+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:54:40.154167+0000","last_undegraded":"2023-09-23T22:54:40.154167+0000","last_fullsized":"2023-09-23T22:54:40.154167+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":206,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688391+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:54:46.688391+0000","last_peered":"2023-09-23T22:54:46.688391+0000","last_clean":"2023-09-23T22:54:46.688391+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:54:46.688391+0000","last_undegraded":"2023-09-23T22:54:46.688391+0000","last_fullsized":"2023-09-23T22:54:46.688391+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":257,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.137928+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:54:40.137928+0000","last_peered":"2023-09-23T22:54:40.137928+0000","last_clean":"2023-09-23T22:54:40.137928+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:54:40.137928+0000","last_undegraded":"2023-09-23T22:54:40.137928+0000","last_fullsized":"2023-09-23T22:54:40.137928+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":242,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688907+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:54:46.688907+0000","last_peered":"2023-09-23T22:54:46.688907+0000","last_clean":"2023-09-23T22:54:46.688907+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:54:46.688907+0000","last_undegraded":"2023-09-23T22:54:46.688907+0000","last_fullsized":"2023-09-23T22:54:46.688907+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134717+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:54:40.134717+0000","last_peered":"2023-09-23T22:54:40.134717+0000","last_clean":"2023-09-23T22:54:40.134717+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:54:40.134717+0000","last_undegraded":"2023-09-23T22:54:40.134717+0000","last_fullsized":"2023-09-23T22:54:40.134717+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":116,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.145287+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:54:40.145287+0000","last_peered":"2023-09-23T22:54:40.145287+0000","last_clean":"2023-09-23T22:54:40.145287+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:54:40.145287+0000","last_undegraded":"2023-09-23T22:54:40.145287+0000","last_fullsized":"2023-09-23T22:54:40.145287+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":315,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138332+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:54:40.138332+0000","last_peered":"2023-09-23T22:54:40.138332+0000","last_clean":"2023-09-23T22:54:40.138332+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:54:40.138332+0000","last_undegraded":"2023-09-23T22:54:40.138332+0000","last_fullsized":"2023-09-23T22:54:40.138332+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":117,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138405+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:54:40.138405+0000","last_peered":"2023-09-23T22:54:40.138405+0000","last_clean":"2023-09-23T22:54:40.138405+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:54:40.138405+0000","last_undegraded":"2023-09-23T22:54:40.138405+0000","last_fullsized":"2023-09-23T22:54:40.138405+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":187,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.138344+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:54:40.138344+0000","last_peered":"2023-09-23T22:54:40.138344+0000","last_clean":"2023-09-23T22:54:40.138344+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:54:40.138344+0000","last_undegraded":"2023-09-23T22:54:40.138344+0000","last_fullsized":"2023-09-23T22:54:40.138344+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":57,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.154055+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:54:40.154055+0000","last_peered":"2023-09-23T22:54:40.154055+0000","last_clean":"2023-09-23T22:54:40.154055+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:54:40.154055+0000","last_undegraded":"2023-09-23T22:54:40.154055+0000","last_fullsized":"2023-09-23T22:54:40.154055+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.134707+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:54:40.134707+0000","last_peered":"2023-09-23T22:54:40.134707+0000","last_clean":"2023-09-23T22:54:40.134707+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:54:40.134707+0000","last_undegraded":"2023-09-23T22:54:40.134707+0000","last_fullsized":"2023-09-23T22:54:40.134707+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":256,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T22:54:46.688154+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:54:46.688154+0000","last_peered":"2023-09-23T22:54:46.688154+0000","last_clean":"2023-09-23T22:54:46.688154+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:54:46.688154+0000","last_undegraded":"2023-09-23T22:54:46.688154+0000","last_fullsized":"2023-09-23T22:54:46.688154+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":272,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T22:54:40.137849+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:54:40.137849+0000","last_peered":"2023-09-23T22:54:40.137849+0000","last_clean":"2023-09-23T22:54:40.137849+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:54:40.137849+0000","last_undegraded":"2023-09-23T22:54:40.137849+0000","last_fullsized":"2023-09-23T22:54:40.137849+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":910753816,"num_objects":13898,"num_object_clones":0,"num_object_copies":41694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5007,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13898,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165380,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11800,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3635032064,"data_stored":3635019848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114230,"ondisk_log_size":114230,"up":48,"acting":44,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430541,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626769,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":780828,"kb_used_data":475348,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92964324,"statfs":{"total":95995035648,"available":95195467776,"internally_reserved":0,"allocated":486756352,"data_stored":486199062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":8.0800000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.087,"5min":243.35900000000001,"15min":243.35900000000001},"last":17.593},{"interface":"front","average":{"1min":3.3559999999999999,"5min":8.8829999999999991,"15min":4.4269999999999996},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":12.081,"5min":243.41499999999999,"15min":243.41499999999999},"last":17.402000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":5.2249999999999996,"15min":3.8079999999999998},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":10.98,"5min":254.62100000000001,"15min":254.62100000000001},"last":3.4889999999999999},{"interface":"front","average":{"1min":3.21,"5min":6.5389999999999997,"15min":4.8079999999999998},"min":{"1min":0.45000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":12.069000000000001,"5min":255.20099999999999,"15min":255.20099999999999},"last":1.4099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":7.5069999999999997,"5min":4.4580000000000002,"15min":3.738},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":81.719999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":3.4430000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":3.1539999999999999,"15min":3.7509999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":9.3529999999999998,"5min":101.13,"15min":185.21799999999999},"last":1.3540000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.1520000000000001,"15min":3.8029999999999999},"min":{"1min":0.309,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":6.5090000000000003,"5min":17.742999999999999,"15min":185.19399999999999},"last":3.339},{"interface":"front","average":{"1min":3.2850000000000001,"5min":2.2799999999999998,"15min":3.4950000000000001},"min":{"1min":0.42599999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":10.378,"5min":10.773999999999999,"15min":128.19800000000001},"last":3.278}]},{"osd":4,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":1.1359999999999999,"15min":0.871},"min":{"1min":0.309,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.317,"5min":10.317,"15min":18.085999999999999},"last":1.4379999999999999},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.107,"15min":0.88500000000000001},"min":{"1min":0.32900000000000001,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":7.4610000000000003,"5min":7.4800000000000004,"15min":18.597000000000001},"last":1.27}]},{"osd":5,"last update":"Sat Sep 23 22:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.339,"15min":0.99099999999999999},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.353999999999999,"5min":10.353999999999999,"15min":18.972000000000001},"last":2.1459999999999999},{"interface":"front","average":{"1min":2.5459999999999998,"5min":1.1779999999999999,"15min":0.93999999999999995},"min":{"1min":0.41599999999999998,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":7.1459999999999999,"5min":7.1459999999999999,"15min":18.625},"last":1.321}]},{"osd":7,"last update":"Sat Sep 23 22:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.123,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":7.8650000000000002,"5min":7.8650000000000002,"15min":12.481999999999999},"last":5.0419999999999998},{"interface":"front","average":{"1min":1.694,"5min":1.21,"15min":0.93600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.774,"5min":6.6269999999999998,"15min":16.661999999999999},"last":3.375}]}]},{"osd":1,"up_from":84,"seq":360777253079,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181036,"kb_used_data":665764,"kb_used_omap":9,"kb_used_meta":515254,"kb_avail":92564116,"statfs":{"total":95995035648,"available":94785654784,"internally_reserved":0,"allocated":681742336,"data_stored":680655534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":527621097},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":4.5519999999999996,"15min":5.4329999999999998},"min":{"1min":0.27700000000000002,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":37.991,"5min":37.991,"15min":101.605},"last":24.527999999999999},{"interface":"front","average":{"1min":6.1550000000000002,"5min":3.819,"15min":5.3019999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":42.191000000000003,"5min":42.191000000000003,"15min":101.892},"last":14.269}]},{"osd":2,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":3.8900000000000001,"15min":3.2549999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":21.215,"5min":26.129000000000001,"15min":37.889000000000003},"last":25.120000000000001},{"interface":"front","average":{"1min":3.9689999999999999,"5min":3.633,"15min":3.3239999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":21.254000000000001,"5min":26.001999999999999,"15min":37.957000000000001},"last":25.067}]},{"osd":3,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.871,"15min":2.645},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.143000000000001,"5min":12.143000000000001,"15min":87.787999999999997},"last":12.997999999999999},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.1899999999999999,"15min":3.1309999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20999999999999999},"max":{"1min":8.0099999999999998,"5min":10.845000000000001,"15min":89.861999999999995},"last":4.4050000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":1.9510000000000001,"15min":2.3900000000000001},"min":{"1min":0.38,"5min":0.35899999999999999,"15min":0.29299999999999998},"max":{"1min":11.388,"5min":11.388,"15min":78.948999999999998},"last":1.5349999999999999},{"interface":"front","average":{"1min":2.3980000000000001,"5min":2.5659999999999998,"15min":2.4020000000000001},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.2579999999999991,"5min":14.058,"15min":78.963999999999999},"last":20.788}]},{"osd":5,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.7110000000000001,"15min":1.8839999999999999},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":12.115,"5min":12.115,"15min":84.254999999999995},"last":12.936},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.78,"15min":2.21},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":8.1050000000000004,"5min":9.8170000000000002,"15min":87.515000000000001},"last":1.577}]},{"osd":6,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":2.6110000000000002,"15min":2.2559999999999998},"min":{"1min":0.40699999999999997,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.6639999999999997,"5min":13.991,"15min":79.311000000000007},"last":20.742000000000001},{"interface":"front","average":{"1min":2.5979999999999999,"5min":1.9810000000000001,"15min":1.9059999999999999},"min":{"1min":0.41899999999999998,"5min":0.40899999999999997,"15min":0.313},"max":{"1min":12.067,"5min":12.067,"15min":79.328999999999994},"last":12.978}]},{"osd":7,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.899,"15min":2.609},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.32800000000000001},"max":{"1min":8.1400000000000006,"5min":9.9329999999999998,"15min":159.018},"last":1.6020000000000001},{"interface":"front","average":{"1min":2.254,"5min":2.6949999999999998,"15min":2.7749999999999999},"min":{"1min":0.443,"5min":0.38900000000000001,"15min":0.37},"max":{"1min":6.6310000000000002,"5min":14.032999999999999,"15min":158.96000000000001},"last":20.902999999999999}]}]},{"osd":0,"up_from":219,"seq":940597837928,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":704144,"kb_used_data":376840,"kb_used_omap":31,"kb_used_meta":327072,"kb_avail":93041008,"statfs":{"total":95995035648,"available":95273992192,"internally_reserved":0,"allocated":385884160,"data_stored":384764951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":334922674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":3.4329999999999998,"15min":3.7410000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.326000000000001,"5min":40.180999999999997,"15min":40.180999999999997},"last":0.71699999999999997},{"interface":"front","average":{"1min":4.2270000000000003,"5min":3.4630000000000001,"15min":3.6499999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.494999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":2.5169999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":5.4139999999999997,"15min":3.5910000000000002},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":89.766000000000005,"5min":111.508,"15min":111.508},"last":3.464},{"interface":"front","average":{"1min":7.617,"5min":5.5449999999999999,"15min":3.2360000000000002},"min":{"1min":0.29499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":89.994,"5min":111.532,"15min":111.532},"last":3.125}]},{"osd":3,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":4.2519999999999998,"15min":3.0510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":8.5380000000000003,"5min":126.73,"15min":126.73},"last":1.637},{"interface":"front","average":{"1min":2.4969999999999999,"5min":3.488,"15min":2.1349999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6440000000000001,"5min":121.483,"15min":121.483},"last":1.573}]},{"osd":4,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.0129999999999999,"15min":2.173},"min":{"1min":0.33000000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.5820000000000007,"5min":11.965999999999999,"15min":18.670999999999999},"last":0.754},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.2400000000000002,"15min":2.036},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.5899999999999999,"5min":16.52,"15min":20.009},"last":1.6699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.8599999999999999,"15min":1.9370000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":6.6630000000000003,"5min":61.756999999999998,"15min":61.756999999999998},"last":1.49},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.1379999999999999,"15min":2.3490000000000002},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4240000000000004,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.78400000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.778,"15min":1.95},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6950000000000003,"5min":59.146999999999998,"15min":59.146999999999998},"last":1.5980000000000001},{"interface":"front","average":{"1min":2.504,"5min":3.069,"15min":2.4550000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.367,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.84099999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.1709999999999998,"15min":2.077},"min":{"1min":0.438,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.6959999999999997,"5min":12.385999999999999,"15min":18.640000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":1.946,"5min":2.0830000000000002,"15min":1.6000000000000001},"min":{"1min":0.42599999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.6779999999999999,"5min":21.954999999999998,"15min":21.954999999999998},"last":1.5409999999999999}]}]},{"osd":2,"up_from":131,"seq":562640715940,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":790184,"kb_used_data":368480,"kb_used_omap":81,"kb_used_meta":421614,"kb_avail":92954968,"statfs":{"total":95995035648,"available":95185887232,"internally_reserved":0,"allocated":377323520,"data_stored":376236142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":431732954},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":6.9960000000000004,"15min":5.734},"min":{"1min":0.22800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":12.76,"5min":132.79499999999999,"15min":132.79499999999999},"last":1},{"interface":"front","average":{"1min":3.286,"5min":6.9400000000000004,"15min":6.0990000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.093,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":3.0470000000000002,"15min":3.173},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":6.3019999999999996,"5min":77.004999999999995,"15min":88.316000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.552,"5min":3.3500000000000001,"15min":3.407},"min":{"1min":0.25700000000000001,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":6.1779999999999999,"5min":76.956000000000003,"15min":88.436999999999998},"last":0.51900000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.2939999999999996,"15min":4.4100000000000001},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.492000000000001,"5min":112.895,"15min":112.895},"last":1.012},{"interface":"front","average":{"1min":2.7869999999999999,"5min":4.109,"15min":3.6219999999999999},"min":{"1min":0.39000000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.1880000000000006,"5min":113.995,"15min":113.995},"last":0.63500000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.6749999999999998,"15min":2.8279999999999998},"min":{"1min":0.60199999999999998,"5min":0.28899999999999998,"15min":0.253},"max":{"1min":9.1219999999999999,"5min":69.113,"15min":108.878},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.4689999999999999,"15min":3.3809999999999998},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2229999999999999,"5min":69.028999999999996,"15min":108.93000000000001},"last":0.36599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.7789999999999999,"15min":2.9940000000000002},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":6.1820000000000004,"5min":22.646000000000001,"15min":97.322999999999993},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.2569999999999999,"5min":2.226,"15min":2.98},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.1420000000000003,"5min":22.501000000000001,"15min":97.301000000000002},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":3.609,"15min":3.3450000000000002},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":4.3040000000000003,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.4670000000000001,"5min":3.4209999999999998,"15min":2.7050000000000001},"min":{"1min":0.48699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":9.2110000000000003,"5min":98.509,"15min":98.509},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":2.0590000000000002,"15min":1.7509999999999999},"min":{"1min":0.48999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.1609999999999996,"5min":10.477,"15min":10.92},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.415,"5min":1.667,"15min":2.3799999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.37},"max":{"1min":6.2720000000000002,"5min":11.311,"15min":88.584000000000003},"last":0.434}]}]},{"osd":3,"up_from":0,"seq":403726926026,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019225,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":595488,"kb_used_data":285976,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93149664,"statfs":{"total":95995035648,"available":95385255936,"internally_reserved":0,"allocated":292839424,"data_stored":292277540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.3140000000000001,"15min":5.9390000000000001},"min":{"1min":0.251,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":5.9509999999999996,"5min":113.35599999999999,"15min":113.35599999999999},"last":4.7530000000000001},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.7229999999999999,"15min":6.5099999999999998},"min":{"1min":0.41199999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.6760000000000002,"5min":113.417,"15min":113.417},"last":4.4859999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":5.0599999999999996,"15min":3.0190000000000001},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":132.304,"15min":132.304},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.7959999999999998,"5min":4.8970000000000002,"15min":2.879},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":10.832000000000001,"5min":132.453,"15min":132.453},"last":10.832000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.3639999999999999,"15min":3.1520000000000001},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.097,"5min":20.308,"15min":84.909000000000006},"last":3.8559999999999999},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.524,"15min":3.137},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.352,"5min":21.704000000000001,"15min":83.840000000000003},"last":4.4690000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":4.1879999999999997,"15min":4.548},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":7.3109999999999999,"5min":164.453,"15min":164.453},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.2170000000000001,"5min":4.343,"15min":4.6689999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.1779999999999999,"15min":1},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.25900000000000001},"max":{"1min":5.0880000000000001,"5min":12.238,"15min":43.189999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1060000000000001,"15min":1},"min":{"1min":0.32900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.3079999999999998,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.0529999999999999,"15min":0.94899999999999995},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9139999999999997,"5min":7.9139999999999997,"15min":27.056000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.417,"5min":1.224,"15min":0.97799999999999998},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":5.298,"5min":12.269,"15min":26.940999999999999},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.2250000000000001,"15min":0.876},"min":{"1min":0.36699999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":5.1449999999999996,"5min":12.311,"15min":12.311},"last":0.498},{"interface":"front","average":{"1min":1.171,"5min":1.0569999999999999,"15min":0.79900000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":4.077,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.60699999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822997,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1180164,"kb_used_data":582268,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92564988,"statfs":{"total":95995035648,"available":94786547712,"internally_reserved":0,"allocated":596242432,"data_stored":595704264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":9.6929999999999996,"5min":8.1189999999999998,"15min":5.133},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":80.450999999999993,"5min":185.62,"15min":185.62},"last":0.46700000000000003},{"interface":"front","average":{"1min":9.734,"5min":8.1829999999999998,"15min":5.1079999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":80.171999999999997,"5min":185.499,"15min":185.499},"last":0.30299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":9.452,"5min":3.9129999999999998,"15min":2.8799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":110.36499999999999,"5min":110.36499999999999,"15min":149.203},"last":0.28799999999999998},{"interface":"front","average":{"1min":9.5340000000000007,"5min":4.3019999999999996,"15min":3.1909999999999998},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":110.89,"5min":110.89,"15min":149.554},"last":2.9940000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":5.54,"15min":5.5140000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.997,"5min":189.126,"15min":189.126},"last":0.33700000000000002},{"interface":"front","average":{"1min":3.5379999999999998,"5min":5.2309999999999999,"15min":5.4059999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":41.070999999999998,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.55700000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.843,"15min":5.6660000000000004},"min":{"1min":0.34899999999999998,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.298999999999999,"5min":98.641000000000005,"15min":257.05000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.085,"5min":4.9130000000000003,"15min":5.516},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27800000000000002},"max":{"1min":11.097,"5min":98.906000000000006,"15min":257.35000000000002},"last":2.7789999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.86899999999999999,"15min":0.73399999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":3.669,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.996,"5min":0.92100000000000004,"15min":0.75700000000000001},"min":{"1min":0.29399999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":5.8019999999999996,"5min":6.3550000000000004,"15min":10.535},"last":0.52100000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1759999999999999,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.3369999999999997,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.139,"15min":1.629},"min":{"1min":0.374,"5min":0.315,"15min":0.254},"max":{"1min":7.9909999999999997,"5min":10.314,"15min":139.739},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.133,"15min":0.88700000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.9509999999999996,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.503},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.99299999999999999,"15min":0.80300000000000005},"min":{"1min":0.378,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.59899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":371720192,"data_stored":371720192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":663621632,"data_stored":663617560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":360251392,"data_stored":360251392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":902234112,"data_stored":902234112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":285085696,"data_stored":285081624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":588124160,"data_stored":588120088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":463994880,"data_stored":463994880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:54:51.711 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:54:51.791 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:51.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:51.804 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:54:51.900 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:51.934 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:51.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:51 smithi052 ceph-mon[180979]: pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 832 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 611 op/s; 5007/39984 objects degraded (12.523%); 0 B/s, 63 objects/s recovering 2023-09-23T22:54:51.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:51 smithi052 ceph-mon[187743]: pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 832 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 611 op/s; 5007/39984 objects degraded (12.523%); 0 B/s, 63 objects/s recovering 2023-09-23T22:54:52.019 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:52.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:51 smithi151 ceph-mon[159210]: pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 832 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 611 op/s; 5007/39984 objects degraded (12.523%); 0 B/s, 63 objects/s recovering 2023-09-23T22:54:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:52.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:52.125 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:52.129 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:52.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:52.323 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:54:52.340 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:54:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:52.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:54:52.459 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:52.506 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:52.583 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:52.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:52.688 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:52.736 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:52.811 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:54:52.881 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:54:52.898 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:52.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:52.937 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:54:52.967 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:54:53.004 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:53.007 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:54:53.044 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:54:53.051 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:53.066 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:54:53.088 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.094 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:54:53.100 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:53.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:53.205 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:53.229 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:53.270 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:53.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:53.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:53.373 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:53.421 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:53.680 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:53.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:53.732 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:54:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:54:53] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.33.4" 2023-09-23T22:54:53.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:53 smithi052 ceph-mon[180979]: pgmap v900: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 745 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 625 op/s; 5007/35829 objects degraded (13.975%); 0 B/s, 63 objects/s recovering 2023-09-23T22:54:53.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1977179367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:53.738 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:53 smithi052 ceph-mon[187743]: pgmap v900: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 745 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 625 op/s; 5007/35829 objects degraded (13.975%); 0 B/s, 63 objects/s recovering 2023-09-23T22:54:53.738 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1977179367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:53.800 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:53.899 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:53.930 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:53 smithi151 ceph-mon[159210]: pgmap v900: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 745 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 625 op/s; 5007/35829 objects degraded (13.975%); 0 B/s, 63 objects/s recovering 2023-09-23T22:54:53.930 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1977179367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:54:54.185 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:54:54.186 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:54:54.369 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:54.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:54.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:54.473 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:54.486 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:54.516 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:54.587 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:54.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:54.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:54.691 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:54.787 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:54.953 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:54.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:55.043 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:54:55.060 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:54:55.146 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:55.246 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:54:55.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:55.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:54:55.358 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:55.428 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:55.576 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:54:55.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:55.755 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:55.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:55.856 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:55.927 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[180979]: pgmap v901: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 703 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 469 op/s; 4719/33792 objects degraded (13.965%); 0 B/s, 86 objects/s recovering 2023-09-23T22:54:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4719/33792 objects degraded (13.965%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1445487043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[187743]: pgmap v901: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 703 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 469 op/s; 4719/33792 objects degraded (13.965%); 0 B/s, 86 objects/s recovering 2023-09-23T22:54:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4719/33792 objects degraded (13.965%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1445487043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:55.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:55 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:56.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:55 smithi151 ceph-mon[159210]: pgmap v901: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 703 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 469 op/s; 4719/33792 objects degraded (13.965%); 0 B/s, 86 objects/s recovering 2023-09-23T22:54:56.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4719/33792 objects degraded (13.965%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:54:56.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1445487043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:56.046 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:55 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:54:56.148 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:54:56.164 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:56.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:56.268 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:56.353 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:56.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:54:56.658 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:56.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:56.769 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:54:56.806 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:56.882 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:54:56.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:56.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:54:56.998 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:54:57.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:56 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:54:57.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:56 smithi151 ceph-mon[159210]: failed to encode map e311 with expected crc 2023-09-23T22:54:57.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:56 smithi151 ceph-mon[159210]: failed to encode map e311 with expected crc 2023-09-23T22:54:57.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:56 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:57.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:56 smithi151 ceph-mon[159210]: osdmap e311: 8 total, 8 up, 6 in 2023-09-23T22:54:57.029 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:57.152 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:54:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:57.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:54:57.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[180979]: failed to encode map e311 with expected crc 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[180979]: failed to encode map e311 with expected crc 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[180979]: osdmap e311: 8 total, 8 up, 6 in 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[187743]: failed to encode map e311 with expected crc 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[187743]: failed to encode map e311 with expected crc 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:54:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:56 smithi052 ceph-mon[187743]: osdmap e311: 8 total, 8 up, 6 in 2023-09-23T22:54:57.260 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:57.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:57.472 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:54:57.472 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:54:57.821 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:57.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:57.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:57.931 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:54:58.017 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:58.034 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:54:58.134 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:54:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:58.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:54:58.141 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:57 smithi052 ceph-mon[180979]: failed to encode map e311 with expected crc 2023-09-23T22:54:58.141 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:57 smithi052 ceph-mon[180979]: pgmap v903: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 604 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 4719/29025 objects degraded (16.258%); 0 B/s, 35 objects/s recovering 2023-09-23T22:54:58.141 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:57 smithi052 ceph-mon[187743]: failed to encode map e311 with expected crc 2023-09-23T22:54:58.141 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:57 smithi052 ceph-mon[187743]: pgmap v903: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 604 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 4719/29025 objects degraded (16.258%); 0 B/s, 35 objects/s recovering 2023-09-23T22:54:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:57 smithi151 ceph-mon[159210]: failed to encode map e311 with expected crc 2023-09-23T22:54:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:57 smithi151 ceph-mon[159210]: pgmap v903: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 604 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 4719/29025 objects degraded (16.258%); 0 B/s, 35 objects/s recovering 2023-09-23T22:54:58.241 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:54:58.288 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:54:58.336 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:58.406 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:54:58.407 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 4719/29025 objects degraded (16.258%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1231,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":311,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":6,"osd_in_since":1695509493,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":9675,"data_bytes":633018296,"bytes_used":4515684352,"bytes_avail":571454529536,"bytes_total":575970213888,"degraded_objects":4719,"degraded_total":29025,"degraded_ratio":0.16258397932816535,"recovering_objects_per_sec":35,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":288,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":530},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T22:53:02.381180+0000","services":{}},"progress_events":{"6b27ada5-699f-4815-a812-68a0dbe12316":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 9s)","progress":0.95505619049072266,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:54:58.565 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:54:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:58.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:54:58.669 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:54:58.742 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:58.748 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2604462181' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:54:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2604462181' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:54:59.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2604462181' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:54:59.479 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:54:59.605 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:54:59.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:54:59.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:54:59.713 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:54:59.827 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:54:59.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:59 smithi052 ceph-mon[180979]: pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 502 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 4506/24126 objects degraded (18.677%); 0 B/s, 51 objects/s recovering 2023-09-23T22:54:59.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:59.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:54:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:59.883 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:59 smithi052 ceph-mon[187743]: pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 502 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 4506/24126 objects degraded (18.677%); 0 B/s, 51 objects/s recovering 2023-09-23T22:54:59.883 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:54:59.883 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:54:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:59 smithi151 ceph-mon[159210]: pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 502 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 4506/24126 objects degraded (18.677%); 0 B/s, 51 objects/s recovering 2023-09-23T22:55:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:54:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:00.340 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:55:00.340 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:55:00.494 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:00.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:00.514 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:55:00.514 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:55:00.603 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:00.700 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:00.891 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:00.903 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:00.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:00.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4506/24126 objects degraded (18.677%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:00.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[180979]: pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 450 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 610 op/s; 4506/21648 objects degraded (20.815%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:00.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:55:00.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:55:00.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:00.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4506/24126 objects degraded (18.677%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:00.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[187743]: pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 450 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 610 op/s; 4506/21648 objects degraded (20.815%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:00.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:55:00.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:55:00.907 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:00 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:00.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:01.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:01.108 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4506/24126 objects degraded (18.677%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:00 smithi151 ceph-mon[159210]: pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 450 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 610 op/s; 4506/21648 objects degraded (20.815%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:55:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:55:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:00 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:01.245 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:55:01.401 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:01.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:01.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:01.462 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:01.514 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:01.517 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:01.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:55:01.568 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:01.598 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:01.631 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:01.647 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:01.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:01.673 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:01.681 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:01.712 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:01.758 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:01.763 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.766 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:55:01.793 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:01.793 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:01.877 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:01.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:01.983 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:02.063 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:02.158 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2043974284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:02.499 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2043974284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:02.499 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2043974284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:03.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:03 smithi151 ceph-mon[159210]: pgmap v906: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 354 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 625 op/s; 4506/17052 objects degraded (26.425%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:03.395 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:03.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:03.429 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:03 smithi052 ceph-mon[180979]: pgmap v906: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 354 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 625 op/s; 4506/17052 objects degraded (26.425%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:03.429 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:55:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:55:03] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.33.4" 2023-09-23T22:55:03.434 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:03 smithi052 ceph-mon[187743]: pgmap v906: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 354 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 625 op/s; 4506/17052 objects degraded (26.425%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:03.507 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:03.539 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:55:03.540 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:55:03.540 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:55:03.540 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:55:03.540 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:55:03.601 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:03.969 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:03.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:04.075 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:04.121 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:04.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:04.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:04.276 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:04.325 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:55:04.332 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:55:04.340 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:04.429 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:04.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:04.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:04.545 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:04.609 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3271682592' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:55:04.620 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:04.708 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3271682592' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:55:04.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3271682592' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:55:04.723 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:04.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:04.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:04.830 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:04.859 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:04.868 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:04.914 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:04.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:05.020 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:05.060 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:05.139 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:05.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:05.248 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:05.284 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:05.385 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:55:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3271682592' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:55:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: osdmap e312: 8 total, 8 up, 6 in 2023-09-23T22:55:05.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: failed to encode map e312 with expected crc 2023-09-23T22:55:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 304 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 4262/14640 objects degraded (29.112%); 0 B/s, 55 objects/s recovering 2023-09-23T22:55:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:05.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4506/17052 objects degraded (26.425%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:05.505 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:05.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:05.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:05.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:05.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3271682592' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:55:05.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: osdmap e312: 8 total, 8 up, 6 in 2023-09-23T22:55:05.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: failed to encode map e312 with expected crc 2023-09-23T22:55:05.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 304 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 4262/14640 objects degraded (29.112%); 0 B/s, 55 objects/s recovering 2023-09-23T22:55:05.582 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:05.583 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4506/17052 objects degraded (26.425%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:05.583 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:05.583 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3271682592' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:55:05.583 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: osdmap e312: 8 total, 8 up, 6 in 2023-09-23T22:55:05.583 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: failed to encode map e312 with expected crc 2023-09-23T22:55:05.583 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 304 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 4262/14640 objects degraded (29.112%); 0 B/s, 55 objects/s recovering 2023-09-23T22:55:05.584 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:05.584 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4506/17052 objects degraded (26.425%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:05.609 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:05.756 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:05.888 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:05.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:05.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:05.999 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:06.119 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:06.184 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:55:06.192 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:06.216 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:55:06.252 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:06.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:06.356 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:06.435 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:06.547 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:06.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:06.623 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:55:06.623 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:55:06.623 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:55:06.623 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:55:06.624 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.624 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164705 7.0 7:0e09ba28:::benchmark_data_smithi052_277165_object72213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:55:06.624 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:55:06.624 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 585.22357690399997, 2023-09-23T22:55:06.624 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31253646400000001, 2023-09-23T22:55:06.624 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164705 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.625 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.626 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396397+0000", 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396395+0000", 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4019999999999999e-06 2023-09-23T22:55:06.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396402+0000", 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1674999999999999e-05 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396414+0000", 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6180000000000003e-06 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396419+0000", 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4564e-05 2023-09-23T22:55:06.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396494+0000", 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013145799999999999 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396625+0000", 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000161607 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396787+0000", 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013774191999999999 2023-09-23T22:55:06.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410561+0000", 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0606999999999998e-05 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410612+0000", 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29822478200000002 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708836+0000", 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1789999999999998e-05 2023-09-23T22:55:06.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708878+0000", 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4976999999999999e-05 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.708933+0000", 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31253646400000001 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.635 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164707 7.0 7:017e1e7c:::benchmark_data_smithi052_277165_object72215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 585.22354088700001, 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31516750300000002, 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:55:06.636 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:55:06.637 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:55:06.637 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164707 2023-09-23T22:55:06.637 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.637 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.637 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.637 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396433+0000", 2023-09-23T22:55:06.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:06.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396432+0000", 2023-09-23T22:55:06.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.994e-06 2023-09-23T22:55:06.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396435+0000", 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0390000000000001e-06 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396441+0000", 2023-09-23T22:55:06.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.155e-06 2023-09-23T22:55:06.641 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.641 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.641 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396445+0000", 2023-09-23T22:55:06.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052570480000000003 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401702+0000", 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000104561 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401807+0000", 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012558299999999999 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:55:06.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.401933+0000", 2023-09-23T22:55:06.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0087188879999999993 2023-09-23T22:55:06.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:06.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410651+0000", 2023-09-23T22:55:06.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2320000000000004e-06 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410659+0000", 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30079543800000003 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711454+0000", 2023-09-23T22:55:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100597 2023-09-23T22:55:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.647 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711555+0000", 2023-09-23T22:55:06.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5464999999999999e-05 2023-09-23T22:55:06.647 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.647 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.647 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.647 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711600+0000", 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31516750300000002 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.648 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.35001.0:164711 7.0 7:0f1a7674:::benchmark_data_smithi052_277165_object72219:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 585.22352367099995, 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31527841600000001, 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:55:06.649 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.35001", 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/799321256", 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 164711 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.650 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396450+0000", 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:06.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396449+0000", 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6979999999999998e-06 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396452+0000", 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0100000000000001e-06 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396458+0000", 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3000000000000002e-06 2023-09-23T22:55:06.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.654 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.654 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.396461+0000", 2023-09-23T22:55:06.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099721069999999992 2023-09-23T22:55:06.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406434+0000", 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012163399999999999 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406555+0000", 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000215744 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.406771+0000", 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039129949999999998 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.656 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410684+0000", 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8912999999999998e-05 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:55:06.657 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.410723+0000", 2023-09-23T22:55:06.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30092187100000001 2023-09-23T22:55:06.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:06.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711645+0000", 2023-09-23T22:55:06.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9745e-05 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711685+0000", 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3800000000000001e-05 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:45:21.711728+0000", 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31527841600000001 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:80359 9.7 e285/271 9:ebab4714:::benchmark_data_smithi052_402718_object80358:head v 285'5134, mlcod=0'0)", 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 150.16152522300001, 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056466570000000001, 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.661 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:55:06.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.663 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458448+0000", 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9662e-05 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458497+0000", 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4939999999999997e-06 2023-09-23T22:55:06.664 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458505+0000", 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.817e-06 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458509+0000", 2023-09-23T22:55:06.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5979000000000001e-05 2023-09-23T22:55:06.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458565+0000", 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2603999999999999e-05 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.458587+0000", 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056246605999999998 2023-09-23T22:55:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514834+0000", 2023-09-23T22:55:06.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0945999999999994e-05 2023-09-23T22:55:06.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.669 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.669 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514915+0000", 2023-09-23T22:55:06.669 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056466570000000001 2023-09-23T22:55:06.669 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.669 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.669 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:80360 9.8 e285/271 9:1e29dae4:::benchmark_data_smithi052_402718_object80359:head v 285'4950, mlcod=0'0)", 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:36.459432+0000", 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 150.160541999, 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055518602, 2023-09-23T22:55:06.670 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.671 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.671 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.671 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.671 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.671 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459432+0000", 2023-09-23T22:55:06.671 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459432+0000", 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.672 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459431+0000", 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9472999999999999e-05 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.673 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459481+0000", 2023-09-23T22:55:06.674 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7770000000000002e-06 2023-09-23T22:55:06.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.674 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.674 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.674 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459488+0000", 2023-09-23T22:55:06.674 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.746e-06 2023-09-23T22:55:06.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.675 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.675 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.675 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459493+0000", 2023-09-23T22:55:06.675 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3930000000000001e-05 2023-09-23T22:55:06.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459537+0000", 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2201999999999999e-05 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.676 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.677 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.459559+0000", 2023-09-23T22:55:06.677 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055311410999999998 2023-09-23T22:55:06.677 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.677 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514871+0000", 2023-09-23T22:55:06.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9516000000000002e-05 2023-09-23T22:55:06.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.678 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:36.514950+0000", 2023-09-23T22:55:06.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.055518602 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89608 9.3 e287/271 9:c63bb180:::benchmark_data_smithi052_402718_object89607:head v 287'5589, mlcod=0'0)", 2023-09-23T22:55:06.679 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.395287+0000", 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 137.224686784, 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11530741, 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395287+0000", 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395287+0000", 2023-09-23T22:55:06.681 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395286+0000", 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5213999999999998e-05 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.682 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395331+0000", 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3318e-05 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395344+0000", 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5339999999999996e-06 2023-09-23T22:55:06.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395351+0000", 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7935000000000001e-05 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395399+0000", 2023-09-23T22:55:06.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5122000000000001e-05 2023-09-23T22:55:06.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.395424+0000", 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11509657299999999 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510520+0000", 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3898000000000003e-05 2023-09-23T22:55:06.686 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510594+0000", 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11530741 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89611 9.6 e287/264 9:6fa34574:::benchmark_data_smithi052_402718_object89610:head v 287'5657, mlcod=287'5654)", 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:55:06.687 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 137.22293308600001, 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11885746699999999, 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:55:06.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397040+0000", 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0765000000000004e-05 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397110+0000", 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4840000000000004e-06 2023-09-23T22:55:06.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397119+0000", 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397123+0000", 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3129000000000001e-05 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397157+0000", 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2439999999999999e-05 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.397169+0000", 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.118685622 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515855+0000", 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3374999999999997e-05 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515898+0000", 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11885746699999999 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.692 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89612 9.8 e287/271 9:161009cf:::benchmark_data_smithi052_402718_object89611:head v 287'5522, mlcod=0'0)", 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.400046+0000", 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 137.219927619, 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.115819034, 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400046+0000", 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400046+0000", 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400045+0000", 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4580999999999998e-05 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400100+0000", 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.110411013 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510511+0000", 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.218e-05 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510523+0000", 2023-09-23T22:55:06.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8108000000000001e-05 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510571+0000", 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5058999999999998e-05 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510606+0000", 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052171370000000002 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515823+0000", 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1754e-05 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.515865+0000", 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.115819034 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89619 9.6 e287/264 9:69e57cf2:::benchmark_data_smithi052_402718_object89618:head v 287'5658, mlcod=287'5654)", 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.400616+0000", 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 137.219357812, 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11554326400000001, 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400616+0000", 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400616+0000", 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400615+0000", 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4066000000000002e-05 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.400649+0000", 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10986414999999999 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510513+0000", 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0089999999999996e-06 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510522+0000", 2023-09-23T22:55:06.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002766569 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.513289+0000", 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9255000000000001e-05 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.513308+0000", 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028208360000000002 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.516129+0000", 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0105000000000001e-05 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.516159+0000", 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.11554326400000001 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:89623 9.f e287/271 9:faa01721:::benchmark_data_smithi052_402718_object89622:head v 287'5673, mlcod=0'0)", 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:52:49.406207+0000", 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 137.21376628600001, 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10779472499999999, 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406207+0000", 2023-09-23T22:55:06.703 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406207+0000", 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406206+0000", 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7041999999999999e-05 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.406284+0000", 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10422577399999999 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510509+0000", 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4123e-05 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510523+0000", 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3522999999999995e-05 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510587+0000", 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7676000000000001e-05 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.510615+0000", 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033438729999999998 2023-09-23T22:55:06.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.513958+0000", 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3506e-05 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:52:49.514002+0000", 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.10779472499999999 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.707 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:99235 9.a e291/271 9:55e66bf1:::benchmark_data_smithi052_402718_object9594:head v 291'6214, mlcod=0'0)", 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:04.198434+0000", 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 122.421539694, 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.120070366, 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198434+0000", 2023-09-23T22:55:06.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198434+0000", 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198433+0000", 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7019999999999998e-06 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198439+0000", 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.06e-05 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198449+0000", 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6500000000000004e-06 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198454+0000", 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2846000000000003e-05 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198497+0000", 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.194e-05 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198519+0000", 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.119938472 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.318457+0000", 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6894000000000002e-05 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.318504+0000", 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.120070366 2023-09-23T22:55:06.712 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:99240 9.f e291/271 9:fd2e385e:::benchmark_data_smithi052_402718_object9599:head v 291'6242, mlcod=0'0)", 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:04.198710+0000", 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 122.42126343, 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12607479299999999, 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.713 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198710+0000", 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198710+0000", 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198709+0000", 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6079999999999998e-06 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198717+0000", 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7909999999999993e-06 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198726+0000", 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.417e-06 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198731+0000", 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7988999999999998e-05 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198769+0000", 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6543999999999999e-05 2023-09-23T22:55:06.716 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.198786+0000", 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12595572299999999 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.324741+0000", 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3596000000000001e-05 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.324785+0000", 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12607479299999999 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:99243 9.5 e291/264 9:ad6e6023:::benchmark_data_smithi052_402718_object9602:head v 291'6174, mlcod=291'6172)", 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:04.201852+0000", 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 122.41812166, 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12300446399999999, 2023-09-23T22:55:06.718 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201852+0000", 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201852+0000", 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201851+0000", 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1185e-05 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201862+0000", 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.116941014 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.318803+0000", 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5119999999999997e-06 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.318813+0000", 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035650699999999999 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.319169+0000", 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1636000000000001e-05 2023-09-23T22:55:06.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.319191+0000", 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056363740000000004 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.324827+0000", 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9201e-05 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.324856+0000", 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12300446399999999 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:99242 9.3 e291/271 9:ce411852:::benchmark_data_smithi052_402718_object9601:head v 291'6180, mlcod=0'0)", 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:04.201893+0000", 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 122.41808036099999, 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.120630508, 2023-09-23T22:55:06.723 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201893+0000", 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201893+0000", 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201893+0000", 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4499999999999997e-06 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.201897+0000", 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.115210933 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.317108+0000", 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5155e-05 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.317123+0000", 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00092100000000000005 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.318044+0000", 2023-09-23T22:55:06.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9984999999999998e-05 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.318074+0000", 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044003640000000004 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.322475+0000", 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8971999999999998e-05 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:04.322524+0000", 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.120630508 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:102634 9.f e292/271 9:fea18f71:::benchmark_data_smithi052_402718_object12993:head v 292'6447, mlcod=0'0)", 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:09.389435+0000", 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 117.230538814, 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.081547735999999996, 2023-09-23T22:55:06.728 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389435+0000", 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389435+0000", 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:06.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389433+0000", 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1554e-05 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389445+0000", 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7768999999999999e-05 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389463+0000", 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3400000000000004e-06 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389472+0000", 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9654999999999997e-05 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389532+0000", 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0419000000000001e-05 2023-09-23T22:55:06.731 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.389562+0000", 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023115369999999998 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391874+0000", 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.079108795999999995 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.470982+0000", 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.081547735999999996 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:102636 9.a e292/271 9:53627422:::benchmark_data_smithi052_402718_object12995:head v 292'6420, mlcod=0'0)", 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:09.391652+0000", 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 117.228321764, 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.080586346000000003, 2023-09-23T22:55:06.733 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391652+0000", 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391652+0000", 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391651+0000", 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4110999999999999e-05 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391665+0000", 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2123999999999999e-05 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391677+0000", 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.25e-06 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391684+0000", 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2787000000000001e-05 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391737+0000", 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6718000000000001e-05 2023-09-23T22:55:06.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.391764+0000", 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.080431929999999999 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.472196+0000", 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2410000000000002e-05 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.472238+0000", 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.080586346000000003 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:102637 9.f e292/271 9:f639225d:::benchmark_data_smithi052_402718_object12996:head v 292'6448, mlcod=0'0)", 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:09.392816+0000", 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 117.227157515, 2023-09-23T22:55:06.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.081284586000000006, 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.392816+0000", 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.392816+0000", 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:06.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.392815+0000", 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.392822+0000", 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.074283099000000005 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.467105+0000", 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8500000000000006e-06 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.467115+0000", 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040025900000000003 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.471118+0000", 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6336999999999999e-05 2023-09-23T22:55:06.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.471144+0000", 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029156339999999998 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.474060+0000", 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0893999999999998e-05 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.742 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.474101+0000", 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.081284586000000006 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:102639 9.9 e292/282 9:962e02d4:::benchmark_data_smithi052_402718_object12998:head v 292'6425, mlcod=292'6423)", 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:09.395767+0000", 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 117.224206345, 2023-09-23T22:55:06.743 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.076605005000000004, 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.395767+0000", 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.395767+0000", 2023-09-23T22:55:06.744 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.395766+0000", 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5370999999999999e-05 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.395781+0000", 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.073104798999999998 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.745 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.468886+0000", 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9200000000000004e-06 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.468894+0000", 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015989899999999999 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.469054+0000", 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3156e-05 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.469077+0000", 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003266026 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.472343+0000", 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9450000000000001e-05 2023-09-23T22:55:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:09.472372+0000", 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.076605005000000004 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:164576 9.f e307/271 9:f4724b11:::benchmark_data_smithi052_402718_object74935:head v 307'10405, mlcod=294'7208)", 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:54:45.304414+0000", 2023-09-23T22:55:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 21.315559549, 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.060280633, 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.304414+0000", 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.304414+0000", 2023-09-23T22:55:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.304413+0000", 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7409999999999992e-06 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.304421+0000", 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.045244448999999999 2023-09-23T22:55:06.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349665+0000", 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7861000000000001e-05 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349683+0000", 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2259000000000002e-05 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349735+0000", 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3009999999999997e-05 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349768+0000", 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014898672999999999 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.752 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.364667+0000", 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7562000000000001e-05 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.364695+0000", 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.060280633 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.753 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45593.0:164595 9.5 e307/264 9:a948c32c:::benchmark_data_smithi052_402718_object74954:head v 307'10192, mlcod=307'10190)", 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:54:45.309460+0000", 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 21.310513710999999, 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.054238432000000003, 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.309460+0000", 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.309460+0000", 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.309459+0000", 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2559999999999996e-06 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:55:06.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.309466+0000", 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040207913999999997 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349674+0000", 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3030000000000005e-06 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349684+0000", 2023-09-23T22:55:06.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016920100000000001 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349853+0000", 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8198000000000001e-05 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.349871+0000", 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013763986000000001 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.363635+0000", 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3357999999999999e-05 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:54:45.363698+0000", 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.054238432000000003 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:55:06.758 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:55:06.759 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:55:06.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:06.790 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:06.835 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:06.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:06.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:06.938 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:06.984 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:07.069 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:07.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:07.177 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:07.222 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:07.315 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:55:07.484 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:07.485 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:55:07.488 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":908,"stamp":"2023-09-23T22:55:04.448761+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":318773176,"num_objects":4880,"num_object_clones":0,"num_object_copies":14640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4262,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4880,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":174682,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12575,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115066,"ondisk_log_size":115066,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3479028,"kb_used_data":1000196,"kb_used_omap":121,"kb_used_meta":2478406,"kb_avail":558991884,"statfs":{"total":575970213888,"available":572407689216,"internally_reserved":0,"allocated":1024200704,"data_stored":1019216323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2537887861},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-314245120,"num_objects":-4795,"num_object_clones":0,"num_object_copies":-14385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4795,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.306273"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":308,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780421+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:54:48.780421+0000","last_peered":"2023-09-23T22:54:48.780421+0000","last_clean":"2023-09-23T22:54:48.780421+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:54:48.780421+0000","last_undegraded":"2023-09-23T22:54:48.780421+0000","last_fullsized":"2023-09-23T22:54:48.780421+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":107,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.782085+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:54:48.782085+0000","last_peered":"2023-09-23T22:54:48.782085+0000","last_clean":"2023-09-23T22:54:48.782085+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:54:48.782085+0000","last_undegraded":"2023-09-23T22:54:48.782085+0000","last_fullsized":"2023-09-23T22:54:48.782085+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":93,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.782634+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:54:48.782634+0000","last_peered":"2023-09-23T22:54:48.782634+0000","last_clean":"2023-09-23T22:54:48.782634+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:54:48.782634+0000","last_undegraded":"2023-09-23T22:54:48.782634+0000","last_fullsized":"2023-09-23T22:54:48.782634+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":290,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719184+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:54:48.719184+0000","last_peered":"2023-09-23T22:54:48.719184+0000","last_clean":"2023-09-23T22:54:48.719184+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:54:48.719184+0000","last_undegraded":"2023-09-23T22:54:48.719184+0000","last_fullsized":"2023-09-23T22:54:48.719184+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":294,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718709+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:54:48.718709+0000","last_peered":"2023-09-23T22:54:48.718709+0000","last_clean":"2023-09-23T22:54:48.718709+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:54:48.718709+0000","last_undegraded":"2023-09-23T22:54:48.718709+0000","last_fullsized":"2023-09-23T22:54:48.718709+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":276,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713480+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:54:48.713480+0000","last_peered":"2023-09-23T22:54:48.713480+0000","last_clean":"2023-09-23T22:54:48.713480+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:54:48.713480+0000","last_undegraded":"2023-09-23T22:54:48.713480+0000","last_fullsized":"2023-09-23T22:54:48.713480+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":221,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719473+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:54:48.719473+0000","last_peered":"2023-09-23T22:54:48.719473+0000","last_clean":"2023-09-23T22:54:48.719473+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:54:48.719473+0000","last_undegraded":"2023-09-23T22:54:48.719473+0000","last_fullsized":"2023-09-23T22:54:48.719473+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710359+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:54:48.710359+0000","last_peered":"2023-09-23T22:54:48.710359+0000","last_clean":"2023-09-23T22:54:48.710359+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:54:48.710359+0000","last_undegraded":"2023-09-23T22:54:48.710359+0000","last_fullsized":"2023-09-23T22:54:48.710359+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":52,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710696+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:54:48.710696+0000","last_peered":"2023-09-23T22:54:48.710696+0000","last_clean":"2023-09-23T22:54:48.710696+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:54:48.710696+0000","last_undegraded":"2023-09-23T22:54:48.710696+0000","last_fullsized":"2023-09-23T22:54:48.710696+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":195,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712410+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:54:48.712410+0000","last_peered":"2023-09-23T22:54:48.712410+0000","last_clean":"2023-09-23T22:54:48.712410+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:54:48.712410+0000","last_undegraded":"2023-09-23T22:54:48.712410+0000","last_fullsized":"2023-09-23T22:54:48.712410+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":69,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781899+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:54:48.781899+0000","last_peered":"2023-09-23T22:54:48.781899+0000","last_clean":"2023-09-23T22:54:48.781899+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:54:48.781899+0000","last_undegraded":"2023-09-23T22:54:48.781899+0000","last_fullsized":"2023-09-23T22:54:48.781899+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":185,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712617+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:54:48.712617+0000","last_peered":"2023-09-23T22:54:48.712617+0000","last_clean":"2023-09-23T22:54:48.712617+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:54:48.712617+0000","last_undegraded":"2023-09-23T22:54:48.712617+0000","last_fullsized":"2023-09-23T22:54:48.712617+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":330,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781936+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:54:48.781936+0000","last_peered":"2023-09-23T22:54:48.781936+0000","last_clean":"2023-09-23T22:54:48.781936+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:54:48.781936+0000","last_undegraded":"2023-09-23T22:54:48.781936+0000","last_fullsized":"2023-09-23T22:54:48.781936+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":276,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780408+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:54:48.780408+0000","last_peered":"2023-09-23T22:54:48.780408+0000","last_clean":"2023-09-23T22:54:48.780408+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:54:48.780408+0000","last_undegraded":"2023-09-23T22:54:48.780408+0000","last_fullsized":"2023-09-23T22:54:48.780408+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":169,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780411+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:54:48.780411+0000","last_peered":"2023-09-23T22:54:48.780411+0000","last_clean":"2023-09-23T22:54:48.780411+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:54:48.780411+0000","last_undegraded":"2023-09-23T22:54:48.780411+0000","last_fullsized":"2023-09-23T22:54:48.780411+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":258,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710450+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:54:48.710450+0000","last_peered":"2023-09-23T22:54:48.710450+0000","last_clean":"2023-09-23T22:54:48.710450+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:54:48.710450+0000","last_undegraded":"2023-09-23T22:54:48.710450+0000","last_fullsized":"2023-09-23T22:54:48.710450+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":277,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718641+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:54:48.718641+0000","last_peered":"2023-09-23T22:54:48.718641+0000","last_clean":"2023-09-23T22:54:48.718641+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:54:48.718641+0000","last_undegraded":"2023-09-23T22:54:48.718641+0000","last_fullsized":"2023-09-23T22:54:48.718641+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":234,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713094+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:54:48.713094+0000","last_peered":"2023-09-23T22:54:48.713094+0000","last_clean":"2023-09-23T22:54:48.713094+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:54:48.713094+0000","last_undegraded":"2023-09-23T22:54:48.713094+0000","last_fullsized":"2023-09-23T22:54:48.713094+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":161,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.782688+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:54:48.782688+0000","last_peered":"2023-09-23T22:54:48.782688+0000","last_clean":"2023-09-23T22:54:48.782688+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:54:48.782688+0000","last_undegraded":"2023-09-23T22:54:48.782688+0000","last_fullsized":"2023-09-23T22:54:48.782688+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":278,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719244+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:54:48.719244+0000","last_peered":"2023-09-23T22:54:48.719244+0000","last_clean":"2023-09-23T22:54:48.719244+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:54:48.719244+0000","last_undegraded":"2023-09-23T22:54:48.719244+0000","last_fullsized":"2023-09-23T22:54:48.719244+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":355,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712176+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:54:48.712176+0000","last_peered":"2023-09-23T22:54:48.712176+0000","last_clean":"2023-09-23T22:54:48.712176+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:54:48.712176+0000","last_undegraded":"2023-09-23T22:54:48.712176+0000","last_fullsized":"2023-09-23T22:54:48.712176+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":238,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712203+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:54:48.712203+0000","last_peered":"2023-09-23T22:54:48.712203+0000","last_clean":"2023-09-23T22:54:48.712203+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:54:48.712203+0000","last_undegraded":"2023-09-23T22:54:48.712203+0000","last_fullsized":"2023-09-23T22:54:48.712203+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"311'10882","reported_seq":10939,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:03.354608+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:55:03.354608+0000","last_peered":"2023-09-23T22:55:03.354608+0000","last_clean":"2023-09-23T22:55:03.354608+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:55:03.354608+0000","last_undegraded":"2023-09-23T22:55:03.354608+0000","last_fullsized":"2023-09-23T22:55:03.354608+0000","mapping_epoch":264,"log_start":"277'3300","ondisk_log_start":"277'3300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16384000,"num_objects":250,"num_object_clones":0,"num_object_copies":750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10882,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":180,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781167+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:54:48.781167+0000","last_peered":"2023-09-23T22:54:48.781167+0000","last_clean":"2023-09-23T22:54:48.781167+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:54:48.781167+0000","last_undegraded":"2023-09-23T22:54:48.781167+0000","last_fullsized":"2023-09-23T22:54:48.781167+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":275,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712258+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:54:48.712258+0000","last_peered":"2023-09-23T22:54:48.712258+0000","last_clean":"2023-09-23T22:54:48.712258+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:54:48.712258+0000","last_undegraded":"2023-09-23T22:54:48.712258+0000","last_fullsized":"2023-09-23T22:54:48.712258+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"311'11197","reported_seq":11259,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:55:01.648095+0000","last_change":"2023-09-23T22:51:37.067119+0000","last_active":"2023-09-23T22:55:01.648095+0000","last_peered":"2023-09-23T22:55:01.648095+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:55:01.648095+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"279'4300","ondisk_log_start":"279'4300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11197,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":-1366}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":190,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781265+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:54:48.781265+0000","last_peered":"2023-09-23T22:54:48.781265+0000","last_clean":"2023-09-23T22:54:48.781265+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:54:48.781265+0000","last_undegraded":"2023-09-23T22:54:48.781265+0000","last_fullsized":"2023-09-23T22:54:48.781265+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":300,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.782640+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:54:48.782640+0000","last_peered":"2023-09-23T22:54:48.782640+0000","last_clean":"2023-09-23T22:54:48.782640+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:54:48.782640+0000","last_undegraded":"2023-09-23T22:54:48.782640+0000","last_fullsized":"2023-09-23T22:54:48.782640+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"311'10974","reported_seq":11037,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:55:00.206126+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:55:00.206126+0000","last_peered":"2023-09-23T22:55:00.206126+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:55:00.206126+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"273'2400","ondisk_log_start":"273'2400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8574,"log_dups_size":0,"ondisk_log_size":8574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20578304,"num_objects":314,"num_object_clones":0,"num_object_copies":942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10974,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-1457}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713050+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:54:48.713050+0000","last_peered":"2023-09-23T22:54:48.713050+0000","last_clean":"2023-09-23T22:54:48.713050+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:54:48.713050+0000","last_undegraded":"2023-09-23T22:54:48.713050+0000","last_fullsized":"2023-09-23T22:54:48.713050+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":303,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713562+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:54:48.713562+0000","last_peered":"2023-09-23T22:54:48.713562+0000","last_clean":"2023-09-23T22:54:48.713562+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:54:48.713562+0000","last_undegraded":"2023-09-23T22:54:48.713562+0000","last_fullsized":"2023-09-23T22:54:48.713562+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"311'10933","reported_seq":10990,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:00.121787+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:55:00.121787+0000","last_peered":"2023-09-23T22:55:00.121787+0000","last_clean":"2023-09-23T22:55:00.121787+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:55:00.121787+0000","last_undegraded":"2023-09-23T22:55:00.121787+0000","last_fullsized":"2023-09-23T22:55:00.121787+0000","mapping_epoch":264,"log_start":"273'2300","ondisk_log_start":"273'2300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8633,"log_dups_size":0,"ondisk_log_size":8633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25100288,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10933,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712912+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:54:48.712912+0000","last_peered":"2023-09-23T22:54:48.712912+0000","last_clean":"2023-09-23T22:54:48.712912+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:54:48.712912+0000","last_undegraded":"2023-09-23T22:54:48.712912+0000","last_fullsized":"2023-09-23T22:54:48.712912+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":205,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718735+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:54:48.718735+0000","last_peered":"2023-09-23T22:54:48.718735+0000","last_clean":"2023-09-23T22:54:48.718735+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:54:48.718735+0000","last_undegraded":"2023-09-23T22:54:48.718735+0000","last_fullsized":"2023-09-23T22:54:48.718735+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"311'10691","reported_seq":14112,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:00.990099+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:55:00.990099+0000","last_peered":"2023-09-23T22:55:00.990099+0000","last_clean":"2023-09-23T22:55:00.990099+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:55:00.990099+0000","last_undegraded":"2023-09-23T22:55:00.990099+0000","last_fullsized":"2023-09-23T22:55:00.990099+0000","mapping_epoch":289,"log_start":"280'4300","ondisk_log_start":"280'4300","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6391,"log_dups_size":0,"ondisk_log_size":6391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10691,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":77,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718834+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:54:48.718834+0000","last_peered":"2023-09-23T22:54:48.718834+0000","last_clean":"2023-09-23T22:54:48.718834+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:54:48.718834+0000","last_undegraded":"2023-09-23T22:54:48.718834+0000","last_fullsized":"2023-09-23T22:54:48.718834+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":284,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781904+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:54:48.781904+0000","last_peered":"2023-09-23T22:54:48.781904+0000","last_clean":"2023-09-23T22:54:48.781904+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:54:48.781904+0000","last_undegraded":"2023-09-23T22:54:48.781904+0000","last_fullsized":"2023-09-23T22:54:48.781904+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"311'10742","reported_seq":10797,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:01.002805+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:55:01.002805+0000","last_peered":"2023-09-23T22:55:01.002805+0000","last_clean":"2023-09-23T22:55:01.002805+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:55:01.002805+0000","last_undegraded":"2023-09-23T22:55:01.002805+0000","last_fullsized":"2023-09-23T22:55:01.002805+0000","mapping_epoch":264,"log_start":"280'4300","ondisk_log_start":"280'4300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6442,"log_dups_size":0,"ondisk_log_size":6442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22282240,"num_objects":340,"num_object_clones":0,"num_object_copies":1020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10742,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":295,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712435+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:54:48.712435+0000","last_peered":"2023-09-23T22:54:48.712435+0000","last_clean":"2023-09-23T22:54:48.712435+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:54:48.712435+0000","last_undegraded":"2023-09-23T22:54:48.712435+0000","last_fullsized":"2023-09-23T22:54:48.712435+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":306,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781721+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:54:48.781721+0000","last_peered":"2023-09-23T22:54:48.781721+0000","last_clean":"2023-09-23T22:54:48.781721+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:54:48.781721+0000","last_undegraded":"2023-09-23T22:54:48.781721+0000","last_fullsized":"2023-09-23T22:54:48.781721+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"311'10874","reported_seq":14283,"reported_epoch":311,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:55:00.997714+0000","last_change":"2023-09-23T22:54:01.194508+0000","last_active":"2023-09-23T22:55:00.997714+0000","last_peered":"2023-09-23T22:55:00.997714+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:54:01.140849+0000","last_became_peered":"2023-09-23T22:54:01.140849+0000","last_unstale":"2023-09-23T22:55:00.997714+0000","last_undegraded":"2023-09-23T22:55:00.997714+0000","last_fullsized":"2023-09-23T22:55:00.997714+0000","mapping_epoch":299,"log_start":"280'4400","ondisk_log_start":"280'4400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20054016,"num_objects":306,"num_object_clones":0,"num_object_copies":918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10874,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1668,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":["6","1","3","5"],"object_location_counts":[{"shards":"1,5,6","objects":306}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":319,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.782536+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:54:48.782536+0000","last_peered":"2023-09-23T22:54:48.782536+0000","last_clean":"2023-09-23T22:54:48.782536+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:54:48.782536+0000","last_undegraded":"2023-09-23T22:54:48.782536+0000","last_fullsized":"2023-09-23T22:54:48.782536+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":294,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781329+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:54:48.781329+0000","last_peered":"2023-09-23T22:54:48.781329+0000","last_clean":"2023-09-23T22:54:48.781329+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:54:48.781329+0000","last_undegraded":"2023-09-23T22:54:48.781329+0000","last_fullsized":"2023-09-23T22:54:48.781329+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"311'10705","reported_seq":14007,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:00.233400+0000","last_change":"2023-09-23T22:54:41.191574+0000","last_active":"2023-09-23T22:55:00.233400+0000","last_peered":"2023-09-23T22:55:00.233400+0000","last_clean":"2023-09-23T22:55:00.233400+0000","last_became_active":"2023-09-23T22:54:41.158218+0000","last_became_peered":"2023-09-23T22:54:41.158218+0000","last_unstale":"2023-09-23T22:55:00.233400+0000","last_undegraded":"2023-09-23T22:55:00.233400+0000","last_fullsized":"2023-09-23T22:55:00.233400+0000","mapping_epoch":306,"log_start":"273'2100","ondisk_log_start":"273'2100","created":264,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:01.903022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23920640,"num_objects":365,"num_object_clones":0,"num_object_copies":1095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10705,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1615,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":239,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780087+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:54:48.780087+0000","last_peered":"2023-09-23T22:54:48.780087+0000","last_clean":"2023-09-23T22:54:48.780087+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:54:48.780087+0000","last_undegraded":"2023-09-23T22:54:48.780087+0000","last_fullsized":"2023-09-23T22:54:48.780087+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":249,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712338+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:54:48.712338+0000","last_peered":"2023-09-23T22:54:48.712338+0000","last_clean":"2023-09-23T22:54:48.712338+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:54:48.712338+0000","last_undegraded":"2023-09-23T22:54:48.712338+0000","last_fullsized":"2023-09-23T22:54:48.712338+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"311'10998","reported_seq":14331,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:00.055305+0000","last_change":"2023-09-23T22:54:48.758724+0000","last_active":"2023-09-23T22:55:00.055305+0000","last_peered":"2023-09-23T22:55:00.055305+0000","last_clean":"2023-09-23T22:55:00.055305+0000","last_became_active":"2023-09-23T22:54:48.714547+0000","last_became_peered":"2023-09-23T22:54:48.714547+0000","last_unstale":"2023-09-23T22:55:00.055305+0000","last_undegraded":"2023-09-23T22:55:00.055305+0000","last_fullsized":"2023-09-23T22:55:00.055305+0000","mapping_epoch":309,"log_start":"286'5400","ondisk_log_start":"286'5400","created":264,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5598,"log_dups_size":0,"ondisk_log_size":5598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:00.801012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10998,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1630,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":349,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719074+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:54:48.719074+0000","last_peered":"2023-09-23T22:54:48.719074+0000","last_clean":"2023-09-23T22:54:48.719074+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:54:48.719074+0000","last_undegraded":"2023-09-23T22:54:48.719074+0000","last_fullsized":"2023-09-23T22:54:48.719074+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":174,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710504+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:54:48.710504+0000","last_peered":"2023-09-23T22:54:48.710504+0000","last_clean":"2023-09-23T22:54:48.710504+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:54:48.710504+0000","last_undegraded":"2023-09-23T22:54:48.710504+0000","last_fullsized":"2023-09-23T22:54:48.710504+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780325+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:54:48.780325+0000","last_peered":"2023-09-23T22:54:48.780325+0000","last_clean":"2023-09-23T22:54:48.780325+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:54:48.780325+0000","last_undegraded":"2023-09-23T22:54:48.780325+0000","last_fullsized":"2023-09-23T22:54:48.780325+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"311'10962","reported_seq":14421,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:03.341411+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:55:03.341411+0000","last_peered":"2023-09-23T22:55:03.341411+0000","last_clean":"2023-09-23T22:55:03.341411+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:55:03.341411+0000","last_undegraded":"2023-09-23T22:55:03.341411+0000","last_fullsized":"2023-09-23T22:55:03.341411+0000","mapping_epoch":282,"log_start":"277'3400","ondisk_log_start":"277'3400","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7562,"log_dups_size":0,"ondisk_log_size":7562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14155776,"num_objects":216,"num_object_clones":0,"num_object_copies":648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10962,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":145,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712455+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:54:48.712455+0000","last_peered":"2023-09-23T22:54:48.712455+0000","last_clean":"2023-09-23T22:54:48.712455+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:54:48.712455+0000","last_undegraded":"2023-09-23T22:54:48.712455+0000","last_fullsized":"2023-09-23T22:54:48.712455+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":307,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780557+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:54:48.780557+0000","last_peered":"2023-09-23T22:54:48.780557+0000","last_clean":"2023-09-23T22:54:48.780557+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:54:48.780557+0000","last_undegraded":"2023-09-23T22:54:48.780557+0000","last_fullsized":"2023-09-23T22:54:48.780557+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":238,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710818+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:54:48.710818+0000","last_peered":"2023-09-23T22:54:48.710818+0000","last_clean":"2023-09-23T22:54:48.710818+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:54:48.710818+0000","last_undegraded":"2023-09-23T22:54:48.710818+0000","last_fullsized":"2023-09-23T22:54:48.710818+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"311'10933","reported_seq":14319,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:01.031668+0000","last_change":"2023-09-23T22:54:21.209191+0000","last_active":"2023-09-23T22:55:01.031668+0000","last_peered":"2023-09-23T22:55:01.031668+0000","last_clean":"2023-09-23T22:55:01.031668+0000","last_became_active":"2023-09-23T22:54:21.131570+0000","last_became_peered":"2023-09-23T22:54:21.131570+0000","last_unstale":"2023-09-23T22:55:01.031668+0000","last_undegraded":"2023-09-23T22:55:01.031668+0000","last_fullsized":"2023-09-23T22:55:01.031668+0000","mapping_epoch":302,"log_start":"281'4550","ondisk_log_start":"281'4550","created":264,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6383,"log_dups_size":0,"ondisk_log_size":6383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22478848,"num_objects":343,"num_object_clones":0,"num_object_copies":1029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10933,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1655,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":313,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710749+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:54:48.710749+0000","last_peered":"2023-09-23T22:54:48.710749+0000","last_clean":"2023-09-23T22:54:48.710749+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:54:48.710749+0000","last_undegraded":"2023-09-23T22:54:48.710749+0000","last_fullsized":"2023-09-23T22:54:48.710749+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":193,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710272+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:54:48.710272+0000","last_peered":"2023-09-23T22:54:48.710272+0000","last_clean":"2023-09-23T22:54:48.710272+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:54:48.710272+0000","last_undegraded":"2023-09-23T22:54:48.710272+0000","last_fullsized":"2023-09-23T22:54:48.710272+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":318,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719389+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:54:48.719389+0000","last_peered":"2023-09-23T22:54:48.719389+0000","last_clean":"2023-09-23T22:54:48.719389+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:54:48.719389+0000","last_undegraded":"2023-09-23T22:54:48.719389+0000","last_fullsized":"2023-09-23T22:54:48.719389+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"311'10811","reported_seq":12640,"reported_epoch":311,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:55:03.355095+0000","last_change":"2023-09-23T22:54:46.998497+0000","last_active":"2023-09-23T22:55:03.355095+0000","last_peered":"2023-09-23T22:55:03.355095+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:55:03.355095+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"276'3300","ondisk_log_start":"276'3300","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15663104,"num_objects":239,"num_object_clones":0,"num_object_copies":717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":830,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10811,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":882,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":-591}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":323,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710585+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:54:48.710585+0000","last_peered":"2023-09-23T22:54:48.710585+0000","last_clean":"2023-09-23T22:54:48.710585+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:54:48.710585+0000","last_undegraded":"2023-09-23T22:54:48.710585+0000","last_fullsized":"2023-09-23T22:54:48.710585+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":400,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718629+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:54:48.718629+0000","last_peered":"2023-09-23T22:54:48.718629+0000","last_clean":"2023-09-23T22:54:48.718629+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:54:48.718629+0000","last_undegraded":"2023-09-23T22:54:48.718629+0000","last_fullsized":"2023-09-23T22:54:48.718629+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712536+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:54:48.712536+0000","last_peered":"2023-09-23T22:54:48.712536+0000","last_clean":"2023-09-23T22:54:48.712536+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:54:48.712536+0000","last_undegraded":"2023-09-23T22:54:48.712536+0000","last_fullsized":"2023-09-23T22:54:48.712536+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":303,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.712934+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:54:48.712934+0000","last_peered":"2023-09-23T22:54:48.712934+0000","last_clean":"2023-09-23T22:54:48.712934+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:54:48.712934+0000","last_undegraded":"2023-09-23T22:54:48.712934+0000","last_fullsized":"2023-09-23T22:54:48.712934+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"311'10988","reported_seq":11045,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:03.329303+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:55:03.329303+0000","last_peered":"2023-09-23T22:55:03.329303+0000","last_clean":"2023-09-23T22:55:03.329303+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:55:03.329303+0000","last_undegraded":"2023-09-23T22:55:03.329303+0000","last_fullsized":"2023-09-23T22:55:03.329303+0000","mapping_epoch":264,"log_start":"277'3400","ondisk_log_start":"277'3400","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7588,"log_dups_size":0,"ondisk_log_size":7588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10988,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":288,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718784+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:54:48.718784+0000","last_peered":"2023-09-23T22:54:48.718784+0000","last_clean":"2023-09-23T22:54:48.718784+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:54:48.718784+0000","last_undegraded":"2023-09-23T22:54:48.718784+0000","last_fullsized":"2023-09-23T22:54:48.718784+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":259,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719028+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:54:48.719028+0000","last_peered":"2023-09-23T22:54:48.719028+0000","last_clean":"2023-09-23T22:54:48.719028+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:54:48.719028+0000","last_undegraded":"2023-09-23T22:54:48.719028+0000","last_fullsized":"2023-09-23T22:54:48.719028+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":237,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:53.356322+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:54:53.356322+0000","last_peered":"2023-09-23T22:54:53.356322+0000","last_clean":"2023-09-23T22:54:53.356322+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:54:53.356322+0000","last_undegraded":"2023-09-23T22:54:53.356322+0000","last_fullsized":"2023-09-23T22:54:53.356322+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"311'10687","reported_seq":10742,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:01.604333+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:55:01.604333+0000","last_peered":"2023-09-23T22:55:01.604333+0000","last_clean":"2023-09-23T22:55:01.604333+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:55:01.604333+0000","last_undegraded":"2023-09-23T22:55:01.604333+0000","last_fullsized":"2023-09-23T22:55:01.604333+0000","mapping_epoch":264,"log_start":"279'3800","ondisk_log_start":"279'3800","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18808832,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10687,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":242,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719471+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:54:48.719471+0000","last_peered":"2023-09-23T22:54:48.719471+0000","last_clean":"2023-09-23T22:54:48.719471+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:54:48.719471+0000","last_undegraded":"2023-09-23T22:54:48.719471+0000","last_fullsized":"2023-09-23T22:54:48.719471+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":191,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710665+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:54:48.710665+0000","last_peered":"2023-09-23T22:54:48.710665+0000","last_clean":"2023-09-23T22:54:48.710665+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:54:48.710665+0000","last_undegraded":"2023-09-23T22:54:48.710665+0000","last_fullsized":"2023-09-23T22:54:48.710665+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":242,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719081+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:54:48.719081+0000","last_peered":"2023-09-23T22:54:48.719081+0000","last_clean":"2023-09-23T22:54:48.719081+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:54:48.719081+0000","last_undegraded":"2023-09-23T22:54:48.719081+0000","last_fullsized":"2023-09-23T22:54:48.719081+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"311'11067","reported_seq":11123,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:01.662613+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:55:01.662613+0000","last_peered":"2023-09-23T22:55:01.662613+0000","last_clean":"2023-09-23T22:55:01.662613+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:55:01.662613+0000","last_undegraded":"2023-09-23T22:55:01.662613+0000","last_fullsized":"2023-09-23T22:55:01.662613+0000","mapping_epoch":264,"log_start":"279'4200","ondisk_log_start":"279'4200","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267608,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11067,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":68,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719575+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:54:48.719575+0000","last_peered":"2023-09-23T22:54:48.719575+0000","last_clean":"2023-09-23T22:54:48.719575+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:54:48.719575+0000","last_undegraded":"2023-09-23T22:54:48.719575+0000","last_fullsized":"2023-09-23T22:54:48.719575+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":109,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713255+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:54:48.713255+0000","last_peered":"2023-09-23T22:54:48.713255+0000","last_clean":"2023-09-23T22:54:48.713255+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:54:48.713255+0000","last_undegraded":"2023-09-23T22:54:48.713255+0000","last_fullsized":"2023-09-23T22:54:48.713255+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":288,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718541+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:54:48.718541+0000","last_peered":"2023-09-23T22:54:48.718541+0000","last_clean":"2023-09-23T22:54:48.718541+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:54:48.718541+0000","last_undegraded":"2023-09-23T22:54:48.718541+0000","last_fullsized":"2023-09-23T22:54:48.718541+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"311'10978","reported_seq":14510,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:55:01.612003+0000","last_change":"2023-09-23T22:53:44.765920+0000","last_active":"2023-09-23T22:55:01.612003+0000","last_peered":"2023-09-23T22:55:01.612003+0000","last_clean":"2023-09-23T22:55:01.612003+0000","last_became_active":"2023-09-23T22:53:44.707193+0000","last_became_peered":"2023-09-23T22:53:44.707193+0000","last_unstale":"2023-09-23T22:55:01.612003+0000","last_undegraded":"2023-09-23T22:55:01.612003+0000","last_fullsized":"2023-09-23T22:55:01.612003+0000","mapping_epoch":296,"log_start":"279'4100","ondisk_log_start":"279'4100","created":264,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20578304,"num_objects":314,"num_object_clones":0,"num_object_copies":942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10978,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":352,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713361+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:54:48.713361+0000","last_peered":"2023-09-23T22:54:48.713361+0000","last_clean":"2023-09-23T22:54:48.713361+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:54:48.713361+0000","last_undegraded":"2023-09-23T22:54:48.713361+0000","last_fullsized":"2023-09-23T22:54:48.713361+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":209,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:53.356342+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:54:53.356342+0000","last_peered":"2023-09-23T22:54:53.356342+0000","last_clean":"2023-09-23T22:54:53.356342+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:54:53.356342+0000","last_undegraded":"2023-09-23T22:54:53.356342+0000","last_fullsized":"2023-09-23T22:54:53.356342+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":261,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.782566+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:54:48.782566+0000","last_peered":"2023-09-23T22:54:48.782566+0000","last_clean":"2023-09-23T22:54:48.782566+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:54:48.782566+0000","last_undegraded":"2023-09-23T22:54:48.782566+0000","last_fullsized":"2023-09-23T22:54:48.782566+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":244,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.718926+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:54:48.718926+0000","last_peered":"2023-09-23T22:54:48.718926+0000","last_clean":"2023-09-23T22:54:48.718926+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:54:48.718926+0000","last_undegraded":"2023-09-23T22:54:48.718926+0000","last_fullsized":"2023-09-23T22:54:48.718926+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710578+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:54:48.710578+0000","last_peered":"2023-09-23T22:54:48.710578+0000","last_clean":"2023-09-23T22:54:48.710578+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:54:48.710578+0000","last_undegraded":"2023-09-23T22:54:48.710578+0000","last_fullsized":"2023-09-23T22:54:48.710578+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":120,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781827+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:54:48.781827+0000","last_peered":"2023-09-23T22:54:48.781827+0000","last_clean":"2023-09-23T22:54:48.781827+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:54:48.781827+0000","last_undegraded":"2023-09-23T22:54:48.781827+0000","last_fullsized":"2023-09-23T22:54:48.781827+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":319,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780507+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:54:48.780507+0000","last_peered":"2023-09-23T22:54:48.780507+0000","last_clean":"2023-09-23T22:54:48.780507+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:54:48.780507+0000","last_undegraded":"2023-09-23T22:54:48.780507+0000","last_fullsized":"2023-09-23T22:54:48.780507+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":121,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.781809+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:54:48.781809+0000","last_peered":"2023-09-23T22:54:48.781809+0000","last_clean":"2023-09-23T22:54:48.781809+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:54:48.781809+0000","last_undegraded":"2023-09-23T22:54:48.781809+0000","last_fullsized":"2023-09-23T22:54:48.781809+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":191,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780528+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:54:48.780528+0000","last_peered":"2023-09-23T22:54:48.780528+0000","last_clean":"2023-09-23T22:54:48.780528+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:54:48.780528+0000","last_undegraded":"2023-09-23T22:54:48.780528+0000","last_fullsized":"2023-09-23T22:54:48.780528+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":61,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.713183+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:54:48.713183+0000","last_peered":"2023-09-23T22:54:48.713183+0000","last_clean":"2023-09-23T22:54:48.713183+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:54:48.713183+0000","last_undegraded":"2023-09-23T22:54:48.713183+0000","last_fullsized":"2023-09-23T22:54:48.713183+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":294,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.710587+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:54:48.710587+0000","last_peered":"2023-09-23T22:54:48.710587+0000","last_clean":"2023-09-23T22:54:48.710587+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:54:48.710587+0000","last_undegraded":"2023-09-23T22:54:48.710587+0000","last_fullsized":"2023-09-23T22:54:48.710587+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":258,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.719478+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:54:48.719478+0000","last_peered":"2023-09-23T22:54:48.719478+0000","last_clean":"2023-09-23T22:54:48.719478+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:54:48.719478+0000","last_undegraded":"2023-09-23T22:54:48.719478+0000","last_fullsized":"2023-09-23T22:54:48.719478+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":276,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-09-23T22:54:48.780228+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:54:48.780228+0000","last_peered":"2023-09-23T22:54:48.780228+0000","last_clean":"2023-09-23T22:54:48.780228+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:54:48.780228+0000","last_undegraded":"2023-09-23T22:54:48.780228+0000","last_fullsized":"2023-09-23T22:54:48.780228+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":318177304,"num_objects":4856,"num_object_clones":0,"num_object_copies":14568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4262,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4856,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174422,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12545,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1672359936,"data_stored":1672347720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114872,"ondisk_log_size":114872,"up":48,"acting":45,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430544,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626772,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":493756,"kb_used_data":188340,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93251396,"statfs":{"total":95995035648,"available":95489429504,"internally_reserved":0,"allocated":192860160,"data_stored":192302179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":8.0800000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.087,"5min":243.35900000000001,"15min":243.35900000000001},"last":3.5550000000000002},{"interface":"front","average":{"1min":3.3559999999999999,"5min":8.8829999999999991,"15min":4.4269999999999996},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":12.081,"5min":243.41499999999999,"15min":243.41499999999999},"last":3.3900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":5.6390000000000002,"15min":4.1959999999999997},"min":{"1min":0.32100000000000001,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":74.899000000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":6.4550000000000001,"5min":6.923,"15min":5.2009999999999996},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":74.956000000000003,"5min":255.20099999999999,"15min":255.20099999999999},"last":9.7739999999999991}]},{"osd":2,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":7.5069999999999997,"5min":4.4580000000000002,"15min":3.738},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":81.719999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":5.0899999999999999},{"interface":"front","average":{"1min":2.1970000000000001,"5min":3.1539999999999999,"15min":3.7509999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":9.3529999999999998,"5min":101.13,"15min":185.21799999999999},"last":4.085}]},{"osd":3,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":1.845,"15min":3.8879999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":4.1079999999999997,"5min":7.734,"15min":185.19399999999999},"last":4.3120000000000003},{"interface":"front","average":{"1min":3.262,"5min":2.4279999999999999,"15min":3.6709999999999998},"min":{"1min":0.438,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.3109999999999999,"5min":10.378,"15min":128.19800000000001},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.3049999999999999,"15min":0.93700000000000006},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.9240000000000004,"5min":10.317,"15min":18.085999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.302,"15min":0.94299999999999995},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":10.122999999999999,"5min":10.122999999999999,"15min":18.597000000000001},"last":0.39600000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.4910000000000001,"15min":1.0549999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.24399999999999999},"max":{"1min":5.351,"5min":10.353999999999999,"15min":18.972000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.325,"15min":1.0049999999999999},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":10.199,"5min":10.199,"15min":18.625},"last":1.3220000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:54:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.123,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":7.8650000000000002,"5min":7.8650000000000002,"15min":12.481999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.694,"5min":1.21,"15min":0.93600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.774,"5min":6.6269999999999998,"15min":16.661999999999999},"last":1.3089999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253082,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":746768,"kb_used_data":221384,"kb_used_omap":9,"kb_used_meta":525110,"kb_avail":92998384,"statfs":{"total":95995035648,"available":95230345216,"internally_reserved":0,"allocated":226697216,"data_stored":225598703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":537713641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":4.5519999999999996,"15min":5.4329999999999998},"min":{"1min":0.27700000000000002,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":37.991,"5min":37.991,"15min":101.605},"last":0.41199999999999998},{"interface":"front","average":{"1min":6.1550000000000002,"5min":3.819,"15min":5.3019999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":42.191000000000003,"5min":42.191000000000003,"15min":101.892},"last":0.40100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":3.8900000000000001,"15min":3.2549999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":21.215,"5min":26.129000000000001,"15min":37.889000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":3.9689999999999999,"5min":3.633,"15min":3.3239999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":21.254000000000001,"5min":26.001999999999999,"15min":37.957000000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.871,"15min":2.645},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.143000000000001,"5min":12.143000000000001,"15min":87.787999999999997},"last":0.27300000000000002},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.1899999999999999,"15min":3.1309999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20999999999999999},"max":{"1min":8.0099999999999998,"5min":10.845000000000001,"15min":89.861999999999995},"last":0.98599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":1.9510000000000001,"15min":2.3900000000000001},"min":{"1min":0.38,"5min":0.35899999999999999,"15min":0.29299999999999998},"max":{"1min":11.388,"5min":11.388,"15min":78.948999999999998},"last":6.9790000000000001},{"interface":"front","average":{"1min":2.3980000000000001,"5min":2.5659999999999998,"15min":2.4020000000000001},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.2579999999999991,"5min":14.058,"15min":78.963999999999999},"last":6.9459999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.7110000000000001,"15min":1.8839999999999999},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":12.115,"5min":12.115,"15min":84.254999999999995},"last":0.433},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.78,"15min":2.21},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":8.1050000000000004,"5min":9.8170000000000002,"15min":87.515000000000001},"last":1.0269999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":2.6110000000000002,"15min":2.2559999999999998},"min":{"1min":0.40699999999999997,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.6639999999999997,"5min":13.991,"15min":79.311000000000007},"last":19.545999999999999},{"interface":"front","average":{"1min":2.5979999999999999,"5min":1.9810000000000001,"15min":1.9059999999999999},"min":{"1min":0.41899999999999998,"5min":0.40899999999999997,"15min":0.313},"max":{"1min":12.067,"5min":12.067,"15min":79.328999999999994},"last":20.614000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.899,"15min":2.609},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.32800000000000001},"max":{"1min":8.1400000000000006,"5min":9.9329999999999998,"15min":159.018},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.254,"5min":2.6949999999999998,"15min":2.7749999999999999},"min":{"1min":0.443,"5min":0.38900000000000001,"15min":0.37},"max":{"1min":6.6310000000000002,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.65000000000000002}]}]},{"osd":0,"up_from":219,"seq":940597837931,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":446068,"kb_used_data":123052,"kb_used_omap":31,"kb_used_meta":322976,"kb_avail":93299084,"statfs":{"total":95995035648,"available":95538262016,"internally_reserved":0,"allocated":126005248,"data_stored":124874328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":330728370},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":3.4329999999999998,"15min":3.7410000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.326000000000001,"5min":40.180999999999997,"15min":40.180999999999997},"last":1.0960000000000001},{"interface":"front","average":{"1min":4.2270000000000003,"5min":3.4630000000000001,"15min":3.6499999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.494999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":1.6839999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":5.4139999999999997,"15min":3.5910000000000002},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":89.766000000000005,"5min":111.508,"15min":111.508},"last":7.7919999999999998},{"interface":"front","average":{"1min":7.617,"5min":5.5449999999999999,"15min":3.2360000000000002},"min":{"1min":0.29499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":89.994,"5min":111.532,"15min":111.532},"last":6.6529999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":4.2519999999999998,"15min":3.0510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":8.5380000000000003,"5min":126.73,"15min":126.73},"last":1.3029999999999999},{"interface":"front","average":{"1min":2.4969999999999999,"5min":3.488,"15min":2.1349999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6440000000000001,"5min":121.483,"15min":121.483},"last":0.28899999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.0129999999999999,"15min":2.173},"min":{"1min":0.33000000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.5820000000000007,"5min":11.965999999999999,"15min":18.670999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.2400000000000002,"15min":2.036},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.5899999999999999,"5min":16.52,"15min":20.009},"last":1.345}]},{"osd":5,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.8599999999999999,"15min":1.9370000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":6.6630000000000003,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.37},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.1379999999999999,"15min":2.3490000000000002},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4240000000000004,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.90000000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.778,"15min":1.95},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6950000000000003,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.504,"5min":3.069,"15min":2.4550000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.367,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.82999999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.1709999999999998,"15min":2.077},"min":{"1min":0.438,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.6959999999999997,"5min":12.385999999999999,"15min":18.640000000000001},"last":2.4319999999999999},{"interface":"front","average":{"1min":1.946,"5min":2.0830000000000002,"15min":1.6000000000000001},"min":{"1min":0.42599999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.6779999999999999,"5min":21.954999999999998,"15min":21.954999999999998},"last":2.355}]}]},{"osd":2,"up_from":131,"seq":562640715943,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":583568,"kb_used_data":149576,"kb_used_omap":81,"kb_used_meta":433902,"kb_avail":93161584,"statfs":{"total":95995035648,"available":95397462016,"internally_reserved":0,"allocated":153165824,"data_stored":152070291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":444315866},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":6.9960000000000004,"15min":5.734},"min":{"1min":0.22800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":12.76,"5min":132.79499999999999,"15min":132.79499999999999},"last":49.741},{"interface":"front","average":{"1min":3.286,"5min":6.9400000000000004,"15min":6.0990000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.093,"5min":132.58099999999999,"15min":132.58099999999999},"last":49.645000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":3.0470000000000002,"15min":3.173},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":6.3019999999999996,"5min":77.004999999999995,"15min":88.316000000000003},"last":19.504999999999999},{"interface":"front","average":{"1min":1.552,"5min":3.3500000000000001,"15min":3.407},"min":{"1min":0.25700000000000001,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":6.1779999999999999,"5min":76.956000000000003,"15min":88.436999999999998},"last":19.523}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.2939999999999996,"15min":4.4100000000000001},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.492000000000001,"5min":112.895,"15min":112.895},"last":19.579999999999998},{"interface":"front","average":{"1min":2.7869999999999999,"5min":4.109,"15min":3.6219999999999999},"min":{"1min":0.39000000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.1880000000000006,"5min":113.995,"15min":113.995},"last":19.542000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.6749999999999998,"15min":2.8279999999999998},"min":{"1min":0.60199999999999998,"5min":0.28899999999999998,"15min":0.253},"max":{"1min":9.1219999999999999,"5min":69.113,"15min":108.878},"last":19.635999999999999},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.4689999999999999,"15min":3.3809999999999998},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2229999999999999,"5min":69.028999999999996,"15min":108.93000000000001},"last":19.594999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.7789999999999999,"15min":2.9940000000000002},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":6.1820000000000004,"5min":22.646000000000001,"15min":97.322999999999993},"last":19.652000000000001},{"interface":"front","average":{"1min":1.2569999999999999,"5min":2.226,"15min":2.98},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.1420000000000003,"5min":22.501000000000001,"15min":97.301000000000002},"last":19.614999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":3.609,"15min":3.3450000000000002},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":4.3040000000000003,"5min":98.597999999999999,"15min":98.597999999999999},"last":19.690999999999999},{"interface":"front","average":{"1min":2.4670000000000001,"5min":3.4209999999999998,"15min":2.7050000000000001},"min":{"1min":0.48699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":9.2110000000000003,"5min":98.509,"15min":98.509},"last":19.675000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":2.0590000000000002,"15min":1.7509999999999999},"min":{"1min":0.48999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.1609999999999996,"5min":10.477,"15min":10.92},"last":19.756},{"interface":"front","average":{"1min":1.415,"5min":1.667,"15min":2.3799999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.37},"max":{"1min":6.2720000000000002,"5min":11.311,"15min":88.584000000000003},"last":19.765999999999998}]}]},{"osd":3,"up_from":0,"seq":403726926029,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019228,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409600,"kb_used_data":104184,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93335552,"statfs":{"total":95995035648,"available":95575605248,"internally_reserved":0,"allocated":106684416,"data_stored":106121841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.3140000000000001,"15min":5.9390000000000001},"min":{"1min":0.251,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":5.9509999999999996,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.7229999999999999,"15min":6.5099999999999998},"min":{"1min":0.41199999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.6760000000000002,"5min":113.417,"15min":113.417},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":5.0599999999999996,"15min":3.0190000000000001},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":132.304,"15min":132.304},"last":0.377},{"interface":"front","average":{"1min":2.7959999999999998,"5min":4.8970000000000002,"15min":2.879},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":10.832000000000001,"5min":132.453,"15min":132.453},"last":1.1120000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.3639999999999999,"15min":3.1520000000000001},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.097,"5min":20.308,"15min":84.909000000000006},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.524,"15min":3.137},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.352,"5min":21.704000000000001,"15min":83.840000000000003},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":4.1879999999999997,"15min":4.548},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":7.3109999999999999,"5min":164.453,"15min":164.453},"last":1.0800000000000001},{"interface":"front","average":{"1min":2.2170000000000001,"5min":4.343,"15min":4.6689999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.72699999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.1779999999999999,"15min":1},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.25900000000000001},"max":{"1min":5.0880000000000001,"5min":12.238,"15min":43.189999999999998},"last":2.028},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1060000000000001,"15min":1},"min":{"1min":0.32900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.3079999999999998,"5min":7.5460000000000003,"15min":42.450000000000003},"last":1.9730000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.0529999999999999,"15min":0.94899999999999995},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9139999999999997,"5min":7.9139999999999997,"15min":27.056000000000001},"last":5.7480000000000002},{"interface":"front","average":{"1min":1.417,"5min":1.224,"15min":0.97799999999999998},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":5.298,"5min":12.269,"15min":26.940999999999999},"last":0.70099999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:54:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.2250000000000001,"15min":0.876},"min":{"1min":0.36699999999999999,"5min":0.36499999999999999,"15min":0.29899999999999999},"max":{"1min":5.1449999999999996,"5min":12.311,"15min":12.311},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.171,"5min":1.0569999999999999,"15min":0.79900000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":4.077,"5min":8.6739999999999995,"15min":8.6739999999999995},"last":0.61799999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823000,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":799268,"kb_used_data":213660,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":92945884,"statfs":{"total":95995035648,"available":95176585216,"internally_reserved":0,"allocated":218787840,"data_stored":218248981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":9.6929999999999996,"5min":8.1189999999999998,"15min":5.133},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":80.450999999999993,"5min":185.62,"15min":185.62},"last":1.5429999999999999},{"interface":"front","average":{"1min":9.734,"5min":8.1829999999999998,"15min":5.1079999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":80.171999999999997,"5min":185.499,"15min":185.499},"last":1.4139999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":9.452,"5min":3.9129999999999998,"15min":2.8799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":110.36499999999999,"5min":110.36499999999999,"15min":149.203},"last":2.9990000000000001},{"interface":"front","average":{"1min":9.5340000000000007,"5min":4.3019999999999996,"15min":3.1909999999999998},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":110.89,"5min":110.89,"15min":149.554},"last":3.4500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":5.54,"15min":5.5140000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.997,"5min":189.126,"15min":189.126},"last":3.1640000000000001},{"interface":"front","average":{"1min":3.5379999999999998,"5min":5.2309999999999999,"15min":5.4059999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":41.070999999999998,"5min":188.74799999999999,"15min":188.74799999999999},"last":3.2109999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.843,"15min":5.6660000000000004},"min":{"1min":0.34899999999999998,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.298999999999999,"5min":98.641000000000005,"15min":257.05000000000001},"last":0.748},{"interface":"front","average":{"1min":2.085,"5min":4.9130000000000003,"15min":5.516},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27800000000000002},"max":{"1min":11.097,"5min":98.906000000000006,"15min":257.35000000000002},"last":2.9119999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.86899999999999999,"15min":0.73399999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":3.669,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.996,"5min":0.92100000000000004,"15min":0.75700000000000001},"min":{"1min":0.29399999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":5.8019999999999996,"5min":6.3550000000000004,"15min":10.535},"last":0.32500000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1759999999999999,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.3369999999999997,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.139,"15min":1.629},"min":{"1min":0.374,"5min":0.315,"15min":0.254},"max":{"1min":7.9909999999999997,"5min":10.314,"15min":139.739},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:54:05 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.133,"15min":0.88700000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.9509999999999996,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.5},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.99299999999999999,"15min":0.80300000000000005},"min":{"1min":0.378,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":6.7149999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.38100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":108527616,"data_stored":108527616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":211881984,"data_stored":211877912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":139198464,"data_stored":139198464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":729284608,"data_stored":729284608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":96276480,"data_stored":96272408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":214700032,"data_stored":214695960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":172490752,"data_stored":172490752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:55:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:07.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:07.595 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:07.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:07.709 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:07 smithi052 ceph-mon[187743]: from='client.35772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:07.709 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:07 smithi052 ceph-mon[187743]: pgmap v909: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 207 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 633 op/s; 4262/10005 objects degraded (42.599%); 0 B/s, 45 objects/s recovering 2023-09-23T22:55:07.710 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:07 smithi052 ceph-mon[180979]: from='client.35772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:07.710 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:07 smithi052 ceph-mon[180979]: pgmap v909: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 207 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 633 op/s; 4262/10005 objects degraded (42.599%); 0 B/s, 45 objects/s recovering 2023-09-23T22:55:07.776 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:07.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:07 smithi151 ceph-mon[159210]: from='client.35772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:07.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:07 smithi151 ceph-mon[159210]: pgmap v909: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 207 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 633 op/s; 4262/10005 objects degraded (42.599%); 0 B/s, 45 objects/s recovering 2023-09-23T22:55:07.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:07.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:07.883 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:07.959 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:08.039 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:08.041 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:08.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:08.145 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:08.175 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:08.219 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:08.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:08.323 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:08.376 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:08.663 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:08.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:08.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:08.769 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:08.798 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:08.836 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:08.951 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:08.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:08.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:09.055 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:09.127 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:09.257 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Removed 89638 objects 2023-09-23T22:55:09.257 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Clean up completed and total clean up time :139.669 2023-09-23T22:55:09.306 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:09.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:09.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:09.399 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:09.410 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:09.425 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:09.442 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:09.448 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:55:09.478 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:09.500 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:09.519 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:09.540 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:09.559 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:09.560 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.560 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.561 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.561 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.561 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.561 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.561 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.561 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:55:09.604 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:09.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:09.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:09.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:09 smithi151 ceph-mon[159210]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 109 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 627 op/s; 4002/5292 objects degraded (75.624%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:09.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/718462337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:09.739 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:09.788 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:09.790 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:09 smithi052 ceph-mon[187743]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 109 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 627 op/s; 4002/5292 objects degraded (75.624%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:09.790 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/718462337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:09.791 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:09 smithi052 ceph-mon[180979]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 109 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 627 op/s; 4002/5292 objects degraded (75.624%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:09.791 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/718462337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:09.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:09.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:09.886 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:55:09.886 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:55:09.893 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:09.921 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:10.491 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:55:10.494 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:55:10.538 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:10.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:10.642 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:10.670 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:10.716 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:10.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:10.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:10.821 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:10.846 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:10.847 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 4002/5292 objects degraded (75.624%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:10.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 4002/5292 objects degraded (75.624%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:10.851 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 4002/5292 objects degraded (75.624%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:11.158 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:11.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:11.174 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T22:55:11.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T22:55:11.261 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:11.289 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:11.395 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:11.436 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:11.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:11.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:11.539 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:11.566 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:11.608 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:11.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:11.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:11 smithi151 ceph-mon[159210]: pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 70 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 608 op/s; 4002/3387 objects degraded (118.158%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:11.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:11.736 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:11.791 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:11 smithi052 ceph-mon[187743]: pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 70 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 608 op/s; 4002/3387 objects degraded (118.158%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:11.791 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:11 smithi052 ceph-mon[180979]: pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 70 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 608 op/s; 4002/3387 objects degraded (118.158%); 0 B/s, 50 objects/s recovering 2023-09-23T22:55:11.943 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:11.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:12.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:12.080 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:12.126 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:12.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:12.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:12.229 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:12.258 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:12.309 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:12.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:12.341 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:55:12.341 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:55:12.413 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:12.424 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:12.447 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:12.504 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:12.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:12.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:12.542 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:55:12.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:12.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: osdmap e313: 8 total, 8 up, 6 in 2023-09-23T22:55:12.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.631 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/37337950' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:12.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:12.632 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:12 smithi151 ceph-mon[159210]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.634 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:12.678 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:12.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: osdmap e313: 8 total, 8 up, 6 in 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/37337950' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:12.704 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[187743]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: osdmap e313: 8 total, 8 up, 6 in 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/37337950' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:12.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:12 smithi052 ceph-mon[180979]: failed to encode map e313 with expected crc 2023-09-23T22:55:12.781 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:12.806 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:13.064 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:13.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:13.115 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:55:13.154 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:55:13.168 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:13.194 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:13.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:13.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:13.345 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:13.374 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:13.421 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:13.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:13.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:13.490 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:13.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:55:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:55:13] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-09-23T22:55:13.525 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:13.554 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:13.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: pgmap v913: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 9.8 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 3984/513 objects degraded (776.608%); 0 B/s, 34 objects/s recovering 2023-09-23T22:55:13.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:13.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:13.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: osdmap e314: 8 total, 8 up, 6 in 2023-09-23T22:55:13.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:13 smithi151 ceph-mon[159210]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: pgmap v913: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 9.8 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 3984/513 objects degraded (776.608%); 0 B/s, 34 objects/s recovering 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: osdmap e314: 8 total, 8 up, 6 in 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[187743]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: pgmap v913: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 9.8 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 577 op/s; 3984/513 objects degraded (776.608%); 0 B/s, 34 objects/s recovering 2023-09-23T22:55:13.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:13.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: failed to encode map e314 with expected crc 2023-09-23T22:55:13.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:13.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: osdmap e314: 8 total, 8 up, 6 in 2023-09-23T22:55:13.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:13 smithi052 ceph-mon[180979]: failed to encode map e314 with expected crc 2023-09-23T22:55:14.166 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:14.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:14.271 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:14.300 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:55:14.302 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:14.302 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:55:14.580 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:14.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:14.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:14.683 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:14.710 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:14.760 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:14.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:14.864 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:14.897 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:15.017 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:15.042 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":914,"stamp":"2023-09-23T22:55:12.534104+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":10229664,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":179391,"num_write_kb":5739278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12853,"num_bytes_recovered":313206838,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115575,"ondisk_log_size":115575,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2518692,"kb_used_data":36020,"kb_used_omap":121,"kb_used_meta":2482502,"kb_avail":559952220,"statfs":{"total":575970213888,"available":573391073280,"internally_reserved":0,"allocated":36884480,"data_stored":31907319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2542082165},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,1,2,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-308543512,"num_objects":-4709,"num_object_clones":0,"num_object_copies":-14127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4709,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":278,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.085342"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":310,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.318357+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:55:04.318357+0000","last_peered":"2023-09-23T22:55:04.318357+0000","last_clean":"2023-09-23T22:55:04.318357+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:55:04.318357+0000","last_undegraded":"2023-09-23T22:55:04.318357+0000","last_fullsized":"2023-09-23T22:55:04.318357+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":109,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319231+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:55:04.319231+0000","last_peered":"2023-09-23T22:55:04.319231+0000","last_clean":"2023-09-23T22:55:04.319231+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:55:04.319231+0000","last_undegraded":"2023-09-23T22:55:04.319231+0000","last_fullsized":"2023-09-23T22:55:04.319231+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317776+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:55:04.317776+0000","last_peered":"2023-09-23T22:55:04.317776+0000","last_clean":"2023-09-23T22:55:04.317776+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:55:04.317776+0000","last_undegraded":"2023-09-23T22:55:04.317776+0000","last_fullsized":"2023-09-23T22:55:04.317776+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":291,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.227032+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:54:56.227032+0000","last_peered":"2023-09-23T22:54:56.227032+0000","last_clean":"2023-09-23T22:54:56.227032+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:54:56.227032+0000","last_undegraded":"2023-09-23T22:54:56.227032+0000","last_fullsized":"2023-09-23T22:54:56.227032+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":295,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.228976+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:54:56.228976+0000","last_peered":"2023-09-23T22:54:56.228976+0000","last_clean":"2023-09-23T22:54:56.228976+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:54:56.228976+0000","last_undegraded":"2023-09-23T22:54:56.228976+0000","last_fullsized":"2023-09-23T22:54:56.228976+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":277,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.147524+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:54:56.147524+0000","last_peered":"2023-09-23T22:54:56.147524+0000","last_clean":"2023-09-23T22:54:56.147524+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:54:56.147524+0000","last_undegraded":"2023-09-23T22:54:56.147524+0000","last_fullsized":"2023-09-23T22:54:56.147524+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":222,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.226524+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:54:56.226524+0000","last_peered":"2023-09-23T22:54:56.226524+0000","last_clean":"2023-09-23T22:54:56.226524+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:54:56.226524+0000","last_undegraded":"2023-09-23T22:54:56.226524+0000","last_fullsized":"2023-09-23T22:54:56.226524+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":174,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191455+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:54:56.191455+0000","last_peered":"2023-09-23T22:54:56.191455+0000","last_clean":"2023-09-23T22:54:56.191455+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:54:56.191455+0000","last_undegraded":"2023-09-23T22:54:56.191455+0000","last_fullsized":"2023-09-23T22:54:56.191455+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":53,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191789+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:54:56.191789+0000","last_peered":"2023-09-23T22:54:56.191789+0000","last_clean":"2023-09-23T22:54:56.191789+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:54:56.191789+0000","last_undegraded":"2023-09-23T22:54:56.191789+0000","last_fullsized":"2023-09-23T22:54:56.191789+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":196,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146361+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:54:56.146361+0000","last_peered":"2023-09-23T22:54:56.146361+0000","last_clean":"2023-09-23T22:54:56.146361+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:54:56.146361+0000","last_undegraded":"2023-09-23T22:54:56.146361+0000","last_fullsized":"2023-09-23T22:54:56.146361+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":71,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319062+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:55:04.319062+0000","last_peered":"2023-09-23T22:55:04.319062+0000","last_clean":"2023-09-23T22:55:04.319062+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:55:04.319062+0000","last_undegraded":"2023-09-23T22:55:04.319062+0000","last_fullsized":"2023-09-23T22:55:04.319062+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":186,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146690+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:54:56.146690+0000","last_peered":"2023-09-23T22:54:56.146690+0000","last_clean":"2023-09-23T22:54:56.146690+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:54:56.146690+0000","last_undegraded":"2023-09-23T22:54:56.146690+0000","last_fullsized":"2023-09-23T22:54:56.146690+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":332,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319780+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:55:04.319780+0000","last_peered":"2023-09-23T22:55:04.319780+0000","last_clean":"2023-09-23T22:55:04.319780+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:55:04.319780+0000","last_undegraded":"2023-09-23T22:55:04.319780+0000","last_fullsized":"2023-09-23T22:55:04.319780+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":278,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317735+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:55:04.317735+0000","last_peered":"2023-09-23T22:55:04.317735+0000","last_clean":"2023-09-23T22:55:04.317735+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:55:04.317735+0000","last_undegraded":"2023-09-23T22:55:04.317735+0000","last_fullsized":"2023-09-23T22:55:04.317735+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":171,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317800+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:55:04.317800+0000","last_peered":"2023-09-23T22:55:04.317800+0000","last_clean":"2023-09-23T22:55:04.317800+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:55:04.317800+0000","last_undegraded":"2023-09-23T22:55:04.317800+0000","last_fullsized":"2023-09-23T22:55:04.317800+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":259,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191539+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:54:56.191539+0000","last_peered":"2023-09-23T22:54:56.191539+0000","last_clean":"2023-09-23T22:54:56.191539+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:54:56.191539+0000","last_undegraded":"2023-09-23T22:54:56.191539+0000","last_fullsized":"2023-09-23T22:54:56.191539+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":278,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.227360+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:54:56.227360+0000","last_peered":"2023-09-23T22:54:56.227360+0000","last_clean":"2023-09-23T22:54:56.227360+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:54:56.227360+0000","last_undegraded":"2023-09-23T22:54:56.227360+0000","last_fullsized":"2023-09-23T22:54:56.227360+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":235,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146557+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:54:56.146557+0000","last_peered":"2023-09-23T22:54:56.146557+0000","last_clean":"2023-09-23T22:54:56.146557+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:54:56.146557+0000","last_undegraded":"2023-09-23T22:54:56.146557+0000","last_fullsized":"2023-09-23T22:54:56.146557+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":163,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317976+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:55:04.317976+0000","last_peered":"2023-09-23T22:55:04.317976+0000","last_clean":"2023-09-23T22:55:04.317976+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:55:04.317976+0000","last_undegraded":"2023-09-23T22:55:04.317976+0000","last_fullsized":"2023-09-23T22:55:04.317976+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":279,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.227176+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:54:56.227176+0000","last_peered":"2023-09-23T22:54:56.227176+0000","last_clean":"2023-09-23T22:54:56.227176+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:54:56.227176+0000","last_undegraded":"2023-09-23T22:54:56.227176+0000","last_fullsized":"2023-09-23T22:54:56.227176+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":356,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146197+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:54:56.146197+0000","last_peered":"2023-09-23T22:54:56.146197+0000","last_clean":"2023-09-23T22:54:56.146197+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:54:56.146197+0000","last_undegraded":"2023-09-23T22:54:56.146197+0000","last_fullsized":"2023-09-23T22:54:56.146197+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":239,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.192039+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:54:56.192039+0000","last_peered":"2023-09-23T22:54:56.192039+0000","last_clean":"2023-09-23T22:54:56.192039+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:54:56.192039+0000","last_undegraded":"2023-09-23T22:54:56.192039+0000","last_fullsized":"2023-09-23T22:54:56.192039+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"312'11091","reported_seq":11149,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:08.338788+0000","last_change":"2023-09-23T22:51:10.575705+0000","last_active":"2023-09-23T22:55:08.338788+0000","last_peered":"2023-09-23T22:55:08.338788+0000","last_clean":"2023-09-23T22:55:08.338788+0000","last_became_active":"2023-09-23T22:51:10.575383+0000","last_became_peered":"2023-09-23T22:51:10.575383+0000","last_unstale":"2023-09-23T22:55:08.338788+0000","last_undegraded":"2023-09-23T22:55:08.338788+0000","last_fullsized":"2023-09-23T22:55:08.338788+0000","mapping_epoch":264,"log_start":"277'3500","ondisk_log_start":"277'3500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7591,"log_dups_size":0,"ondisk_log_size":7591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:20.454093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2686976,"num_objects":41,"num_object_clones":0,"num_object_copies":123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11091,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":182,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.318117+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:55:04.318117+0000","last_peered":"2023-09-23T22:55:04.318117+0000","last_clean":"2023-09-23T22:55:04.318117+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:55:04.318117+0000","last_undegraded":"2023-09-23T22:55:04.318117+0000","last_fullsized":"2023-09-23T22:55:04.318117+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146291+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:54:56.146291+0000","last_peered":"2023-09-23T22:54:56.146291+0000","last_clean":"2023-09-23T22:54:56.146291+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:54:56.146291+0000","last_undegraded":"2023-09-23T22:54:56.146291+0000","last_fullsized":"2023-09-23T22:54:56.146291+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"312'11492","reported_seq":11593,"reported_epoch":313,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:55:11.679122+0000","last_change":"2023-09-23T22:55:10.943652+0000","last_active":"2023-09-23T22:55:11.679122+0000","last_peered":"2023-09-23T22:55:11.679122+0000","last_clean":"2023-09-23T22:51:33.369927+0000","last_became_active":"2023-09-23T22:51:37.047098+0000","last_became_peered":"2023-09-23T22:51:37.047098+0000","last_unstale":"2023-09-23T22:55:11.679122+0000","last_undegraded":"2023-09-23T22:51:36.596054+0000","last_fullsized":"2023-09-23T22:51:36.590175+0000","mapping_epoch":271,"log_start":"281'4600","ondisk_log_start":"281'4600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11492,"num_write_kb":367744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,4","objects":-1643}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":192,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.318945+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:55:04.318945+0000","last_peered":"2023-09-23T22:55:04.318945+0000","last_clean":"2023-09-23T22:55:04.318945+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:55:04.318945+0000","last_undegraded":"2023-09-23T22:55:04.318945+0000","last_fullsized":"2023-09-23T22:55:04.318945+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":302,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317872+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:55:04.317872+0000","last_peered":"2023-09-23T22:55:04.317872+0000","last_clean":"2023-09-23T22:55:04.317872+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:55:04.317872+0000","last_undegraded":"2023-09-23T22:55:04.317872+0000","last_fullsized":"2023-09-23T22:55:04.317872+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"312'11288","reported_seq":11352,"reported_epoch":312,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:55:09.188680+0000","last_change":"2023-09-23T22:51:35.542041+0000","last_active":"2023-09-23T22:55:09.188680+0000","last_peered":"2023-09-23T22:55:09.188680+0000","last_clean":"2023-09-23T22:51:33.378002+0000","last_became_active":"2023-09-23T22:51:35.539001+0000","last_became_peered":"2023-09-23T22:51:35.539001+0000","last_unstale":"2023-09-23T22:55:09.188680+0000","last_undegraded":"2023-09-23T22:51:35.425874+0000","last_fullsized":"2023-09-23T22:51:35.413997+0000","mapping_epoch":271,"log_start":"275'2700","ondisk_log_start":"275'2700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:37.020245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11288,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-1771}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":135,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146524+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:54:56.146524+0000","last_peered":"2023-09-23T22:54:56.146524+0000","last_clean":"2023-09-23T22:54:56.146524+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:54:56.146524+0000","last_undegraded":"2023-09-23T22:54:56.146524+0000","last_fullsized":"2023-09-23T22:54:56.146524+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":304,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146532+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:54:56.146532+0000","last_peered":"2023-09-23T22:54:56.146532+0000","last_clean":"2023-09-23T22:54:56.146532+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:54:56.146532+0000","last_undegraded":"2023-09-23T22:54:56.146532+0000","last_fullsized":"2023-09-23T22:54:56.146532+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"312'11316","reported_seq":11374,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.221148+0000","last_change":"2023-09-23T22:51:10.574018+0000","last_active":"2023-09-23T22:55:09.221148+0000","last_peered":"2023-09-23T22:55:09.221148+0000","last_clean":"2023-09-23T22:55:09.221148+0000","last_became_active":"2023-09-23T22:51:10.573744+0000","last_became_peered":"2023-09-23T22:51:10.573744+0000","last_unstale":"2023-09-23T22:55:09.221148+0000","last_undegraded":"2023-09-23T22:55:09.221148+0000","last_fullsized":"2023-09-23T22:55:09.221148+0000","mapping_epoch":264,"log_start":"275'2700","ondisk_log_start":"275'2700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:13.696889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11316,"num_write_kb":362112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":107,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146553+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:54:56.146553+0000","last_peered":"2023-09-23T22:54:56.146553+0000","last_clean":"2023-09-23T22:54:56.146553+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:54:56.146553+0000","last_undegraded":"2023-09-23T22:54:56.146553+0000","last_fullsized":"2023-09-23T22:54:56.146553+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":206,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.230147+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:54:56.230147+0000","last_peered":"2023-09-23T22:54:56.230147+0000","last_clean":"2023-09-23T22:54:56.230147+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:54:56.230147+0000","last_undegraded":"2023-09-23T22:54:56.230147+0000","last_fullsized":"2023-09-23T22:54:56.230147+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"312'11008","reported_seq":14430,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.172903+0000","last_change":"2023-09-23T22:52:57.565908+0000","last_active":"2023-09-23T22:55:09.172903+0000","last_peered":"2023-09-23T22:55:09.172903+0000","last_clean":"2023-09-23T22:55:09.172903+0000","last_became_active":"2023-09-23T22:52:57.525760+0000","last_became_peered":"2023-09-23T22:52:57.525760+0000","last_unstale":"2023-09-23T22:55:09.172903+0000","last_undegraded":"2023-09-23T22:55:09.172903+0000","last_fullsized":"2023-09-23T22:55:09.172903+0000","mapping_epoch":289,"log_start":"284'4600","ondisk_log_start":"284'4600","created":264,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11008,"num_write_kb":352256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109707264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":78,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.230360+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:54:56.230360+0000","last_peered":"2023-09-23T22:54:56.230360+0000","last_clean":"2023-09-23T22:54:56.230360+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:54:56.230360+0000","last_undegraded":"2023-09-23T22:54:56.230360+0000","last_fullsized":"2023-09-23T22:54:56.230360+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":286,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319648+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:55:04.319648+0000","last_peered":"2023-09-23T22:55:04.319648+0000","last_clean":"2023-09-23T22:55:04.319648+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:55:04.319648+0000","last_undegraded":"2023-09-23T22:55:04.319648+0000","last_fullsized":"2023-09-23T22:55:04.319648+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"312'11082","reported_seq":11138,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.248046+0000","last_change":"2023-09-23T22:51:10.575178+0000","last_active":"2023-09-23T22:55:09.248046+0000","last_peered":"2023-09-23T22:55:09.248046+0000","last_clean":"2023-09-23T22:55:09.248046+0000","last_became_active":"2023-09-23T22:51:10.574789+0000","last_became_peered":"2023-09-23T22:51:10.574789+0000","last_unstale":"2023-09-23T22:55:09.248046+0000","last_undegraded":"2023-09-23T22:55:09.248046+0000","last_fullsized":"2023-09-23T22:55:09.248046+0000","mapping_epoch":264,"log_start":"284'4600","ondisk_log_start":"284'4600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6482,"log_dups_size":0,"ondisk_log_size":6482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11082,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":296,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146274+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:54:56.146274+0000","last_peered":"2023-09-23T22:54:56.146274+0000","last_clean":"2023-09-23T22:54:56.146274+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:54:56.146274+0000","last_undegraded":"2023-09-23T22:54:56.146274+0000","last_fullsized":"2023-09-23T22:54:56.146274+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":308,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.318903+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:55:04.318903+0000","last_peered":"2023-09-23T22:55:04.318903+0000","last_clean":"2023-09-23T22:55:04.318903+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:55:04.318903+0000","last_undegraded":"2023-09-23T22:55:04.318903+0000","last_fullsized":"2023-09-23T22:55:04.318903+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"312'11180","reported_seq":14590,"reported_epoch":312,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:55:09.213486+0000","last_change":"2023-09-23T22:54:01.194508+0000","last_active":"2023-09-23T22:55:09.213486+0000","last_peered":"2023-09-23T22:55:09.213486+0000","last_clean":"2023-09-23T22:51:33.388845+0000","last_became_active":"2023-09-23T22:54:01.140849+0000","last_became_peered":"2023-09-23T22:54:01.140849+0000","last_unstale":"2023-09-23T22:55:09.213486+0000","last_undegraded":"2023-09-23T22:55:09.213486+0000","last_fullsized":"2023-09-23T22:55:09.213486+0000","mapping_epoch":299,"log_start":"284'4700","ondisk_log_start":"284'4700","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6480,"log_dups_size":0,"ondisk_log_size":6480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11180,"num_write_kb":357760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1668,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":["6","1","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":321,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317635+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:55:04.317635+0000","last_peered":"2023-09-23T22:55:04.317635+0000","last_clean":"2023-09-23T22:55:04.317635+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:55:04.317635+0000","last_undegraded":"2023-09-23T22:55:04.317635+0000","last_fullsized":"2023-09-23T22:55:04.317635+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":296,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319019+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:55:04.319019+0000","last_peered":"2023-09-23T22:55:04.319019+0000","last_clean":"2023-09-23T22:55:04.319019+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:55:04.319019+0000","last_undegraded":"2023-09-23T22:55:04.319019+0000","last_fullsized":"2023-09-23T22:55:04.319019+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"312'11070","reported_seq":14374,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.143576+0000","last_change":"2023-09-23T22:54:41.191574+0000","last_active":"2023-09-23T22:55:09.143576+0000","last_peered":"2023-09-23T22:55:09.143576+0000","last_clean":"2023-09-23T22:55:09.143576+0000","last_became_active":"2023-09-23T22:54:41.158218+0000","last_became_peered":"2023-09-23T22:54:41.158218+0000","last_unstale":"2023-09-23T22:55:09.143576+0000","last_undegraded":"2023-09-23T22:55:09.143576+0000","last_fullsized":"2023-09-23T22:55:09.143576+0000","mapping_epoch":306,"log_start":"274'2400","ondisk_log_start":"274'2400","created":264,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":8670,"log_dups_size":0,"ondisk_log_size":8670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:01.903022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11070,"num_write_kb":354240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1615,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":241,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317116+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:55:04.317116+0000","last_peered":"2023-09-23T22:55:04.317116+0000","last_clean":"2023-09-23T22:55:04.317116+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:55:04.317116+0000","last_undegraded":"2023-09-23T22:55:04.317116+0000","last_fullsized":"2023-09-23T22:55:04.317116+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":250,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.192175+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:54:56.192175+0000","last_peered":"2023-09-23T22:54:56.192175+0000","last_clean":"2023-09-23T22:54:56.192175+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:54:56.192175+0000","last_undegraded":"2023-09-23T22:54:56.192175+0000","last_fullsized":"2023-09-23T22:54:56.192175+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"312'11352","reported_seq":14686,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.257336+0000","last_change":"2023-09-23T22:54:48.758724+0000","last_active":"2023-09-23T22:55:09.257336+0000","last_peered":"2023-09-23T22:55:09.257336+0000","last_clean":"2023-09-23T22:55:09.257336+0000","last_became_active":"2023-09-23T22:54:48.714547+0000","last_became_peered":"2023-09-23T22:54:48.714547+0000","last_unstale":"2023-09-23T22:55:09.257336+0000","last_undegraded":"2023-09-23T22:55:09.257336+0000","last_fullsized":"2023-09-23T22:55:09.257336+0000","mapping_epoch":309,"log_start":"286'5400","ondisk_log_start":"286'5400","created":264,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:00.801012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11352,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1630,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":351,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319566+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:55:04.319566+0000","last_peered":"2023-09-23T22:55:04.319566+0000","last_clean":"2023-09-23T22:55:04.319566+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:55:04.319566+0000","last_undegraded":"2023-09-23T22:55:04.319566+0000","last_fullsized":"2023-09-23T22:55:04.319566+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":175,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191720+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:54:56.191720+0000","last_peered":"2023-09-23T22:54:56.191720+0000","last_clean":"2023-09-23T22:54:56.191720+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:54:56.191720+0000","last_undegraded":"2023-09-23T22:54:56.191720+0000","last_fullsized":"2023-09-23T22:54:56.191720+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":144,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317392+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:55:04.317392+0000","last_peered":"2023-09-23T22:55:04.317392+0000","last_clean":"2023-09-23T22:55:04.317392+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:55:04.317392+0000","last_undegraded":"2023-09-23T22:55:04.317392+0000","last_fullsized":"2023-09-23T22:55:04.317392+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"312'11129","reported_seq":14589,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:08.329481+0000","last_change":"2023-09-23T22:52:27.545600+0000","last_active":"2023-09-23T22:55:08.329481+0000","last_peered":"2023-09-23T22:55:08.329481+0000","last_clean":"2023-09-23T22:55:08.329481+0000","last_became_active":"2023-09-23T22:52:27.538417+0000","last_became_peered":"2023-09-23T22:52:27.538417+0000","last_unstale":"2023-09-23T22:55:08.329481+0000","last_undegraded":"2023-09-23T22:55:08.329481+0000","last_fullsized":"2023-09-23T22:55:08.329481+0000","mapping_epoch":282,"log_start":"277'3600","ondisk_log_start":"277'3600","created":264,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7529,"log_dups_size":0,"ondisk_log_size":7529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:55.244907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3211264,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11129,"num_write_kb":357696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":146,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146451+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:54:56.146451+0000","last_peered":"2023-09-23T22:54:56.146451+0000","last_clean":"2023-09-23T22:54:56.146451+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:54:56.146451+0000","last_undegraded":"2023-09-23T22:54:56.146451+0000","last_fullsized":"2023-09-23T22:54:56.146451+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":309,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317846+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:55:04.317846+0000","last_peered":"2023-09-23T22:55:04.317846+0000","last_clean":"2023-09-23T22:55:04.317846+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:55:04.317846+0000","last_undegraded":"2023-09-23T22:55:04.317846+0000","last_fullsized":"2023-09-23T22:55:04.317846+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":239,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191633+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:54:56.191633+0000","last_peered":"2023-09-23T22:54:56.191633+0000","last_clean":"2023-09-23T22:54:56.191633+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:54:56.191633+0000","last_undegraded":"2023-09-23T22:54:56.191633+0000","last_fullsized":"2023-09-23T22:54:56.191633+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"312'11276","reported_seq":14663,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.232829+0000","last_change":"2023-09-23T22:54:21.209191+0000","last_active":"2023-09-23T22:55:09.232829+0000","last_peered":"2023-09-23T22:55:09.232829+0000","last_clean":"2023-09-23T22:55:09.232829+0000","last_became_active":"2023-09-23T22:54:21.131570+0000","last_became_peered":"2023-09-23T22:54:21.131570+0000","last_unstale":"2023-09-23T22:55:09.232829+0000","last_undegraded":"2023-09-23T22:55:09.232829+0000","last_fullsized":"2023-09-23T22:55:09.232829+0000","mapping_epoch":302,"log_start":"284'4850","ondisk_log_start":"284'4850","created":264,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6426,"log_dups_size":0,"ondisk_log_size":6426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11276,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1655,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":314,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191587+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:54:56.191587+0000","last_peered":"2023-09-23T22:54:56.191587+0000","last_clean":"2023-09-23T22:54:56.191587+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:54:56.191587+0000","last_undegraded":"2023-09-23T22:54:56.191587+0000","last_fullsized":"2023-09-23T22:54:56.191587+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":194,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191350+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:54:56.191350+0000","last_peered":"2023-09-23T22:54:56.191350+0000","last_clean":"2023-09-23T22:54:56.191350+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:54:56.191350+0000","last_undegraded":"2023-09-23T22:54:56.191350+0000","last_fullsized":"2023-09-23T22:54:56.191350+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":319,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.226434+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:54:56.226434+0000","last_peered":"2023-09-23T22:54:56.226434+0000","last_clean":"2023-09-23T22:54:56.226434+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:54:56.226434+0000","last_undegraded":"2023-09-23T22:54:56.226434+0000","last_fullsized":"2023-09-23T22:54:56.226434+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.8","version":"312'11022","reported_seq":13372,"reported_epoch":312,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:55:08.362228+0000","last_change":"2023-09-23T22:54:46.998497+0000","last_active":"2023-09-23T22:55:08.362228+0000","last_peered":"2023-09-23T22:55:08.362228+0000","last_clean":"2023-09-23T22:51:33.385505+0000","last_became_active":"2023-09-23T22:51:35.527594+0000","last_became_peered":"2023-09-23T22:51:35.527594+0000","last_unstale":"2023-09-23T22:55:08.362228+0000","last_undegraded":"2023-09-23T22:51:35.424431+0000","last_fullsized":"2023-09-23T22:51:35.414979+0000","mapping_epoch":271,"log_start":"277'3500","ondisk_log_start":"277'3500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:16.951388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1835008,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11022,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1142,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":-542}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":324,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191558+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:54:56.191558+0000","last_peered":"2023-09-23T22:54:56.191558+0000","last_clean":"2023-09-23T22:54:56.191558+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:54:56.191558+0000","last_undegraded":"2023-09-23T22:54:56.191558+0000","last_fullsized":"2023-09-23T22:54:56.191558+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":401,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.228887+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:54:56.228887+0000","last_peered":"2023-09-23T22:54:56.228887+0000","last_clean":"2023-09-23T22:54:56.228887+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:54:56.228887+0000","last_undegraded":"2023-09-23T22:54:56.228887+0000","last_fullsized":"2023-09-23T22:54:56.228887+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":66,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146616+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:54:56.146616+0000","last_peered":"2023-09-23T22:54:56.146616+0000","last_clean":"2023-09-23T22:54:56.146616+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:54:56.146616+0000","last_undegraded":"2023-09-23T22:54:56.146616+0000","last_fullsized":"2023-09-23T22:54:56.146616+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":304,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146421+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:54:56.146421+0000","last_peered":"2023-09-23T22:54:56.146421+0000","last_clean":"2023-09-23T22:54:56.146421+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:54:56.146421+0000","last_undegraded":"2023-09-23T22:54:56.146421+0000","last_fullsized":"2023-09-23T22:54:56.146421+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"312'11197","reported_seq":11255,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:08.357070+0000","last_change":"2023-09-23T22:51:10.509420+0000","last_active":"2023-09-23T22:55:08.357070+0000","last_peered":"2023-09-23T22:55:08.357070+0000","last_clean":"2023-09-23T22:55:08.357070+0000","last_became_active":"2023-09-23T22:51:10.509111+0000","last_became_peered":"2023-09-23T22:51:10.509111+0000","last_unstale":"2023-09-23T22:55:08.357070+0000","last_undegraded":"2023-09-23T22:55:08.357070+0000","last_fullsized":"2023-09-23T22:55:08.357070+0000","mapping_epoch":264,"log_start":"277'3600","ondisk_log_start":"277'3600","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":7597,"log_dups_size":0,"ondisk_log_size":7597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:33.397391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1900544,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11197,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":289,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.229070+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:54:56.229070+0000","last_peered":"2023-09-23T22:54:56.229070+0000","last_clean":"2023-09-23T22:54:56.229070+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:54:56.229070+0000","last_undegraded":"2023-09-23T22:54:56.229070+0000","last_fullsized":"2023-09-23T22:54:56.229070+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":260,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.229086+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:54:56.229086+0000","last_peered":"2023-09-23T22:54:56.229086+0000","last_clean":"2023-09-23T22:54:56.229086+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:54:56.229086+0000","last_undegraded":"2023-09-23T22:54:56.229086+0000","last_fullsized":"2023-09-23T22:54:56.229086+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":238,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.226885+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:54:56.226885+0000","last_peered":"2023-09-23T22:54:56.226885+0000","last_clean":"2023-09-23T22:54:56.226885+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:54:56.226885+0000","last_undegraded":"2023-09-23T22:54:56.226885+0000","last_fullsized":"2023-09-23T22:54:56.226885+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"312'10974","reported_seq":11031,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.244934+0000","last_change":"2023-09-23T22:51:10.576804+0000","last_active":"2023-09-23T22:55:09.244934+0000","last_peered":"2023-09-23T22:55:09.244934+0000","last_clean":"2023-09-23T22:55:09.244934+0000","last_became_active":"2023-09-23T22:51:10.576247+0000","last_became_peered":"2023-09-23T22:51:10.576247+0000","last_unstale":"2023-09-23T22:55:09.244934+0000","last_undegraded":"2023-09-23T22:55:09.244934+0000","last_fullsized":"2023-09-23T22:55:09.244934+0000","mapping_epoch":264,"log_start":"279'4100","ondisk_log_start":"279'4100","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10974,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":243,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.232963+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:54:56.232963+0000","last_peered":"2023-09-23T22:54:56.232963+0000","last_clean":"2023-09-23T22:54:56.232963+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:54:56.232963+0000","last_undegraded":"2023-09-23T22:54:56.232963+0000","last_fullsized":"2023-09-23T22:54:56.232963+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":193,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.316324+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:55:04.316324+0000","last_peered":"2023-09-23T22:55:04.316324+0000","last_clean":"2023-09-23T22:55:04.316324+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:55:04.316324+0000","last_undegraded":"2023-09-23T22:55:04.316324+0000","last_fullsized":"2023-09-23T22:55:04.316324+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":243,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.230592+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:54:56.230592+0000","last_peered":"2023-09-23T22:54:56.230592+0000","last_clean":"2023-09-23T22:54:56.230592+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:54:56.230592+0000","last_undegraded":"2023-09-23T22:54:56.230592+0000","last_fullsized":"2023-09-23T22:54:56.230592+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"312'11362","reported_seq":11420,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.261320+0000","last_change":"2023-09-23T22:51:10.574069+0000","last_active":"2023-09-23T22:55:09.261320+0000","last_peered":"2023-09-23T22:55:09.261320+0000","last_clean":"2023-09-23T22:55:09.261320+0000","last_became_active":"2023-09-23T22:51:10.573750+0000","last_became_peered":"2023-09-23T22:51:10.573750+0000","last_unstale":"2023-09-23T22:55:09.261320+0000","last_undegraded":"2023-09-23T22:55:09.261320+0000","last_fullsized":"2023-09-23T22:55:09.261320+0000","mapping_epoch":264,"log_start":"280'4500","ondisk_log_start":"280'4500","created":264,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6862,"log_dups_size":0,"ondisk_log_size":6862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11362,"num_write_kb":363521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.226746+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:54:56.226746+0000","last_peered":"2023-09-23T22:54:56.226746+0000","last_clean":"2023-09-23T22:54:56.226746+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:54:56.226746+0000","last_undegraded":"2023-09-23T22:54:56.226746+0000","last_fullsized":"2023-09-23T22:54:56.226746+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":110,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146824+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:54:56.146824+0000","last_peered":"2023-09-23T22:54:56.146824+0000","last_clean":"2023-09-23T22:54:56.146824+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:54:56.146824+0000","last_undegraded":"2023-09-23T22:54:56.146824+0000","last_fullsized":"2023-09-23T22:54:56.146824+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":289,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.227942+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:54:56.227942+0000","last_peered":"2023-09-23T22:54:56.227942+0000","last_clean":"2023-09-23T22:54:56.227942+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:54:56.227942+0000","last_undegraded":"2023-09-23T22:54:56.227942+0000","last_fullsized":"2023-09-23T22:54:56.227942+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"312'11292","reported_seq":14826,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:09.205811+0000","last_change":"2023-09-23T22:53:44.765920+0000","last_active":"2023-09-23T22:55:09.205811+0000","last_peered":"2023-09-23T22:55:09.205811+0000","last_clean":"2023-09-23T22:55:09.205811+0000","last_became_active":"2023-09-23T22:53:44.707193+0000","last_became_peered":"2023-09-23T22:53:44.707193+0000","last_unstale":"2023-09-23T22:55:09.205811+0000","last_undegraded":"2023-09-23T22:55:09.205811+0000","last_fullsized":"2023-09-23T22:55:09.205811+0000","mapping_epoch":296,"log_start":"280'4400","ondisk_log_start":"280'4400","created":264,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:51:09.495811+0000","last_clean_scrub_stamp":"2023-09-23T22:51:09.495811+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11292,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1729,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":353,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.147493+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:54:56.147493+0000","last_peered":"2023-09-23T22:54:56.147493+0000","last_clean":"2023-09-23T22:54:56.147493+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:54:56.147493+0000","last_undegraded":"2023-09-23T22:54:56.147493+0000","last_fullsized":"2023-09-23T22:54:56.147493+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":210,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.226961+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:54:56.226961+0000","last_peered":"2023-09-23T22:54:56.226961+0000","last_clean":"2023-09-23T22:54:56.226961+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:54:56.226961+0000","last_undegraded":"2023-09-23T22:54:56.226961+0000","last_fullsized":"2023-09-23T22:54:56.226961+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":263,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317706+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:55:04.317706+0000","last_peered":"2023-09-23T22:55:04.317706+0000","last_clean":"2023-09-23T22:55:04.317706+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:55:04.317706+0000","last_undegraded":"2023-09-23T22:55:04.317706+0000","last_fullsized":"2023-09-23T22:55:04.317706+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":245,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.230451+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:54:56.230451+0000","last_peered":"2023-09-23T22:54:56.230451+0000","last_clean":"2023-09-23T22:54:56.230451+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:54:56.230451+0000","last_undegraded":"2023-09-23T22:54:56.230451+0000","last_fullsized":"2023-09-23T22:54:56.230451+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191645+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:54:56.191645+0000","last_peered":"2023-09-23T22:54:56.191645+0000","last_clean":"2023-09-23T22:54:56.191645+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:54:56.191645+0000","last_undegraded":"2023-09-23T22:54:56.191645+0000","last_fullsized":"2023-09-23T22:54:56.191645+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":122,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.319567+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:55:04.319567+0000","last_peered":"2023-09-23T22:55:04.319567+0000","last_clean":"2023-09-23T22:55:04.319567+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:55:04.319567+0000","last_undegraded":"2023-09-23T22:55:04.319567+0000","last_fullsized":"2023-09-23T22:55:04.319567+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":321,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317891+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:55:04.317891+0000","last_peered":"2023-09-23T22:55:04.317891+0000","last_clean":"2023-09-23T22:55:04.317891+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:55:04.317891+0000","last_undegraded":"2023-09-23T22:55:04.317891+0000","last_fullsized":"2023-09-23T22:55:04.317891+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":123,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.318980+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:55:04.318980+0000","last_peered":"2023-09-23T22:55:04.318980+0000","last_clean":"2023-09-23T22:55:04.318980+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:55:04.318980+0000","last_undegraded":"2023-09-23T22:55:04.318980+0000","last_fullsized":"2023-09-23T22:55:04.318980+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":193,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.318450+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:55:04.318450+0000","last_peered":"2023-09-23T22:55:04.318450+0000","last_clean":"2023-09-23T22:55:04.318450+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:55:04.318450+0000","last_undegraded":"2023-09-23T22:55:04.318450+0000","last_fullsized":"2023-09-23T22:55:04.318450+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":62,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.146714+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:54:56.146714+0000","last_peered":"2023-09-23T22:54:56.146714+0000","last_clean":"2023-09-23T22:54:56.146714+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:54:56.146714+0000","last_undegraded":"2023-09-23T22:54:56.146714+0000","last_fullsized":"2023-09-23T22:54:56.146714+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":295,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.191419+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:54:56.191419+0000","last_peered":"2023-09-23T22:54:56.191419+0000","last_clean":"2023-09-23T22:54:56.191419+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:54:56.191419+0000","last_undegraded":"2023-09-23T22:54:56.191419+0000","last_fullsized":"2023-09-23T22:54:56.191419+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":259,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T22:54:56.226604+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:54:56.226604+0000","last_peered":"2023-09-23T22:54:56.226604+0000","last_clean":"2023-09-23T22:54:56.226604+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:54:56.226604+0000","last_undegraded":"2023-09-23T22:54:56.226604+0000","last_fullsized":"2023-09-23T22:54:56.226604+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":278,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T22:55:04.317275+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:55:04.317275+0000","last_peered":"2023-09-23T22:55:04.317275+0000","last_clean":"2023-09-23T22:55:04.317275+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:55:04.317275+0000","last_undegraded":"2023-09-23T22:55:04.317275+0000","last_fullsized":"2023-09-23T22:55:04.317275+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":9633792,"num_objects":147,"num_object_clones":0,"num_object_copies":441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":147,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179131,"num_write_kb":5736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12823,"num_bytes_recovered":313196544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":627572736,"data_stored":627572736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115381,"ondisk_log_size":115381,"up":48,"acting":45,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430546,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626774,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300548,"kb_used_data":3324,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93444604,"statfs":{"total":95995035648,"available":95687274496,"internally_reserved":0,"allocated":3403776,"data_stored":2845615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":8.0800000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.087,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.315},{"interface":"front","average":{"1min":3.3559999999999999,"5min":8.8829999999999991,"15min":4.4269999999999996},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":12.081,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":5.6390000000000002,"15min":4.1959999999999997},"min":{"1min":0.32100000000000001,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":74.899000000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.376},{"interface":"front","average":{"1min":6.4550000000000001,"5min":6.923,"15min":5.2009999999999996},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":74.956000000000003,"5min":255.20099999999999,"15min":255.20099999999999},"last":0.53900000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":7.5069999999999997,"5min":4.4580000000000002,"15min":3.738},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":81.719999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.435},{"interface":"front","average":{"1min":2.1970000000000001,"5min":3.1539999999999999,"15min":3.7509999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":9.3529999999999998,"5min":101.13,"15min":185.21799999999999},"last":0.42499999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":1.845,"15min":3.8879999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":4.1079999999999997,"5min":7.734,"15min":185.19399999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":3.262,"5min":2.4279999999999999,"15min":3.6709999999999998},"min":{"1min":0.438,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.3109999999999999,"5min":10.378,"15min":128.19800000000001},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.3049999999999999,"15min":0.93700000000000006},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.9240000000000004,"5min":10.317,"15min":18.085999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.302,"15min":0.94299999999999995},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":10.122999999999999,"5min":10.122999999999999,"15min":18.597000000000001},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.4910000000000001,"15min":1.0549999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.24399999999999999},"max":{"1min":5.351,"5min":10.353999999999999,"15min":18.972000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.325,"15min":1.0049999999999999},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":10.199,"5min":10.199,"15min":18.625},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:55:02 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.298,"15min":0.98599999999999999},"min":{"1min":0.42099999999999999,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":10.225,"5min":10.225,"15min":12.481999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.4550000000000001,"15min":1.028},"min":{"1min":0.40500000000000003,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.5549999999999997,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.50600000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253084,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529056,"kb_used_data":3928,"kb_used_omap":9,"kb_used_meta":525110,"kb_avail":93216096,"statfs":{"total":95995035648,"available":95453282304,"internally_reserved":0,"allocated":4022272,"data_stored":2923735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":537713641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":4.5519999999999996,"15min":5.4329999999999998},"min":{"1min":0.27700000000000002,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":37.991,"5min":37.991,"15min":101.605},"last":64.796999999999997},{"interface":"front","average":{"1min":6.1550000000000002,"5min":3.819,"15min":5.3019999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":42.191000000000003,"5min":42.191000000000003,"15min":101.892},"last":64.432000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":3.8900000000000001,"15min":3.2549999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":21.215,"5min":26.129000000000001,"15min":37.889000000000003},"last":0.33600000000000002},{"interface":"front","average":{"1min":3.9689999999999999,"5min":3.633,"15min":3.3239999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":21.254000000000001,"5min":26.001999999999999,"15min":37.957000000000001},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.871,"15min":2.645},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.143000000000001,"5min":12.143000000000001,"15min":87.787999999999997},"last":0.313},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.1899999999999999,"15min":3.1309999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20999999999999999},"max":{"1min":8.0099999999999998,"5min":10.845000000000001,"15min":89.861999999999995},"last":0.27900000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":1.9510000000000001,"15min":2.3900000000000001},"min":{"1min":0.38,"5min":0.35899999999999999,"15min":0.29299999999999998},"max":{"1min":11.388,"5min":11.388,"15min":78.948999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.3980000000000001,"5min":2.5659999999999998,"15min":2.4020000000000001},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.2579999999999991,"5min":14.058,"15min":78.963999999999999},"last":0.56200000000000006}]},{"osd":5,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.7110000000000001,"15min":1.8839999999999999},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":12.115,"5min":12.115,"15min":84.254999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.78,"15min":2.21},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":8.1050000000000004,"5min":9.8170000000000002,"15min":87.515000000000001},"last":0.504}]},{"osd":6,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":2.6110000000000002,"15min":2.2559999999999998},"min":{"1min":0.40699999999999997,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.6639999999999997,"5min":13.991,"15min":79.311000000000007},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.5979999999999999,"5min":1.9810000000000001,"15min":1.9059999999999999},"min":{"1min":0.41899999999999998,"5min":0.40899999999999997,"15min":0.313},"max":{"1min":12.067,"5min":12.067,"15min":79.328999999999994},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.899,"15min":2.609},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.32800000000000001},"max":{"1min":8.1400000000000006,"5min":9.9329999999999998,"15min":159.018},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.254,"5min":2.6949999999999998,"15min":2.7749999999999999},"min":{"1min":0.443,"5min":0.38900000000000001,"15min":0.37},"max":{"1min":6.6310000000000002,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.48099999999999998}]}]},{"osd":0,"up_from":219,"seq":940597837932,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347068,"kb_used_data":19956,"kb_used_omap":31,"kb_used_meta":327072,"kb_avail":93398084,"statfs":{"total":95995035648,"available":95639638016,"internally_reserved":0,"allocated":20434944,"data_stored":19303844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":334922674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":3.4329999999999998,"15min":3.7410000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.326000000000001,"5min":40.180999999999997,"15min":40.180999999999997},"last":3.48},{"interface":"front","average":{"1min":4.2270000000000003,"5min":3.4630000000000001,"15min":3.6499999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.494999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":5.8520000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":5.4139999999999997,"15min":3.5910000000000002},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":89.766000000000005,"5min":111.508,"15min":111.508},"last":15.683},{"interface":"front","average":{"1min":7.617,"5min":5.5449999999999999,"15min":3.2360000000000002},"min":{"1min":0.29499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":89.994,"5min":111.532,"15min":111.532},"last":7.1619999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":4.2519999999999998,"15min":3.0510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":8.5380000000000003,"5min":126.73,"15min":126.73},"last":5.6820000000000004},{"interface":"front","average":{"1min":2.4969999999999999,"5min":3.488,"15min":2.1349999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6440000000000001,"5min":121.483,"15min":121.483},"last":3.573}]},{"osd":4,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.0129999999999999,"15min":2.173},"min":{"1min":0.33000000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.5820000000000007,"5min":11.965999999999999,"15min":18.670999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.2400000000000002,"15min":2.036},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.5899999999999999,"5min":16.52,"15min":20.009},"last":5.7229999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.8599999999999999,"15min":1.9370000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":6.6630000000000003,"5min":61.756999999999998,"15min":61.756999999999998},"last":3.4910000000000001},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.1379999999999999,"15min":2.3490000000000002},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4240000000000004,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.778,"15min":1.95},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6950000000000003,"5min":59.146999999999998,"15min":59.146999999999998},"last":3.5390000000000001},{"interface":"front","average":{"1min":2.504,"5min":3.069,"15min":2.4550000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.367,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.1709999999999998,"15min":2.077},"min":{"1min":0.438,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.6959999999999997,"5min":12.385999999999999,"15min":18.640000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.946,"5min":2.0830000000000002,"15min":1.6000000000000001},"min":{"1min":0.42599999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.6779999999999999,"5min":21.954999999999998,"15min":21.954999999999998},"last":10.525}]}]},{"osd":2,"up_from":131,"seq":562640715945,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429144,"kb_used_data":3344,"kb_used_omap":81,"kb_used_meta":425710,"kb_avail":93316008,"statfs":{"total":95995035648,"available":95555592192,"internally_reserved":0,"allocated":3424256,"data_stored":2328543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":435927258},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":6.9960000000000004,"15min":5.734},"min":{"1min":0.22800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":12.76,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.311},{"interface":"front","average":{"1min":3.286,"5min":6.9400000000000004,"15min":6.0990000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.093,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.24299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":3.0470000000000002,"15min":3.173},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":6.3019999999999996,"5min":77.004999999999995,"15min":88.316000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.552,"5min":3.3500000000000001,"15min":3.407},"min":{"1min":0.25700000000000001,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":6.1779999999999999,"5min":76.956000000000003,"15min":88.436999999999998},"last":0.70299999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.2939999999999996,"15min":4.4100000000000001},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.492000000000001,"5min":112.895,"15min":112.895},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.7869999999999999,"5min":4.109,"15min":3.6219999999999999},"min":{"1min":0.39000000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.1880000000000006,"5min":113.995,"15min":113.995},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.6749999999999998,"15min":2.8279999999999998},"min":{"1min":0.60199999999999998,"5min":0.28899999999999998,"15min":0.253},"max":{"1min":9.1219999999999999,"5min":69.113,"15min":108.878},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.4689999999999999,"15min":3.3809999999999998},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2229999999999999,"5min":69.028999999999996,"15min":108.93000000000001},"last":0.36399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.7789999999999999,"15min":2.9940000000000002},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":6.1820000000000004,"5min":22.646000000000001,"15min":97.322999999999993},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.2569999999999999,"5min":2.226,"15min":2.98},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.1420000000000003,"5min":22.501000000000001,"15min":97.301000000000002},"last":0.50700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":3.609,"15min":3.3450000000000002},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":4.3040000000000003,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.4670000000000001,"5min":3.4209999999999998,"15min":2.7050000000000001},"min":{"1min":0.48699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":9.2110000000000003,"5min":98.509,"15min":98.509},"last":3.613}]},{"osd":7,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":2.0590000000000002,"15min":1.7509999999999999},"min":{"1min":0.48999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.1609999999999996,"5min":10.477,"15min":10.92},"last":0.624},{"interface":"front","average":{"1min":1.415,"5min":1.667,"15min":2.3799999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.37},"max":{"1min":6.2720000000000002,"5min":11.311,"15min":88.584000000000003},"last":0.45800000000000002}]}]},{"osd":3,"up_from":0,"seq":403726926031,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019230,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312260,"kb_used_data":2748,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93432892,"statfs":{"total":95995035648,"available":95675281408,"internally_reserved":0,"allocated":2813952,"data_stored":2255269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.3140000000000001,"15min":5.9390000000000001},"min":{"1min":0.251,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":5.9509999999999996,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.7229999999999999,"15min":6.5099999999999998},"min":{"1min":0.41199999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.6760000000000002,"5min":113.417,"15min":113.417},"last":0.33200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":5.0599999999999996,"15min":3.0190000000000001},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":132.304,"15min":132.304},"last":0.308},{"interface":"front","average":{"1min":2.7959999999999998,"5min":4.8970000000000002,"15min":2.879},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":10.832000000000001,"5min":132.453,"15min":132.453},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.3639999999999999,"15min":3.1520000000000001},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.097,"5min":20.308,"15min":84.909000000000006},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.524,"15min":3.137},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.352,"5min":21.704000000000001,"15min":83.840000000000003},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":4.1879999999999997,"15min":4.548},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":7.3109999999999999,"5min":164.453,"15min":164.453},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.2170000000000001,"5min":4.343,"15min":4.6689999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.1779999999999999,"15min":1},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.25900000000000001},"max":{"1min":5.0880000000000001,"5min":12.238,"15min":43.189999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1060000000000001,"15min":1},"min":{"1min":0.32900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.3079999999999998,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.0529999999999999,"15min":0.94899999999999995},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9139999999999997,"5min":7.9139999999999997,"15min":27.056000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.417,"5min":1.224,"15min":0.97799999999999998},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":5.298,"5min":12.269,"15min":26.940999999999999},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.2130000000000001,"15min":0.89400000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":3.544,"5min":12.311,"15min":12.311},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.95499999999999996,"15min":0.80600000000000005},"min":{"1min":0.36399999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":3.0169999999999999,"5min":6.7919999999999998,"15min":8.6739999999999995},"last":0.63200000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823002,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":600616,"kb_used_data":2720,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":93144536,"statfs":{"total":95995035648,"available":95380004864,"internally_reserved":0,"allocated":2785280,"data_stored":2250313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":9.6929999999999996,"5min":8.1189999999999998,"15min":5.133},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":80.450999999999993,"5min":185.62,"15min":185.62},"last":0.29999999999999999},{"interface":"front","average":{"1min":9.734,"5min":8.1829999999999998,"15min":5.1079999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":80.171999999999997,"5min":185.499,"15min":185.499},"last":4.3140000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":9.452,"5min":3.9129999999999998,"15min":2.8799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":110.36499999999999,"5min":110.36499999999999,"15min":149.203},"last":0.38400000000000001},{"interface":"front","average":{"1min":9.5340000000000007,"5min":4.3019999999999996,"15min":3.1909999999999998},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":110.89,"5min":110.89,"15min":149.554},"last":4.2629999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":5.54,"15min":5.5140000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.997,"5min":189.126,"15min":189.126},"last":4.2880000000000003},{"interface":"front","average":{"1min":3.5379999999999998,"5min":5.2309999999999999,"15min":5.4059999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":41.070999999999998,"5min":188.74799999999999,"15min":188.74799999999999},"last":3.9670000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.843,"15min":5.6660000000000004},"min":{"1min":0.34899999999999998,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.298999999999999,"5min":98.641000000000005,"15min":257.05000000000001},"last":4},{"interface":"front","average":{"1min":2.085,"5min":4.9130000000000003,"15min":5.516},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27800000000000002},"max":{"1min":11.097,"5min":98.906000000000006,"15min":257.35000000000002},"last":4.0940000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.86899999999999999,"15min":0.73399999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":3.669,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":4.2400000000000002},{"interface":"front","average":{"1min":0.996,"5min":0.92100000000000004,"15min":0.75700000000000001},"min":{"1min":0.29399999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":5.8019999999999996,"5min":6.3550000000000004,"15min":10.535},"last":5.9370000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1759999999999999,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.3369999999999997,"5min":8.0830000000000002,"15min":140.07499999999999},"last":4.3360000000000003},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.139,"15min":1.629},"min":{"1min":0.374,"5min":0.315,"15min":0.254},"max":{"1min":7.9909999999999997,"5min":10.314,"15min":139.739},"last":4.1609999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:55:06 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.2749999999999999,"15min":0.94999999999999996},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.29999999999999999},"max":{"1min":6.3579999999999997,"5min":10.367000000000001,"15min":10.367000000000001},"last":4.1390000000000002},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.125,"15min":0.85999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36199999999999999,"15min":0.311},"max":{"1min":5.8719999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":4.3570000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16252928,"data_stored":16252928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":611319808,"data_stored":611319808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:55:15.048 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:55:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:15.050 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:15.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:15.059 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:55:15.059 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:55:15.152 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:15.157 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_3' removed 2023-09-23T22:55:15.193 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:15.403 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:15.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:15.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:15.506 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:15.532 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: failed to encode map e314 with expected crc 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: pgmap v915: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 413 op/s; 3414/72 objects degraded (4741.667%); 0 B/s, 105 objects/s recovering 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3175817966' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 3414/72 objects degraded (4741.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: failed to encode map e315 with expected crc 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:55:15.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[187743]: osdmap e315: 8 total, 8 up, 6 in 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: failed to encode map e314 with expected crc 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: pgmap v915: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 413 op/s; 3414/72 objects degraded (4741.667%); 0 B/s, 105 objects/s recovering 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3175817966' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 3414/72 objects degraded (4741.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: failed to encode map e315 with expected crc 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:55:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:15 smithi052 ceph-mon[180979]: osdmap e315: 8 total, 8 up, 6 in 2023-09-23T22:55:15.778 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:15.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:15.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:15.825 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: failed to encode map e314 with expected crc 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: pgmap v915: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 413 op/s; 3414/72 objects degraded (4741.667%); 0 B/s, 105 objects/s recovering 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3175817966' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 3414/72 objects degraded (4741.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: failed to encode map e315 with expected crc 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:55:15.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:15 smithi151 ceph-mon[159210]: osdmap e315: 8 total, 8 up, 6 in 2023-09-23T22:55:15.838 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:55:15.882 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:15.998 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:16.165 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:55:16.217 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:16.453 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:16.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:16.466 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:55:16.466 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:55:16.467 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T22:55:16.467 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T22:55:16.557 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:16.593 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:16.696 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:16 smithi151 ceph-mon[159210]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:16 smithi151 ceph-mon[159210]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:16 smithi151 ceph-mon[159210]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.700 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:16 smithi052 ceph-mon[187743]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.700 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:16 smithi052 ceph-mon[187743]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.701 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:16 smithi052 ceph-mon[187743]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.701 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:16 smithi052 ceph-mon[180979]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.701 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:16 smithi052 ceph-mon[180979]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.701 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:16 smithi052 ceph-mon[180979]: failed to encode map e315 with expected crc 2023-09-23T22:55:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:16.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:16.723 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:16.803 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:16.832 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:16.877 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:16.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:16.966 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:16.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:17.016 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:17.027 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:17.054 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:17.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:17.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:17.080 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:55:17.100 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:17.118 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:17.135 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:17.154 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:17.173 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:17.174 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.175 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:55:17.180 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:17.209 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:17.649 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:55:17.650 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:55:17.650 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:55:17.650 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:55:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:17 smithi052 ceph-mon[187743]: pgmap v917: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:17 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3414/72 objects degraded (4741.667%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T22:55:17.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/492822966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:17.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:17 smithi052 ceph-mon[180979]: pgmap v917: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:17 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3414/72 objects degraded (4741.667%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T22:55:17.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/492822966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:17.807 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:17 smithi151 ceph-mon[159210]: pgmap v917: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:17.807 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:17 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3414/72 objects degraded (4741.667%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T22:55:17.807 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/492822966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:17.828 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:17.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:17.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:17.932 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:17.963 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:18.114 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:18.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:18.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:18.218 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:18.249 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:18.322 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:55:18.344 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:18.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:18.449 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:18.473 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:18.482 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:18.689 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:19.218 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:19.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:19.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:19.322 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:19.350 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:19.594 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:19.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:19.698 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:19.725 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:19.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:19 smithi052 ceph-mon[180979]: pgmap v918: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:19.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:19.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:19.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:19 smithi052 ceph-mon[187743]: pgmap v918: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:19.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:19.730 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:19.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:19 smithi151 ceph-mon[159210]: pgmap v918: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:19.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:19.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:19.818 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:19.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:19.891 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:55:19.891 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1252,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":315,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509173,"num_in_osds":6,"osd_in_since":1695509493,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":2567106560,"bytes_avail":573403107328,"bytes_total":575970213888},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T22:53:02.381180+0000","services":{}},"progress_events":{"6b27ada5-699f-4815-a812-68a0dbe12316":{"message":"Global Recovery Event (3m)\n [===========================.] (remaining: 7s)","progress":0.96629214286804199,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:55:19.923 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:19.951 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:19.997 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:20.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:20.025 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:55:20.025 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:55:20.025 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:55:20.025 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:55:20.025 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:55:20.101 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:20.129 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:20.222 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:20.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:20.326 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:20.358 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:20.639 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:55:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2784264614' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:55:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2830234855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:55:20.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:20 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:55:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2784264614' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:55:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2830234855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:55:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:20 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:55:20.909 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T22:55:20.910 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:55:20.919 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:20.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:20.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2784264614' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:55:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2830234855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:55:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:20 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:55:20.980 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:55:21.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:21.053 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:21.119 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:21.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:21.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:21.135 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:55:21.162 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:55:21.223 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:21.270 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:21.824 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:21.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[187743]: pgmap v919: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:21.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:21.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:55:21.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[187743]: osdmap e316: 8 total, 8 up, 6 in 2023-09-23T22:55:21.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[180979]: pgmap v919: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:21.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:21.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:55:21.826 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:21 smithi052 ceph-mon[180979]: osdmap e316: 8 total, 8 up, 6 in 2023-09-23T22:55:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:21.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:21.928 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:21.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:21 smithi151 ceph-mon[159210]: pgmap v919: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:21.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:21 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:21.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:21 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:55:21.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:21 smithi151 ceph-mon[159210]: osdmap e316: 8 total, 8 up, 6 in 2023-09-23T22:55:21.956 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:21.997 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:22.004 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:22.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:22.109 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:22.137 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:22.151 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:22.184 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:22.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:22.287 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:22.317 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:22.361 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:22.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:22.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:22.394 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:55:22.394 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:55:22.394 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:55:22.394 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [] 2023-09-23T22:55:22.394 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:55:22.469 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:22.502 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:22.559 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_4' created 2023-09-23T22:55:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:22 smithi151 ceph-mon[159210]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:22 smithi151 ceph-mon[159210]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:22 smithi151 ceph-mon[159210]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:22 smithi151 ceph-mon[159210]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/936652597' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:55:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[187743]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[187743]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[187743]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[187743]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/936652597' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:55:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[180979]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[180979]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[180979]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[180979]: failed to encode map e316 with expected crc 2023-09-23T22:55:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/936652597' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:55:23.015 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:55:23.256 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:23.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:23.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:23.388 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:55:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:55:23] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.33.4" 2023-09-23T22:55:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[187743]: pgmap v921: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[187743]: failed to encode map e317 with expected crc 2023-09-23T22:55:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/936652597' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T22:55:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[187743]: osdmap e317: 8 total, 8 up, 6 in 2023-09-23T22:55:23.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[187743]: failed to encode map e317 with expected crc 2023-09-23T22:55:23.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[180979]: pgmap v921: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:23.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[180979]: failed to encode map e317 with expected crc 2023-09-23T22:55:23.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/936652597' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T22:55:23.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[180979]: osdmap e317: 8 total, 8 up, 6 in 2023-09-23T22:55:23.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:23 smithi052 ceph-mon[180979]: failed to encode map e317 with expected crc 2023-09-23T22:55:23.918 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:55:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:23 smithi151 ceph-mon[159210]: pgmap v921: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:23 smithi151 ceph-mon[159210]: failed to encode map e317 with expected crc 2023-09-23T22:55:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/936652597' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T22:55:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:23 smithi151 ceph-mon[159210]: osdmap e317: 8 total, 8 up, 6 in 2023-09-23T22:55:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:23 smithi151 ceph-mon[159210]: failed to encode map e317 with expected crc 2023-09-23T22:55:23.953 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:23.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:23.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:24.057 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:24.090 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:24.160 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T22:55:24.248 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:24.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:24.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:24.353 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:24.372 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:24.389 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:24.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:24.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:24.512 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:24.530 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:24.547 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:24.550 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:55:24.575 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:24.581 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:24.595 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:24.619 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:24.643 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:24.671 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:24.672 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.673 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:55:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:24 smithi151 ceph-mon[159210]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:24 smithi151 ceph-mon[159210]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:24 smithi151 ceph-mon[159210]: osdmap e318: 8 total, 8 up, 6 in 2023-09-23T22:55:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:24 smithi151 ceph-mon[159210]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:24 smithi151 ceph-mon[159210]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3035800574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:24.704 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:24.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:24.810 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:24.841 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:24.887 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:24.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:24.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:24.931 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:55:24.932 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:55:24.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[187743]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[187743]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[187743]: osdmap e318: 8 total, 8 up, 6 in 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[187743]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[187743]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3035800574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[180979]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[180979]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[180979]: osdmap e318: 8 total, 8 up, 6 in 2023-09-23T22:55:24.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[180979]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[180979]: failed to encode map e318 with expected crc 2023-09-23T22:55:24.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3035800574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:24.992 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:25.020 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:25.568 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:25.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:25.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:25.672 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:25.701 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:25 smithi151 ceph-mon[159210]: pgmap v924: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:25 smithi151 ceph-mon[159210]: failed to encode map e318 with expected crc 2023-09-23T22:55:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:25 smithi151 ceph-mon[159210]: failed to encode map e319 with expected crc 2023-09-23T22:55:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:25 smithi151 ceph-mon[159210]: osdmap e319: 8 total, 8 up, 6 in 2023-09-23T22:55:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[187743]: pgmap v924: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[187743]: failed to encode map e318 with expected crc 2023-09-23T22:55:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[187743]: failed to encode map e319 with expected crc 2023-09-23T22:55:25.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[187743]: osdmap e319: 8 total, 8 up, 6 in 2023-09-23T22:55:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[180979]: pgmap v924: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[180979]: failed to encode map e318 with expected crc 2023-09-23T22:55:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[180979]: failed to encode map e319 with expected crc 2023-09-23T22:55:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[180979]: osdmap e319: 8 total, 8 up, 6 in 2023-09-23T22:55:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:26.048 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":925,"stamp":"2023-09-23T22:55:24.563087+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":228,"acting":228,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2507004,"kb_used_data":20236,"kb_used_omap":121,"kb_used_meta":2486598,"kb_avail":559963908,"statfs":{"total":575970213888,"available":573403041792,"internally_reserved":0,"allocated":20721664,"data_stored":15677634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2546276469},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":24,"apply_latency_ms":24,"commit_latency_ns":24000000,"apply_latency_ns":24000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.109827"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":313,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850948+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:55:15.850948+0000","last_peered":"2023-09-23T22:55:15.850948+0000","last_clean":"2023-09-23T22:55:15.850948+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:55:15.850948+0000","last_undegraded":"2023-09-23T22:55:15.850948+0000","last_fullsized":"2023-09-23T22:55:15.850948+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":112,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.852121+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:55:15.852121+0000","last_peered":"2023-09-23T22:55:15.852121+0000","last_clean":"2023-09-23T22:55:15.852121+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:55:15.852121+0000","last_undegraded":"2023-09-23T22:55:15.852121+0000","last_fullsized":"2023-09-23T22:55:15.852121+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":98,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850902+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:55:15.850902+0000","last_peered":"2023-09-23T22:55:15.850902+0000","last_clean":"2023-09-23T22:55:15.850902+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:55:15.850902+0000","last_undegraded":"2023-09-23T22:55:15.850902+0000","last_fullsized":"2023-09-23T22:55:15.850902+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":296,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646799+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:55:21.646799+0000","last_peered":"2023-09-23T22:55:21.646799+0000","last_clean":"2023-09-23T22:55:21.646799+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:55:21.646799+0000","last_undegraded":"2023-09-23T22:55:21.646799+0000","last_fullsized":"2023-09-23T22:55:21.646799+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":300,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646573+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:55:21.646573+0000","last_peered":"2023-09-23T22:55:21.646573+0000","last_clean":"2023-09-23T22:55:21.646573+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:55:21.646573+0000","last_undegraded":"2023-09-23T22:55:21.646573+0000","last_fullsized":"2023-09-23T22:55:21.646573+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":280,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139526+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:55:13.139526+0000","last_peered":"2023-09-23T22:55:13.139526+0000","last_clean":"2023-09-23T22:55:13.139526+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:55:13.139526+0000","last_undegraded":"2023-09-23T22:55:13.139526+0000","last_fullsized":"2023-09-23T22:55:13.139526+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":227,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647126+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:55:21.647126+0000","last_peered":"2023-09-23T22:55:21.647126+0000","last_clean":"2023-09-23T22:55:21.647126+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:55:21.647126+0000","last_undegraded":"2023-09-23T22:55:21.647126+0000","last_fullsized":"2023-09-23T22:55:21.647126+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":177,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196157+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:55:13.196157+0000","last_peered":"2023-09-23T22:55:13.196157+0000","last_clean":"2023-09-23T22:55:13.196157+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:55:13.196157+0000","last_undegraded":"2023-09-23T22:55:13.196157+0000","last_fullsized":"2023-09-23T22:55:13.196157+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":56,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196467+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:55:13.196467+0000","last_peered":"2023-09-23T22:55:13.196467+0000","last_clean":"2023-09-23T22:55:13.196467+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:55:13.196467+0000","last_undegraded":"2023-09-23T22:55:13.196467+0000","last_fullsized":"2023-09-23T22:55:13.196467+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139021+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:55:13.139021+0000","last_peered":"2023-09-23T22:55:13.139021+0000","last_clean":"2023-09-23T22:55:13.139021+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:55:13.139021+0000","last_undegraded":"2023-09-23T22:55:13.139021+0000","last_fullsized":"2023-09-23T22:55:13.139021+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.852013+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:55:15.852013+0000","last_peered":"2023-09-23T22:55:15.852013+0000","last_clean":"2023-09-23T22:55:15.852013+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:55:15.852013+0000","last_undegraded":"2023-09-23T22:55:15.852013+0000","last_fullsized":"2023-09-23T22:55:15.852013+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":189,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139224+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:55:13.139224+0000","last_peered":"2023-09-23T22:55:13.139224+0000","last_clean":"2023-09-23T22:55:13.139224+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:55:13.139224+0000","last_undegraded":"2023-09-23T22:55:13.139224+0000","last_fullsized":"2023-09-23T22:55:13.139224+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":335,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.854449+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:55:15.854449+0000","last_peered":"2023-09-23T22:55:15.854449+0000","last_clean":"2023-09-23T22:55:15.854449+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:55:15.854449+0000","last_undegraded":"2023-09-23T22:55:15.854449+0000","last_fullsized":"2023-09-23T22:55:15.854449+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":281,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851421+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:55:15.851421+0000","last_peered":"2023-09-23T22:55:15.851421+0000","last_clean":"2023-09-23T22:55:15.851421+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:55:15.851421+0000","last_undegraded":"2023-09-23T22:55:15.851421+0000","last_fullsized":"2023-09-23T22:55:15.851421+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":174,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851464+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:55:15.851464+0000","last_peered":"2023-09-23T22:55:15.851464+0000","last_clean":"2023-09-23T22:55:15.851464+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:55:15.851464+0000","last_undegraded":"2023-09-23T22:55:15.851464+0000","last_fullsized":"2023-09-23T22:55:15.851464+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":262,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196369+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:55:13.196369+0000","last_peered":"2023-09-23T22:55:13.196369+0000","last_clean":"2023-09-23T22:55:13.196369+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:55:13.196369+0000","last_undegraded":"2023-09-23T22:55:13.196369+0000","last_fullsized":"2023-09-23T22:55:13.196369+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":283,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646491+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:55:21.646491+0000","last_peered":"2023-09-23T22:55:21.646491+0000","last_clean":"2023-09-23T22:55:21.646491+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:55:21.646491+0000","last_undegraded":"2023-09-23T22:55:21.646491+0000","last_fullsized":"2023-09-23T22:55:21.646491+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":238,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139458+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:55:13.139458+0000","last_peered":"2023-09-23T22:55:13.139458+0000","last_clean":"2023-09-23T22:55:13.139458+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:55:13.139458+0000","last_undegraded":"2023-09-23T22:55:13.139458+0000","last_fullsized":"2023-09-23T22:55:13.139458+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":166,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851055+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:55:15.851055+0000","last_peered":"2023-09-23T22:55:15.851055+0000","last_clean":"2023-09-23T22:55:15.851055+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:55:15.851055+0000","last_undegraded":"2023-09-23T22:55:15.851055+0000","last_fullsized":"2023-09-23T22:55:15.851055+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":284,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646873+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:55:21.646873+0000","last_peered":"2023-09-23T22:55:21.646873+0000","last_clean":"2023-09-23T22:55:21.646873+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:55:21.646873+0000","last_undegraded":"2023-09-23T22:55:21.646873+0000","last_fullsized":"2023-09-23T22:55:21.646873+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":359,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.138699+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:55:13.138699+0000","last_peered":"2023-09-23T22:55:13.138699+0000","last_clean":"2023-09-23T22:55:13.138699+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:55:13.138699+0000","last_undegraded":"2023-09-23T22:55:13.138699+0000","last_fullsized":"2023-09-23T22:55:13.138699+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":242,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196800+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:55:13.196800+0000","last_peered":"2023-09-23T22:55:13.196800+0000","last_clean":"2023-09-23T22:55:13.196800+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:55:13.196800+0000","last_undegraded":"2023-09-23T22:55:13.196800+0000","last_fullsized":"2023-09-23T22:55:13.196800+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":1,"reported_epoch":317,"state":"creating+peering","last_fresh":"2023-09-23T22:55:22.989253+0000","last_change":"2023-09-23T22:55:22.989253+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.989253+0000","last_undegraded":"2023-09-23T22:55:22.989253+0000","last_fullsized":"2023-09-23T22:55:22.989253+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:40.611676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":185,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.852220+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:55:15.852220+0000","last_peered":"2023-09-23T22:55:15.852220+0000","last_clean":"2023-09-23T22:55:15.852220+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:55:15.852220+0000","last_undegraded":"2023-09-23T22:55:15.852220+0000","last_fullsized":"2023-09-23T22:55:15.852220+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.138788+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:55:13.138788+0000","last_peered":"2023-09-23T22:55:13.138788+0000","last_clean":"2023-09-23T22:55:13.138788+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:55:13.138788+0000","last_undegraded":"2023-09-23T22:55:13.138788+0000","last_fullsized":"2023-09-23T22:55:13.138788+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":195,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851861+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:55:15.851861+0000","last_peered":"2023-09-23T22:55:15.851861+0000","last_clean":"2023-09-23T22:55:15.851861+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:55:15.851861+0000","last_undegraded":"2023-09-23T22:55:15.851861+0000","last_fullsized":"2023-09-23T22:55:15.851861+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":305,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850976+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:55:15.850976+0000","last_peered":"2023-09-23T22:55:15.850976+0000","last_clean":"2023-09-23T22:55:15.850976+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:55:15.850976+0000","last_undegraded":"2023-09-23T22:55:15.850976+0000","last_fullsized":"2023-09-23T22:55:15.850976+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":1,"reported_epoch":317,"state":"creating+peering","last_fresh":"2023-09-23T22:55:22.990948+0000","last_change":"2023-09-23T22:55:22.990948+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.990948+0000","last_undegraded":"2023-09-23T22:55:22.990948+0000","last_fullsized":"2023-09-23T22:55:22.990948+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:46.749922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":138,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139462+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:55:13.139462+0000","last_peered":"2023-09-23T22:55:13.139462+0000","last_clean":"2023-09-23T22:55:13.139462+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:55:13.139462+0000","last_undegraded":"2023-09-23T22:55:13.139462+0000","last_fullsized":"2023-09-23T22:55:13.139462+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":307,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139148+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:55:13.139148+0000","last_peered":"2023-09-23T22:55:13.139148+0000","last_clean":"2023-09-23T22:55:13.139148+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:55:13.139148+0000","last_undegraded":"2023-09-23T22:55:13.139148+0000","last_fullsized":"2023-09-23T22:55:13.139148+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139293+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:55:13.139293+0000","last_peered":"2023-09-23T22:55:13.139293+0000","last_clean":"2023-09-23T22:55:13.139293+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:55:13.139293+0000","last_undegraded":"2023-09-23T22:55:13.139293+0000","last_fullsized":"2023-09-23T22:55:13.139293+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":212,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647317+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:55:21.647317+0000","last_peered":"2023-09-23T22:55:21.647317+0000","last_clean":"2023-09-23T22:55:21.647317+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:55:21.647317+0000","last_undegraded":"2023-09-23T22:55:21.647317+0000","last_fullsized":"2023-09-23T22:55:21.647317+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":83,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647363+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:55:21.647363+0000","last_peered":"2023-09-23T22:55:21.647363+0000","last_clean":"2023-09-23T22:55:21.647363+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:55:21.647363+0000","last_undegraded":"2023-09-23T22:55:21.647363+0000","last_fullsized":"2023-09-23T22:55:21.647363+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":289,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.854378+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:55:15.854378+0000","last_peered":"2023-09-23T22:55:15.854378+0000","last_clean":"2023-09-23T22:55:15.854378+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:55:15.854378+0000","last_undegraded":"2023-09-23T22:55:15.854378+0000","last_fullsized":"2023-09-23T22:55:15.854378+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":299,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.138988+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:55:13.138988+0000","last_peered":"2023-09-23T22:55:13.138988+0000","last_clean":"2023-09-23T22:55:13.138988+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:55:13.138988+0000","last_undegraded":"2023-09-23T22:55:13.138988+0000","last_fullsized":"2023-09-23T22:55:13.138988+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":311,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851844+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:55:15.851844+0000","last_peered":"2023-09-23T22:55:15.851844+0000","last_clean":"2023-09-23T22:55:15.851844+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:55:15.851844+0000","last_undegraded":"2023-09-23T22:55:15.851844+0000","last_fullsized":"2023-09-23T22:55:15.851844+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":324,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850815+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:55:15.850815+0000","last_peered":"2023-09-23T22:55:15.850815+0000","last_clean":"2023-09-23T22:55:15.850815+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:55:15.850815+0000","last_undegraded":"2023-09-23T22:55:15.850815+0000","last_fullsized":"2023-09-23T22:55:15.850815+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":299,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851930+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:55:15.851930+0000","last_peered":"2023-09-23T22:55:15.851930+0000","last_clean":"2023-09-23T22:55:15.851930+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:55:15.851930+0000","last_undegraded":"2023-09-23T22:55:15.851930+0000","last_fullsized":"2023-09-23T22:55:15.851930+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850731+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:55:15.850731+0000","last_peered":"2023-09-23T22:55:15.850731+0000","last_clean":"2023-09-23T22:55:15.850731+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:55:15.850731+0000","last_undegraded":"2023-09-23T22:55:15.850731+0000","last_fullsized":"2023-09-23T22:55:15.850731+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":253,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196943+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:55:13.196943+0000","last_peered":"2023-09-23T22:55:13.196943+0000","last_clean":"2023-09-23T22:55:13.196943+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:55:13.196943+0000","last_undegraded":"2023-09-23T22:55:13.196943+0000","last_fullsized":"2023-09-23T22:55:13.196943+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":356,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647871+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:55:21.647871+0000","last_peered":"2023-09-23T22:55:21.647871+0000","last_clean":"2023-09-23T22:55:21.647871+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:55:21.647871+0000","last_undegraded":"2023-09-23T22:55:21.647871+0000","last_fullsized":"2023-09-23T22:55:21.647871+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196528+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:55:13.196528+0000","last_peered":"2023-09-23T22:55:13.196528+0000","last_clean":"2023-09-23T22:55:13.196528+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:55:13.196528+0000","last_undegraded":"2023-09-23T22:55:13.196528+0000","last_fullsized":"2023-09-23T22:55:13.196528+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":147,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851304+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:55:15.851304+0000","last_peered":"2023-09-23T22:55:15.851304+0000","last_clean":"2023-09-23T22:55:15.851304+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:55:15.851304+0000","last_undegraded":"2023-09-23T22:55:15.851304+0000","last_fullsized":"2023-09-23T22:55:15.851304+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":149,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139068+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:55:13.139068+0000","last_peered":"2023-09-23T22:55:13.139068+0000","last_clean":"2023-09-23T22:55:13.139068+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:55:13.139068+0000","last_undegraded":"2023-09-23T22:55:13.139068+0000","last_fullsized":"2023-09-23T22:55:13.139068+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":312,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851504+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:55:15.851504+0000","last_peered":"2023-09-23T22:55:15.851504+0000","last_clean":"2023-09-23T22:55:15.851504+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:55:15.851504+0000","last_undegraded":"2023-09-23T22:55:15.851504+0000","last_fullsized":"2023-09-23T22:55:15.851504+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":242,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196389+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:55:13.196389+0000","last_peered":"2023-09-23T22:55:13.196389+0000","last_clean":"2023-09-23T22:55:13.196389+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:55:13.196389+0000","last_undegraded":"2023-09-23T22:55:13.196389+0000","last_fullsized":"2023-09-23T22:55:13.196389+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":317,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196242+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:55:13.196242+0000","last_peered":"2023-09-23T22:55:13.196242+0000","last_clean":"2023-09-23T22:55:13.196242+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:55:13.196242+0000","last_undegraded":"2023-09-23T22:55:13.196242+0000","last_fullsized":"2023-09-23T22:55:13.196242+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":197,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196077+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:55:13.196077+0000","last_peered":"2023-09-23T22:55:13.196077+0000","last_clean":"2023-09-23T22:55:13.196077+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:55:13.196077+0000","last_undegraded":"2023-09-23T22:55:13.196077+0000","last_fullsized":"2023-09-23T22:55:13.196077+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":324,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647026+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:55:21.647026+0000","last_peered":"2023-09-23T22:55:21.647026+0000","last_clean":"2023-09-23T22:55:21.647026+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:55:21.647026+0000","last_undegraded":"2023-09-23T22:55:21.647026+0000","last_fullsized":"2023-09-23T22:55:21.647026+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":327,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196338+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:55:13.196338+0000","last_peered":"2023-09-23T22:55:13.196338+0000","last_clean":"2023-09-23T22:55:13.196338+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:55:13.196338+0000","last_undegraded":"2023-09-23T22:55:13.196338+0000","last_fullsized":"2023-09-23T22:55:13.196338+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":406,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646490+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:55:21.646490+0000","last_peered":"2023-09-23T22:55:21.646490+0000","last_clean":"2023-09-23T22:55:21.646490+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:55:21.646490+0000","last_undegraded":"2023-09-23T22:55:21.646490+0000","last_fullsized":"2023-09-23T22:55:21.646490+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139202+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:55:13.139202+0000","last_peered":"2023-09-23T22:55:13.139202+0000","last_clean":"2023-09-23T22:55:13.139202+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:55:13.139202+0000","last_undegraded":"2023-09-23T22:55:13.139202+0000","last_fullsized":"2023-09-23T22:55:13.139202+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139361+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:55:13.139361+0000","last_peered":"2023-09-23T22:55:13.139361+0000","last_clean":"2023-09-23T22:55:13.139361+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:55:13.139361+0000","last_undegraded":"2023-09-23T22:55:13.139361+0000","last_fullsized":"2023-09-23T22:55:13.139361+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":1,"reported_epoch":317,"state":"creating+peering","last_fresh":"2023-09-23T22:55:22.991327+0000","last_change":"2023-09-23T22:55:22.991327+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.991327+0000","last_undegraded":"2023-09-23T22:55:22.991327+0000","last_fullsized":"2023-09-23T22:55:22.991327+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:37.219673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":294,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646664+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:55:21.646664+0000","last_peered":"2023-09-23T22:55:21.646664+0000","last_clean":"2023-09-23T22:55:21.646664+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:55:21.646664+0000","last_undegraded":"2023-09-23T22:55:21.646664+0000","last_fullsized":"2023-09-23T22:55:21.646664+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":265,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647555+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:55:21.647555+0000","last_peered":"2023-09-23T22:55:21.647555+0000","last_clean":"2023-09-23T22:55:21.647555+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:55:21.647555+0000","last_undegraded":"2023-09-23T22:55:21.647555+0000","last_fullsized":"2023-09-23T22:55:21.647555+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":243,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646637+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:55:21.646637+0000","last_peered":"2023-09-23T22:55:21.646637+0000","last_clean":"2023-09-23T22:55:21.646637+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:55:21.646637+0000","last_undegraded":"2023-09-23T22:55:21.646637+0000","last_fullsized":"2023-09-23T22:55:21.646637+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":248,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647760+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:55:21.647760+0000","last_peered":"2023-09-23T22:55:21.647760+0000","last_clean":"2023-09-23T22:55:21.647760+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:55:21.647760+0000","last_undegraded":"2023-09-23T22:55:21.647760+0000","last_fullsized":"2023-09-23T22:55:21.647760+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":196,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196481+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:55:13.196481+0000","last_peered":"2023-09-23T22:55:13.196481+0000","last_clean":"2023-09-23T22:55:13.196481+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:55:13.196481+0000","last_undegraded":"2023-09-23T22:55:13.196481+0000","last_fullsized":"2023-09-23T22:55:13.196481+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":248,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647474+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:55:21.647474+0000","last_peered":"2023-09-23T22:55:21.647474+0000","last_clean":"2023-09-23T22:55:21.647474+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:55:21.647474+0000","last_undegraded":"2023-09-23T22:55:21.647474+0000","last_fullsized":"2023-09-23T22:55:21.647474+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647267+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:55:21.647267+0000","last_peered":"2023-09-23T22:55:21.647267+0000","last_clean":"2023-09-23T22:55:21.647267+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:55:21.647267+0000","last_undegraded":"2023-09-23T22:55:21.647267+0000","last_fullsized":"2023-09-23T22:55:21.647267+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":113,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139428+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:55:13.139428+0000","last_peered":"2023-09-23T22:55:13.139428+0000","last_clean":"2023-09-23T22:55:13.139428+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:55:13.139428+0000","last_undegraded":"2023-09-23T22:55:13.139428+0000","last_fullsized":"2023-09-23T22:55:13.139428+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":294,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646317+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:55:21.646317+0000","last_peered":"2023-09-23T22:55:21.646317+0000","last_clean":"2023-09-23T22:55:21.646317+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:55:21.646317+0000","last_undegraded":"2023-09-23T22:55:21.646317+0000","last_fullsized":"2023-09-23T22:55:21.646317+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:55:22.547749+0000","last_change":"2023-09-23T22:55:22.547749+0000","last_active":"2023-09-23T22:55:22.547749+0000","last_peered":"2023-09-23T22:55:22.547749+0000","last_clean":"2023-09-23T22:55:22.547749+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:55:22.547749+0000","last_undegraded":"2023-09-23T22:55:22.547749+0000","last_fullsized":"2023-09-23T22:55:22.547749+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":356,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139452+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:55:13.139452+0000","last_peered":"2023-09-23T22:55:13.139452+0000","last_clean":"2023-09-23T22:55:13.139452+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:55:13.139452+0000","last_undegraded":"2023-09-23T22:55:13.139452+0000","last_fullsized":"2023-09-23T22:55:13.139452+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":215,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.646718+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:55:21.646718+0000","last_peered":"2023-09-23T22:55:21.646718+0000","last_clean":"2023-09-23T22:55:21.646718+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:55:21.646718+0000","last_undegraded":"2023-09-23T22:55:21.646718+0000","last_fullsized":"2023-09-23T22:55:21.646718+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":266,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850659+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:55:15.850659+0000","last_peered":"2023-09-23T22:55:15.850659+0000","last_clean":"2023-09-23T22:55:15.850659+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:55:15.850659+0000","last_undegraded":"2023-09-23T22:55:15.850659+0000","last_fullsized":"2023-09-23T22:55:15.850659+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":250,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647407+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:55:21.647407+0000","last_peered":"2023-09-23T22:55:21.647407+0000","last_clean":"2023-09-23T22:55:21.647407+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:55:21.647407+0000","last_undegraded":"2023-09-23T22:55:21.647407+0000","last_fullsized":"2023-09-23T22:55:21.647407+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":98,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196466+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:55:13.196466+0000","last_peered":"2023-09-23T22:55:13.196466+0000","last_clean":"2023-09-23T22:55:13.196466+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:55:13.196466+0000","last_undegraded":"2023-09-23T22:55:13.196466+0000","last_fullsized":"2023-09-23T22:55:13.196466+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":125,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.854311+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:55:15.854311+0000","last_peered":"2023-09-23T22:55:15.854311+0000","last_clean":"2023-09-23T22:55:15.854311+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:55:15.854311+0000","last_undegraded":"2023-09-23T22:55:15.854311+0000","last_fullsized":"2023-09-23T22:55:15.854311+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":324,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851611+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:55:15.851611+0000","last_peered":"2023-09-23T22:55:15.851611+0000","last_clean":"2023-09-23T22:55:15.851611+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:55:15.851611+0000","last_undegraded":"2023-09-23T22:55:15.851611+0000","last_fullsized":"2023-09-23T22:55:15.851611+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":126,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851927+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:55:15.851927+0000","last_peered":"2023-09-23T22:55:15.851927+0000","last_clean":"2023-09-23T22:55:15.851927+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:55:15.851927+0000","last_undegraded":"2023-09-23T22:55:15.851927+0000","last_fullsized":"2023-09-23T22:55:15.851927+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":196,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.851353+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:55:15.851353+0000","last_peered":"2023-09-23T22:55:15.851353+0000","last_clean":"2023-09-23T22:55:15.851353+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:55:15.851353+0000","last_undegraded":"2023-09-23T22:55:15.851353+0000","last_fullsized":"2023-09-23T22:55:15.851353+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":65,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.139365+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:55:13.139365+0000","last_peered":"2023-09-23T22:55:13.139365+0000","last_clean":"2023-09-23T22:55:13.139365+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:55:13.139365+0000","last_undegraded":"2023-09-23T22:55:13.139365+0000","last_fullsized":"2023-09-23T22:55:13.139365+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":298,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-09-23T22:55:13.196160+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:55:13.196160+0000","last_peered":"2023-09-23T22:55:13.196160+0000","last_clean":"2023-09-23T22:55:13.196160+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:55:13.196160+0000","last_undegraded":"2023-09-23T22:55:13.196160+0000","last_fullsized":"2023-09-23T22:55:13.196160+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":264,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-09-23T22:55:21.647181+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:55:21.647181+0000","last_peered":"2023-09-23T22:55:21.647181+0000","last_clean":"2023-09-23T22:55:21.647181+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:55:21.647181+0000","last_undegraded":"2023-09-23T22:55:21.647181+0000","last_fullsized":"2023-09-23T22:55:21.647181+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":281,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-09-23T22:55:15.850807+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:55:15.850807+0000","last_peered":"2023-09-23T22:55:15.850807+0000","last_clean":"2023-09-23T22:55:15.850807+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:55:15.850807+0000","last_undegraded":"2023-09-23T22:55:15.850807+0000","last_fullsized":"2023-09-23T22:55:15.850807+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":9,"acting":9,"num_store_stats":1},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430548,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626776,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300576,"kb_used_data":3352,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93444576,"statfs":{"total":95995035648,"available":95687245824,"internally_reserved":0,"allocated":3432448,"data_stored":2869506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":8.0800000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.087,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.496},{"interface":"front","average":{"1min":3.3559999999999999,"5min":8.8829999999999991,"15min":4.4269999999999996},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":12.081,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":5.6390000000000002,"15min":4.1959999999999997},"min":{"1min":0.32100000000000001,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":74.899000000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.433},{"interface":"front","average":{"1min":6.4550000000000001,"5min":6.923,"15min":5.2009999999999996},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":74.956000000000003,"5min":255.20099999999999,"15min":255.20099999999999},"last":0.52300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:26 2023","interfaces":[{"interface":"back","average":{"1min":7.5069999999999997,"5min":4.4580000000000002,"15min":3.738},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":81.719999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.1970000000000001,"5min":3.1539999999999999,"15min":3.7509999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":9.3529999999999998,"5min":101.13,"15min":185.21799999999999},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":1.845,"15min":3.8879999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":4.1079999999999997,"5min":7.734,"15min":185.19399999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":3.262,"5min":2.4279999999999999,"15min":3.6709999999999998},"min":{"1min":0.438,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.3109999999999999,"5min":10.378,"15min":128.19800000000001},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.3049999999999999,"15min":0.93700000000000006},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.9240000000000004,"5min":10.317,"15min":18.085999999999999},"last":0.379},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.302,"15min":0.94299999999999995},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":10.122999999999999,"5min":10.122999999999999,"15min":18.597000000000001},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.4910000000000001,"15min":1.0549999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.24399999999999999},"max":{"1min":5.351,"5min":10.353999999999999,"15min":18.972000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.325,"15min":1.0049999999999999},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":10.199,"5min":10.199,"15min":18.625},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:55:02 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.298,"15min":0.98599999999999999},"min":{"1min":0.42099999999999999,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":10.225,"5min":10.225,"15min":12.481999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.4550000000000001,"15min":1.028},"min":{"1min":0.40500000000000003,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.5549999999999997,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.36099999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253086,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529096,"kb_used_data":3968,"kb_used_omap":9,"kb_used_meta":525110,"kb_avail":93216056,"statfs":{"total":95995035648,"available":95453241344,"internally_reserved":0,"allocated":4063232,"data_stored":2948341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":537713641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":4.5519999999999996,"15min":5.4329999999999998},"min":{"1min":0.27700000000000002,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":37.991,"5min":37.991,"15min":101.605},"last":0.27800000000000002},{"interface":"front","average":{"1min":6.1550000000000002,"5min":3.819,"15min":5.3019999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":42.191000000000003,"5min":42.191000000000003,"15min":101.892},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":3.8900000000000001,"15min":3.2549999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":21.215,"5min":26.129000000000001,"15min":37.889000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":3.9689999999999999,"5min":3.633,"15min":3.3239999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":21.254000000000001,"5min":26.001999999999999,"15min":37.957000000000001},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.871,"15min":2.645},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.143000000000001,"5min":12.143000000000001,"15min":87.787999999999997},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.1899999999999999,"15min":3.1309999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20999999999999999},"max":{"1min":8.0099999999999998,"5min":10.845000000000001,"15min":89.861999999999995},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":1.9510000000000001,"15min":2.3900000000000001},"min":{"1min":0.38,"5min":0.35899999999999999,"15min":0.29299999999999998},"max":{"1min":11.388,"5min":11.388,"15min":78.948999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.3980000000000001,"5min":2.5659999999999998,"15min":2.4020000000000001},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.2579999999999991,"5min":14.058,"15min":78.963999999999999},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.7110000000000001,"15min":1.8839999999999999},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":12.115,"5min":12.115,"15min":84.254999999999995},"last":0.498},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.78,"15min":2.21},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":8.1050000000000004,"5min":9.8170000000000002,"15min":87.515000000000001},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":2.6110000000000002,"15min":2.2559999999999998},"min":{"1min":0.40699999999999997,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.6639999999999997,"5min":13.991,"15min":79.311000000000007},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.5979999999999999,"5min":1.9810000000000001,"15min":1.9059999999999999},"min":{"1min":0.41899999999999998,"5min":0.40899999999999997,"15min":0.313},"max":{"1min":12.067,"5min":12.067,"15min":79.328999999999994},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.899,"15min":2.609},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.32800000000000001},"max":{"1min":8.1400000000000006,"5min":9.9329999999999998,"15min":159.018},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.254,"5min":2.6949999999999998,"15min":2.7749999999999999},"min":{"1min":0.443,"5min":0.38900000000000001,"15min":0.37},"max":{"1min":6.6310000000000002,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.53300000000000003}]}]},{"osd":0,"up_from":219,"seq":940597837935,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":335216,"kb_used_data":4008,"kb_used_omap":31,"kb_used_meta":331168,"kb_avail":93409936,"statfs":{"total":95995035648,"available":95651774464,"internally_reserved":0,"allocated":4104192,"data_stored":2952822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":339116978},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":3.4329999999999998,"15min":3.7410000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.326000000000001,"5min":40.180999999999997,"15min":40.180999999999997},"last":0.46600000000000003},{"interface":"front","average":{"1min":4.2270000000000003,"5min":3.4630000000000001,"15min":3.6499999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.494999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":0.52000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":5.4139999999999997,"15min":3.5910000000000002},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":89.766000000000005,"5min":111.508,"15min":111.508},"last":0.48499999999999999},{"interface":"front","average":{"1min":7.617,"5min":5.5449999999999999,"15min":3.2360000000000002},"min":{"1min":0.29499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":89.994,"5min":111.532,"15min":111.532},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":4.2519999999999998,"15min":3.0510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":8.5380000000000003,"5min":126.73,"15min":126.73},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.4969999999999999,"5min":3.488,"15min":2.1349999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6440000000000001,"5min":121.483,"15min":121.483},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.0129999999999999,"15min":2.173},"min":{"1min":0.33000000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.5820000000000007,"5min":11.965999999999999,"15min":18.670999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.2400000000000002,"15min":2.036},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.5899999999999999,"5min":16.52,"15min":20.009},"last":0.60799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.8599999999999999,"15min":1.9370000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":6.6630000000000003,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.1379999999999999,"15min":2.3490000000000002},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4240000000000004,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.778,"15min":1.95},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6950000000000003,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.62},{"interface":"front","average":{"1min":2.504,"5min":3.069,"15min":2.4550000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.367,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.81299999999999994}]},{"osd":7,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.1709999999999998,"15min":2.077},"min":{"1min":0.438,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.6959999999999997,"5min":12.385999999999999,"15min":18.640000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.946,"5min":2.0830000000000002,"15min":1.6000000000000001},"min":{"1min":0.42599999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.6779999999999999,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.78800000000000003}]}]},{"osd":2,"up_from":131,"seq":562640715947,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429184,"kb_used_data":3384,"kb_used_omap":81,"kb_used_meta":425710,"kb_avail":93315968,"statfs":{"total":95995035648,"available":95555551232,"internally_reserved":0,"allocated":3465216,"data_stored":2353601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":435927258},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":24,"apply_latency_ms":24,"commit_latency_ns":24000000,"apply_latency_ns":24000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":6.9960000000000004,"15min":5.734},"min":{"1min":0.22800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":12.76,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.373},{"interface":"front","average":{"1min":3.286,"5min":6.9400000000000004,"15min":6.0990000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.093,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":3.0470000000000002,"15min":3.173},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":6.3019999999999996,"5min":77.004999999999995,"15min":88.316000000000003},"last":0.31},{"interface":"front","average":{"1min":1.552,"5min":3.3500000000000001,"15min":3.407},"min":{"1min":0.25700000000000001,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":6.1779999999999999,"5min":76.956000000000003,"15min":88.436999999999998},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.2939999999999996,"15min":4.4100000000000001},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.492000000000001,"5min":112.895,"15min":112.895},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.7869999999999999,"5min":4.109,"15min":3.6219999999999999},"min":{"1min":0.39000000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.1880000000000006,"5min":113.995,"15min":113.995},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.6749999999999998,"15min":2.8279999999999998},"min":{"1min":0.60199999999999998,"5min":0.28899999999999998,"15min":0.253},"max":{"1min":9.1219999999999999,"5min":69.113,"15min":108.878},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.4689999999999999,"15min":3.3809999999999998},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2229999999999999,"5min":69.028999999999996,"15min":108.93000000000001},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.7789999999999999,"15min":2.9940000000000002},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":6.1820000000000004,"5min":22.646000000000001,"15min":97.322999999999993},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.2569999999999999,"5min":2.226,"15min":2.98},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.1420000000000003,"5min":22.501000000000001,"15min":97.301000000000002},"last":0.504}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":3.609,"15min":3.3450000000000002},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":4.3040000000000003,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.4670000000000001,"5min":3.4209999999999998,"15min":2.7050000000000001},"min":{"1min":0.48699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":9.2110000000000003,"5min":98.509,"15min":98.509},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":2.0590000000000002,"15min":1.7509999999999999},"min":{"1min":0.48999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.1609999999999996,"5min":10.477,"15min":10.92},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.415,"5min":1.667,"15min":2.3799999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.37},"max":{"1min":6.2720000000000002,"5min":11.311,"15min":88.584000000000003},"last":0.56299999999999994}]}]},{"osd":3,"up_from":0,"seq":403726926033,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019232,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312288,"kb_used_data":2776,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93432864,"statfs":{"total":95995035648,"available":95675252736,"internally_reserved":0,"allocated":2842624,"data_stored":2279160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.3140000000000001,"15min":5.9390000000000001},"min":{"1min":0.251,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":5.9509999999999996,"5min":113.35599999999999,"15min":113.35599999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.7229999999999999,"15min":6.5099999999999998},"min":{"1min":0.41199999999999998,"5min":0.217,"15min":0.217},"max":{"1min":5.6760000000000002,"5min":113.417,"15min":113.417},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":5.0599999999999996,"15min":3.0190000000000001},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":132.304,"15min":132.304},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.7959999999999998,"5min":4.8970000000000002,"15min":2.879},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":10.832000000000001,"5min":132.453,"15min":132.453},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.3639999999999999,"15min":3.1520000000000001},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.097,"5min":20.308,"15min":84.909000000000006},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.524,"15min":3.137},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.352,"5min":21.704000000000001,"15min":83.840000000000003},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":4.1879999999999997,"15min":4.548},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":7.3109999999999999,"5min":164.453,"15min":164.453},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.2170000000000001,"5min":4.343,"15min":4.6689999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.1779999999999999,"15min":1},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.25900000000000001},"max":{"1min":5.0880000000000001,"5min":12.238,"15min":43.189999999999998},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1060000000000001,"15min":1},"min":{"1min":0.32900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.3079999999999998,"5min":7.5460000000000003,"15min":42.450000000000003},"last":0.40100000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.0529999999999999,"15min":0.94899999999999995},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9139999999999997,"5min":7.9139999999999997,"15min":27.056000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.417,"5min":1.224,"15min":0.97799999999999998},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":5.298,"5min":12.269,"15min":26.940999999999999},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.2130000000000001,"15min":0.89400000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":3.544,"5min":12.311,"15min":12.311},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.95499999999999996,"15min":0.80600000000000005},"min":{"1min":0.36399999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":3.0169999999999999,"5min":6.7919999999999998,"15min":8.6739999999999995},"last":0.44700000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823004,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":600644,"kb_used_data":2748,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":93144508,"statfs":{"total":95995035648,"available":95379976192,"internally_reserved":0,"allocated":2813952,"data_stored":2274204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":9.6929999999999996,"5min":8.1189999999999998,"15min":5.133},"min":{"1min":0.32500000000000001,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":80.450999999999993,"5min":185.62,"15min":185.62},"last":0.55000000000000004},{"interface":"front","average":{"1min":9.734,"5min":8.1829999999999998,"15min":5.1079999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":80.171999999999997,"5min":185.499,"15min":185.499},"last":0.496}]},{"osd":1,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":9.452,"5min":3.9129999999999998,"15min":2.8799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":110.36499999999999,"5min":110.36499999999999,"15min":149.203},"last":0.63600000000000001},{"interface":"front","average":{"1min":9.5340000000000007,"5min":4.3019999999999996,"15min":3.1909999999999998},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":110.89,"5min":110.89,"15min":149.554},"last":0.57999999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":5.54,"15min":5.5140000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.997,"5min":189.126,"15min":189.126},"last":0.441},{"interface":"front","average":{"1min":3.5379999999999998,"5min":5.2309999999999999,"15min":5.4059999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":41.070999999999998,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.843,"15min":5.6660000000000004},"min":{"1min":0.34899999999999998,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.298999999999999,"5min":98.641000000000005,"15min":257.05000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.085,"5min":4.9130000000000003,"15min":5.516},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27800000000000002},"max":{"1min":11.097,"5min":98.906000000000006,"15min":257.35000000000002},"last":0.60799999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.86899999999999999,"15min":0.73399999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":3.669,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.996,"5min":0.92100000000000004,"15min":0.75700000000000001},"min":{"1min":0.29399999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":5.8019999999999996,"5min":6.3550000000000004,"15min":10.535},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1759999999999999,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.3369999999999997,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.139,"15min":1.629},"min":{"1min":0.374,"5min":0.315,"15min":0.254},"max":{"1min":7.9909999999999997,"5min":10.314,"15min":139.739},"last":0.34000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:55:06 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.2749999999999999,"15min":0.94999999999999996},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.29999999999999999},"max":{"1min":6.3579999999999997,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.125,"15min":0.85999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36199999999999999,"15min":0.311},"max":{"1min":5.8719999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:55:26.100 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:26.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:26.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:26.121 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:26.203 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:26.231 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:26.712 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:26 smithi052 ceph-mon[180979]: from='client.45932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:26.712 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:26 smithi052 ceph-mon[187743]: from='client.45932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:26.829 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:26.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:26.933 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:26.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:26 smithi151 ceph-mon[159210]: from='client.45932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:26.965 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:27.062 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:27.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:27.166 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:27.200 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:27.250 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:27.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:27.356 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:27.384 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:27.426 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:27.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:27.530 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:27.560 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:27.574 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:27.667 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:27.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:27.772 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:27.798 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:27.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:27.841 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:55:27.841 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:55:27.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:27 smithi151 ceph-mon[159210]: pgmap v926: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:27.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:27.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:27.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:27 smithi052 ceph-mon[180979]: pgmap v926: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:27.865 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:27 smithi052 ceph-mon[187743]: pgmap v926: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:27.947 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:27.978 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:28.091 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:55:28.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:28.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:28.335 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:28.363 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:28.405 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:28.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:28.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:28.542 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:28.805 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:55:28.994 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:28.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:29.054 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:55:29.098 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:29.127 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:29.319 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:29.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:29.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:29.422 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:29.457 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:29.584 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:55:29.637 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:29.642 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:29 smithi052 ceph-mon[180979]: pgmap v927: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:29.642 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/856745311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:29.642 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:29 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:29.642 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:29 smithi052 ceph-mon[187743]: pgmap v927: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:29.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/856745311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:29.643 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:29 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:29.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:29.686 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:29.744 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:29.776 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:29.876 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:29.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:29 smithi151 ceph-mon[159210]: pgmap v927: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/856745311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:29 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:29.979 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:30.007 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:30.279 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:30.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:30.363 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:30.379 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:30.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:30.416 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:30.465 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:30.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:30.574 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: osdmap e320: 8 total, 8 up, 6 in 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.576 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3256527120' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:55:30.583 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T22:55:30.608 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:55:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T22:55:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:30.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: osdmap e320: 8 total, 8 up, 6 in 2023-09-23T22:55:30.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3256527120' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: osdmap e320: 8 total, 8 up, 6 in 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: failed to encode map e320 with expected crc 2023-09-23T22:55:30.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3256527120' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:55:31.712 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:31.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:31.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[187743]: pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3256527120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[187743]: osdmap e321: 8 total, 8 up, 6 in 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[187743]: failed to encode map e321 with expected crc 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[187743]: failed to encode map e321 with expected crc 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[180979]: pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3256527120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:55:31.837 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[180979]: osdmap e321: 8 total, 8 up, 6 in 2023-09-23T22:55:31.838 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[180979]: failed to encode map e321 with expected crc 2023-09-23T22:55:31.838 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:31 smithi052 ceph-mon[180979]: failed to encode map e321 with expected crc 2023-09-23T22:55:31.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:31 smithi151 ceph-mon[159210]: pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:31.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3256527120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:55:31.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:31 smithi151 ceph-mon[159210]: osdmap e321: 8 total, 8 up, 6 in 2023-09-23T22:55:31.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:31 smithi151 ceph-mon[159210]: failed to encode map e321 with expected crc 2023-09-23T22:55:31.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:31 smithi151 ceph-mon[159210]: failed to encode map e321 with expected crc 2023-09-23T22:55:31.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:31.896 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:31.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:32.001 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:32.034 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:32.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:32.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:32.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:32.336 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:32.366 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:32.393 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:55:32.412 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:32.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:32.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:32.518 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:32.546 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:32 smithi151 ceph-mon[159210]: failed to encode map e321 with expected crc 2023-09-23T22:55:32.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:32 smithi151 ceph-mon[159210]: failed to encode map e321 with expected crc 2023-09-23T22:55:32.708 DEBUG:teuthology.orchestra.run.smithi052:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T22:55:32.711 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:55:32.711 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:55:32.910 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:32 smithi052 ceph-mon[180979]: failed to encode map e321 with expected crc 2023-09-23T22:55:32.910 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:32 smithi052 ceph-mon[180979]: failed to encode map e321 with expected crc 2023-09-23T22:55:32.910 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:32 smithi052 ceph-mon[187743]: failed to encode map e321 with expected crc 2023-09-23T22:55:32.910 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:32 smithi052 ceph-mon[187743]: failed to encode map e321 with expected crc 2023-09-23T22:55:32.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:hints = 1 2023-09-23T22:55:32.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:55:32.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object prefix: benchmark_data_smithi052_461816 2023-09-23T22:55:32.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:55:32.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:55:33.460 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:33.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:33.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:33.571 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:33.613 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:33.678 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:33.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:33.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:55:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:55:33] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.33.4" 2023-09-23T22:55:33.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:33 smithi052 ceph-mon[187743]: pgmap v931: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:33.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:33 smithi052 ceph-mon[180979]: pgmap v931: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:33.785 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:33.831 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:33.916 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:33.920 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:33 smithi151 ceph-mon[159210]: pgmap v931: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:55:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:33.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:33.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 1 16 3501 3485 217.8 217.812 0.00536671 0.00457516 2023-09-23T22:55:34.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:34.112 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:55:34.115 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:34.122 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:55:34.141 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:34.192 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:34.221 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:34.265 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:55:34.313 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:34.344 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:34.373 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:34.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:34.402 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:34.409 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:55:34.410 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:55:34.410 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:55:34.410 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:55:34.410 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:55:34.436 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:34.477 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:34.481 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:34.486 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.486 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.486 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.486 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.487 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.487 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.487 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.487 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:55:34.530 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:34.600 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2094198049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:34.600 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:34.600 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:34.622 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:34.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:34.731 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:34.760 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:34.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 2 16 6389 6373 199.141 180.5 0.00441656 0.00501261 2023-09-23T22:55:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2094198049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2094198049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:34.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:35.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 3 16 9636 9620 200.399 202.938 0.00434399 0.00498228 2023-09-23T22:55:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:35 smithi151 ceph-mon[159210]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:35 smithi151 ceph-mon[159210]: pgmap v932: 89 pgs: 89 active+clean; 23 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 2.8 MiB/s wr, 44 op/s 2023-09-23T22:55:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:35.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:35 smithi052 ceph-mon[180979]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:35.980 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:35 smithi052 ceph-mon[180979]: pgmap v932: 89 pgs: 89 active+clean; 23 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 2.8 MiB/s wr, 44 op/s 2023-09-23T22:55:35.980 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:35.980 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:35 smithi052 ceph-mon[187743]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:55:35.980 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:35 smithi052 ceph-mon[187743]: pgmap v932: 89 pgs: 89 active+clean; 23 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 2.8 MiB/s wr, 44 op/s 2023-09-23T22:55:35.980 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:36.403 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:36.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:36.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:36.508 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:36.547 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:36.589 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:36.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:36.694 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:36.743 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:36.924 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 4 16 12630 12614 197.077 187.125 0.0039233 0.00506652 2023-09-23T22:55:37.434 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":931,"stamp":"2023-09-23T22:55:32.455956+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":260,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2511324,"kb_used_data":20460,"kb_used_omap":121,"kb_used_meta":2490694,"kb_avail":559959588,"statfs":{"total":575970213888,"available":573398618112,"internally_reserved":0,"allocated":20951040,"data_stored":15883607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2550470773},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.892868"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":318,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752906+0000","last_change":"2023-09-23T22:41:10.929214+0000","last_active":"2023-09-23T22:55:29.752906+0000","last_peered":"2023-09-23T22:55:29.752906+0000","last_clean":"2023-09-23T22:55:29.752906+0000","last_became_active":"2023-09-23T22:41:10.928823+0000","last_became_peered":"2023-09-23T22:41:10.928823+0000","last_unstale":"2023-09-23T22:55:29.752906+0000","last_undegraded":"2023-09-23T22:55:29.752906+0000","last_fullsized":"2023-09-23T22:55:29.752906+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752798+0000","last_change":"2023-09-23T22:51:34.423131+0000","last_active":"2023-09-23T22:55:29.752798+0000","last_peered":"2023-09-23T22:55:29.752798+0000","last_clean":"2023-09-23T22:55:29.752798+0000","last_became_active":"2023-09-23T22:51:34.422769+0000","last_became_peered":"2023-09-23T22:51:34.422769+0000","last_unstale":"2023-09-23T22:55:29.752798+0000","last_undegraded":"2023-09-23T22:55:29.752798+0000","last_fullsized":"2023-09-23T22:55:29.752798+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":103,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752757+0000","last_change":"2023-09-23T22:51:34.425083+0000","last_active":"2023-09-23T22:55:29.752757+0000","last_peered":"2023-09-23T22:55:29.752757+0000","last_clean":"2023-09-23T22:55:29.752757+0000","last_became_active":"2023-09-23T22:51:34.424707+0000","last_became_peered":"2023-09-23T22:51:34.424707+0000","last_unstale":"2023-09-23T22:55:29.752757+0000","last_undegraded":"2023-09-23T22:55:29.752757+0000","last_fullsized":"2023-09-23T22:55:29.752757+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560528+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:55:23.560528+0000","last_peered":"2023-09-23T22:55:23.560528+0000","last_clean":"2023-09-23T22:55:23.560528+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:55:23.560528+0000","last_undegraded":"2023-09-23T22:55:23.560528+0000","last_fullsized":"2023-09-23T22:55:23.560528+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":302,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560238+0000","last_change":"2023-09-23T22:51:34.423227+0000","last_active":"2023-09-23T22:55:23.560238+0000","last_peered":"2023-09-23T22:55:23.560238+0000","last_clean":"2023-09-23T22:55:23.560238+0000","last_became_active":"2023-09-23T22:51:34.422849+0000","last_became_peered":"2023-09-23T22:51:34.422849+0000","last_unstale":"2023-09-23T22:55:23.560238+0000","last_undegraded":"2023-09-23T22:55:23.560238+0000","last_fullsized":"2023-09-23T22:55:23.560238+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:29.252641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":285,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567798+0000","last_change":"2023-09-23T22:37:03.800646+0000","last_active":"2023-09-23T22:55:24.567798+0000","last_peered":"2023-09-23T22:55:24.567798+0000","last_clean":"2023-09-23T22:55:24.567798+0000","last_became_active":"2023-09-23T22:37:03.800283+0000","last_became_peered":"2023-09-23T22:37:03.800283+0000","last_unstale":"2023-09-23T22:55:24.567798+0000","last_undegraded":"2023-09-23T22:55:24.567798+0000","last_fullsized":"2023-09-23T22:55:24.567798+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":229,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560702+0000","last_change":"2023-09-23T22:51:34.417771+0000","last_active":"2023-09-23T22:55:23.560702+0000","last_peered":"2023-09-23T22:55:23.560702+0000","last_clean":"2023-09-23T22:55:23.560702+0000","last_became_active":"2023-09-23T22:51:34.417496+0000","last_became_peered":"2023-09-23T22:51:34.417496+0000","last_unstale":"2023-09-23T22:55:23.560702+0000","last_undegraded":"2023-09-23T22:55:23.560702+0000","last_fullsized":"2023-09-23T22:55:23.560702+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:43.719513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":182,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751183+0000","last_change":"2023-09-23T22:51:34.425766+0000","last_active":"2023-09-23T22:55:24.751183+0000","last_peered":"2023-09-23T22:55:24.751183+0000","last_clean":"2023-09-23T22:55:24.751183+0000","last_became_active":"2023-09-23T22:51:34.425487+0000","last_became_peered":"2023-09-23T22:51:34.425487+0000","last_unstale":"2023-09-23T22:55:24.751183+0000","last_undegraded":"2023-09-23T22:55:24.751183+0000","last_fullsized":"2023-09-23T22:55:24.751183+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:29.065967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751488+0000","last_change":"2023-09-23T22:51:34.427405+0000","last_active":"2023-09-23T22:55:24.751488+0000","last_peered":"2023-09-23T22:55:24.751488+0000","last_clean":"2023-09-23T22:55:24.751488+0000","last_became_active":"2023-09-23T22:51:34.427107+0000","last_became_peered":"2023-09-23T22:51:34.427107+0000","last_unstale":"2023-09-23T22:55:24.751488+0000","last_undegraded":"2023-09-23T22:55:24.751488+0000","last_fullsized":"2023-09-23T22:55:24.751488+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:13.555237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":204,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567058+0000","last_change":"2023-09-23T22:51:34.418805+0000","last_active":"2023-09-23T22:55:24.567058+0000","last_peered":"2023-09-23T22:55:24.567058+0000","last_clean":"2023-09-23T22:55:24.567058+0000","last_became_active":"2023-09-23T22:51:34.418397+0000","last_became_peered":"2023-09-23T22:51:34.418397+0000","last_unstale":"2023-09-23T22:55:24.567058+0000","last_undegraded":"2023-09-23T22:55:24.567058+0000","last_fullsized":"2023-09-23T22:55:24.567058+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":79,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752605+0000","last_change":"2023-09-23T22:51:34.511966+0000","last_active":"2023-09-23T22:55:29.752605+0000","last_peered":"2023-09-23T22:55:29.752605+0000","last_clean":"2023-09-23T22:55:29.752605+0000","last_became_active":"2023-09-23T22:51:34.419318+0000","last_became_peered":"2023-09-23T22:51:34.419318+0000","last_unstale":"2023-09-23T22:55:29.752605+0000","last_undegraded":"2023-09-23T22:55:29.752605+0000","last_fullsized":"2023-09-23T22:55:29.752605+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":194,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567225+0000","last_change":"2023-09-23T22:46:44.025105+0000","last_active":"2023-09-23T22:55:24.567225+0000","last_peered":"2023-09-23T22:55:24.567225+0000","last_clean":"2023-09-23T22:55:24.567225+0000","last_became_active":"2023-09-23T22:46:44.024585+0000","last_became_peered":"2023-09-23T22:46:44.024585+0000","last_unstale":"2023-09-23T22:55:24.567225+0000","last_undegraded":"2023-09-23T22:55:24.567225+0000","last_fullsized":"2023-09-23T22:55:24.567225+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":340,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752774+0000","last_change":"2023-09-23T22:44:53.855308+0000","last_active":"2023-09-23T22:55:29.752774+0000","last_peered":"2023-09-23T22:55:29.752774+0000","last_clean":"2023-09-23T22:55:29.752774+0000","last_became_active":"2023-09-23T22:44:53.854892+0000","last_became_peered":"2023-09-23T22:44:53.854892+0000","last_unstale":"2023-09-23T22:55:29.752774+0000","last_undegraded":"2023-09-23T22:55:29.752774+0000","last_fullsized":"2023-09-23T22:55:29.752774+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":286,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.751826+0000","last_change":"2023-09-23T22:37:03.800813+0000","last_active":"2023-09-23T22:55:29.751826+0000","last_peered":"2023-09-23T22:55:29.751826+0000","last_clean":"2023-09-23T22:55:29.751826+0000","last_became_active":"2023-09-23T22:37:03.800463+0000","last_became_peered":"2023-09-23T22:37:03.800463+0000","last_unstale":"2023-09-23T22:55:29.751826+0000","last_undegraded":"2023-09-23T22:55:29.751826+0000","last_fullsized":"2023-09-23T22:55:29.751826+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":179,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.751963+0000","last_change":"2023-09-23T22:44:53.908470+0000","last_active":"2023-09-23T22:55:29.751963+0000","last_peered":"2023-09-23T22:55:29.751963+0000","last_clean":"2023-09-23T22:55:29.751963+0000","last_became_active":"2023-09-23T22:44:53.857240+0000","last_became_peered":"2023-09-23T22:44:53.857240+0000","last_unstale":"2023-09-23T22:55:29.751963+0000","last_undegraded":"2023-09-23T22:55:29.751963+0000","last_fullsized":"2023-09-23T22:55:29.751963+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":267,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751292+0000","last_change":"2023-09-23T22:51:34.427352+0000","last_active":"2023-09-23T22:55:24.751292+0000","last_peered":"2023-09-23T22:55:24.751292+0000","last_clean":"2023-09-23T22:55:24.751292+0000","last_became_active":"2023-09-23T22:51:34.427027+0000","last_became_peered":"2023-09-23T22:51:34.427027+0000","last_unstale":"2023-09-23T22:55:24.751292+0000","last_undegraded":"2023-09-23T22:55:24.751292+0000","last_fullsized":"2023-09-23T22:55:24.751292+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:35.794025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":285,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560398+0000","last_change":"2023-09-23T22:51:34.420735+0000","last_active":"2023-09-23T22:55:23.560398+0000","last_peered":"2023-09-23T22:55:23.560398+0000","last_clean":"2023-09-23T22:55:23.560398+0000","last_became_active":"2023-09-23T22:51:34.420411+0000","last_became_peered":"2023-09-23T22:51:34.420411+0000","last_unstale":"2023-09-23T22:55:23.560398+0000","last_undegraded":"2023-09-23T22:55:23.560398+0000","last_fullsized":"2023-09-23T22:55:23.560398+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:20.387609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":243,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567795+0000","last_change":"2023-09-23T22:51:34.418921+0000","last_active":"2023-09-23T22:55:24.567795+0000","last_peered":"2023-09-23T22:55:24.567795+0000","last_clean":"2023-09-23T22:55:24.567795+0000","last_became_active":"2023-09-23T22:51:34.418504+0000","last_became_peered":"2023-09-23T22:51:34.418504+0000","last_unstale":"2023-09-23T22:55:24.567795+0000","last_undegraded":"2023-09-23T22:55:24.567795+0000","last_fullsized":"2023-09-23T22:55:24.567795+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752939+0000","last_change":"2023-09-23T22:46:14.247422+0000","last_active":"2023-09-23T22:55:29.752939+0000","last_peered":"2023-09-23T22:55:29.752939+0000","last_clean":"2023-09-23T22:55:29.752939+0000","last_became_active":"2023-09-23T22:46:14.247111+0000","last_became_peered":"2023-09-23T22:46:14.247111+0000","last_unstale":"2023-09-23T22:55:29.752939+0000","last_undegraded":"2023-09-23T22:55:29.752939+0000","last_fullsized":"2023-09-23T22:55:29.752939+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":286,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560572+0000","last_change":"2023-09-23T22:51:34.417743+0000","last_active":"2023-09-23T22:55:23.560572+0000","last_peered":"2023-09-23T22:55:23.560572+0000","last_clean":"2023-09-23T22:55:23.560572+0000","last_became_active":"2023-09-23T22:51:34.417439+0000","last_became_peered":"2023-09-23T22:51:34.417439+0000","last_unstale":"2023-09-23T22:55:23.560572+0000","last_undegraded":"2023-09-23T22:55:23.560572+0000","last_fullsized":"2023-09-23T22:55:23.560572+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:06.850233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":364,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.566903+0000","last_change":"2023-09-23T22:46:14.254608+0000","last_active":"2023-09-23T22:55:24.566903+0000","last_peered":"2023-09-23T22:55:24.566903+0000","last_clean":"2023-09-23T22:55:24.566903+0000","last_became_active":"2023-09-23T22:46:14.254216+0000","last_became_peered":"2023-09-23T22:46:14.254216+0000","last_unstale":"2023-09-23T22:55:24.566903+0000","last_undegraded":"2023-09-23T22:55:24.566903+0000","last_fullsized":"2023-09-23T22:55:24.566903+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":247,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751917+0000","last_change":"2023-09-23T22:51:34.427212+0000","last_active":"2023-09-23T22:55:24.751917+0000","last_peered":"2023-09-23T22:55:24.751917+0000","last_clean":"2023-09-23T22:55:24.751917+0000","last_became_active":"2023-09-23T22:51:34.426879+0000","last_became_peered":"2023-09-23T22:51:34.426879+0000","last_unstale":"2023-09-23T22:55:24.751917+0000","last_undegraded":"2023-09-23T22:55:24.751917+0000","last_fullsized":"2023-09-23T22:55:24.751917+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:49.284504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":11,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.669673+0000","last_change":"2023-09-23T22:55:23.669240+0000","last_active":"2023-09-23T22:55:23.669673+0000","last_peered":"2023-09-23T22:55:23.669673+0000","last_clean":"2023-09-23T22:55:23.669673+0000","last_became_active":"2023-09-23T22:55:23.668982+0000","last_became_peered":"2023-09-23T22:55:23.668982+0000","last_unstale":"2023-09-23T22:55:23.669673+0000","last_undegraded":"2023-09-23T22:55:23.669673+0000","last_fullsized":"2023-09-23T22:55:23.669673+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:40.611676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":190,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.751990+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:55:29.751990+0000","last_peered":"2023-09-23T22:55:29.751990+0000","last_clean":"2023-09-23T22:55:29.751990+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:55:29.751990+0000","last_undegraded":"2023-09-23T22:55:29.751990+0000","last_fullsized":"2023-09-23T22:55:29.751990+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":284,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567009+0000","last_change":"2023-09-23T22:45:51.508501+0000","last_active":"2023-09-23T22:55:24.567009+0000","last_peered":"2023-09-23T22:55:24.567009+0000","last_clean":"2023-09-23T22:55:24.567009+0000","last_became_active":"2023-09-23T22:45:51.508189+0000","last_became_peered":"2023-09-23T22:45:51.508189+0000","last_unstale":"2023-09-23T22:55:24.567009+0000","last_undegraded":"2023-09-23T22:55:24.567009+0000","last_fullsized":"2023-09-23T22:55:24.567009+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":11,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567757+0000","last_change":"2023-09-23T22:55:23.669375+0000","last_active":"2023-09-23T22:55:24.567757+0000","last_peered":"2023-09-23T22:55:24.567757+0000","last_clean":"2023-09-23T22:55:24.567757+0000","last_became_active":"2023-09-23T22:55:23.669048+0000","last_became_peered":"2023-09-23T22:55:23.669048+0000","last_unstale":"2023-09-23T22:55:24.567757+0000","last_undegraded":"2023-09-23T22:55:24.567757+0000","last_fullsized":"2023-09-23T22:55:24.567757+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":200,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752130+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:55:29.752130+0000","last_peered":"2023-09-23T22:55:29.752130+0000","last_clean":"2023-09-23T22:55:29.752130+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:55:29.752130+0000","last_undegraded":"2023-09-23T22:55:29.752130+0000","last_fullsized":"2023-09-23T22:55:29.752130+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":310,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752859+0000","last_change":"2023-09-23T22:51:34.425204+0000","last_active":"2023-09-23T22:55:29.752859+0000","last_peered":"2023-09-23T22:55:29.752859+0000","last_clean":"2023-09-23T22:55:29.752859+0000","last_became_active":"2023-09-23T22:51:34.424828+0000","last_became_peered":"2023-09-23T22:51:34.424828+0000","last_unstale":"2023-09-23T22:55:29.752859+0000","last_undegraded":"2023-09-23T22:55:29.752859+0000","last_fullsized":"2023-09-23T22:55:29.752859+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":11,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.311330+0000","last_change":"2023-09-23T22:55:24.310230+0000","last_active":"2023-09-23T22:55:24.311330+0000","last_peered":"2023-09-23T22:55:24.311330+0000","last_clean":"2023-09-23T22:55:24.311330+0000","last_became_active":"2023-09-23T22:55:24.309921+0000","last_became_peered":"2023-09-23T22:55:24.309921+0000","last_unstale":"2023-09-23T22:55:24.311330+0000","last_undegraded":"2023-09-23T22:55:24.311330+0000","last_fullsized":"2023-09-23T22:55:24.311330+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:46.749922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":143,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567791+0000","last_change":"2023-09-23T22:51:34.434315+0000","last_active":"2023-09-23T22:55:24.567791+0000","last_peered":"2023-09-23T22:55:24.567791+0000","last_clean":"2023-09-23T22:55:24.567791+0000","last_became_active":"2023-09-23T22:51:34.417371+0000","last_became_peered":"2023-09-23T22:51:34.417371+0000","last_unstale":"2023-09-23T22:55:24.567791+0000","last_undegraded":"2023-09-23T22:55:24.567791+0000","last_fullsized":"2023-09-23T22:55:24.567791+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":312,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567199+0000","last_change":"2023-09-23T22:44:53.863184+0000","last_active":"2023-09-23T22:55:24.567199+0000","last_peered":"2023-09-23T22:55:24.567199+0000","last_clean":"2023-09-23T22:55:24.567199+0000","last_became_active":"2023-09-23T22:44:53.862732+0000","last_became_peered":"2023-09-23T22:44:53.862732+0000","last_unstale":"2023-09-23T22:55:24.567199+0000","last_undegraded":"2023-09-23T22:55:24.567199+0000","last_fullsized":"2023-09-23T22:55:24.567199+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":12,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:25.247662+0000","last_change":"2023-09-23T22:55:23.668859+0000","last_active":"2023-09-23T22:55:25.247662+0000","last_peered":"2023-09-23T22:55:25.247662+0000","last_clean":"2023-09-23T22:55:25.247662+0000","last_became_active":"2023-09-23T22:55:23.668590+0000","last_became_peered":"2023-09-23T22:55:23.668590+0000","last_unstale":"2023-09-23T22:55:25.247662+0000","last_undegraded":"2023-09-23T22:55:25.247662+0000","last_fullsized":"2023-09-23T22:55:25.247662+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.963824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":115,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567479+0000","last_change":"2023-09-23T22:51:34.417767+0000","last_active":"2023-09-23T22:55:24.567479+0000","last_peered":"2023-09-23T22:55:24.567479+0000","last_clean":"2023-09-23T22:55:24.567479+0000","last_became_active":"2023-09-23T22:51:34.417369+0000","last_became_peered":"2023-09-23T22:51:34.417369+0000","last_unstale":"2023-09-23T22:55:24.567479+0000","last_undegraded":"2023-09-23T22:55:24.567479+0000","last_fullsized":"2023-09-23T22:55:24.567479+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":214,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560190+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:55:23.560190+0000","last_peered":"2023-09-23T22:55:23.560190+0000","last_clean":"2023-09-23T22:55:23.560190+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:55:23.560190+0000","last_undegraded":"2023-09-23T22:55:23.560190+0000","last_fullsized":"2023-09-23T22:55:23.560190+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":12,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752252+0000","last_change":"2023-09-23T22:55:24.311230+0000","last_active":"2023-09-23T22:55:29.752252+0000","last_peered":"2023-09-23T22:55:29.752252+0000","last_clean":"2023-09-23T22:55:29.752252+0000","last_became_active":"2023-09-23T22:55:24.309796+0000","last_became_peered":"2023-09-23T22:55:24.309796+0000","last_unstale":"2023-09-23T22:55:29.752252+0000","last_undegraded":"2023-09-23T22:55:29.752252+0000","last_fullsized":"2023-09-23T22:55:29.752252+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":85,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560291+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:55:23.560291+0000","last_peered":"2023-09-23T22:55:23.560291+0000","last_clean":"2023-09-23T22:55:23.560291+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:55:23.560291+0000","last_undegraded":"2023-09-23T22:55:23.560291+0000","last_fullsized":"2023-09-23T22:55:23.560291+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":294,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752640+0000","last_change":"2023-09-23T22:51:34.423216+0000","last_active":"2023-09-23T22:55:29.752640+0000","last_peered":"2023-09-23T22:55:29.752640+0000","last_clean":"2023-09-23T22:55:29.752640+0000","last_became_active":"2023-09-23T22:51:34.422786+0000","last_became_peered":"2023-09-23T22:51:34.422786+0000","last_unstale":"2023-09-23T22:55:29.752640+0000","last_undegraded":"2023-09-23T22:55:29.752640+0000","last_fullsized":"2023-09-23T22:55:29.752640+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":12,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:25.247612+0000","last_change":"2023-09-23T22:55:23.667518+0000","last_active":"2023-09-23T22:55:25.247612+0000","last_peered":"2023-09-23T22:55:25.247612+0000","last_clean":"2023-09-23T22:55:25.247612+0000","last_became_active":"2023-09-23T22:55:23.667219+0000","last_became_peered":"2023-09-23T22:55:23.667219+0000","last_unstale":"2023-09-23T22:55:25.247612+0000","last_undegraded":"2023-09-23T22:55:25.247612+0000","last_fullsized":"2023-09-23T22:55:25.247612+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:01.639738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":304,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567050+0000","last_change":"2023-09-23T22:40:13.762496+0000","last_active":"2023-09-23T22:55:24.567050+0000","last_peered":"2023-09-23T22:55:24.567050+0000","last_clean":"2023-09-23T22:55:24.567050+0000","last_became_active":"2023-09-23T22:40:13.762129+0000","last_became_peered":"2023-09-23T22:40:13.762129+0000","last_unstale":"2023-09-23T22:55:24.567050+0000","last_undegraded":"2023-09-23T22:55:24.567050+0000","last_fullsized":"2023-09-23T22:55:24.567050+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":316,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752136+0000","last_change":"2023-09-23T22:41:10.925953+0000","last_active":"2023-09-23T22:55:29.752136+0000","last_peered":"2023-09-23T22:55:29.752136+0000","last_clean":"2023-09-23T22:55:29.752136+0000","last_became_active":"2023-09-23T22:41:10.925532+0000","last_became_peered":"2023-09-23T22:41:10.925532+0000","last_unstale":"2023-09-23T22:55:29.752136+0000","last_undegraded":"2023-09-23T22:55:29.752136+0000","last_fullsized":"2023-09-23T22:55:29.752136+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":11,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.566901+0000","last_change":"2023-09-23T22:55:24.310069+0000","last_active":"2023-09-23T22:55:24.566901+0000","last_peered":"2023-09-23T22:55:24.566901+0000","last_clean":"2023-09-23T22:55:24.566901+0000","last_became_active":"2023-09-23T22:55:24.309699+0000","last_became_peered":"2023-09-23T22:55:24.309699+0000","last_unstale":"2023-09-23T22:55:24.566901+0000","last_undegraded":"2023-09-23T22:55:24.566901+0000","last_fullsized":"2023-09-23T22:55:24.566901+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":329,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752613+0000","last_change":"2023-09-23T22:44:53.856174+0000","last_active":"2023-09-23T22:55:29.752613+0000","last_peered":"2023-09-23T22:55:29.752613+0000","last_clean":"2023-09-23T22:55:29.752613+0000","last_became_active":"2023-09-23T22:44:53.855729+0000","last_became_peered":"2023-09-23T22:44:53.855729+0000","last_unstale":"2023-09-23T22:55:29.752613+0000","last_undegraded":"2023-09-23T22:55:29.752613+0000","last_fullsized":"2023-09-23T22:55:29.752613+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":304,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752316+0000","last_change":"2023-09-23T22:40:13.762051+0000","last_active":"2023-09-23T22:55:29.752316+0000","last_peered":"2023-09-23T22:55:29.752316+0000","last_clean":"2023-09-23T22:55:29.752316+0000","last_became_active":"2023-09-23T22:40:13.761723+0000","last_became_peered":"2023-09-23T22:40:13.761723+0000","last_unstale":"2023-09-23T22:55:29.752316+0000","last_undegraded":"2023-09-23T22:55:29.752316+0000","last_fullsized":"2023-09-23T22:55:29.752316+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":12,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752483+0000","last_change":"2023-09-23T22:55:24.310261+0000","last_active":"2023-09-23T22:55:29.752483+0000","last_peered":"2023-09-23T22:55:29.752483+0000","last_clean":"2023-09-23T22:55:29.752483+0000","last_became_active":"2023-09-23T22:55:24.309692+0000","last_became_peered":"2023-09-23T22:55:24.309692+0000","last_unstale":"2023-09-23T22:55:29.752483+0000","last_undegraded":"2023-09-23T22:55:29.752483+0000","last_fullsized":"2023-09-23T22:55:29.752483+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752098+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:55:29.752098+0000","last_peered":"2023-09-23T22:55:29.752098+0000","last_clean":"2023-09-23T22:55:29.752098+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:55:29.752098+0000","last_undegraded":"2023-09-23T22:55:29.752098+0000","last_fullsized":"2023-09-23T22:55:29.752098+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":258,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751946+0000","last_change":"2023-09-23T22:51:34.415931+0000","last_active":"2023-09-23T22:55:24.751946+0000","last_peered":"2023-09-23T22:55:24.751946+0000","last_clean":"2023-09-23T22:55:24.751946+0000","last_became_active":"2023-09-23T22:51:34.415589+0000","last_became_peered":"2023-09-23T22:51:34.415589+0000","last_unstale":"2023-09-23T22:55:24.751946+0000","last_undegraded":"2023-09-23T22:55:24.751946+0000","last_fullsized":"2023-09-23T22:55:24.751946+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:53.311678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":12,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752351+0000","last_change":"2023-09-23T22:55:23.671717+0000","last_active":"2023-09-23T22:55:29.752351+0000","last_peered":"2023-09-23T22:55:29.752351+0000","last_clean":"2023-09-23T22:55:29.752351+0000","last_became_active":"2023-09-23T22:55:23.671376+0000","last_became_peered":"2023-09-23T22:55:23.671376+0000","last_unstale":"2023-09-23T22:55:29.752351+0000","last_undegraded":"2023-09-23T22:55:29.752351+0000","last_fullsized":"2023-09-23T22:55:29.752351+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":358,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560359+0000","last_change":"2023-09-23T22:46:44.032034+0000","last_active":"2023-09-23T22:55:23.560359+0000","last_peered":"2023-09-23T22:55:23.560359+0000","last_clean":"2023-09-23T22:55:23.560359+0000","last_became_active":"2023-09-23T22:46:44.031755+0000","last_became_peered":"2023-09-23T22:46:44.031755+0000","last_unstale":"2023-09-23T22:55:23.560359+0000","last_undegraded":"2023-09-23T22:55:23.560359+0000","last_fullsized":"2023-09-23T22:55:23.560359+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:02.229204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751801+0000","last_change":"2023-09-23T22:51:34.418928+0000","last_active":"2023-09-23T22:55:24.751801+0000","last_peered":"2023-09-23T22:55:24.751801+0000","last_clean":"2023-09-23T22:55:24.751801+0000","last_became_active":"2023-09-23T22:51:34.418572+0000","last_became_peered":"2023-09-23T22:51:34.418572+0000","last_unstale":"2023-09-23T22:55:24.751801+0000","last_undegraded":"2023-09-23T22:55:24.751801+0000","last_fullsized":"2023-09-23T22:55:24.751801+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:44.957836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":152,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752844+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:55:29.752844+0000","last_peered":"2023-09-23T22:55:29.752844+0000","last_clean":"2023-09-23T22:55:29.752844+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:55:29.752844+0000","last_undegraded":"2023-09-23T22:55:29.752844+0000","last_fullsized":"2023-09-23T22:55:29.752844+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":11,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567089+0000","last_change":"2023-09-23T22:55:23.670762+0000","last_active":"2023-09-23T22:55:24.567089+0000","last_peered":"2023-09-23T22:55:24.567089+0000","last_clean":"2023-09-23T22:55:24.567089+0000","last_became_active":"2023-09-23T22:55:23.670402+0000","last_became_peered":"2023-09-23T22:55:23.670402+0000","last_unstale":"2023-09-23T22:55:24.567089+0000","last_undegraded":"2023-09-23T22:55:24.567089+0000","last_fullsized":"2023-09-23T22:55:24.567089+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":154,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567124+0000","last_change":"2023-09-23T22:46:44.024905+0000","last_active":"2023-09-23T22:55:24.567124+0000","last_peered":"2023-09-23T22:55:24.567124+0000","last_clean":"2023-09-23T22:55:24.567124+0000","last_became_active":"2023-09-23T22:46:44.024192+0000","last_became_peered":"2023-09-23T22:46:44.024192+0000","last_unstale":"2023-09-23T22:55:24.567124+0000","last_undegraded":"2023-09-23T22:55:24.567124+0000","last_fullsized":"2023-09-23T22:55:24.567124+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":317,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752136+0000","last_change":"2023-09-23T22:46:14.247395+0000","last_active":"2023-09-23T22:55:29.752136+0000","last_peered":"2023-09-23T22:55:29.752136+0000","last_clean":"2023-09-23T22:55:29.752136+0000","last_became_active":"2023-09-23T22:46:14.247072+0000","last_became_peered":"2023-09-23T22:46:14.247072+0000","last_unstale":"2023-09-23T22:55:29.752136+0000","last_undegraded":"2023-09-23T22:55:29.752136+0000","last_fullsized":"2023-09-23T22:55:29.752136+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751444+0000","last_change":"2023-09-23T22:51:34.418935+0000","last_active":"2023-09-23T22:55:24.751444+0000","last_peered":"2023-09-23T22:55:24.751444+0000","last_clean":"2023-09-23T22:55:24.751444+0000","last_became_active":"2023-09-23T22:51:34.418608+0000","last_became_peered":"2023-09-23T22:51:34.418608+0000","last_unstale":"2023-09-23T22:55:24.751444+0000","last_undegraded":"2023-09-23T22:55:24.751444+0000","last_fullsized":"2023-09-23T22:55:24.751444+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.944795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":12,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:25.247582+0000","last_change":"2023-09-23T22:55:23.667252+0000","last_active":"2023-09-23T22:55:25.247582+0000","last_peered":"2023-09-23T22:55:25.247582+0000","last_clean":"2023-09-23T22:55:25.247582+0000","last_became_active":"2023-09-23T22:55:23.666965+0000","last_became_peered":"2023-09-23T22:55:23.666965+0000","last_unstale":"2023-09-23T22:55:25.247582+0000","last_undegraded":"2023-09-23T22:55:25.247582+0000","last_fullsized":"2023-09-23T22:55:25.247582+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:16.632323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":322,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751282+0000","last_change":"2023-09-23T22:51:34.426302+0000","last_active":"2023-09-23T22:55:24.751282+0000","last_peered":"2023-09-23T22:55:24.751282+0000","last_clean":"2023-09-23T22:55:24.751282+0000","last_became_active":"2023-09-23T22:51:34.426025+0000","last_became_peered":"2023-09-23T22:51:34.426025+0000","last_unstale":"2023-09-23T22:55:24.751282+0000","last_undegraded":"2023-09-23T22:55:24.751282+0000","last_fullsized":"2023-09-23T22:55:24.751282+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:10.091081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":202,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751094+0000","last_change":"2023-09-23T22:46:44.024699+0000","last_active":"2023-09-23T22:55:24.751094+0000","last_peered":"2023-09-23T22:55:24.751094+0000","last_clean":"2023-09-23T22:55:24.751094+0000","last_became_active":"2023-09-23T22:46:44.024436+0000","last_became_peered":"2023-09-23T22:46:44.024436+0000","last_unstale":"2023-09-23T22:55:24.751094+0000","last_undegraded":"2023-09-23T22:55:24.751094+0000","last_fullsized":"2023-09-23T22:55:24.751094+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:34.398700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":327,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.749097+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:55:24.749097+0000","last_peered":"2023-09-23T22:55:24.749097+0000","last_clean":"2023-09-23T22:55:24.749097+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:55:24.749097+0000","last_undegraded":"2023-09-23T22:55:24.749097+0000","last_fullsized":"2023-09-23T22:55:24.749097+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":11,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567142+0000","last_change":"2023-09-23T22:55:24.310888+0000","last_active":"2023-09-23T22:55:24.567142+0000","last_peered":"2023-09-23T22:55:24.567142+0000","last_clean":"2023-09-23T22:55:24.567142+0000","last_became_active":"2023-09-23T22:55:24.310396+0000","last_became_peered":"2023-09-23T22:55:24.310396+0000","last_unstale":"2023-09-23T22:55:24.567142+0000","last_undegraded":"2023-09-23T22:55:24.567142+0000","last_fullsized":"2023-09-23T22:55:24.567142+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":332,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751302+0000","last_change":"2023-09-23T22:51:34.425305+0000","last_active":"2023-09-23T22:55:24.751302+0000","last_peered":"2023-09-23T22:55:24.751302+0000","last_clean":"2023-09-23T22:55:24.751302+0000","last_became_active":"2023-09-23T22:51:34.425003+0000","last_became_peered":"2023-09-23T22:51:34.425003+0000","last_unstale":"2023-09-23T22:55:24.751302+0000","last_undegraded":"2023-09-23T22:55:24.751302+0000","last_fullsized":"2023-09-23T22:55:24.751302+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:56.885651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":408,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560147+0000","last_change":"2023-09-23T22:46:15.240791+0000","last_active":"2023-09-23T22:55:23.560147+0000","last_peered":"2023-09-23T22:55:23.560147+0000","last_clean":"2023-09-23T22:55:23.560147+0000","last_became_active":"2023-09-23T22:46:15.240361+0000","last_became_peered":"2023-09-23T22:46:15.240361+0000","last_unstale":"2023-09-23T22:55:23.560147+0000","last_undegraded":"2023-09-23T22:55:23.560147+0000","last_fullsized":"2023-09-23T22:55:23.560147+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:39.145343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":74,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567178+0000","last_change":"2023-09-23T22:51:34.421088+0000","last_active":"2023-09-23T22:55:24.567178+0000","last_peered":"2023-09-23T22:55:24.567178+0000","last_clean":"2023-09-23T22:55:24.567178+0000","last_became_active":"2023-09-23T22:51:34.420645+0000","last_became_peered":"2023-09-23T22:51:34.420645+0000","last_unstale":"2023-09-23T22:55:24.567178+0000","last_undegraded":"2023-09-23T22:55:24.567178+0000","last_fullsized":"2023-09-23T22:55:24.567178+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":312,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567663+0000","last_change":"2023-09-23T22:46:14.249253+0000","last_active":"2023-09-23T22:55:24.567663+0000","last_peered":"2023-09-23T22:55:24.567663+0000","last_clean":"2023-09-23T22:55:24.567663+0000","last_became_active":"2023-09-23T22:46:14.248929+0000","last_became_peered":"2023-09-23T22:46:14.248929+0000","last_unstale":"2023-09-23T22:55:24.567663+0000","last_undegraded":"2023-09-23T22:55:24.567663+0000","last_fullsized":"2023-09-23T22:55:24.567663+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":11,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.670528+0000","last_change":"2023-09-23T22:55:23.669810+0000","last_active":"2023-09-23T22:55:23.670528+0000","last_peered":"2023-09-23T22:55:23.670528+0000","last_clean":"2023-09-23T22:55:23.670528+0000","last_became_active":"2023-09-23T22:55:23.669515+0000","last_became_peered":"2023-09-23T22:55:23.669515+0000","last_unstale":"2023-09-23T22:55:23.670528+0000","last_undegraded":"2023-09-23T22:55:23.670528+0000","last_fullsized":"2023-09-23T22:55:23.670528+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:37.219673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":296,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.666913+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:55:23.666913+0000","last_peered":"2023-09-23T22:55:23.666913+0000","last_clean":"2023-09-23T22:55:23.666913+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:55:23.666913+0000","last_undegraded":"2023-09-23T22:55:23.666913+0000","last_fullsized":"2023-09-23T22:55:23.666913+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":267,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560089+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:55:23.560089+0000","last_peered":"2023-09-23T22:55:23.560089+0000","last_clean":"2023-09-23T22:55:23.560089+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:55:23.560089+0000","last_undegraded":"2023-09-23T22:55:23.560089+0000","last_fullsized":"2023-09-23T22:55:23.560089+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":245,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560358+0000","last_change":"2023-09-23T22:46:14.245641+0000","last_active":"2023-09-23T22:55:23.560358+0000","last_peered":"2023-09-23T22:55:23.560358+0000","last_clean":"2023-09-23T22:55:23.560358+0000","last_became_active":"2023-09-23T22:46:14.245375+0000","last_became_peered":"2023-09-23T22:46:14.245375+0000","last_unstale":"2023-09-23T22:55:23.560358+0000","last_undegraded":"2023-09-23T22:55:23.560358+0000","last_fullsized":"2023-09-23T22:55:23.560358+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:17.087730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":11,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567479+0000","last_change":"2023-09-23T22:55:23.670955+0000","last_active":"2023-09-23T22:55:24.567479+0000","last_peered":"2023-09-23T22:55:24.567479+0000","last_clean":"2023-09-23T22:55:24.567479+0000","last_became_active":"2023-09-23T22:55:23.670584+0000","last_became_peered":"2023-09-23T22:55:23.670584+0000","last_unstale":"2023-09-23T22:55:24.567479+0000","last_undegraded":"2023-09-23T22:55:24.567479+0000","last_fullsized":"2023-09-23T22:55:24.567479+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":250,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.667112+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:55:23.667112+0000","last_peered":"2023-09-23T22:55:23.667112+0000","last_clean":"2023-09-23T22:55:23.667112+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:55:23.667112+0000","last_undegraded":"2023-09-23T22:55:23.667112+0000","last_fullsized":"2023-09-23T22:55:23.667112+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":201,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751603+0000","last_change":"2023-09-23T22:46:44.023634+0000","last_active":"2023-09-23T22:55:24.751603+0000","last_peered":"2023-09-23T22:55:24.751603+0000","last_clean":"2023-09-23T22:55:24.751603+0000","last_became_active":"2023-09-23T22:46:44.023339+0000","last_became_peered":"2023-09-23T22:46:44.023339+0000","last_unstale":"2023-09-23T22:55:24.751603+0000","last_undegraded":"2023-09-23T22:55:24.751603+0000","last_fullsized":"2023-09-23T22:55:24.751603+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:02.537526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":250,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560531+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:55:23.560531+0000","last_peered":"2023-09-23T22:55:23.560531+0000","last_clean":"2023-09-23T22:55:23.560531+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:55:23.560531+0000","last_undegraded":"2023-09-23T22:55:23.560531+0000","last_fullsized":"2023-09-23T22:55:23.560531+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":12,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752638+0000","last_change":"2023-09-23T22:55:23.671279+0000","last_active":"2023-09-23T22:55:29.752638+0000","last_peered":"2023-09-23T22:55:29.752638+0000","last_clean":"2023-09-23T22:55:29.752638+0000","last_became_active":"2023-09-23T22:55:23.671010+0000","last_became_peered":"2023-09-23T22:55:23.671010+0000","last_unstale":"2023-09-23T22:55:29.752638+0000","last_undegraded":"2023-09-23T22:55:29.752638+0000","last_fullsized":"2023-09-23T22:55:29.752638+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":76,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.666875+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:55:23.666875+0000","last_peered":"2023-09-23T22:55:23.666875+0000","last_clean":"2023-09-23T22:55:23.666875+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:55:23.666875+0000","last_undegraded":"2023-09-23T22:55:23.666875+0000","last_fullsized":"2023-09-23T22:55:23.666875+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":118,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567701+0000","last_change":"2023-09-23T22:46:44.024485+0000","last_active":"2023-09-23T22:55:24.567701+0000","last_peered":"2023-09-23T22:55:24.567701+0000","last_clean":"2023-09-23T22:55:24.567701+0000","last_became_active":"2023-09-23T22:46:44.024217+0000","last_became_peered":"2023-09-23T22:46:44.024217+0000","last_unstale":"2023-09-23T22:55:24.567701+0000","last_undegraded":"2023-09-23T22:55:24.567701+0000","last_fullsized":"2023-09-23T22:55:24.567701+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":296,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.559961+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:55:23.559961+0000","last_peered":"2023-09-23T22:55:23.559961+0000","last_clean":"2023-09-23T22:55:23.559961+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:55:23.559961+0000","last_undegraded":"2023-09-23T22:55:23.559961+0000","last_fullsized":"2023-09-23T22:55:23.559961+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":11,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567661+0000","last_change":"2023-09-23T22:55:23.668632+0000","last_active":"2023-09-23T22:55:24.567661+0000","last_peered":"2023-09-23T22:55:24.567661+0000","last_clean":"2023-09-23T22:55:24.567661+0000","last_became_active":"2023-09-23T22:55:23.668276+0000","last_became_peered":"2023-09-23T22:55:23.668276+0000","last_unstale":"2023-09-23T22:55:24.567661+0000","last_undegraded":"2023-09-23T22:55:24.567661+0000","last_fullsized":"2023-09-23T22:55:24.567661+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":361,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567707+0000","last_change":"2023-09-23T22:51:34.415722+0000","last_active":"2023-09-23T22:55:24.567707+0000","last_peered":"2023-09-23T22:55:24.567707+0000","last_clean":"2023-09-23T22:55:24.567707+0000","last_became_active":"2023-09-23T22:51:34.415272+0000","last_became_peered":"2023-09-23T22:51:34.415272+0000","last_unstale":"2023-09-23T22:55:24.567707+0000","last_undegraded":"2023-09-23T22:55:24.567707+0000","last_fullsized":"2023-09-23T22:55:24.567707+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":217,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560461+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:55:23.560461+0000","last_peered":"2023-09-23T22:55:23.560461+0000","last_clean":"2023-09-23T22:55:23.560461+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:55:23.560461+0000","last_undegraded":"2023-09-23T22:55:23.560461+0000","last_fullsized":"2023-09-23T22:55:23.560461+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":271,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752660+0000","last_change":"2023-09-23T22:28:43.678897+0000","last_active":"2023-09-23T22:55:29.752660+0000","last_peered":"2023-09-23T22:55:29.752660+0000","last_clean":"2023-09-23T22:55:29.752660+0000","last_became_active":"2023-09-23T22:28:43.678586+0000","last_became_peered":"2023-09-23T22:28:43.678586+0000","last_unstale":"2023-09-23T22:55:29.752660+0000","last_undegraded":"2023-09-23T22:55:29.752660+0000","last_fullsized":"2023-09-23T22:55:29.752660+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":252,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.560389+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:55:23.560389+0000","last_peered":"2023-09-23T22:55:23.560389+0000","last_clean":"2023-09-23T22:55:23.560389+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:55:23.560389+0000","last_undegraded":"2023-09-23T22:55:23.560389+0000","last_fullsized":"2023-09-23T22:55:23.560389+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":104,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:25.247636+0000","last_change":"2023-09-23T22:51:34.427246+0000","last_active":"2023-09-23T22:55:25.247636+0000","last_peered":"2023-09-23T22:55:25.247636+0000","last_clean":"2023-09-23T22:55:25.247636+0000","last_became_active":"2023-09-23T22:51:34.426812+0000","last_became_peered":"2023-09-23T22:51:34.426812+0000","last_unstale":"2023-09-23T22:55:25.247636+0000","last_undegraded":"2023-09-23T22:55:25.247636+0000","last_fullsized":"2023-09-23T22:55:25.247636+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:39.734696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":130,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752542+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:55:29.752542+0000","last_peered":"2023-09-23T22:55:29.752542+0000","last_clean":"2023-09-23T22:55:29.752542+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:55:29.752542+0000","last_undegraded":"2023-09-23T22:55:29.752542+0000","last_fullsized":"2023-09-23T22:55:29.752542+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":329,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752358+0000","last_change":"2023-09-23T22:51:34.428088+0000","last_active":"2023-09-23T22:55:29.752358+0000","last_peered":"2023-09-23T22:55:29.752358+0000","last_clean":"2023-09-23T22:55:29.752358+0000","last_became_active":"2023-09-23T22:51:34.427263+0000","last_became_peered":"2023-09-23T22:51:34.427263+0000","last_unstale":"2023-09-23T22:55:29.752358+0000","last_undegraded":"2023-09-23T22:55:29.752358+0000","last_fullsized":"2023-09-23T22:55:29.752358+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":131,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752489+0000","last_change":"2023-09-23T22:45:51.513456+0000","last_active":"2023-09-23T22:55:29.752489+0000","last_peered":"2023-09-23T22:55:29.752489+0000","last_clean":"2023-09-23T22:55:29.752489+0000","last_became_active":"2023-09-23T22:45:51.513008+0000","last_became_peered":"2023-09-23T22:45:51.513008+0000","last_unstale":"2023-09-23T22:55:29.752489+0000","last_undegraded":"2023-09-23T22:55:29.752489+0000","last_fullsized":"2023-09-23T22:55:29.752489+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":201,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752957+0000","last_change":"2023-09-23T22:51:34.427423+0000","last_active":"2023-09-23T22:55:29.752957+0000","last_peered":"2023-09-23T22:55:29.752957+0000","last_clean":"2023-09-23T22:55:29.752957+0000","last_became_active":"2023-09-23T22:51:34.427057+0000","last_became_peered":"2023-09-23T22:51:34.427057+0000","last_unstale":"2023-09-23T22:55:29.752957+0000","last_undegraded":"2023-09-23T22:55:29.752957+0000","last_fullsized":"2023-09-23T22:55:29.752957+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":70,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.567585+0000","last_change":"2023-09-23T22:51:34.420527+0000","last_active":"2023-09-23T22:55:24.567585+0000","last_peered":"2023-09-23T22:55:24.567585+0000","last_clean":"2023-09-23T22:55:24.567585+0000","last_became_active":"2023-09-23T22:51:34.420142+0000","last_became_peered":"2023-09-23T22:51:34.420142+0000","last_unstale":"2023-09-23T22:55:24.567585+0000","last_undegraded":"2023-09-23T22:55:24.567585+0000","last_fullsized":"2023-09-23T22:55:24.567585+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":303,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-09-23T22:55:24.751132+0000","last_change":"2023-09-23T22:51:34.425505+0000","last_active":"2023-09-23T22:55:24.751132+0000","last_peered":"2023-09-23T22:55:24.751132+0000","last_clean":"2023-09-23T22:55:24.751132+0000","last_became_active":"2023-09-23T22:51:34.424555+0000","last_became_peered":"2023-09-23T22:51:34.424555+0000","last_unstale":"2023-09-23T22:55:24.751132+0000","last_undegraded":"2023-09-23T22:55:24.751132+0000","last_fullsized":"2023-09-23T22:55:24.751132+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:57.365641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":266,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-09-23T22:55:23.666696+0000","last_change":"2023-09-23T22:47:51.507147+0000","last_active":"2023-09-23T22:55:23.666696+0000","last_peered":"2023-09-23T22:55:23.666696+0000","last_clean":"2023-09-23T22:55:23.666696+0000","last_became_active":"2023-09-23T22:47:51.503679+0000","last_became_peered":"2023-09-23T22:47:51.503679+0000","last_unstale":"2023-09-23T22:55:23.666696+0000","last_undegraded":"2023-09-23T22:55:23.666696+0000","last_fullsized":"2023-09-23T22:55:23.666696+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:34.581686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":286,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T22:55:29.752191+0000","last_change":"2023-09-23T22:37:03.800723+0000","last_active":"2023-09-23T22:55:29.752191+0000","last_peered":"2023-09-23T22:55:29.752191+0000","last_clean":"2023-09-23T22:55:29.752191+0000","last_became_active":"2023-09-23T22:37:03.800385+0000","last_became_peered":"2023-09-23T22:37:03.800385+0000","last_unstale":"2023-09-23T22:55:29.752191+0000","last_undegraded":"2023-09-23T22:55:29.752191+0000","last_fullsized":"2023-09-23T22:55:29.752191+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10294,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430550,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626778,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304712,"kb_used_data":3392,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93440440,"statfs":{"total":95995035648,"available":95683010560,"internally_reserved":0,"allocated":3473408,"data_stored":2908734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":8.1950000000000003,"15min":4.3280000000000003},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":17.593,"5min":243.35900000000001,"15min":243.35900000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.7170000000000001,"5min":8.7170000000000005,"15min":4.476},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":17.402000000000001,"5min":243.41499999999999,"15min":243.41499999999999},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":6.3300000000000001,"5min":5.6390000000000002,"15min":4.1959999999999997},"min":{"1min":0.32100000000000001,"5min":0.255,"15min":0.22800000000000001},"max":{"1min":74.899000000000001,"5min":254.62100000000001,"15min":254.62100000000001},"last":0.437},{"interface":"front","average":{"1min":6.4550000000000001,"5min":6.923,"15min":5.2009999999999996},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":74.956000000000003,"5min":255.20099999999999,"15min":255.20099999999999},"last":0.53100000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:55:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":4.1909999999999998,"15min":3.6339999999999999},"min":{"1min":0.35499999999999998,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":5.0899999999999999,"5min":100.85299999999999,"15min":100.85299999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.181,"5min":3.226,"15min":3.7149999999999999},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":15.875999999999999,"5min":101.13,"15min":185.21799999999999},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8009999999999999,"5min":1.845,"15min":3.8879999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.27100000000000002},"max":{"1min":4.1079999999999997,"5min":7.734,"15min":185.19399999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.262,"5min":2.4279999999999999,"15min":3.6709999999999998},"min":{"1min":0.438,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.3109999999999999,"5min":10.378,"15min":128.19800000000001},"last":0.58999999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.3049999999999999,"15min":0.93700000000000006},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.9240000000000004,"5min":10.317,"15min":18.085999999999999},"last":0.502},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.302,"15min":0.94299999999999995},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":10.122999999999999,"5min":10.122999999999999,"15min":18.597000000000001},"last":0.498}]},{"osd":5,"last update":"Sat Sep 23 22:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5580000000000001,"5min":1.4910000000000001,"15min":1.0549999999999999},"min":{"1min":0.33100000000000002,"5min":0.27600000000000002,"15min":0.24399999999999999},"max":{"1min":5.351,"5min":10.353999999999999,"15min":18.972000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.325,"15min":1.0049999999999999},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26200000000000001},"max":{"1min":10.199,"5min":10.199,"15min":18.625},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:55:02 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.298,"15min":0.98599999999999999},"min":{"1min":0.42099999999999999,"5min":0.34799999999999998,"15min":0.316},"max":{"1min":10.225,"5min":10.225,"15min":12.481999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.4550000000000001,"15min":1.028},"min":{"1min":0.40500000000000003,"5min":0.33200000000000002,"15min":0.27100000000000002},"max":{"1min":5.5549999999999997,"5min":6.6269999999999998,"15min":16.661999999999999},"last":0.52000000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253087,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529124,"kb_used_data":3996,"kb_used_omap":9,"kb_used_meta":525110,"kb_avail":93216028,"statfs":{"total":95995035648,"available":95453212672,"internally_reserved":0,"allocated":4091904,"data_stored":2972489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":537713641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9489999999999998,"5min":4.5519999999999996,"15min":5.4329999999999998},"min":{"1min":0.27700000000000002,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":37.991,"5min":37.991,"15min":101.605},"last":0.317},{"interface":"front","average":{"1min":6.1550000000000002,"5min":3.819,"15min":5.3019999999999996},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":42.191000000000003,"5min":42.191000000000003,"15min":101.892},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:28 2023","interfaces":[{"interface":"back","average":{"1min":4.2309999999999999,"5min":3.8900000000000001,"15min":3.2549999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":21.215,"5min":26.129000000000001,"15min":37.889000000000003},"last":0.436},{"interface":"front","average":{"1min":3.9689999999999999,"5min":3.633,"15min":3.3239999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":21.254000000000001,"5min":26.001999999999999,"15min":37.957000000000001},"last":0.373}]},{"osd":3,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.871,"15min":2.645},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.143000000000001,"5min":12.143000000000001,"15min":87.787999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.1899999999999999,"15min":3.1309999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20999999999999999},"max":{"1min":8.0099999999999998,"5min":10.845000000000001,"15min":89.861999999999995},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":1.9510000000000001,"15min":2.3900000000000001},"min":{"1min":0.38,"5min":0.35899999999999999,"15min":0.29299999999999998},"max":{"1min":11.388,"5min":11.388,"15min":78.948999999999998},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.3980000000000001,"5min":2.5659999999999998,"15min":2.4020000000000001},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":9.2579999999999991,"5min":14.058,"15min":78.963999999999999},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.7110000000000001,"15min":1.8839999999999999},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.27900000000000003},"max":{"1min":12.115,"5min":12.115,"15min":84.254999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.78,"15min":2.21},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":8.1050000000000004,"5min":9.8170000000000002,"15min":87.515000000000001},"last":0.497}]},{"osd":6,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":2.6110000000000002,"15min":2.2559999999999998},"min":{"1min":0.40699999999999997,"5min":0.33800000000000002,"15min":0.317},"max":{"1min":6.6639999999999997,"5min":13.991,"15min":79.311000000000007},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.5979999999999999,"5min":1.9810000000000001,"15min":1.9059999999999999},"min":{"1min":0.41899999999999998,"5min":0.40899999999999997,"15min":0.313},"max":{"1min":12.067,"5min":12.067,"15min":79.328999999999994},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:54:37 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.899,"15min":2.609},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.32800000000000001},"max":{"1min":8.1400000000000006,"5min":9.9329999999999998,"15min":159.018},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.254,"5min":2.6949999999999998,"15min":2.7749999999999999},"min":{"1min":0.443,"5min":0.38900000000000001,"15min":0.37},"max":{"1min":6.6310000000000002,"5min":14.032999999999999,"15min":158.96000000000001},"last":0.64900000000000002}]}]},{"osd":0,"up_from":219,"seq":940597837936,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":335244,"kb_used_data":4036,"kb_used_omap":31,"kb_used_meta":331168,"kb_avail":93409908,"statfs":{"total":95995035648,"available":95651745792,"internally_reserved":0,"allocated":4132864,"data_stored":2976970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":339116978},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":3.4329999999999998,"15min":3.7410000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.326000000000001,"5min":40.180999999999997,"15min":40.180999999999997},"last":0.46600000000000003},{"interface":"front","average":{"1min":4.2270000000000003,"5min":3.4630000000000001,"15min":3.6499999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.494999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":0.52000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":5.4139999999999997,"15min":3.5910000000000002},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":89.766000000000005,"5min":111.508,"15min":111.508},"last":0.48499999999999999},{"interface":"front","average":{"1min":7.617,"5min":5.5449999999999999,"15min":3.2360000000000002},"min":{"1min":0.29499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":89.994,"5min":111.532,"15min":111.532},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":4.2519999999999998,"15min":3.0510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":8.5380000000000003,"5min":126.73,"15min":126.73},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.4969999999999999,"5min":3.488,"15min":2.1349999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6440000000000001,"5min":121.483,"15min":121.483},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.0129999999999999,"15min":2.173},"min":{"1min":0.33000000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.5820000000000007,"5min":11.965999999999999,"15min":18.670999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.2400000000000002,"15min":2.036},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.5899999999999999,"5min":16.52,"15min":20.009},"last":0.60799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.8599999999999999,"15min":1.9370000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.318},"max":{"1min":6.6630000000000003,"5min":61.756999999999998,"15min":61.756999999999998},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.1379999999999999,"15min":2.3490000000000002},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4240000000000004,"5min":61.777000000000001,"15min":61.777000000000001},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.778,"15min":1.95},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6950000000000003,"5min":59.146999999999998,"15min":59.146999999999998},"last":0.62},{"interface":"front","average":{"1min":2.504,"5min":3.069,"15min":2.4550000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":7.367,"5min":59.164000000000001,"15min":59.164000000000001},"last":0.81299999999999994}]},{"osd":7,"last update":"Sat Sep 23 22:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.1709999999999998,"15min":2.077},"min":{"1min":0.438,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":8.6959999999999997,"5min":12.385999999999999,"15min":18.640000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.946,"5min":2.0830000000000002,"15min":1.6000000000000001},"min":{"1min":0.42599999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.6779999999999999,"5min":21.954999999999998,"15min":21.954999999999998},"last":0.78800000000000003}]}]},{"osd":2,"up_from":131,"seq":562640715949,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429224,"kb_used_data":3424,"kb_used_omap":81,"kb_used_meta":425710,"kb_avail":93315928,"statfs":{"total":95995035648,"available":95555510272,"internally_reserved":0,"allocated":3506176,"data_stored":2385669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":435927258},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":6.9960000000000004,"15min":5.734},"min":{"1min":0.22800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":12.76,"5min":132.79499999999999,"15min":132.79499999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":3.286,"5min":6.9400000000000004,"15min":6.0990000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.093,"5min":132.58099999999999,"15min":132.58099999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":3.0470000000000002,"15min":3.173},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":6.3019999999999996,"5min":77.004999999999995,"15min":88.316000000000003},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.552,"5min":3.3500000000000001,"15min":3.407},"min":{"1min":0.25700000000000001,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":6.1779999999999999,"5min":76.956000000000003,"15min":88.436999999999998},"last":0.23999999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":4.2939999999999996,"15min":4.4100000000000001},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":12.492000000000001,"5min":112.895,"15min":112.895},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.7869999999999999,"5min":4.109,"15min":3.6219999999999999},"min":{"1min":0.39000000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.1880000000000006,"5min":113.995,"15min":113.995},"last":0.317}]},{"osd":4,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.6749999999999998,"15min":2.8279999999999998},"min":{"1min":0.60199999999999998,"5min":0.28899999999999998,"15min":0.253},"max":{"1min":9.1219999999999999,"5min":69.113,"15min":108.878},"last":0.436},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.4689999999999999,"15min":3.3809999999999998},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2229999999999999,"5min":69.028999999999996,"15min":108.93000000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.7789999999999999,"15min":2.9940000000000002},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":6.1820000000000004,"5min":22.646000000000001,"15min":97.322999999999993},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.2569999999999999,"5min":2.226,"15min":2.98},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":4.1420000000000003,"5min":22.501000000000001,"15min":97.301000000000002},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":3.609,"15min":3.3450000000000002},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":4.3040000000000003,"5min":98.597999999999999,"15min":98.597999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.4670000000000001,"5min":3.4209999999999998,"15min":2.7050000000000001},"min":{"1min":0.48699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":9.2110000000000003,"5min":98.509,"15min":98.509},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":2.0590000000000002,"15min":1.7509999999999999},"min":{"1min":0.48999999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.1609999999999996,"5min":10.477,"15min":10.92},"last":0.503},{"interface":"front","average":{"1min":1.415,"5min":1.667,"15min":2.3799999999999999},"min":{"1min":0.378,"5min":0.371,"15min":0.37},"max":{"1min":6.2720000000000002,"5min":11.311,"15min":88.584000000000003},"last":0.56499999999999995}]}]},{"osd":3,"up_from":0,"seq":403726926035,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019234,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312336,"kb_used_data":2824,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93432816,"statfs":{"total":95995035648,"available":95675203584,"internally_reserved":0,"allocated":2891776,"data_stored":2326313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:24 2023","interfaces":[{"interface":"back","average":{"1min":26.692,"5min":9.3520000000000003,"15min":7.1840000000000002},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":159.62200000000001,"5min":159.62200000000001,"15min":159.62200000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":26.629000000000001,"5min":9.5440000000000005,"15min":7.7160000000000002},"min":{"1min":0.33100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":159.51400000000001,"5min":159.51400000000001,"15min":159.51400000000001},"last":0.64200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":5.0599999999999996,"15min":3.0190000000000001},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":132.304,"15min":132.304},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.7959999999999998,"5min":4.8970000000000002,"15min":2.879},"min":{"1min":0.31,"5min":0.253,"15min":0.253},"max":{"1min":10.832000000000001,"5min":132.453,"15min":132.453},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:54:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.3639999999999999,"15min":3.1520000000000001},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.097,"5min":20.308,"15min":84.909000000000006},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.524,"15min":3.137},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.352,"5min":21.704000000000001,"15min":83.840000000000003},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":4.1879999999999997,"15min":4.548},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":7.3109999999999999,"5min":164.453,"15min":164.453},"last":0.65900000000000003},{"interface":"front","average":{"1min":2.2170000000000001,"5min":4.343,"15min":4.6689999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.27200000000000002},"max":{"1min":5.5549999999999997,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.67400000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.0529999999999999,"15min":1.012},"min":{"1min":0.29399999999999998,"5min":0.28999999999999998,"15min":0.25900000000000001},"max":{"1min":3.0960000000000001,"5min":10.321999999999999,"15min":43.189999999999998},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.0129999999999999,"15min":1.0149999999999999},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.4119999999999999,"5min":6.4349999999999996,"15min":42.450000000000003},"last":0.54400000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:54:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.0529999999999999,"15min":0.94899999999999995},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9139999999999997,"5min":7.9139999999999997,"15min":27.056000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.417,"5min":1.224,"15min":0.97799999999999998},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":5.298,"5min":12.269,"15min":26.940999999999999},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 23 22:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.2130000000000001,"15min":0.89400000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":3.544,"5min":12.311,"15min":12.311},"last":0.629},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.95499999999999996,"15min":0.80600000000000005},"min":{"1min":0.36399999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":3.0169999999999999,"5min":6.7919999999999998,"15min":8.6739999999999995},"last":0.69799999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823006,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":600684,"kb_used_data":2788,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":93144468,"statfs":{"total":95995035648,"available":95379935232,"internally_reserved":0,"allocated":2854912,"data_stored":2313432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:25 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":6.4390000000000001,"15min":5.1269999999999998},"min":{"1min":0.26800000000000002,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":6.3230000000000004,"5min":185.62,"15min":185.62},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.3720000000000001,"5min":6.3860000000000001,"15min":5.0899999999999999},"min":{"1min":0.29399999999999998,"5min":0.248,"15min":0.248},"max":{"1min":4.3140000000000001,"5min":185.499,"15min":185.499},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":9.452,"5min":3.9129999999999998,"15min":2.8799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":110.36499999999999,"5min":110.36499999999999,"15min":149.203},"last":0.50900000000000001},{"interface":"front","average":{"1min":9.5340000000000007,"5min":4.3019999999999996,"15min":3.1909999999999998},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":110.89,"5min":110.89,"15min":149.554},"last":0.71199999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0220000000000002,"5min":5.54,"15min":5.5140000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.997,"5min":189.126,"15min":189.126},"last":0.48399999999999999},{"interface":"front","average":{"1min":3.5379999999999998,"5min":5.2309999999999999,"15min":5.4059999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":41.070999999999998,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.843,"15min":5.6660000000000004},"min":{"1min":0.34899999999999998,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":11.298999999999999,"5min":98.641000000000005,"15min":257.05000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.085,"5min":4.9130000000000003,"15min":5.516},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27800000000000002},"max":{"1min":11.097,"5min":98.906000000000006,"15min":257.35000000000002},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:55:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.0109999999999999,"15min":0.79600000000000004},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":5.7619999999999996,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.544,"5min":1.109,"15min":0.83199999999999996},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":10.535},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:54:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1759999999999999,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.3369999999999997,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.139,"15min":1.629},"min":{"1min":0.374,"5min":0.315,"15min":0.254},"max":{"1min":7.9909999999999997,"5min":10.314,"15min":139.739},"last":0.60999999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:55:06 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.2749999999999999,"15min":0.94999999999999996},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.29999999999999999},"max":{"1min":6.3579999999999997,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.125,"15min":0.85999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36199999999999999,"15min":0.311},"max":{"1min":5.8719999999999999,"5min":8.0459999999999994,"15min":8.0459999999999994},"last":0.73699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:55:37.440 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:55:37.440 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:55:37.440 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:55:37.440 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:55:37.440 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:55:37.441 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:55:37.529 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:37.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:37.635 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:37.689 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:37 smithi052 ceph-mon[180979]: pgmap v933: 89 pgs: 89 active+clean; 345 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 43 MiB/s wr, 688 op/s 2023-09-23T22:55:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:37 smithi052 ceph-mon[187743]: pgmap v933: 89 pgs: 89 active+clean; 345 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 43 MiB/s wr, 688 op/s 2023-09-23T22:55:37.765 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:37.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:37.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:37.796 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:55:37.873 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:37.928 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 5 16 14948 14932 186.551 144.875 0.012456 0.00534993 2023-09-23T22:55:37.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:37 smithi151 ceph-mon[159210]: pgmap v933: 89 pgs: 89 active+clean; 345 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 43 MiB/s wr, 688 op/s 2023-09-23T22:55:37.954 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:38.178 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:38.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:38.283 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:38.330 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:38.421 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:38.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:38.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:38.531 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:38.578 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:38.624 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:38.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:38.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:38.728 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:38.754 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:38.794 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:38.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:38.902 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:38.927 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 6 16 16930 16914 176.107 123.875 0.00310722 0.00567405 2023-09-23T22:55:38.956 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:39.062 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:39.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:39.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:39.251 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:39.312 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:39.394 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:39.402 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:39.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:39.512 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:39.559 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:39.649 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:39.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:39.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:39.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:39.769 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:39 smithi052 ceph-mon[180979]: pgmap v934: 89 pgs: 89 active+clean; 695 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T22:55:39.769 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:39 smithi052 ceph-mon[187743]: pgmap v934: 89 pgs: 89 active+clean; 695 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T22:55:39.805 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:39 smithi151 ceph-mon[159210]: pgmap v934: 89 pgs: 89 active+clean; 695 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T22:55:39.808 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:39.890 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:39.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:39.928 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 7 16 19199 19183 171.189 141.812 0.00410986 0.00583447 2023-09-23T22:55:39.929 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:55:39.999 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:40.102 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:40.928 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 8 16 21441 21425 167.306 140.125 0.00521431 0.00597064 2023-09-23T22:55:41.176 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:55:41.417 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2977053224' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:55:41.417 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:41 smithi052 ceph-mon[187743]: pgmap v935: 89 pgs: 89 active+clean; 841 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:55:41.421 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2977053224' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:55:41.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:41 smithi052 ceph-mon[180979]: pgmap v935: 89 pgs: 89 active+clean; 841 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:55:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2977053224' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:55:41.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:41 smithi151 ceph-mon[159210]: pgmap v935: 89 pgs: 89 active+clean; 841 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:55:41.928 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 9 16 23172 23156 160.726 108.188 0.0078643 0.00621636 2023-09-23T22:55:42.046 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:42.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:42.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:42.155 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:42.197 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:42.285 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:42.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:42.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:42.394 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:42.421 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:42.421 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[187743]: failed to encode map e322 with expected crc 2023-09-23T22:55:42.421 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[187743]: failed to encode map e322 with expected crc 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2977053224' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[187743]: osdmap e322: 8 total, 8 up, 6 in 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[180979]: failed to encode map e322 with expected crc 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[180979]: failed to encode map e322 with expected crc 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2977053224' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:55:42.422 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:42 smithi052 ceph-mon[180979]: osdmap e322: 8 total, 8 up, 6 in 2023-09-23T22:55:42.424 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:42.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:42 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:42 smithi151 ceph-mon[159210]: failed to encode map e322 with expected crc 2023-09-23T22:55:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:42 smithi151 ceph-mon[159210]: failed to encode map e322 with expected crc 2023-09-23T22:55:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2977053224' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:55:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:42 smithi151 ceph-mon[159210]: osdmap e322: 8 total, 8 up, 6 in 2023-09-23T22:55:42.877 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:42.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:42.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:42.928 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 10 16 25126 25110 156.866 122.125 0.00294696 0.00636915 2023-09-23T22:55:42.982 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:43.059 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:43.224 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:43.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:43.329 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:43.382 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:43.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:43 smithi151 ceph-mon[159210]: pgmap v937: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T22:55:43.453 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:43.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:43.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:43.457 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:43.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:55:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:55:43] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.33.4" 2023-09-23T22:55:43.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:43 smithi052 ceph-mon[187743]: pgmap v937: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T22:55:43.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:43 smithi052 ceph-mon[180979]: pgmap v937: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T22:55:43.505 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:55:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:55:43.505 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:48295 8.d 8:ba748e4c:::benchmark_data_smithi052_337357_object48294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 532.55294692699999, 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29879999400000001, 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:55:43.506 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48295 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.507 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946848+0000", 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946847+0000", 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946865+0000", 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8448999999999999e-05 2023-09-23T22:55:43.508 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946874+0000", 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3960000000000006e-06 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946879+0000", 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:55:43.509 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.946921+0000", 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2422000000000001e-05 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.947042+0000", 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120188 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.510 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.947180+0000", 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013842699999999999 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948502+0000", 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001321979 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.511 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948841+0000", 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033914399999999999 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245568+0000", 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29672640900000002 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245607+0000", 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9313000000000001e-05 2023-09-23T22:55:43.512 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245648+0000", 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0911999999999997e-05 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45206.0:48295 8.d e228/217 ondisk, result = 0)", 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:55:43.513 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 532.55092200900003, 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29678497500000001, 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.514 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948873+0000", 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948872+0000", 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:50.948876+0000", 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1010000000000002e-06 2023-09-23T22:55:43.515 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245491+0000", 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29661482700000003 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245512+0000", 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1078e-05 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.516 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245551+0000", 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9286999999999998e-05 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245561+0000", 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7019999999999996e-06 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:46:51.245658+0000", 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6485999999999997e-05 2023-09-23T22:55:43.517 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166076 9.f 9:f351c9e7:::benchmark_data_smithi052_402718_object76435:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.727880+0000", 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.771915094999997, 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0372312260000001, 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.518 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166076 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727880+0000", 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727880+0000", 2023-09-23T22:55:43.519 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727878+0000", 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727884+0000", 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4299999999999997e-06 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727892+0000", 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3899999999999993e-06 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727898+0000", 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1279999999999996e-06 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727952+0000", 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3717000000000001e-05 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.727964+0000", 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1868999999999999e-05 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.755130+0000", 2023-09-23T22:55:43.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0271657869999999 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.755289+0000", 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015896199999999999 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.755412+0000", 2023-09-23T22:55:43.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000123628 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.758804+0000", 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033918160000000002 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.758854+0000", 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0377000000000002e-05 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765011+0000", 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0061563019999999998 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765071+0000", 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0194e-05 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765111+0000", 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9864000000000003e-05 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166082 9.f 9:fe0d287e:::benchmark_data_smithi052_402718_object76441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.738164+0000", 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.761630361999998, 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0461191599999999, 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.527 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166082 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738164+0000", 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738164+0000", 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738162+0000", 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738170+0000", 2023-09-23T22:55:43.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3469999999999999e-06 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738185+0000", 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5155e-05 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738196+0000", 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0835e-05 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738262+0000", 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5461000000000002e-05 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.738276+0000", 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4177e-05 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.758890+0000", 2023-09-23T22:55:43.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.020614245 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.759001+0000", 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110908 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.759138+0000", 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013666199999999999 2023-09-23T22:55:43.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765148+0000", 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.006009955 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765206+0000", 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.834e-05 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784208+0000", 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01900164 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784248+0000", 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0068000000000003e-05 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784283+0000", 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5722999999999997e-05 2023-09-23T22:55:43.534 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166101 9.f 9:f39a191d:::benchmark_data_smithi052_402718_object76460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.761437+0000", 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.738357336, 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0229151830000001, 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.535 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166101 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761437+0000", 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761437+0000", 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761437+0000", 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761440+0000", 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6569999999999999e-06 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761448+0000", 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7670000000000003e-06 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761454+0000", 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6450000000000002e-06 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761545+0000", 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0872999999999995e-05 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.761561+0000", 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.649e-05 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.761951+0000", 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.000389679 2023-09-23T22:55:43.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.762077+0000", 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012638100000000001 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.762189+0000", 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011201699999999999 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765176+0000", 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029870579999999999 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784129+0000", 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018952632000000001 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784291+0000", 2023-09-23T22:55:43.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016175799999999999 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784323+0000", 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2329999999999997e-05 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784353+0000", 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9471e-05 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.542 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166113 9.f 9:f851c1d5:::benchmark_data_smithi052_402718_object76472:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.782284+0000", 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.717510920999999, 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0022682409999999, 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.543 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166113 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782284+0000", 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782284+0000", 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782282+0000", 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782290+0000", 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1400000000000002e-06 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782307+0000", 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7754e-05 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782315+0000", 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9869999999999999e-06 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782381+0000", 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5852000000000006e-05 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.782392+0000", 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0403e-05 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765227+0000", 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98283567500000002 2023-09-23T22:55:43.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765346+0000", 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119294 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.765460+0000", 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113513 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784358+0000", 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018898102999999999 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784412+0000", 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4230000000000001e-05 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784478+0000", 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5994999999999994e-05 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784512+0000", 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3967999999999997e-05 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784552+0000", 2023-09-23T22:55:43.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9719999999999999e-05 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166158 9.f 9:fb3f4b83:::benchmark_data_smithi052_402718_object76517:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.837209+0000", 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.662585548000003, 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.95353122499999998, 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.551 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166158 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.837209+0000", 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.837209+0000", 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.837208+0000", 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.837214+0000", 2023-09-23T22:55:43.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7350000000000001e-06 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.837227+0000", 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3176000000000001e-05 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.837235+0000", 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5229999999999995e-06 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.839160+0000", 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0019250090000000001 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.839179+0000", 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8768000000000001e-05 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.768538+0000", 2023-09-23T22:55:43.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.92935890200000004 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.768665+0000", 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012729700000000001 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.768792+0000", 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012709900000000001 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784363+0000", 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015570934 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784446+0000", 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2913999999999999e-05 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790645+0000", 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0061983769999999997 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790696+0000", 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1717e-05 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790740+0000", 2023-09-23T22:55:43.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4023999999999997e-05 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166163 9.f 9:fe37d41a:::benchmark_data_smithi052_402718_object76522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.843988+0000", 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.655806761999997, 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.95961031600000002, 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.559 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166163 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.843988+0000", 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.843988+0000", 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.843987+0000", 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.843990+0000", 2023-09-23T22:55:43.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.129e-06 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.843996+0000", 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7590000000000003e-06 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.843999+0000", 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2119999999999999e-06 2023-09-23T22:55:43.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.844066+0000", 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6705999999999995e-05 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.844078+0000", 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1996e-05 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.771828+0000", 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.92774944800000003 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.771936+0000", 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010805 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.772010+0000", 2023-09-23T22:55:43.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4418000000000005e-05 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784368+0000", 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012358185000000001 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790772+0000", 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0064040420000000004 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803486+0000", 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012713502 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803547+0000", 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0850999999999997e-05 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803598+0000", 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1677e-05 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166164 9.f 9:f3e1f229:::benchmark_data_smithi052_402718_object76523:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.848691+0000", 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.651103194999997, 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.95503521999999996, 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.567 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166164 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848691+0000", 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848691+0000", 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848690+0000", 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848698+0000", 2023-09-23T22:55:43.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4699999999999996e-06 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848714+0000", 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6460000000000002e-05 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848721+0000", 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8510000000000001e-06 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848760+0000", 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9022e-05 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.848769+0000", 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0520000000000007e-06 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.774971+0000", 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.92620194600000005 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.775121+0000", 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014957499999999999 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.775251+0000", 2023-09-23T22:55:43.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013042300000000001 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784373+0000", 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0091220769999999993 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790790+0000", 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0064167449999999997 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.573 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803636+0000", 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012845897 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803684+0000", 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8485000000000001e-05 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803727+0000", 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2435999999999998e-05 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166206 9.f 9:f461288e:::benchmark_data_smithi052_402718_object76565:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.904334+0000", 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.595460944000003, 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.89952350700000006, 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.575 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166206 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904334+0000", 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904334+0000", 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904332+0000", 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904339+0000", 2023-09-23T22:55:43.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0589999999999997e-06 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904353+0000", 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3369e-05 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904360+0000", 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5650000000000004e-06 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904417+0000", 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6848999999999997e-05 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.904427+0000", 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0288e-05 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.777842+0000", 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.87341469500000002 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.777976+0000", 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013364900000000001 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.778094+0000", 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011817899999999999 2023-09-23T22:55:43.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784379+0000", 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062848030000000003 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790807+0000", 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0064284399999999997 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803773+0000", 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012966445 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803818+0000", 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4564999999999998e-05 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803857+0000", 2023-09-23T22:55:43.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9249000000000003e-05 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166210 9.f 9:f386d3ae:::benchmark_data_smithi052_402718_object76569:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.909709+0000", 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.590085848000001, 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.89427182699999996, 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.583 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166210 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909709+0000", 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909709+0000", 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909707+0000", 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909715+0000", 2023-09-23T22:55:43.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4749999999999996e-06 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909739+0000", 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3804000000000001e-05 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909749+0000", 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0246e-05 2023-09-23T22:55:43.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909823+0000", 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3589000000000003e-05 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.909841+0000", 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8349999999999999e-05 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.780962+0000", 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.87112099399999998 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.781120+0000", 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015823899999999999 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.781250+0000", 2023-09-23T22:55:43.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000129901 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784418+0000", 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031683599999999998 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790829+0000", 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0064107440000000003 2023-09-23T22:55:43.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803893+0000", 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013064285 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803936+0000", 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2367999999999998e-05 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.803981+0000", 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4912e-05 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166221 9.f 9:fd298760:::benchmark_data_smithi052_402718_object76580:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.930172+0000", 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.569622187999997, 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.88688842999999995, 2023-09-23T22:55:43.591 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166221 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930172+0000", 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930172+0000", 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930171+0000", 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930180+0000", 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1379999999999996e-06 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930194+0000", 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4073000000000001e-05 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930203+0000", 2023-09-23T22:55:43.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8470000000000002e-06 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930311+0000", 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010785600000000001 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930323+0000", 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2364000000000001e-05 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784609+0000", 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85428532800000001 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784743+0000", 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013459600000000001 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.784861+0000", 2023-09-23T22:55:43.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000117648 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.797084+0000", 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012223018 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.804031+0000", 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0069470269999999997 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.597 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.816975+0000", 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012943837 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817018+0000", 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3679e-05 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817061+0000", 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2605000000000003e-05 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166223 9.f 9:f3577fce:::benchmark_data_smithi052_402718_object76582:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.930228+0000", 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.569566815000002, 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.88713337800000003, 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.599 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166223 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930228+0000", 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930228+0000", 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930227+0000", 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930232+0000", 2023-09-23T22:55:43.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3889999999999998e-06 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930242+0000", 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9869999999999994e-06 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930248+0000", 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7689999999999999e-06 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930341+0000", 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2749000000000003e-05 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.930346+0000", 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.454e-06 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.787944+0000", 2023-09-23T22:55:43.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85759835200000001 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.788061+0000", 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011752399999999999 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.788176+0000", 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000114782 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.604 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.804084+0000", 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015907551999999998 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817068+0000", 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01298413 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.605 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817289+0000", 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022082499999999999 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817328+0000", 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9780000000000002e-05 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817361+0000", 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2762999999999998e-05 2023-09-23T22:55:43.606 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166226 9.f 9:fd3d7cb8:::benchmark_data_smithi052_402718_object76585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.935202+0000", 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.564592707999999, 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.88226088700000005, 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.607 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166226 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935202+0000", 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935202+0000", 2023-09-23T22:55:43.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935201+0000", 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935206+0000", 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4229999999999996e-06 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935216+0000", 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5259999999999999e-06 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935222+0000", 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.711e-06 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935440+0000", 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021827700000000001 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.935450+0000", 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7289999999999995e-06 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790841+0000", 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.85539188899999996 2023-09-23T22:55:43.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790913+0000", 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1699000000000006e-05 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.790983+0000", 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9930000000000003e-05 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.804119+0000", 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013136289000000001 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817073+0000", 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012953522 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817393+0000", 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032033300000000001 2023-09-23T22:55:43.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817432+0000", 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8776999999999998e-05 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817463+0000", 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0877000000000003e-05 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.614 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166245 9.f 9:f235bdf7:::benchmark_data_smithi052_402718_object76604:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.969239+0000", 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.530555827000001, 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.848316246, 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.615 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166245 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969239+0000", 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969239+0000", 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969237+0000", 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969246+0000", 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4149999999999999e-06 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969264+0000", 2023-09-23T22:55:43.618 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8029e-05 2023-09-23T22:55:43.618 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.618 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.618 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.618 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:43.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969275+0000", 2023-09-23T22:55:43.619 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1336e-05 2023-09-23T22:55:43.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.619 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969342+0000", 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6772000000000004e-05 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.969354+0000", 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1946e-05 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.793454+0000", 2023-09-23T22:55:43.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.82410036200000003 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.793575+0000", 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121352 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.793660+0000", 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4734000000000006e-05 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.816917+0000", 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023256365000000001 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817077+0000", 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016086200000000001 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817494+0000", 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000416112 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817526+0000", 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2199000000000001e-05 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817555+0000", 2023-09-23T22:55:43.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9371999999999999e-05 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166246 9.f 9:f0fab35a:::benchmark_data_smithi052_402718_object76605:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.970119+0000", 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.529675656000002, 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.84752781099999996, 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.624 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166246 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970119+0000", 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.625 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970119+0000", 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970118+0000", 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970123+0000", 2023-09-23T22:55:43.626 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6689999999999997e-06 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970132+0000", 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9039999999999998e-06 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970137+0000", 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4859999999999997e-06 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970177+0000", 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9678000000000002e-05 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.970184+0000", 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7599999999999997e-06 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.797113+0000", 2023-09-23T22:55:43.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.82692879799999996 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.797260+0000", 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014715999999999999 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.797371+0000", 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011106400000000001 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.816962+0000", 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.019591224000000001 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817082+0000", 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120195 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817585+0000", 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00050283299999999995 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817618+0000", 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2653000000000001e-05 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817647+0000", 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9085999999999999e-05 2023-09-23T22:55:43.631 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166252 9.f 9:f77e63c3:::benchmark_data_smithi052_402718_object76611:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:47.983901+0000", 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.515894005, 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.83394573100000002, 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.632 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166252 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.983901+0000", 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.983901+0000", 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.983899+0000", 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.983906+0000", 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.241e-06 2023-09-23T22:55:43.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.983916+0000", 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0910999999999999e-05 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.983922+0000", 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7010000000000001e-06 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.984352+0000", 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000429632 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:47.984364+0000", 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2026e-05 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.800013+0000", 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.81564953200000001 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.800122+0000", 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000108187 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.800242+0000", 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000120198 2023-09-23T22:55:43.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817087+0000", 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016845064999999999 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817120+0000", 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2919000000000001e-05 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.638 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817760+0000", 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00064007300000000001 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817807+0000", 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7219999999999999e-05 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817846+0000", 2023-09-23T22:55:43.639 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9388999999999998e-05 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:166259 9.f 9:f4338da7:::benchmark_data_smithi052_402718_object76618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:54:48.012396+0000", 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.487399144000001, 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.82038414000000004, 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.640 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 166259 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012396+0000", 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012396+0000", 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012394+0000", 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012402+0000", 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6300000000000004e-06 2023-09-23T22:55:43.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012418+0000", 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5840000000000001e-05 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012426+0000", 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2520000000000002e-06 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012499+0000", 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3102000000000006e-05 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.012511+0000", 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2473999999999999e-05 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.804144+0000", 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.79163242599999994 2023-09-23T22:55:43.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.804256+0000", 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000112518 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.804353+0000", 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7365e-05 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817126+0000", 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012772428000000001 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.817155+0000", 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9048999999999999e-05 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.832673+0000", 2023-09-23T22:55:43.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015518021 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.832736+0000", 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3155000000000001e-05 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:54:48.832780+0000", 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3606000000000002e-05 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.647 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:16594 10.f e321/317 10:f445366d:::benchmark_data_smithi052_461816_object16593:head v 321'1041, mlcod=321'1038)", 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:38.579815+0000", 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4.9199792559999995, 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24760716499999999, 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579815+0000", 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579815+0000", 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579815+0000", 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:55:43.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579869+0000", 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3964e-05 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579879+0000", 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0701e-05 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579886+0000", 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7020000000000004e-06 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579929+0000", 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2956e-05 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.579947+0000", 2023-09-23T22:55:43.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8338999999999998e-05 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.827378+0000", 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.247430766 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.827423+0000", 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4483000000000001e-05 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.652 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:16597 10.7 e321/317 10:e222d696:::benchmark_data_smithi052_461816_object16596:head v 321'1019, mlcod=321'1017)", 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:38.580534+0000", 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4.9192607710000003, 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24692345900000001, 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:55:43.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.580534+0000", 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.580534+0000", 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.580533+0000", 2023-09-23T22:55:43.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.580582+0000", 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9013e-05 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.823822+0000", 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24323936400000001 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.823829+0000", 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8359999999999992e-06 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.823911+0000", 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1032999999999995e-05 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:55:43.656 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.823931+0000", 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0068000000000002e-05 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.827430+0000", 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034991169999999999 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:38.827457+0000", 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7651999999999999e-05 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.657 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "" 2023-09-23T22:55:43.658 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:55:43.713 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:43.930 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 11 16 26953 26937 152.973 114.188 0.00712143 0.00653209 2023-09-23T22:55:44.930 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 12 16 28134 28118 146.377 73.8125 0.004936 0.00677768 2023-09-23T22:55:45.050 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:55:45.329 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:55:45.371 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:45.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:45.481 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:45.559 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:45.696 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:45 smithi052 ceph-mon[180979]: pgmap v938: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-09-23T22:55:45.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:45 smithi052 ceph-mon[187743]: pgmap v938: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-09-23T22:55:45.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 13 16 29754 29738 142.896 101.25 0.0714474 0.00696666 2023-09-23T22:55:45.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:45.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:45.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:45 smithi151 ceph-mon[159210]: pgmap v938: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-09-23T22:55:46.038 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:46.066 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:46.111 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:46.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:46.214 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:46.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:46.497 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:46.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:46.568 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:46.605 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:46.650 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:46.742 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:46.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:46.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:46.804 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:46.851 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:46.896 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:46.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 14 16 31362 31346 139.869 100.5 0.0136251 0.00714373 2023-09-23T22:55:46.995 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:47.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:47.102 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:47.182 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:47.428 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:47.465 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:47.504 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:55:47.539 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:47.573 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:47.602 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:47.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:47.610 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:47.649 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:47.683 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:47.686 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.687 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:55:47.710 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:47.713 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:47 smithi052 ceph-mon[180979]: pgmap v939: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T22:55:47.713 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1318757806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:47.713 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:47 smithi052 ceph-mon[187743]: pgmap v939: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T22:55:47.714 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1318757806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:47.753 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:47.795 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:47.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:47 smithi151 ceph-mon[159210]: pgmap v939: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T22:55:47.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1318757806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:47.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:47.903 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:47.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 15 16 32920 32904 137.036 97.375 0.00416446 0.00720032 2023-09-23T22:55:47.933 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:48.455 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:48.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:48.561 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:48.623 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:48.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 16 16 34343 34327 134.031 88.9375 0.00362755 0.00742952 2023-09-23T22:55:49.291 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:49.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:49.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:49.403 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:49.458 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:49.517 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:49.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:49.622 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:49.648 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:49.690 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:49.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:49 smithi052 ceph-mon[187743]: pgmap v940: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T22:55:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:49.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:49 smithi052 ceph-mon[180979]: pgmap v940: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T22:55:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:49.798 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:49.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:49 smithi151 ceph-mon[159210]: pgmap v940: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T22:55:49.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:55:49.844 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:55:49.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:49.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 17 16 35544 35528 130.563 75.0625 0.00916907 0.00765277 2023-09-23T22:55:49.935 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:49.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:49.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:50.044 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:50.105 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:50.191 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:50.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:50.297 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:55:50.357 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:50.380 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [7, 3] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:55:50.380 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:55:50.380 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [1, 3, 4, 5, 6, 7, 2, 0] 2023-09-23T22:55:50.380 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T22:55:50.381 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:55:50.514 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:55:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:50.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:55:50.623 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:50.692 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:55:50 smithi052 systemd[1]: Stopping Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:55:50.698 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:50.799 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:50.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:50.820 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:55:50.820 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:55:50.907 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:50.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 18 16 37224 37208 129.142 105 0.00907064 0.00773779 2023-09-23T22:55:50.980 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:51.070 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:51.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:51.177 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:51.201 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:55:51.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1243433503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:51.230 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:51 smithi052 ceph-mon[187743]: pgmap v941: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T22:55:51.230 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1243433503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:51.230 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:51 smithi052 ceph-mon[180979]: pgmap v941: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T22:55:51.234 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:51.337 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:51.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:51.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:51.444 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:51.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1243433503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:55:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:51 smithi151 ceph-mon[159210]: pgmap v941: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-09-23T22:55:51.489 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:55:51 smithi052 conmon[264731]: 2023-09-23T22:55:51.376+0000 7fef0ae92700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:55:51.489 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:55:51 smithi052 conmon[264731]: 2023-09-23T22:55:51.376+0000 7fef0ae92700 -1 osd.2 323 *** Got signal Terminated *** 2023-09-23T22:55:51.490 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:55:51 smithi052 conmon[264731]: 2023-09-23T22:55:51.376+0000 7fef0ae92700 -1 osd.2 323 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:55:51.493 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:51.861 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:55:51.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 19 16 39049 39033 128.348 114.062 0.0150001 0.00778644 2023-09-23T22:55:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:52 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:52 smithi151 ceph-mon[159210]: failed to encode map e323 with expected crc 2023-09-23T22:55:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:52 smithi151 ceph-mon[159210]: failed to encode map e323 with expected crc 2023-09-23T22:55:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1243433503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:52 smithi151 ceph-mon[159210]: osdmap e323: 8 total, 8 up, 6 in 2023-09-23T22:55:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:52 smithi151 ceph-mon[159210]: osd.2 marked itself down and dead 2023-09-23T22:55:52.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[187743]: failed to encode map e323 with expected crc 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[187743]: failed to encode map e323 with expected crc 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1243433503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[187743]: osdmap e323: 8 total, 8 up, 6 in 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[187743]: osd.2 marked itself down and dead 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[180979]: failed to encode map e323 with expected crc 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[180979]: failed to encode map e323 with expected crc 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1243433503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:55:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[180979]: osdmap e323: 8 total, 8 up, 6 in 2023-09-23T22:55:52.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:52 smithi052 ceph-mon[180979]: osd.2 marked itself down and dead 2023-09-23T22:55:52.631 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:55:52.675 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:52.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:52.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:52.774 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:52.780 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:52.817 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:52.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:55:52.932113+0000 min lat: 0.00121105 max lat: 0.252928 avg lat: 0.00782319 2023-09-23T22:55:52.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:55:52.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 20 16 39432 39416 123.129 23.9375 0.0029126 0.00782319 2023-09-23T22:55:52.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:52.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:52.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:53.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:53.119 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:53.162 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:53.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[187743]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:55:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[187743]: failed to encode map e324 with expected crc 2023-09-23T22:55:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[187743]: osdmap e324: 8 total, 7 up, 6 in 2023-09-23T22:55:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[187743]: pgmap v944: 89 pgs: 16 stale+active+clean, 73 active+clean; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T22:55:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[180979]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:55:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[180979]: failed to encode map e324 with expected crc 2023-09-23T22:55:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[180979]: osdmap e324: 8 total, 7 up, 6 in 2023-09-23T22:55:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:53 smithi052 ceph-mon[180979]: pgmap v944: 89 pgs: 16 stale+active+clean, 73 active+clean; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T22:55:53.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:55:53 smithi052 bash[465553]: d77ee82fce9a6952037c717636df1ebd96f68fc4c493d1b73c852d6dac94f0d6 2023-09-23T22:55:53.270 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:53.320 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:53.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:53 smithi151 ceph-mon[159210]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:55:53.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:53 smithi151 ceph-mon[159210]: failed to encode map e324 with expected crc 2023-09-23T22:55:53.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:53 smithi151 ceph-mon[159210]: osdmap e324: 8 total, 7 up, 6 in 2023-09-23T22:55:53.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:53 smithi151 ceph-mon[159210]: pgmap v944: 89 pgs: 16 stale+active+clean, 73 active+clean; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T22:55:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:53.654 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:55:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:55:53] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T22:55:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:53.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:53.756 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:53.812 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:53.930 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:53.933 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 21 16 40795 40779 121.322 85.1875 0.00750509 0.0082363 2023-09-23T22:55:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:53.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:54.035 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:54.085 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:54.184 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:54.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:54.292 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:54.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:54 smithi151 ceph-mon[159210]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:54 smithi151 ceph-mon[159210]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:54 smithi151 ceph-mon[159210]: osdmap e325: 8 total, 7 up, 6 in 2023-09-23T22:55:54.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:54 smithi151 ceph-mon[159210]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.351 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:54 smithi151 ceph-mon[159210]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.352 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:54 smithi151 ceph-mon[159210]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.352 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:54.451 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:54.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:54.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[187743]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[187743]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[187743]: osdmap e325: 8 total, 7 up, 6 in 2023-09-23T22:55:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[187743]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[187743]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[187743]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[180979]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[180979]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[180979]: osdmap e325: 8 total, 7 up, 6 in 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[180979]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[180979]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:54 smithi052 ceph-mon[180979]: failed to encode map e325 with expected crc 2023-09-23T22:55:54.563 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:54.623 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:54.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:55:54.693 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:54.724 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:54.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:54.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:54.917 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:54.931 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 22 16 42510 42494 120.679 107.188 0.00593417 0.00828174 2023-09-23T22:55:55.087 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:55.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:55.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:55.197 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:55:55.256 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:55.378 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:55:55.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:55.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:55:55.486 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:55:55.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:55 smithi052 ceph-mon[180979]: pgmap v946: 89 pgs: 3 active+undersized+degraded, 8 active+undersized, 12 stale+active+clean, 66 active+clean; 2.3 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5064/114534 objects degraded (4.421%) 2023-09-23T22:55:55.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:55 smithi052 ceph-mon[187743]: pgmap v946: 89 pgs: 3 active+undersized+degraded, 8 active+undersized, 12 stale+active+clean, 66 active+clean; 2.3 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5064/114534 objects degraded (4.421%) 2023-09-23T22:55:55.553 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:55 smithi151 ceph-mon[159210]: pgmap v946: 89 pgs: 3 active+undersized+degraded, 8 active+undersized, 12 stale+active+clean, 66 active+clean; 2.3 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 5064/114534 objects degraded (4.421%) 2023-09-23T22:55:55.557 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:55.658 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:55:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:55.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:55:55.681 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:55:55.714 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:55:55.750 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:55:55.767 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:55.781 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:55:55.818 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:55:55.819 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:55.853 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.888 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:55:55.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 23 16 44190 44174 119.998 105 0.00635776 0.00832918 2023-09-23T22:55:56.476 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:56 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 5064/114534 objects degraded (4.421%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:55:56.476 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3606426928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:56.476 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:56 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 5064/114534 objects degraded (4.421%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:55:56.476 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3606426928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:56.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:56 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 5064/114534 objects degraded (4.421%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:55:56.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3606426928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:55:56.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:56.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:56.876 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:55:56.907 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:56.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 24 16 45565 45549 118.578 85.9375 0.00349618 0.00839064 2023-09-23T22:55:57.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:57 smithi052 ceph-mon[180979]: pgmap v947: 89 pgs: 5 active+undersized+degraded, 18 active+undersized, 5 stale+active+clean, 61 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 90 MiB/s wr, 1.45k op/s; 10568/122304 objects degraded (8.641%) 2023-09-23T22:55:57.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:57 smithi052 ceph-mon[187743]: pgmap v947: 89 pgs: 5 active+undersized+degraded, 18 active+undersized, 5 stale+active+clean, 61 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 90 MiB/s wr, 1.45k op/s; 10568/122304 objects degraded (8.641%) 2023-09-23T22:55:57.671 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:55:57.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:57.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:55:57.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:57 smithi151 ceph-mon[159210]: pgmap v947: 89 pgs: 5 active+undersized+degraded, 18 active+undersized, 5 stale+active+clean, 61 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 90 MiB/s wr, 1.45k op/s; 10568/122304 objects degraded (8.641%) 2023-09-23T22:55:57.775 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:57.785 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:55:57.786 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:55:57.833 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:57.834 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T22:55:57.874 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:55:57.911 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:55:57.911 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T22:55:57.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 25 16 46832 46816 117.003 79.1875 0.0140564 0.00854281 2023-09-23T22:55:57.939 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:57.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:57.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:58.043 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:55:58.093 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:58.275 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:55:58.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:58.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:55:58.310 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:55:58.310 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:55:58.310 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:55:58.310 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:55:58.311 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:55:58.383 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:55:58.441 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:58.561 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:55:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:58.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:55:58.670 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:55:58.720 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:58.932 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 26 16 48239 48223 115.885 87.9375 0.00231824 0.00860929 2023-09-23T22:55:59.674 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:55:59.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:55:59 smithi052 ceph-mon[187743]: pgmap v948: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 91 MiB/s wr, 1.45k op/s; 20106/136611 objects degraded (14.718%) 2023-09-23T22:55:59.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:55:59 smithi052 ceph-mon[180979]: pgmap v948: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 91 MiB/s wr, 1.45k op/s; 20106/136611 objects degraded (14.718%) 2023-09-23T22:55:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:59.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:55:59.769 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:55:59.778 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:55:59.786 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:55:59.825 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:55:59 smithi151 ceph-mon[159210]: pgmap v948: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 91 MiB/s wr, 1.45k op/s; 20106/136611 objects degraded (14.718%) 2023-09-23T22:55:59.829 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:55:59.917 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:55:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:55:59.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:55:59.933 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 27 16 49803 49787 115.208 97.75 0.00770344 0.00867485 2023-09-23T22:56:00.029 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:00.094 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:00.178 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:00.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:00.289 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:00.337 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:00.388 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:00.437 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:00.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:00.470 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:00.548 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:00.603 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:00.705 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:00.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:00.813 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:00.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:00.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 28 16 51270 51254 114.368 91.6875 0.00960617 0.00873932 2023-09-23T22:56:00.987 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:00.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:01.098 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:01.157 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:01.198 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:56:01.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:01 smithi151 ceph-mon[159210]: pgmap v949: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 20106/136611 objects degraded (14.718%) 2023-09-23T22:56:01.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/145427375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:01.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:01 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:01 smithi052 ceph-mon[180979]: pgmap v949: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 20106/136611 objects degraded (14.718%) 2023-09-23T22:56:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/145427375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:01.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:01 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:01.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:01 smithi052 ceph-mon[187743]: pgmap v949: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 20106/136611 objects degraded (14.718%) 2023-09-23T22:56:01.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/145427375' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:01.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:01 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:56:01.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 29 16 52470 52454 113.011 75 0.0125044 0.00884428 2023-09-23T22:56:02.187 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:02.187 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[187743]: failed to encode map e326 with expected crc 2023-09-23T22:56:02.187 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/145427375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:56:02.187 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[187743]: osdmap e326: 8 total, 7 up, 6 in 2023-09-23T22:56:02.188 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:02.188 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[180979]: failed to encode map e326 with expected crc 2023-09-23T22:56:02.188 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/145427375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:56:02.188 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:02 smithi052 ceph-mon[180979]: osdmap e326: 8 total, 7 up, 6 in 2023-09-23T22:56:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:02 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:02 smithi151 ceph-mon[159210]: failed to encode map e326 with expected crc 2023-09-23T22:56:02.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/145427375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:56:02.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:02 smithi151 ceph-mon[159210]: osdmap e326: 8 total, 7 up, 6 in 2023-09-23T22:56:02.481 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:02.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:02.585 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:02.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:02.772 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:02.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:02.881 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:02.911 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [7, 3] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:56:02.911 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:56:02.911 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:56:02.911 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:56:02.912 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:56:02.912 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:56:02.933 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 30 16 53866 53850 112.152 87.25 0.0143341 0.00891189 2023-09-23T22:56:02.974 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:03.122 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:03.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:03.199 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:03 smithi052 ceph-mon[180979]: pgmap v951: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s; 22314/152370 objects degraded (14.645%) 2023-09-23T22:56:03.229 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:03.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:03 smithi151 ceph-mon[159210]: pgmap v951: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s; 22314/152370 objects degraded (14.645%) 2023-09-23T22:56:03.286 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:03.385 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:03.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:03.487 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:03.496 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:56:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:56:03] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-09-23T22:56:03.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:03 smithi052 ceph-mon[187743]: pgmap v951: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s; 22314/152370 objects degraded (14.645%) 2023-09-23T22:56:03.522 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:03.571 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:03.573 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:03.614 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:56:03.615 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:56:03.626 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:03.631 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:03.671 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:03.680 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:03.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:03.709 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:03.749 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:03.780 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:03.780 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.780 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.780 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:03.781 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.781 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.781 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.781 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.781 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:56:03.786 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:03.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:03.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 31 16 55140 55124 111.103 79.625 0.00722962 0.00899612 2023-09-23T22:56:03.988 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:04.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:04.105 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:04.134 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:04.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:04 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 22314/152370 objects degraded (14.645%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:04.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4147788899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:04 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 22314/152370 objects degraded (14.645%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:04.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4147788899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:04 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 22314/152370 objects degraded (14.645%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4147788899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:04.708 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:56:04.753 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:04.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:04.859 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:04.924 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:04.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 32 16 56253 56237 109.805 69.5625 0.00794533 0.00910292 2023-09-23T22:56:05.101 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:56:05.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[180979]: pgmap v952: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 23132/157464 objects degraded (14.690%) 2023-09-23T22:56:05.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:05.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:05.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:05.558 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[187743]: pgmap v952: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 23132/157464 objects degraded (14.690%) 2023-09-23T22:56:05.558 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:05.558 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:05.559 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:05.654 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:05.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:05.758 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:05.821 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:05 smithi151 ceph-mon[159210]: pgmap v952: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 23132/157464 objects degraded (14.690%) 2023-09-23T22:56:05.821 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:05.821 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:05.821 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:05.822 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:05.873 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:05.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:05.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 33 16 57313 57297 108.485 66.25 0.00319884 0.00918363 2023-09-23T22:56:05.980 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:06.063 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:06.098 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:06.165 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:06.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:06.273 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:06.330 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:06.433 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:06.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:06.514 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:06.515 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:06.543 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:06.608 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:06.906 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:06.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:06.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:06.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:06 smithi052 ceph-mon[180979]: pgmap v953: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 23840/163980 objects degraded (14.538%) 2023-09-23T22:56:06.912 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:06.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:06.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:06 smithi052 ceph-mon[187743]: pgmap v953: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 23840/163980 objects degraded (14.538%) 2023-09-23T22:56:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:06.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:06.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 34 16 58618 58602 107.693 81.5625 0.011633 0.00928069 2023-09-23T22:56:07.015 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:07.035 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:56:07.036 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "" 2023-09-23T22:56:07.036 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:56:07.076 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:07.076 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:07.076 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:06 smithi151 ceph-mon[159210]: pgmap v953: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 23840/163980 objects degraded (14.538%) 2023-09-23T22:56:07.077 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:07.212 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:07.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:07.325 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:07.399 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:07.518 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:07.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:07.629 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:07.655 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:07.698 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:07.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:07.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:07.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:07.851 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:07.934 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 35 16 59717 59701 106.578 68.6875 0.00878278 0.00937781 2023-09-23T22:56:08.099 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:56:08.429 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:08.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:08.535 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:08.585 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:08.685 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:08 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 25786/175533 objects degraded (14.690%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:08.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:08 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 25786/175533 objects degraded (14.690%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:08.723 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:08.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:08.829 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:08.868 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:08.935 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 36 16 61152 61136 106.105 89.6875 0.00980022 0.0094196 2023-09-23T22:56:08.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:08 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 25786/175533 objects degraded (14.690%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:09.143 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:09.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:09.247 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:09.312 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:09.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:09.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:09.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:09.526 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:09.567 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:09.578 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:09.729 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:09.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:09.838 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:09.893 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:09 smithi052 ceph-mon[180979]: pgmap v954: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 25786/175533 objects degraded (14.690%) 2023-09-23T22:56:09.899 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:09 smithi052 ceph-mon[187743]: pgmap v954: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 25786/175533 objects degraded (14.690%) 2023-09-23T22:56:09.901 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:09.935 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 37 16 62521 62505 105.55 85.5625 0.0107985 0.00946952 2023-09-23T22:56:09.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:09 smithi151 ceph-mon[159210]: pgmap v954: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 25786/175533 objects degraded (14.690%) 2023-09-23T22:56:10.155 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:56:10.155 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:56:10.211 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:10.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:10.328 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:10.397 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:10.518 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:10.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:10.627 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:10.650 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1227474365' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:10.650 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:10.652 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:10.680 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:56:10.703 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:10.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:10.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:10.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:10.886 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:10.936 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 38 16 63710 63694 104.728 74.3125 0.0147469 0.00954446 2023-09-23T22:56:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1227474365' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1227474365' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:10.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:11.013 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:11.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:11.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:11.128 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:11.176 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:11.548 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:56:11.645 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:11.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:11.751 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:11.787 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[187743]: pgmap v955: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 25786/175533 objects degraded (14.690%) 2023-09-23T22:56:11.787 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:56:11.787 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:11.787 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[187743]: osdmap e327: 8 total, 7 up, 6 in 2023-09-23T22:56:11.788 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[187743]: failed to encode map e327 with expected crc 2023-09-23T22:56:11.788 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[180979]: pgmap v955: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 25786/175533 objects degraded (14.690%) 2023-09-23T22:56:11.788 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:56:11.788 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:11.788 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[180979]: osdmap e327: 8 total, 7 up, 6 in 2023-09-23T22:56:11.788 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:11 smithi052 ceph-mon[180979]: failed to encode map e327 with expected crc 2023-09-23T22:56:11.790 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:11.906 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:11.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:11 smithi151 ceph-mon[159210]: pgmap v955: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 25786/175533 objects degraded (14.690%) 2023-09-23T22:56:11.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:11 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:56:11.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:11.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:11 smithi151 ceph-mon[159210]: osdmap e327: 8 total, 7 up, 6 in 2023-09-23T22:56:11.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:11 smithi151 ceph-mon[159210]: failed to encode map e327 with expected crc 2023-09-23T22:56:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:11.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:11.936 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 39 16 65006 64990 104.119 81 0.0183579 0.00959971 2023-09-23T22:56:12.021 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:12.089 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:12.158 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:12.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:12.275 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:12.318 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:12.360 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:12.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:12.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:12.472 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:12.520 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:12.741 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:12.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:12.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:12.847 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:12.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:12 smithi052 ceph-mon[180979]: failed to encode map e327 with expected crc 2023-09-23T22:56:12.849 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:12 smithi052 ceph-mon[187743]: failed to encode map e327 with expected crc 2023-09-23T22:56:12.906 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:12.946 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:56:12.946539+0000 min lat: 0.00121105 max lat: 1.08043 avg lat: 0.00965954 2023-09-23T22:56:12.946 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:56:12.946 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 40 16 66257 66241 103.445 78.1875 0.00683342 0.00965954 2023-09-23T22:56:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:12 smithi151 ceph-mon[159210]: failed to encode map e327 with expected crc 2023-09-23T22:56:12.975 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:12.997 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:13.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:13.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:13.102 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:13.163 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:13.266 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:13.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:13.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:13.440 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:56:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:56:13] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-09-23T22:56:13.445 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:13.543 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:13.566 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:13.605 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:13.648 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:13.691 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:13.723 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [7, 3] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:56:13.723 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:56:13.723 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 4, 5, 6] 2023-09-23T22:56:13.723 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T22:56:13.728 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:13 smithi052 ceph-mon[187743]: pgmap v957: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 27794/190083 objects degraded (14.622%) 2023-09-23T22:56:13.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3967170253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:13 smithi052 ceph-mon[180979]: pgmap v957: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 27794/190083 objects degraded (14.622%) 2023-09-23T22:56:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3967170253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:13.764 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:13.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.764 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:13.764 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.765 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:56:13.786 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:56:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:13 smithi151 ceph-mon[159210]: pgmap v957: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 27794/190083 objects degraded (14.622%) 2023-09-23T22:56:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3967170253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:13.949 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 41 16 67534 67518 102.861 79.8125 0.0110646 0.00971663 2023-09-23T22:56:14.909 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:14.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:14 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:14 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:14.949 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 42 16 68679 68663 102.116 71.5625 0.00667694 0.00978749 2023-09-23T22:56:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:14 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:14.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:14 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:14 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:14.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:14 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:15.015 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:15.083 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:15.237 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:15.242 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:15.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:15.348 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:15.394 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:15.528 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:15.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:15.580 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:15.627 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:15.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:15.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[187743]: pgmap v958: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 28601/194925 objects degraded (14.673%) 2023-09-23T22:56:15.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 28601/194925 objects degraded (14.673%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:15.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:15.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:15.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:15.681 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[180979]: pgmap v958: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 28601/194925 objects degraded (14.673%) 2023-09-23T22:56:15.682 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 28601/194925 objects degraded (14.673%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:15.682 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:15.682 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:15.682 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:15 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:15.692 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:15.714 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:56:15.714 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:56:15.714 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:56:15.714 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:56:15.714 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:56:15.801 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:15.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:15.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:15.910 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:15 smithi151 ceph-mon[159210]: pgmap v958: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 28601/194925 objects degraded (14.673%) 2023-09-23T22:56:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 28601/194925 objects degraded (14.673%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:15 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:56:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:15 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:56:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:15 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:15.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 43 16 69897 69881 101.512 76.125 0.0112333 0.00984629 2023-09-23T22:56:15.992 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:16.111 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:16.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:16.215 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:16.250 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:16.312 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:16.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:16.434 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:16.497 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:16.601 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:16.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:16.719 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:16.747 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:56:16.789 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:16.897 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:16.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:16.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 44 16 70879 70863 100.6 61.375 0.00328779 0.00992347 2023-09-23T22:56:17.002 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:17.043 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:17.237 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:56:17.336 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:17.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:17.448 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:17.510 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:17.601 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:17.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:17.712 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:17.754 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:17 smithi151 ceph-mon[159210]: pgmap v959: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 29392/201864 objects degraded (14.560%) 2023-09-23T22:56:17.949 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 45 16 71976 71960 99.888 68.5625 0.00292904 0.0100021 2023-09-23T22:56:17.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:17 smithi052 ceph-mon[180979]: pgmap v959: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 29392/201864 objects degraded (14.560%) 2023-09-23T22:56:17.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:17 smithi052 ceph-mon[187743]: pgmap v959: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 29392/201864 objects degraded (14.560%) 2023-09-23T22:56:18.262 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:18.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:18.369 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:18.449 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:18.533 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:18.537 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:18.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:18.644 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:18.705 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:18.788 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:18.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:18.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:18.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:18.920 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:18.946 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:18.949 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 46 16 73052 73036 99.1787 67.25 0.0128732 0.010077 2023-09-23T22:56:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:18 smithi052 conmon[180954]: 2023-09-23T22:56:18.938+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e327 definitely_dead 0 2023-09-23T22:56:19.042 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:19.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:19.156 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:19.217 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:19.326 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:19.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:19.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:19.598 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:19.650 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:19.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: pgmap v960: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s; 31182/212721 objects degraded (14.659%) 2023-09-23T22:56:19.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/86965179' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:56:19.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:56:19.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T22:56:19.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3462201246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:56:19.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:19.696 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:19.698 INFO:teuthology.orchestra.run.smithi052.stderr:marked out osd.6. 2023-09-23T22:56:19.730 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:56:19.851 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:19.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:19.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 47 16 73753 73737 98.0011 43.8125 0.00299078 0.0101496 2023-09-23T22:56:19.956 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:19.985 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: pgmap v960: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s; 31182/212721 objects degraded (14.659%) 2023-09-23T22:56:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/86965179' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:56:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:56:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T22:56:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3462201246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:56:19.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:19.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: pgmap v960: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s; 31182/212721 objects degraded (14.659%) 2023-09-23T22:56:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/86965179' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:56:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:56:19.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T22:56:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3462201246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:56:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:19.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:20.029 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:20.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:20.133 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:20.163 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:20.225 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:20.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:20.332 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:20.365 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:20.763 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:20.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:20.867 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:20.898 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:20.948 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:20.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: failed to encode map e328 with expected crc 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3462201246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: osdmap e328: 8 total, 7 up, 5 in 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31182/212721 objects degraded (14.659%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: failed to encode map e329 with expected crc 2023-09-23T22:56:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: osdmap e329: 8 total, 7 up, 5 in 2023-09-23T22:56:20.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:20 smithi151 ceph-mon[159210]: failed to encode map e329 with expected crc 2023-09-23T22:56:20.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 48 16 73753 73737 95.9604 0 - 0.0101496 2023-09-23T22:56:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:20.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: failed to encode map e328 with expected crc 2023-09-23T22:56:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T22:56:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3462201246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:56:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: osdmap e328: 8 total, 7 up, 5 in 2023-09-23T22:56:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31182/212721 objects degraded (14.659%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: failed to encode map e329 with expected crc 2023-09-23T22:56:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: osdmap e329: 8 total, 7 up, 5 in 2023-09-23T22:56:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[187743]: failed to encode map e329 with expected crc 2023-09-23T22:56:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: failed to encode map e328 with expected crc 2023-09-23T22:56:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T22:56:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3462201246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:56:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: osdmap e328: 8 total, 7 up, 5 in 2023-09-23T22:56:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31182/212721 objects degraded (14.659%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: failed to encode map e329 with expected crc 2023-09-23T22:56:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: osdmap e329: 8 total, 7 up, 5 in 2023-09-23T22:56:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:20 smithi052 ceph-mon[180979]: failed to encode map e329 with expected crc 2023-09-23T22:56:21.057 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:21.085 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:21.130 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:21.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:21.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:21.236 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:21.293 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:21.355 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:21.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:21.461 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:21.514 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:21.600 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:21.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:21.708 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:21.735 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:21.781 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:21.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:21.887 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:21.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:21 smithi151 ceph-mon[159210]: failed to encode map e329 with expected crc 2023-09-23T22:56:21.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:21 smithi151 ceph-mon[159210]: pgmap v963: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 59 MiB/s wr, 942 op/s; 31182/212721 objects degraded (14.659%) 2023-09-23T22:56:21.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:21 smithi151 ceph-mon[159210]: osdmap e330: 8 total, 7 up, 5 in 2023-09-23T22:56:21.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:21 smithi151 ceph-mon[159210]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:21 smithi151 ceph-mon[159210]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:21 smithi151 ceph-mon[159210]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.914 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:21.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 49 16 73901 73885 94.1915 4.625 0.00388781 0.0104717 2023-09-23T22:56:21.960 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:21.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:21.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[187743]: failed to encode map e329 with expected crc 2023-09-23T22:56:21.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[187743]: pgmap v963: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 59 MiB/s wr, 942 op/s; 31182/212721 objects degraded (14.659%) 2023-09-23T22:56:21.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[187743]: osdmap e330: 8 total, 7 up, 5 in 2023-09-23T22:56:21.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[187743]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[187743]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[187743]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[180979]: failed to encode map e329 with expected crc 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[180979]: pgmap v963: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 4.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 59 MiB/s wr, 942 op/s; 31182/212721 objects degraded (14.659%) 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[180979]: osdmap e330: 8 total, 7 up, 5 in 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[180979]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[180979]: failed to encode map e330 with expected crc 2023-09-23T22:56:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:21 smithi052 ceph-mon[180979]: failed to encode map e330 with expected crc 2023-09-23T22:56:22.044 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:56:22.045 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:56:22.045 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:56:22.045 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:56:22.045 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.045 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110891 9.3 9:c27b0003:::benchmark_data_smithi052_402718_object21250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:56:22.046 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:56:22.046 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 180.985988381, 2023-09-23T22:56:22.046 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25767365199999998, 2023-09-23T22:56:22.046 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.046 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.047 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:22.047 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:22.047 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:22.047 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110891 2023-09-23T22:56:22.047 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.047 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:56:22.048 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.048 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.049 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.049 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:56:22.049 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.049 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.049 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055633+0000", 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055637+0000", 2023-09-23T22:56:22.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.126e-06 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055645+0000", 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9850000000000006e-06 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055650+0000", 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4990000000000002e-06 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055758+0000", 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010737 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.052 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.053 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.053 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055921+0000", 2023-09-23T22:56:22.053 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163437 2023-09-23T22:56:22.053 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.053 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.053 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:56:22.054 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.056098+0000", 2023-09-23T22:56:22.054 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000176563 2023-09-23T22:56:22.054 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.054 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.054 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:22.055 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.059728+0000", 2023-09-23T22:56:22.055 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036304570000000001 2023-09-23T22:56:22.055 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.055 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.055 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.055 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.062797+0000", 2023-09-23T22:56:22.056 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00306851 2023-09-23T22:56:22.056 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.056 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.056 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.056 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312384+0000", 2023-09-23T22:56:22.056 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.249587273 2023-09-23T22:56:22.057 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.057 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.057 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.057 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312438+0000", 2023-09-23T22:56:22.057 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4354999999999997e-05 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313307+0000", 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00086921899999999996 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.058 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110899 9.3 9:c9f9c7b0:::benchmark_data_smithi052_402718_object21258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 180.978325071, 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29968218299999999, 2023-09-23T22:56:22.059 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110899 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.060 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:56:22.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.062 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063299+0000", 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5469999999999999e-06 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063306+0000", 2023-09-23T22:56:22.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6889999999999998e-06 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063312+0000", 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6489999999999996e-06 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066300+0000", 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029886909999999999 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066461+0000", 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016098600000000001 2023-09-23T22:56:22.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066620+0000", 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015845099999999999 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.066 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312307+0000", 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24568741599999999 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312349+0000", 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1686e-05 2023-09-23T22:56:22.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362127+0000", 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049778356000000003 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362181+0000", 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3421999999999998e-05 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362979+0000", 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079871 2023-09-23T22:56:22.069 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110900 9.3 9:cfcfdec5:::benchmark_data_smithi052_402718_object21259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:56:22.070 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 180.97830499400001, 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.300308513, 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:22.071 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110900 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063319+0000", 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.525e-06 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063325+0000", 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5999999999999997e-06 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063329+0000", 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5170000000000001e-06 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069300+0000", 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059708189999999996 2023-09-23T22:56:22.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069454+0000", 2023-09-23T22:56:22.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015409899999999999 2023-09-23T22:56:22.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:56:22.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069610+0000", 2023-09-23T22:56:22.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155332 2023-09-23T22:56:22.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312314+0000", 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24270393900000001 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313359+0000", 2023-09-23T22:56:22.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010457520000000001 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363082+0000", 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049722865999999998 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363133+0000", 2023-09-23T22:56:22.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0757999999999998e-05 2023-09-23T22:56:22.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363626+0000", 2023-09-23T22:56:22.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000492659 2023-09-23T22:56:22.082 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.082 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.082 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.083 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110905 9.3 9:cdd24a36:::benchmark_data_smithi052_402718_object21264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:56:22.083 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:56:22.083 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 180.97742035499999, 2023-09-23T22:56:22.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29997646300000003, 2023-09-23T22:56:22.083 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.084 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.084 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:22.084 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:22.084 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:22.084 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110905 2023-09-23T22:56:22.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.085 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:56:22.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:56:22.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064201+0000", 2023-09-23T22:56:22.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.088 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064206+0000", 2023-09-23T22:56:22.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7419999999999997e-06 2023-09-23T22:56:22.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064217+0000", 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.094e-05 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064224+0000", 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2529999999999998e-06 2023-09-23T22:56:22.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072314+0000", 2023-09-23T22:56:22.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0080899820000000008 2023-09-23T22:56:22.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072465+0000", 2023-09-23T22:56:22.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015051600000000001 2023-09-23T22:56:22.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:56:22.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.309128+0000", 2023-09-23T22:56:22.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23666314399999999 2023-09-23T22:56:22.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.314036+0000", 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049078869999999997 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.320379+0000", 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00634324 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363720+0000", 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.043340956999999999 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363771+0000", 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1734999999999998e-05 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.364178+0000", 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040673700000000001 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44296 10.2 e325/317 10:41443089:::benchmark_data_smithi052_461816_object44295:head v 325'2735, mlcod=325'2732)", 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:55.999388+0000", 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.042234543999999, 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063611318, 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.095 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999388+0000", 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999388+0000", 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999387+0000", 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999450+0000", 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3182000000000006e-05 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999460+0000", 2023-09-23T22:56:22.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7440000000000002e-06 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999465+0000", 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3580000000000002e-06 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999511+0000", 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5528000000000002e-05 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999528+0000", 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7676000000000002e-05 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.062967+0000", 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063438897999999994 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.062999+0000", 2023-09-23T22:56:22.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1738e-05 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44302 10.2 e325/317 10:43e5bdbc:::benchmark_data_smithi052_461816_object44301:head v 325'2736, mlcod=325'2732)", 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:55.999478+0000", 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.042143705000001, 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065492382000000002, 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.100 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999478+0000", 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999478+0000", 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999478+0000", 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999503+0000", 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4944e-05 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999507+0000", 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3599999999999998e-06 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:55.999511+0000", 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8840000000000004e-06 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.060302+0000", 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.060790420999999997 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.060321+0000", 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8893e-05 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064942+0000", 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046212709999999997 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064971+0000", 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.883e-05 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44301 10.b e325/317 10:dc4a60d7:::benchmark_data_smithi052_461816_object44300:head v 325'2832, mlcod=325'2830)", 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.000473+0000", 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.041148926999998, 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063901823999999996, 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.105 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000473+0000", 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000473+0000", 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000472+0000", 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000581+0000", 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010908000000000001 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000592+0000", 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1029999999999999e-05 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000598+0000", 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5210000000000002e-06 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000646+0000", 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8446999999999999e-05 2023-09-23T22:56:22.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000668+0000", 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1364000000000001e-05 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064338+0000", 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063670593999999997 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064375+0000", 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6838000000000002e-05 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44307 10.0 e325/317 10:0bedc234:::benchmark_data_smithi052_461816_object44306:head v 325'2788, mlcod=325'2785)", 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.000613+0000", 2023-09-23T22:56:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.041008976000001, 2023-09-23T22:56:22.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.062626159000000001, 2023-09-23T22:56:22.111 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.111 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.111 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000613+0000", 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000613+0000", 2023-09-23T22:56:22.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000613+0000", 2023-09-23T22:56:22.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000649+0000", 2023-09-23T22:56:22.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6230999999999999e-05 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000655+0000", 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1449999999999996e-06 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000659+0000", 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7780000000000001e-06 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000709+0000", 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9864000000000002e-05 2023-09-23T22:56:22.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000758+0000", 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9288e-05 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.063187+0000", 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.062428751999999997 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.063239+0000", 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2479999999999999e-05 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.118 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.119 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.119 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44303 10.b e325/317 10:d39b2763:::benchmark_data_smithi052_461816_object44302:head v 325'2833, mlcod=325'2830)", 2023-09-23T22:56:22.119 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.000670+0000", 2023-09-23T22:56:22.120 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.040951670999998, 2023-09-23T22:56:22.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063747023, 2023-09-23T22:56:22.120 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.120 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.120 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000670+0000", 2023-09-23T22:56:22.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000670+0000", 2023-09-23T22:56:22.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000670+0000", 2023-09-23T22:56:22.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000700+0000", 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0029999999999999e-05 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000707+0000", 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8580000000000002e-06 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000713+0000", 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6710000000000004e-06 2023-09-23T22:56:22.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.060033+0000", 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.059320718000000001 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.060069+0000", 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5762000000000001e-05 2023-09-23T22:56:22.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064379+0000", 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043096970000000004 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064417+0000", 2023-09-23T22:56:22.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8584999999999997e-05 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44305 10.b e325/317 10:d4d20080:::benchmark_data_smithi052_461816_object44304:head v 325'2834, mlcod=325'2830)", 2023-09-23T22:56:22.128 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.000722+0000", 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.040900418, 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063719360000000003, 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000722+0000", 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000722+0000", 2023-09-23T22:56:22.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000721+0000", 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000771+0000", 2023-09-23T22:56:22.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9159999999999997e-05 2023-09-23T22:56:22.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000778+0000", 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8879999999999997e-06 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.000784+0000", 2023-09-23T22:56:22.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4859999999999997e-06 2023-09-23T22:56:22.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.061733+0000", 2023-09-23T22:56:22.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.060948684000000003 2023-09-23T22:56:22.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.061752+0000", 2023-09-23T22:56:22.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9105000000000001e-05 2023-09-23T22:56:22.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064421+0000", 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026689420000000001 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064441+0000", 2023-09-23T22:56:22.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0353999999999999e-05 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44312 10.4 e325/317 10:2e5d7dd4:::benchmark_data_smithi052_461816_object44311:head v 325'2797, mlcod=325'2795)", 2023-09-23T22:56:22.137 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.002699+0000", 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.038923047000001, 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.059945699999999998, 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.002699+0000", 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.002699+0000", 2023-09-23T22:56:22.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.002698+0000", 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.002718+0000", 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0100000000000001e-05 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.059973+0000", 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.057254899999999997 2023-09-23T22:56:22.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.059995+0000", 2023-09-23T22:56:22.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1226999999999999e-05 2023-09-23T22:56:22.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.060037+0000", 2023-09-23T22:56:22.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1950999999999999e-05 2023-09-23T22:56:22.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.060069+0000", 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2135999999999999e-05 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.062615+0000", 2023-09-23T22:56:22.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025458849999999999 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.062645+0000", 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0230999999999999e-05 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.145 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44318 10.0 e325/317 10:075904ee:::benchmark_data_smithi052_461816_object44317:head v 325'2789, mlcod=325'2786)", 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.003528+0000", 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 26.038094241, 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.060912504999999999, 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.146 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.003528+0000", 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.003528+0000", 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.003520+0000", 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999919 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.003566+0000", 2023-09-23T22:56:22.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5992999999999998e-05 2023-09-23T22:56:22.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.059985+0000", 2023-09-23T22:56:22.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.056419247999999998 2023-09-23T22:56:22.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.059998+0000", 2023-09-23T22:56:22.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2969000000000001e-05 2023-09-23T22:56:22.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.061635+0000", 2023-09-23T22:56:22.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016372439999999999 2023-09-23T22:56:22.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.061659+0000", 2023-09-23T22:56:22.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4102999999999999e-05 2023-09-23T22:56:22.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064405+0000", 2023-09-23T22:56:22.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027456310000000001 2023-09-23T22:56:22.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.064440+0000", 2023-09-23T22:56:22.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5355999999999998e-05 2023-09-23T22:56:22.155 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.155 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.156 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.156 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44390 10.b e325/317 10:d5efb40f:::benchmark_data_smithi052_461816_object44389:head v 325'2840, mlcod=325'2836)", 2023-09-23T22:56:22.156 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.088974+0000", 2023-09-23T22:56:22.156 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 25.952648617000001, 2023-09-23T22:56:22.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.040024156999999998, 2023-09-23T22:56:22.157 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.157 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.157 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.088974+0000", 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.088974+0000", 2023-09-23T22:56:22.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.088973+0000", 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.089010+0000", 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7545000000000001e-05 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.121788+0000", 2023-09-23T22:56:22.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.032777160999999999 2023-09-23T22:56:22.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.121803+0000", 2023-09-23T22:56:22.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5872999999999999e-05 2023-09-23T22:56:22.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.124516+0000", 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002712955 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.124534+0000", 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7172000000000001e-05 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.128967+0000", 2023-09-23T22:56:22.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0044331639999999999 2023-09-23T22:56:22.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.128998+0000", 2023-09-23T22:56:22.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0927999999999999e-05 2023-09-23T22:56:22.164 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44617 10.2 e325/317 10:42f612af:::benchmark_data_smithi052_461816_object44616:head v 325'2757, mlcod=325'2755)", 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.230235+0000", 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 25.811387523, 2023-09-23T22:56:22.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.097530256999999995, 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T22:56:22.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230234+0000", 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230284+0000", 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0538999999999998e-05 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230294+0000", 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2580000000000001e-06 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230301+0000", 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0979999999999996e-06 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230348+0000", 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6904999999999999e-05 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230370+0000", 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2036999999999999e-05 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.231896+0000", 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015261840000000001 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327765+0000", 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095868996999999997 2023-09-23T22:56:22.170 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44626 10.2 e325/317 10:4fdb9bab:::benchmark_data_smithi052_461816_object44625:head v 325'2758, mlcod=325'2755)", 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.234607+0000", 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 25.807015549999999, 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096288367, 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.171 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T22:56:22.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234606+0000", 2023-09-23T22:56:22.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234684+0000", 2023-09-23T22:56:22.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8349999999999994e-05 2023-09-23T22:56:22.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327708+0000", 2023-09-23T22:56:22.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.093023764999999994 2023-09-23T22:56:22.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327737+0000", 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9223000000000001e-05 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327860+0000", 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122982 2023-09-23T22:56:22.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327916+0000", 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5405999999999998e-05 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330864+0000", 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002948492 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330895+0000", 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0898999999999997e-05 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.178 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44632 10.b e325/317 10:d9603e28:::benchmark_data_smithi052_461816_object44631:head v 325'2853, mlcod=325'2851)", 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.237428+0000", 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 25.804193972, 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096280273999999999, 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237427+0000", 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237481+0000", 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3566000000000002e-05 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327715+0000", 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.090234522999999997 2023-09-23T22:56:22.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327736+0000", 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0746000000000001e-05 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327869+0000", 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013274200000000001 2023-09-23T22:56:22.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327926+0000", 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7645999999999997e-05 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333664+0000", 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057375569999999999 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333708+0000", 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4566e-05 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.184 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71905 10.1 e327/317 10:8c42e143:::benchmark_data_smithi052_461816_object71904:head v 327'4349, mlcod=327'4347)", 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.837941+0000", 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4.2036812340000003, 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.064534164000000005, 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.837941+0000", 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.837941+0000", 2023-09-23T22:56:22.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.837940+0000", 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838080+0000", 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013978300000000001 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838092+0000", 2023-09-23T22:56:22.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1885999999999999e-05 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838099+0000", 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0489999999999998e-06 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838149+0000", 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0485999999999997e-05 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838171+0000", 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1979000000000001e-05 2023-09-23T22:56:22.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902406+0000", 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.064235193999999995 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902475+0000", 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8589000000000003e-05 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.191 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71910 10.b e327/317 10:da00891c:::benchmark_data_smithi052_461816_object71909:head v 327'4562, mlcod=327'4560)", 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.838808+0000", 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4.2028143849999999, 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070775292000000004, 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T22:56:22.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838807+0000", 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838881+0000", 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3671999999999995e-05 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838927+0000", 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6146999999999997e-05 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838934+0000", 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3769999999999998e-06 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838979+0000", 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.494e-05 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838998+0000", 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8782000000000001e-05 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909541+0000", 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070543146000000001 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909583+0000", 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1967e-05 2023-09-23T22:56:22.196 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71913 10.1 e327/317 10:868eb1a2:::benchmark_data_smithi052_461816_object71912:head v 327'4350, mlcod=327'4347)", 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.840731+0000", 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4.2008910569999998, 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.066217576, 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.197 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840730+0000", 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840777+0000", 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6655e-05 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061592056999999999 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902407+0000", 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7666000000000002e-05 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902538+0000", 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000131793 2023-09-23T22:56:22.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902555+0000", 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6888000000000002e-05 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906899+0000", 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043436869999999997 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906949+0000", 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9629999999999997e-05 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71918 10.b e327/317 10:d9e3a463:::benchmark_data_smithi052_461816_object71917:head v 327'4563, mlcod=327'4560)", 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.844519+0000", 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 4.1971030579999997, 2023-09-23T22:56:22.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074161447000000005, 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T22:56:22.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844518+0000", 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844600+0000", 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2000000000000001e-05 2023-09-23T22:56:22.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.057768742999999997 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902406+0000", 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7252999999999997e-05 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.904983+0000", 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025770709999999998 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.905005+0000", 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2028e-05 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:22.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918640+0000", 2023-09-23T22:56:22.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01363467 2023-09-23T22:56:22.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:22.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:22.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:22.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918681+0000", 2023-09-23T22:56:22.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0522000000000003e-05 2023-09-23T22:56:22.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.209 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.210 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:22.210 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:22.210 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:56:22.210 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:22.236 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:22.286 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:22.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:22.391 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:22.393 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:22.413 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:22.427 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:22.435 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:22.462 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:22.482 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:22.484 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:22.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:22.501 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:22.521 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:22.522 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.522 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.522 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:22.523 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.523 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.523 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.523 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.523 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:56:22.587 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:22.617 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:22.668 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:22.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:22.772 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:22.801 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:22.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 50 16 73901 73885 92.3085 0 - 0.0104717 2023-09-23T22:56:22.986 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:22 smithi052 ceph-mon[187743]: failed to encode map e330 with expected crc 2023-09-23T22:56:22.987 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:22 smithi052 ceph-mon[187743]: failed to encode map e330 with expected crc 2023-09-23T22:56:22.987 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2784201402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:22.987 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:22 smithi052 ceph-mon[180979]: failed to encode map e330 with expected crc 2023-09-23T22:56:22.987 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:22 smithi052 ceph-mon[180979]: failed to encode map e330 with expected crc 2023-09-23T22:56:22.987 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2784201402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:22 smithi151 ceph-mon[159210]: failed to encode map e330 with expected crc 2023-09-23T22:56:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:22 smithi151 ceph-mon[159210]: failed to encode map e330 with expected crc 2023-09-23T22:56:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2784201402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:23.257 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:56:23.397 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:23.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:23.501 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:23.506 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:56:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:56:23] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-09-23T22:56:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:23.508 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:23.529 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:23.578 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:23.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:23.685 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:23.690 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:23.719 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:23.771 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:23.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:23 smithi052 ceph-mon[187743]: pgmap v965: 89 pgs: 1 activating+degraded+remapped, 9 active+clean+remapped, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s; 32284/219276 objects degraded (14.723%); 18553/219276 objects misplaced (8.461%); 129 KiB/s, 2 objects/s recovering 2023-09-23T22:56:23.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:23 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:56:23.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:23 smithi052 ceph-mon[180979]: pgmap v965: 89 pgs: 1 activating+degraded+remapped, 9 active+clean+remapped, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s; 32284/219276 objects degraded (14.723%); 18553/219276 objects misplaced (8.461%); 129 KiB/s, 2 objects/s recovering 2023-09-23T22:56:23.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:23 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:56:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:23.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:23.870 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:56:23.875 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:23.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:23 smithi151 ceph-mon[159210]: pgmap v965: 89 pgs: 1 activating+degraded+remapped, 9 active+clean+remapped, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s; 32284/219276 objects degraded (14.723%); 18553/219276 objects misplaced (8.461%); 129 KiB/s, 2 objects/s recovering 2023-09-23T22:56:23.899 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:23 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:56:23.902 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:23.947 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:23.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 51 16 73901 73885 90.4993 0 - 0.0104717 2023-09-23T22:56:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:23.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:24.055 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:24.082 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:24.126 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:24.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:24.233 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:24.264 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:24.315 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:24.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:24.419 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:24.445 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:24.492 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:24.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:24.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:24.597 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:24.623 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:24.671 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:24.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:24.777 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:24.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:24.854 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:24.913 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:24.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:24.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:24.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 52 16 73901 73885 88.7596 0 - 0.0104717 2023-09-23T22:56:25.018 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:25.044 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:25.095 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:25.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:25.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:25.139 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:56:25.139 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:56:25.200 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:25.227 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:25.274 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:25.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:25.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:25.381 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:25.415 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:25.509 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:56:25.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:25 smithi052 ceph-mon[187743]: pgmap v966: 89 pgs: 1 activating+degraded+remapped, 10 active+clean+remapped, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 23 MiB/s wr, 364 op/s; 32284/219276 objects degraded (14.723%); 18553/219276 objects misplaced (8.461%); 129 KiB/s, 2 objects/s recovering 2023-09-23T22:56:25.886 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 32284/219276 objects degraded (14.723%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:25.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:25 smithi052 ceph-mon[180979]: pgmap v966: 89 pgs: 1 activating+degraded+remapped, 10 active+clean+remapped, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 23 MiB/s wr, 364 op/s; 32284/219276 objects degraded (14.723%); 18553/219276 objects misplaced (8.461%); 129 KiB/s, 2 objects/s recovering 2023-09-23T22:56:25.886 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 32284/219276 objects degraded (14.723%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:25.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 53 16 73901 73885 87.0856 0 - 0.0104717 2023-09-23T22:56:26.131 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:56:26.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:25 smithi151 ceph-mon[159210]: pgmap v966: 89 pgs: 1 activating+degraded+remapped, 10 active+clean+remapped, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 23 MiB/s wr, 364 op/s; 32284/219276 objects degraded (14.723%); 18553/219276 objects misplaced (8.461%); 129 KiB/s, 2 objects/s recovering 2023-09-23T22:56:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 32284/219276 objects degraded (14.723%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:26.232 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5] out_osds: [7, 3, 6] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:56:26.232 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:56:26.233 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T22:56:26.233 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:56:26.273 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:26.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:26.377 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:26.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:26.436 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:26.448 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:26.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:26.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:26.553 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:26.579 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:26.624 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:26.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:26.730 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:26.756 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:26.804 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:26.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:26.911 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:26.944 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:26.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 54 16 73901 73885 85.4735 0 - 0.0104717 2023-09-23T22:56:26.952 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:27.005 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:26 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:56:27.006 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:26 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:56:27.021 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:27.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:27.125 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:27.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:26 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:56:27.152 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:27.155 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:27.199 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:27.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:27.309 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:27.340 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:27.769 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:56:27.856 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:27 smithi052 ceph-mon[187743]: pgmap v967: 89 pgs: 1 activating+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 20 MiB/s wr, 324 op/s; 36813/219324 objects degraded (16.785%); 22961/219324 objects misplaced (10.469%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:56:27.856 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3775812708' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:27.856 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:27 smithi052 ceph-mon[180979]: pgmap v967: 89 pgs: 1 activating+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 20 MiB/s wr, 324 op/s; 36813/219324 objects degraded (16.785%); 22961/219324 objects misplaced (10.469%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:56:27.856 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3775812708' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:27.920 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:27.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:27.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 55 16 73901 73885 83.92 0 - 0.0104717 2023-09-23T22:56:28.025 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:28.050 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:27 smithi151 ceph-mon[159210]: pgmap v967: 89 pgs: 1 activating+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 20 MiB/s wr, 324 op/s; 36813/219324 objects degraded (16.785%); 22961/219324 objects misplaced (10.469%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:56:28.050 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3775812708' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:28.053 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:28.101 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:28.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:28.209 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:28.241 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:28.924 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:28.924 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3775812708' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:28.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[187743]: osdmap e331: 8 total, 7 up, 5 in 2023-09-23T22:56:28.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[187743]: failed to encode map e331 with expected crc 2023-09-23T22:56:28.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:28.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3775812708' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:28.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[180979]: osdmap e331: 8 total, 7 up, 5 in 2023-09-23T22:56:28.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:28 smithi052 ceph-mon[180979]: failed to encode map e331 with expected crc 2023-09-23T22:56:28.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 56 16 73901 73885 82.422 0 - 0.0104717 2023-09-23T22:56:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:28 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3775812708' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:28 smithi151 ceph-mon[159210]: osdmap e331: 8 total, 7 up, 5 in 2023-09-23T22:56:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:28 smithi151 ceph-mon[159210]: failed to encode map e331 with expected crc 2023-09-23T22:56:29.625 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:29.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:29.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:29.729 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:29.762 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:29.817 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:29.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:29.921 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:29.947 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:29.950 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 57 16 73901 73885 80.9766 0 - 0.0104717 2023-09-23T22:56:29.990 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:29.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:29 smithi052 ceph-mon[187743]: pgmap v969: 89 pgs: 1 activating+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 17 MiB/s wr, 275 op/s; 36813/219324 objects degraded (16.785%); 22497/219324 objects misplaced (10.257%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T22:56:29.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:29 smithi052 ceph-mon[180979]: pgmap v969: 89 pgs: 1 activating+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 17 MiB/s wr, 275 op/s; 36813/219324 objects degraded (16.785%); 22497/219324 objects misplaced (10.257%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T22:56:29.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:29.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:30.048 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:30.067 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:30.090 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:30.095 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:30.108 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:30.119 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:29 smithi151 ceph-mon[159210]: pgmap v969: 89 pgs: 1 activating+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 45 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 17 MiB/s wr, 275 op/s; 36813/219324 objects degraded (16.785%); 22497/219324 objects misplaced (10.257%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T22:56:30.121 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:30.130 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:30.151 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:30.165 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:30.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:30.178 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:30.178 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.179 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:56:30.273 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:30.306 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:30.629 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:56:30.629 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:56:30.629 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:56:30.629 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:56:30.629 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:56:30.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 58 16 73901 73885 79.5809 0 - 0.0104717 2023-09-23T22:56:30.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2418990882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:30.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 36813/219324 objects degraded (16.785%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:30.973 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2418990882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:30.973 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 36813/219324 objects degraded (16.785%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:30.989 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:31.012 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:31.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:31.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:31.116 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:31.139 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2418990882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:31.139 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 36813/219324 objects degraded (16.785%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:31.142 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:31.184 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:31.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:31.295 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:31.312 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:56:31.314 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:56:31.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:31.394 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:56:31.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:56:31.541 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:31.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:31.647 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:31.708 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:31.808 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:31.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:31.913 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:31.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 59 16 74569 74553 78.9399 4.175 0.0175792 0.0126629 2023-09-23T22:56:31.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:31 smithi052 ceph-mon[187743]: pgmap v970: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+remapped, 1 activating+degraded+remapped, 20 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 40 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 19 MiB/s wr, 302 op/s; 41650/221730 objects degraded (18.784%); 27141/221730 objects misplaced (12.241%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:56:31.954 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:31 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:56:31.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:31 smithi052 ceph-mon[180979]: pgmap v970: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+remapped, 1 activating+degraded+remapped, 20 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 40 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 19 MiB/s wr, 302 op/s; 41650/221730 objects degraded (18.784%); 27141/221730 objects misplaced (12.241%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:56:31.955 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:31 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:56:31.958 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:32.181 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":970,"stamp":"2023-09-23T22:56:30.476557+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4842788768,"num_objects":73910,"num_object_clones":0,"num_object_copies":221730,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":41650,"num_objects_misplaced":27141,"num_objects_unfound":0,"num_objects_dirty":73910,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":74146,"num_write_kb":4731149,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":609,"num_bytes_recovered":38156980,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74080,"ondisk_log_size":74080,"up":221,"acting":240,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":257,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":13417692,"kb_used_data":10761716,"kb_used_omap":121,"kb_used_meta":2655814,"kb_avail":455308068,"statfs":{"total":479975178240,"available":466235461632,"internally_reserved":0,"allocated":11019997184,"data_stored":11015396214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2719553653},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,14,2],"upper_bound":16384},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":183762944,"num_objects":2804,"num_object_clones":0,"num_object_copies":8412,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":14782,"num_objects_misplaced":27141,"num_objects_unfound":0,"num_objects_dirty":2804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2804,"num_write_kb":179456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":579,"num_bytes_recovered":38146686,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.283000"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":333,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.200546+0000","last_change":"2023-09-23T22:55:53.200247+0000","last_active":"2023-09-23T22:56:20.200546+0000","last_peered":"2023-09-23T22:56:20.200546+0000","last_clean":"2023-09-23T22:55:51.197224+0000","last_became_active":"2023-09-23T22:55:53.200247+0000","last_became_peered":"2023-09-23T22:55:53.200247+0000","last_unstale":"2023-09-23T22:56:20.200546+0000","last_undegraded":"2023-09-23T22:56:20.200546+0000","last_fullsized":"2023-09-23T22:55:53.196948+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":135,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.213708+0000","last_change":"2023-09-23T22:56:20.211032+0000","last_active":"2023-09-23T22:56:20.213708+0000","last_peered":"2023-09-23T22:56:20.213708+0000","last_clean":"2023-09-23T22:56:20.213708+0000","last_became_active":"2023-09-23T22:56:20.210310+0000","last_became_peered":"2023-09-23T22:56:20.210310+0000","last_unstale":"2023-09-23T22:56:20.213708+0000","last_undegraded":"2023-09-23T22:56:20.213708+0000","last_fullsized":"2023-09-23T22:56:20.213708+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.208064+0000","last_change":"2023-09-23T22:56:21.208064+0000","last_active":"2023-09-23T22:56:21.208064+0000","last_peered":"2023-09-23T22:56:21.208064+0000","last_clean":"2023-09-23T22:56:21.208064+0000","last_became_active":"2023-09-23T22:56:21.207143+0000","last_became_peered":"2023-09-23T22:56:21.207143+0000","last_unstale":"2023-09-23T22:56:21.208064+0000","last_undegraded":"2023-09-23T22:56:21.208064+0000","last_fullsized":"2023-09-23T22:56:21.208064+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":311,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.202867+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:56:21.202867+0000","last_peered":"2023-09-23T22:56:21.202867+0000","last_clean":"2023-09-23T22:56:21.202867+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:56:21.202867+0000","last_undegraded":"2023-09-23T22:56:21.202867+0000","last_fullsized":"2023-09-23T22:56:21.202867+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":321,"reported_epoch":331,"state":"active+undersized","last_fresh":"2023-09-23T22:56:27.765355+0000","last_change":"2023-09-23T22:55:53.198868+0000","last_active":"2023-09-23T22:56:27.765355+0000","last_peered":"2023-09-23T22:56:27.765355+0000","last_clean":"2023-09-23T22:55:51.195765+0000","last_became_active":"2023-09-23T22:55:53.198868+0000","last_became_peered":"2023-09-23T22:55:53.198868+0000","last_unstale":"2023-09-23T22:56:27.765355+0000","last_undegraded":"2023-09-23T22:56:27.765355+0000","last_fullsized":"2023-09-23T22:55:53.195551+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:53.182763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T22:56:27.767104+0000","last_change":"2023-09-23T22:56:20.358021+0000","last_active":"2023-09-23T22:56:27.767104+0000","last_peered":"2023-09-23T22:56:27.767104+0000","last_clean":"2023-09-23T22:56:27.767104+0000","last_became_active":"2023-09-23T22:56:20.357519+0000","last_became_peered":"2023-09-23T22:56:20.357519+0000","last_unstale":"2023-09-23T22:56:27.767104+0000","last_undegraded":"2023-09-23T22:56:27.767104+0000","last_fullsized":"2023-09-23T22:56:27.767104+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:01.542689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":251,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.202610+0000","last_change":"2023-09-23T22:56:20.362169+0000","last_active":"2023-09-23T22:56:21.202610+0000","last_peered":"2023-09-23T22:56:21.202610+0000","last_clean":"2023-09-23T22:56:21.202610+0000","last_became_active":"2023-09-23T22:56:20.361766+0000","last_became_peered":"2023-09-23T22:56:20.361766+0000","last_unstale":"2023-09-23T22:56:21.202610+0000","last_undegraded":"2023-09-23T22:56:21.202610+0000","last_fullsized":"2023-09-23T22:56:21.202610+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:59.224393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":139,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.291040+0000","last_change":"2023-09-23T22:56:21.291040+0000","last_active":"2023-09-23T22:56:21.291040+0000","last_peered":"2023-09-23T22:56:21.291040+0000","last_clean":"2023-09-23T22:56:21.291040+0000","last_became_active":"2023-09-23T22:56:21.290100+0000","last_became_peered":"2023-09-23T22:56:21.290100+0000","last_unstale":"2023-09-23T22:56:21.291040+0000","last_undegraded":"2023-09-23T22:56:21.291040+0000","last_fullsized":"2023-09-23T22:56:21.291040+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.190709+0000","last_change":"2023-09-23T22:55:53.199541+0000","last_active":"2023-09-23T22:56:20.190709+0000","last_peered":"2023-09-23T22:56:20.190709+0000","last_clean":"2023-09-23T22:44:51.837412+0000","last_became_active":"2023-09-23T22:55:53.199541+0000","last_became_peered":"2023-09-23T22:55:53.199541+0000","last_unstale":"2023-09-23T22:56:20.190709+0000","last_undegraded":"2023-09-23T22:56:20.190709+0000","last_fullsized":"2023-09-23T22:55:53.197701+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":161,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.765510+0000","last_change":"2023-09-23T22:56:21.215512+0000","last_active":"2023-09-23T22:56:27.765510+0000","last_peered":"2023-09-23T22:56:27.765510+0000","last_clean":"2023-09-23T22:56:27.765510+0000","last_became_active":"2023-09-23T22:56:21.215127+0000","last_became_peered":"2023-09-23T22:56:21.215127+0000","last_unstale":"2023-09-23T22:56:27.765510+0000","last_undegraded":"2023-09-23T22:56:27.765510+0000","last_fullsized":"2023-09-23T22:56:27.765510+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:33.348767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":98,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.227390+0000","last_change":"2023-09-23T22:56:21.227390+0000","last_active":"2023-09-23T22:56:21.227390+0000","last_peered":"2023-09-23T22:56:21.227390+0000","last_clean":"2023-09-23T22:56:21.227390+0000","last_became_active":"2023-09-23T22:56:21.208490+0000","last_became_peered":"2023-09-23T22:56:21.208490+0000","last_unstale":"2023-09-23T22:56:21.227390+0000","last_undegraded":"2023-09-23T22:56:21.227390+0000","last_fullsized":"2023-09-23T22:56:21.227390+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.765415+0000","last_change":"2023-09-23T22:56:21.209792+0000","last_active":"2023-09-23T22:56:27.765415+0000","last_peered":"2023-09-23T22:56:27.765415+0000","last_clean":"2023-09-23T22:56:27.765415+0000","last_became_active":"2023-09-23T22:56:21.209180+0000","last_became_peered":"2023-09-23T22:56:21.209180+0000","last_unstale":"2023-09-23T22:56:27.765415+0000","last_undegraded":"2023-09-23T22:56:27.765415+0000","last_fullsized":"2023-09-23T22:56:27.765415+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.886895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":375,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.470686+0000","last_change":"2023-09-23T22:56:21.470686+0000","last_active":"2023-09-23T22:56:21.470686+0000","last_peered":"2023-09-23T22:56:21.470686+0000","last_clean":"2023-09-23T22:56:21.470686+0000","last_became_active":"2023-09-23T22:56:21.261869+0000","last_became_peered":"2023-09-23T22:56:21.261869+0000","last_unstale":"2023-09-23T22:56:21.470686+0000","last_undegraded":"2023-09-23T22:56:21.470686+0000","last_fullsized":"2023-09-23T22:56:21.470686+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":304,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.209736+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:56:20.209736+0000","last_peered":"2023-09-23T22:56:20.209736+0000","last_clean":"2023-09-23T22:56:20.209736+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:56:20.209736+0000","last_undegraded":"2023-09-23T22:56:20.209736+0000","last_fullsized":"2023-09-23T22:56:20.209736+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":198,"reported_epoch":330,"state":"activating+degraded+remapped","last_fresh":"2023-09-23T22:56:21.200245+0000","last_change":"2023-09-23T22:56:21.206058+0000","last_active":"2023-09-23T22:56:19.698074+0000","last_peered":"2023-09-23T22:56:10.604360+0000","last_clean":"2023-09-23T22:55:51.197512+0000","last_became_active":"2023-09-23T22:55:53.209298+0000","last_became_peered":"2023-09-23T22:55:53.209298+0000","last_unstale":"2023-09-23T22:56:21.200245+0000","last_undegraded":"2023-09-23T22:56:21.200245+0000","last_fullsized":"2023-09-23T22:56:21.200245+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":221,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.193526+0000","last_change":"2023-09-23T22:55:53.198779+0000","last_active":"2023-09-23T22:56:20.193526+0000","last_peered":"2023-09-23T22:56:20.193526+0000","last_clean":"2023-09-23T22:44:51.858616+0000","last_became_active":"2023-09-23T22:55:53.198779+0000","last_became_peered":"2023-09-23T22:55:53.198779+0000","last_unstale":"2023-09-23T22:56:20.193526+0000","last_undegraded":"2023-09-23T22:56:20.193526+0000","last_fullsized":"2023-09-23T22:55:53.196667+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:46.378185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":304,"reported_epoch":330,"state":"active+undersized","last_fresh":"2023-09-23T22:56:21.205163+0000","last_change":"2023-09-23T22:55:53.200228+0000","last_active":"2023-09-23T22:56:21.205163+0000","last_peered":"2023-09-23T22:56:21.205163+0000","last_clean":"2023-09-23T22:55:51.196266+0000","last_became_active":"2023-09-23T22:55:53.200228+0000","last_became_peered":"2023-09-23T22:55:53.200228+0000","last_unstale":"2023-09-23T22:56:21.205163+0000","last_undegraded":"2023-09-23T22:56:21.205163+0000","last_fullsized":"2023-09-23T22:55:53.195451+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:23.575604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":199,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T22:56:27.766407+0000","last_change":"2023-09-23T22:56:20.364317+0000","last_active":"2023-09-23T22:56:27.766407+0000","last_peered":"2023-09-23T22:56:27.766407+0000","last_clean":"2023-09-23T22:56:27.766407+0000","last_became_active":"2023-09-23T22:56:20.363957+0000","last_became_peered":"2023-09-23T22:56:20.363957+0000","last_unstale":"2023-09-23T22:56:27.766407+0000","last_undegraded":"2023-09-23T22:56:27.766407+0000","last_fullsized":"2023-09-23T22:56:27.766407+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:34.748723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":190,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.209748+0000","last_change":"2023-09-23T22:56:21.209748+0000","last_active":"2023-09-23T22:56:21.209748+0000","last_peered":"2023-09-23T22:56:21.209748+0000","last_clean":"2023-09-23T22:56:21.209748+0000","last_became_active":"2023-09-23T22:56:21.206594+0000","last_became_peered":"2023-09-23T22:56:21.206594+0000","last_unstale":"2023-09-23T22:56:21.209748+0000","last_undegraded":"2023-09-23T22:56:21.209748+0000","last_fullsized":"2023-09-23T22:56:21.209748+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":315,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.215747+0000","last_change":"2023-09-23T22:56:21.215509+0000","last_active":"2023-09-23T22:56:21.215747+0000","last_peered":"2023-09-23T22:56:21.215747+0000","last_clean":"2023-09-23T22:56:21.215747+0000","last_became_active":"2023-09-23T22:56:21.215057+0000","last_became_peered":"2023-09-23T22:56:21.215057+0000","last_unstale":"2023-09-23T22:56:21.215747+0000","last_undegraded":"2023-09-23T22:56:21.215747+0000","last_fullsized":"2023-09-23T22:56:21.215747+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:10.120897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":26,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.765577+0000","last_change":"2023-09-23T22:56:21.213637+0000","last_active":"2023-09-23T22:56:27.765577+0000","last_peered":"2023-09-23T22:56:27.765577+0000","last_clean":"2023-09-23T22:56:27.765577+0000","last_became_active":"2023-09-23T22:56:21.212846+0000","last_became_peered":"2023-09-23T22:56:21.212846+0000","last_unstale":"2023-09-23T22:56:27.765577+0000","last_undegraded":"2023-09-23T22:56:27.765577+0000","last_fullsized":"2023-09-23T22:56:27.765577+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:46.186837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.767105+0000","last_change":"2023-09-23T22:56:21.209659+0000","last_active":"2023-09-23T22:56:27.767105+0000","last_peered":"2023-09-23T22:56:27.767105+0000","last_clean":"2023-09-23T22:56:27.767105+0000","last_became_active":"2023-09-23T22:56:21.209233+0000","last_became_peered":"2023-09-23T22:56:21.209233+0000","last_unstale":"2023-09-23T22:56:27.767105+0000","last_undegraded":"2023-09-23T22:56:27.767105+0000","last_fullsized":"2023-09-23T22:56:27.767105+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:14.844115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.6","version":"330'4616","reported_seq":4639,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.563463+0000","last_change":"2023-09-23T22:55:23.669240+0000","last_active":"2023-09-23T22:56:21.563463+0000","last_peered":"2023-09-23T22:56:21.563463+0000","last_clean":"2023-09-23T22:56:21.563463+0000","last_became_active":"2023-09-23T22:55:23.668982+0000","last_became_peered":"2023-09-23T22:55:23.668982+0000","last_unstale":"2023-09-23T22:56:21.563463+0000","last_undegraded":"2023-09-23T22:56:21.563463+0000","last_fullsized":"2023-09-23T22:56:21.563463+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4616,"log_dups_size":0,"ondisk_log_size":4616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:40.611676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302514176,"num_objects":4616,"num_object_clones":0,"num_object_copies":13848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4616,"num_write_kb":295424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":199,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.201354+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:56:20.201354+0000","last_peered":"2023-09-23T22:56:20.201354+0000","last_clean":"2023-09-23T22:56:20.201354+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:56:20.201354+0000","last_undegraded":"2023-09-23T22:56:20.201354+0000","last_fullsized":"2023-09-23T22:56:20.201354+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.213702+0000","last_change":"2023-09-23T22:56:20.210137+0000","last_active":"2023-09-23T22:56:20.213702+0000","last_peered":"2023-09-23T22:56:20.213702+0000","last_clean":"2023-09-23T22:56:20.213702+0000","last_became_active":"2023-09-23T22:56:20.209865+0000","last_became_peered":"2023-09-23T22:56:20.209865+0000","last_unstale":"2023-09-23T22:56:20.213702+0000","last_undegraded":"2023-09-23T22:56:20.213702+0000","last_fullsized":"2023-09-23T22:56:20.213702+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.7","version":"330'4529","reported_seq":4770,"reported_epoch":330,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:25.986036+0000","last_change":"2023-09-23T22:56:21.508184+0000","last_active":"2023-09-23T22:56:25.986036+0000","last_peered":"2023-09-23T22:56:25.986036+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:56:21.421308+0000","last_became_peered":"2023-09-23T22:56:21.421308+0000","last_unstale":"2023-09-23T22:56:25.986036+0000","last_undegraded":"2023-09-23T22:56:21.233163+0000","last_fullsized":"2023-09-23T22:56:21.205823+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4529,"log_dups_size":0,"ondisk_log_size":4529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296812544,"num_objects":4529,"num_object_clones":0,"num_object_copies":13587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4529,"num_objects_misplaced":4408,"num_objects_unfound":0,"num_objects_dirty":4529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4529,"num_write_kb":289856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":6881280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":121}],"blocked_by":[],"up_primary":4,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":209,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.203393+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:56:20.203393+0000","last_peered":"2023-09-23T22:56:20.203393+0000","last_clean":"2023-09-23T22:56:20.203393+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:56:20.203393+0000","last_undegraded":"2023-09-23T22:56:20.203393+0000","last_fullsized":"2023-09-23T22:56:20.203393+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":328,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.365548+0000","last_change":"2023-09-23T22:56:20.364338+0000","last_active":"2023-09-23T22:56:20.365548+0000","last_peered":"2023-09-23T22:56:20.365548+0000","last_clean":"2023-09-23T22:56:20.365548+0000","last_became_active":"2023-09-23T22:56:20.363441+0000","last_became_peered":"2023-09-23T22:56:20.363441+0000","last_unstale":"2023-09-23T22:56:20.365548+0000","last_undegraded":"2023-09-23T22:56:20.365548+0000","last_fullsized":"2023-09-23T22:56:20.365548+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"330'4622","reported_seq":4645,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.644543+0000","last_change":"2023-09-23T22:55:24.310230+0000","last_active":"2023-09-23T22:56:21.644543+0000","last_peered":"2023-09-23T22:56:21.644543+0000","last_clean":"2023-09-23T22:56:21.644543+0000","last_became_active":"2023-09-23T22:55:24.309921+0000","last_became_peered":"2023-09-23T22:55:24.309921+0000","last_unstale":"2023-09-23T22:56:21.644543+0000","last_undegraded":"2023-09-23T22:56:21.644543+0000","last_fullsized":"2023-09-23T22:56:21.644543+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4622,"log_dups_size":0,"ondisk_log_size":4622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:46.749922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302907392,"num_objects":4622,"num_object_clones":0,"num_object_copies":13866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4622,"num_write_kb":295808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.419024+0000","last_change":"2023-09-23T22:56:20.418726+0000","last_active":"2023-09-23T22:56:20.419024+0000","last_peered":"2023-09-23T22:56:20.419024+0000","last_clean":"2023-09-23T22:56:20.419024+0000","last_became_active":"2023-09-23T22:56:20.363404+0000","last_became_peered":"2023-09-23T22:56:20.363404+0000","last_unstale":"2023-09-23T22:56:20.419024+0000","last_undegraded":"2023-09-23T22:56:20.419024+0000","last_fullsized":"2023-09-23T22:56:20.419024+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T22:56:27.765676+0000","last_change":"2023-09-23T22:56:20.363294+0000","last_active":"2023-09-23T22:56:27.765676+0000","last_peered":"2023-09-23T22:56:27.765676+0000","last_clean":"2023-09-23T22:56:27.765676+0000","last_became_active":"2023-09-23T22:56:20.362304+0000","last_became_peered":"2023-09-23T22:56:20.362304+0000","last_unstale":"2023-09-23T22:56:27.765676+0000","last_undegraded":"2023-09-23T22:56:27.765676+0000","last_fullsized":"2023-09-23T22:56:27.765676+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.994131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.5","version":"330'4611","reported_seq":4639,"reported_epoch":330,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:21.660980+0000","last_change":"2023-09-23T22:55:53.200485+0000","last_active":"2023-09-23T22:56:21.660980+0000","last_peered":"2023-09-23T22:56:21.660980+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:55:53.200485+0000","last_became_peered":"2023-09-23T22:55:53.200485+0000","last_unstale":"2023-09-23T22:56:21.660980+0000","last_undegraded":"2023-09-23T22:55:53.196004+0000","last_fullsized":"2023-09-23T22:55:53.195781+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4611,"log_dups_size":0,"ondisk_log_size":4611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:39.647676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302186496,"num_objects":4611,"num_object_clones":0,"num_object_copies":13833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4611,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4611,"num_write_kb":295104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4611}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T22:56:27.766243+0000","last_change":"2023-09-23T22:56:20.358092+0000","last_active":"2023-09-23T22:56:27.766243+0000","last_peered":"2023-09-23T22:56:27.766243+0000","last_clean":"2023-09-23T22:56:27.766243+0000","last_became_active":"2023-09-23T22:56:20.357773+0000","last_became_peered":"2023-09-23T22:56:20.357773+0000","last_unstale":"2023-09-23T22:56:27.766243+0000","last_undegraded":"2023-09-23T22:56:27.766243+0000","last_fullsized":"2023-09-23T22:56:27.766243+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:14.698016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.355341+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:56:20.355341+0000","last_peered":"2023-09-23T22:56:20.355341+0000","last_clean":"2023-09-23T22:56:20.355341+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:56:20.355341+0000","last_undegraded":"2023-09-23T22:56:20.355341+0000","last_fullsized":"2023-09-23T22:56:20.355341+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"330'4593","reported_seq":4614,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.562549+0000","last_change":"2023-09-23T22:55:24.311230+0000","last_active":"2023-09-23T22:56:21.562549+0000","last_peered":"2023-09-23T22:56:21.562549+0000","last_clean":"2023-09-23T22:56:21.562549+0000","last_became_active":"2023-09-23T22:55:24.309796+0000","last_became_peered":"2023-09-23T22:55:24.309796+0000","last_unstale":"2023-09-23T22:56:21.562549+0000","last_undegraded":"2023-09-23T22:56:21.562549+0000","last_fullsized":"2023-09-23T22:56:21.562549+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4593,"log_dups_size":0,"ondisk_log_size":4593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301006848,"num_objects":4593,"num_object_clones":0,"num_object_copies":13779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4593,"num_write_kb":293952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":97,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.355301+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:56:20.355301+0000","last_peered":"2023-09-23T22:56:20.355301+0000","last_clean":"2023-09-23T22:56:20.355301+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:56:20.355301+0000","last_undegraded":"2023-09-23T22:56:20.355301+0000","last_fullsized":"2023-09-23T22:56:20.355301+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":313,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.262144+0000","last_change":"2023-09-23T22:56:21.262144+0000","last_active":"2023-09-23T22:56:21.262144+0000","last_peered":"2023-09-23T22:56:21.262144+0000","last_clean":"2023-09-23T22:56:21.262144+0000","last_became_active":"2023-09-23T22:56:21.261482+0000","last_became_peered":"2023-09-23T22:56:21.261482+0000","last_unstale":"2023-09-23T22:56:21.262144+0000","last_undegraded":"2023-09-23T22:56:21.262144+0000","last_fullsized":"2023-09-23T22:56:21.262144+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"330'4679","reported_seq":4706,"reported_epoch":330,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:21.562984+0000","last_change":"2023-09-23T22:55:53.199646+0000","last_active":"2023-09-23T22:56:21.562984+0000","last_peered":"2023-09-23T22:56:21.562984+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:55:53.199646+0000","last_became_peered":"2023-09-23T22:55:53.199646+0000","last_unstale":"2023-09-23T22:56:21.562984+0000","last_undegraded":"2023-09-23T22:55:53.196179+0000","last_fullsized":"2023-09-23T22:55:53.196049+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4679,"log_dups_size":0,"ondisk_log_size":4679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306642944,"num_objects":4679,"num_object_clones":0,"num_object_copies":14037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4679,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4679,"num_write_kb":299456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":4679}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":104,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.210280+0000","last_change":"2023-09-23T22:56:20.208394+0000","last_active":"2023-09-23T22:56:20.210280+0000","last_peered":"2023-09-23T22:56:20.210280+0000","last_clean":"2023-09-23T22:56:20.210280+0000","last_became_active":"2023-09-23T22:56:20.208065+0000","last_became_peered":"2023-09-23T22:56:20.208065+0000","last_unstale":"2023-09-23T22:56:20.210280+0000","last_undegraded":"2023-09-23T22:56:20.210280+0000","last_fullsized":"2023-09-23T22:56:20.210280+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":331,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.203875+0000","last_change":"2023-09-23T22:55:53.198961+0000","last_active":"2023-09-23T22:56:20.203875+0000","last_peered":"2023-09-23T22:56:20.203875+0000","last_clean":"2023-09-23T22:55:51.198609+0000","last_became_active":"2023-09-23T22:55:53.198961+0000","last_became_peered":"2023-09-23T22:55:53.198961+0000","last_unstale":"2023-09-23T22:56:20.203875+0000","last_undegraded":"2023-09-23T22:56:20.203875+0000","last_fullsized":"2023-09-23T22:55:53.196914+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"330'4604","reported_seq":5585,"reported_epoch":330,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:56:26.712480+0000","last_change":"2023-09-23T22:56:21.498257+0000","last_active":"2023-09-23T22:56:26.712480+0000","last_peered":"2023-09-23T22:56:26.712480+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:56:21.371406+0000","last_became_peered":"2023-09-23T22:56:21.371406+0000","last_unstale":"2023-09-23T22:56:26.712480+0000","last_undegraded":"2023-09-23T22:56:26.712480+0000","last_fullsized":"2023-09-23T22:56:21.199197+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4604,"log_dups_size":0,"ondisk_log_size":4604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301727744,"num_objects":4604,"num_object_clones":0,"num_object_copies":13812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4120,"num_objects_unfound":0,"num_objects_dirty":4604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4604,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":31195136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4","objects":484},{"shards":"1,4,6","objects":4120}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":344,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.204914+0000","last_change":"2023-09-23T22:55:53.202300+0000","last_active":"2023-09-23T22:56:20.204914+0000","last_peered":"2023-09-23T22:56:20.204914+0000","last_clean":"2023-09-23T22:55:51.197959+0000","last_became_active":"2023-09-23T22:55:53.202300+0000","last_became_peered":"2023-09-23T22:55:53.202300+0000","last_unstale":"2023-09-23T22:56:20.204914+0000","last_undegraded":"2023-09-23T22:56:20.204914+0000","last_fullsized":"2023-09-23T22:55:53.197851+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":322,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.213719+0000","last_change":"2023-09-23T22:56:20.210960+0000","last_active":"2023-09-23T22:56:20.213719+0000","last_peered":"2023-09-23T22:56:20.213719+0000","last_clean":"2023-09-23T22:56:20.213719+0000","last_became_active":"2023-09-23T22:56:20.210745+0000","last_became_peered":"2023-09-23T22:56:20.210745+0000","last_unstale":"2023-09-23T22:56:20.213719+0000","last_undegraded":"2023-09-23T22:56:20.213719+0000","last_fullsized":"2023-09-23T22:56:20.213719+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"330'4479","reported_seq":4500,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.609318+0000","last_change":"2023-09-23T22:55:24.310261+0000","last_active":"2023-09-23T22:56:21.609318+0000","last_peered":"2023-09-23T22:56:21.609318+0000","last_clean":"2023-09-23T22:56:21.609318+0000","last_became_active":"2023-09-23T22:55:24.309692+0000","last_became_peered":"2023-09-23T22:55:24.309692+0000","last_unstale":"2023-09-23T22:56:21.609318+0000","last_undegraded":"2023-09-23T22:56:21.609318+0000","last_fullsized":"2023-09-23T22:56:21.609318+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4479,"log_dups_size":0,"ondisk_log_size":4479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293535744,"num_objects":4479,"num_object_clones":0,"num_object_copies":13437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4479,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":258,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.203478+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:56:20.203478+0000","last_peered":"2023-09-23T22:56:20.203478+0000","last_clean":"2023-09-23T22:56:20.203478+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:56:20.203478+0000","last_undegraded":"2023-09-23T22:56:20.203478+0000","last_fullsized":"2023-09-23T22:56:20.203478+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":212,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.201745+0000","last_change":"2023-09-23T22:55:53.200618+0000","last_active":"2023-09-23T22:56:20.201745+0000","last_peered":"2023-09-23T22:56:20.201745+0000","last_clean":"2023-09-23T22:51:24.617408+0000","last_became_active":"2023-09-23T22:55:53.200618+0000","last_became_peered":"2023-09-23T22:55:53.200618+0000","last_unstale":"2023-09-23T22:56:20.201745+0000","last_undegraded":"2023-09-23T22:56:20.201745+0000","last_fullsized":"2023-09-23T22:55:53.196370+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"330'4579","reported_seq":4608,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:21.668691+0000","last_change":"2023-09-23T22:56:21.423950+0000","last_active":"2023-09-23T22:56:21.668691+0000","last_peered":"2023-09-23T22:56:21.668691+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:56:21.407987+0000","last_became_peered":"2023-09-23T22:56:21.407987+0000","last_unstale":"2023-09-23T22:56:21.668691+0000","last_undegraded":"2023-09-23T22:56:21.255394+0000","last_fullsized":"2023-09-23T22:56:21.199911+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4579,"log_dups_size":0,"ondisk_log_size":4579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300089344,"num_objects":4579,"num_object_clones":0,"num_object_copies":13737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4567,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4579,"num_write_kb":293056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4","objects":12},{"shards":"0,4,6","objects":4567}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":376,"reported_epoch":330,"state":"active+undersized","last_fresh":"2023-09-23T22:56:21.202481+0000","last_change":"2023-09-23T22:55:53.203502+0000","last_active":"2023-09-23T22:56:21.202481+0000","last_peered":"2023-09-23T22:56:21.202481+0000","last_clean":"2023-09-23T22:55:51.196240+0000","last_became_active":"2023-09-23T22:55:53.203502+0000","last_became_peered":"2023-09-23T22:55:53.203502+0000","last_unstale":"2023-09-23T22:56:21.202481+0000","last_undegraded":"2023-09-23T22:56:21.202481+0000","last_fullsized":"2023-09-23T22:55:53.196522+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:59.555018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":220,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.214721+0000","last_change":"2023-09-23T22:56:21.214721+0000","last_active":"2023-09-23T22:56:21.214721+0000","last_peered":"2023-09-23T22:56:21.214721+0000","last_clean":"2023-09-23T22:56:21.214721+0000","last_became_active":"2023-09-23T22:56:21.211329+0000","last_became_peered":"2023-09-23T22:56:21.211329+0000","last_unstale":"2023-09-23T22:56:21.214721+0000","last_undegraded":"2023-09-23T22:56:21.214721+0000","last_fullsized":"2023-09-23T22:56:21.214721+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,6],"avail_no_missing":["5","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":161,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.203527+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:56:20.203527+0000","last_peered":"2023-09-23T22:56:20.203527+0000","last_clean":"2023-09-23T22:56:20.203527+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:56:20.203527+0000","last_undegraded":"2023-09-23T22:56:20.203527+0000","last_fullsized":"2023-09-23T22:56:20.203527+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.a","version":"330'4638","reported_seq":4673,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.765907+0000","last_change":"2023-09-23T22:56:21.236419+0000","last_active":"2023-09-23T22:56:27.765907+0000","last_peered":"2023-09-23T22:56:27.765907+0000","last_clean":"2023-09-23T22:56:27.765907+0000","last_became_active":"2023-09-23T22:56:21.215225+0000","last_became_peered":"2023-09-23T22:56:21.215225+0000","last_unstale":"2023-09-23T22:56:27.765907+0000","last_undegraded":"2023-09-23T22:56:27.765907+0000","last_fullsized":"2023-09-23T22:56:27.765907+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4638,"log_dups_size":0,"ondisk_log_size":4638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:15.354698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303955968,"num_objects":4638,"num_object_clones":0,"num_object_copies":13914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4638,"num_objects_unfound":0,"num_objects_dirty":4638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4638,"num_write_kb":296832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4638}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":13,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T22:56:27.766037+0000","last_change":"2023-09-23T22:56:20.363703+0000","last_active":"2023-09-23T22:56:27.766037+0000","last_peered":"2023-09-23T22:56:27.766037+0000","last_clean":"2023-09-23T22:56:27.766037+0000","last_became_active":"2023-09-23T22:56:20.363272+0000","last_became_peered":"2023-09-23T22:56:20.363272+0000","last_unstale":"2023-09-23T22:56:27.766037+0000","last_undegraded":"2023-09-23T22:56:27.766037+0000","last_fullsized":"2023-09-23T22:56:27.766037+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:06.482210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":335,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.364781+0000","last_change":"2023-09-23T22:56:20.363589+0000","last_active":"2023-09-23T22:56:20.364781+0000","last_peered":"2023-09-23T22:56:20.364781+0000","last_clean":"2023-09-23T22:56:20.364781+0000","last_became_active":"2023-09-23T22:56:20.363246+0000","last_became_peered":"2023-09-23T22:56:20.363246+0000","last_unstale":"2023-09-23T22:56:20.364781+0000","last_undegraded":"2023-09-23T22:56:20.364781+0000","last_fullsized":"2023-09-23T22:56:20.364781+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.214338+0000","last_change":"2023-09-23T22:56:21.211430+0000","last_active":"2023-09-23T22:56:21.214338+0000","last_peered":"2023-09-23T22:56:21.214338+0000","last_clean":"2023-09-23T22:56:21.214338+0000","last_became_active":"2023-09-23T22:56:21.211069+0000","last_became_peered":"2023-09-23T22:56:21.211069+0000","last_unstale":"2023-09-23T22:56:21.214338+0000","last_undegraded":"2023-09-23T22:56:21.214338+0000","last_fullsized":"2023-09-23T22:56:21.214338+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:11:40.687951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"330'4605","reported_seq":4632,"reported_epoch":330,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:21.632214+0000","last_change":"2023-09-23T22:55:53.199706+0000","last_active":"2023-09-23T22:56:21.632214+0000","last_peered":"2023-09-23T22:56:21.632214+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:55:53.199706+0000","last_became_peered":"2023-09-23T22:55:53.199706+0000","last_unstale":"2023-09-23T22:56:21.632214+0000","last_undegraded":"2023-09-23T22:55:53.196616+0000","last_fullsized":"2023-09-23T22:55:53.196454+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4605,"log_dups_size":0,"ondisk_log_size":4605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301793280,"num_objects":4605,"num_object_clones":0,"num_object_copies":13815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4605,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4605,"num_write_kb":294720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":4605}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":277,"reported_epoch":330,"state":"active+undersized","last_fresh":"2023-09-23T22:56:21.201329+0000","last_change":"2023-09-23T22:55:53.199820+0000","last_active":"2023-09-23T22:56:21.201329+0000","last_peered":"2023-09-23T22:56:21.201329+0000","last_clean":"2023-09-23T22:51:24.616651+0000","last_became_active":"2023-09-23T22:55:53.199820+0000","last_became_peered":"2023-09-23T22:55:53.199820+0000","last_unstale":"2023-09-23T22:56:21.201329+0000","last_undegraded":"2023-09-23T22:56:21.201329+0000","last_fullsized":"2023-09-23T22:55:53.195330+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:52.779172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.765213+0000","last_change":"2023-09-23T22:56:21.212434+0000","last_active":"2023-09-23T22:56:27.765213+0000","last_peered":"2023-09-23T22:56:27.765213+0000","last_clean":"2023-09-23T22:56:27.765213+0000","last_became_active":"2023-09-23T22:56:21.209765+0000","last_became_peered":"2023-09-23T22:56:21.209765+0000","last_unstale":"2023-09-23T22:56:27.765213+0000","last_undegraded":"2023-09-23T22:56:27.765213+0000","last_fullsized":"2023-09-23T22:56:27.765213+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:35.292030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":339,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.202557+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:56:21.202557+0000","last_peered":"2023-09-23T22:56:21.202557+0000","last_clean":"2023-09-23T22:56:21.202557+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:56:21.202557+0000","last_undegraded":"2023-09-23T22:56:21.202557+0000","last_fullsized":"2023-09-23T22:56:21.202557+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"327'4680","reported_seq":4704,"reported_epoch":331,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:28.546129+0000","last_change":"2023-09-23T22:56:28.534601+0000","last_active":"2023-09-23T22:56:21.199804+0000","last_peered":"2023-09-23T22:56:19.646980+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:55:24.310396+0000","last_became_peered":"2023-09-23T22:55:24.310396+0000","last_unstale":"2023-09-23T22:56:28.546129+0000","last_undegraded":"2023-09-23T22:56:28.518043+0000","last_fullsized":"2023-09-23T22:56:21.199804+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4680,"log_dups_size":0,"ondisk_log_size":4680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:01.685913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306708480,"num_objects":4680,"num_object_clones":0,"num_object_copies":14040,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":4681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4680,"num_write_kb":299520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"0,1","objects":4679},{"shards":"1","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":331,"state":"activating+remapped","last_fresh":"2023-09-23T22:56:28.543817+0000","last_change":"2023-09-23T22:56:21.206017+0000","last_active":"2023-09-23T22:51:33.395844+0000","last_peered":"2023-09-23T22:51:24.616865+0000","last_clean":"2023-09-23T22:51:24.616865+0000","last_became_active":"2023-09-23T22:45:51.502185+0000","last_became_peered":"2023-09-23T22:45:51.502185+0000","last_unstale":"2023-09-23T22:56:28.543817+0000","last_undegraded":"2023-09-23T22:56:28.543817+0000","last_fullsized":"2023-09-23T22:56:28.543817+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:28.175175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":431,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.213783+0000","last_change":"2023-09-23T22:56:21.213296+0000","last_active":"2023-09-23T22:56:21.213783+0000","last_peered":"2023-09-23T22:56:21.213783+0000","last_clean":"2023-09-23T22:56:21.213783+0000","last_became_active":"2023-09-23T22:56:21.209805+0000","last_became_peered":"2023-09-23T22:56:21.209805+0000","last_unstale":"2023-09-23T22:56:21.213783+0000","last_undegraded":"2023-09-23T22:56:21.213783+0000","last_fullsized":"2023-09-23T22:56:21.213783+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:15.073184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.213752+0000","last_change":"2023-09-23T22:56:20.210570+0000","last_active":"2023-09-23T22:56:20.213752+0000","last_peered":"2023-09-23T22:56:20.213752+0000","last_clean":"2023-09-23T22:56:20.213752+0000","last_became_active":"2023-09-23T22:56:20.210257+0000","last_became_peered":"2023-09-23T22:56:20.210257+0000","last_unstale":"2023-09-23T22:56:20.213752+0000","last_undegraded":"2023-09-23T22:56:20.213752+0000","last_fullsized":"2023-09-23T22:56:20.213752+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":216,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.365482+0000","last_change":"2023-09-23T22:56:20.363700+0000","last_active":"2023-09-23T22:56:20.365482+0000","last_peered":"2023-09-23T22:56:20.365482+0000","last_clean":"2023-09-23T22:56:20.365482+0000","last_became_active":"2023-09-23T22:56:20.363347+0000","last_became_peered":"2023-09-23T22:56:20.363347+0000","last_unstale":"2023-09-23T22:56:20.365482+0000","last_undegraded":"2023-09-23T22:56:20.365482+0000","last_fullsized":"2023-09-23T22:56:20.365482+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"330'4690","reported_seq":4717,"reported_epoch":330,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:21.680521+0000","last_change":"2023-09-23T22:55:53.200491+0000","last_active":"2023-09-23T22:56:21.680521+0000","last_peered":"2023-09-23T22:56:21.680521+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:55:53.200491+0000","last_became_peered":"2023-09-23T22:55:53.200491+0000","last_unstale":"2023-09-23T22:56:21.680521+0000","last_undegraded":"2023-09-23T22:55:53.195494+0000","last_fullsized":"2023-09-23T22:55:53.195356+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4690,"log_dups_size":0,"ondisk_log_size":4690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:47.108767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307363840,"num_objects":4690,"num_object_clones":0,"num_object_copies":14070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4690,"num_write_kb":300160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4690}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":308,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.197794+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:56:21.197794+0000","last_peered":"2023-09-23T22:56:21.197794+0000","last_clean":"2023-09-23T22:56:21.197794+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:56:21.197794+0000","last_undegraded":"2023-09-23T22:56:21.197794+0000","last_fullsized":"2023-09-23T22:56:21.197794+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":279,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.201224+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:56:21.201224+0000","last_peered":"2023-09-23T22:56:21.201224+0000","last_clean":"2023-09-23T22:56:21.201224+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:56:21.201224+0000","last_undegraded":"2023-09-23T22:56:21.201224+0000","last_fullsized":"2023-09-23T22:56:21.201224+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":263,"reported_epoch":330,"state":"active+undersized","last_fresh":"2023-09-23T22:56:21.202222+0000","last_change":"2023-09-23T22:55:53.198950+0000","last_active":"2023-09-23T22:56:21.202222+0000","last_peered":"2023-09-23T22:56:21.202222+0000","last_clean":"2023-09-23T22:55:51.196259+0000","last_became_active":"2023-09-23T22:55:53.198950+0000","last_became_peered":"2023-09-23T22:55:53.198950+0000","last_unstale":"2023-09-23T22:56:21.202222+0000","last_undegraded":"2023-09-23T22:56:21.202222+0000","last_fullsized":"2023-09-23T22:55:53.195857+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:23.246246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"330'4664","reported_seq":4693,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.566947+0000","last_change":"2023-09-23T22:56:21.229053+0000","last_active":"2023-09-23T22:56:21.566947+0000","last_peered":"2023-09-23T22:56:21.566947+0000","last_clean":"2023-09-23T22:56:21.566947+0000","last_became_active":"2023-09-23T22:56:21.207374+0000","last_became_peered":"2023-09-23T22:56:21.207374+0000","last_unstale":"2023-09-23T22:56:21.566947+0000","last_undegraded":"2023-09-23T22:56:21.566947+0000","last_fullsized":"2023-09-23T22:56:21.566947+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4664,"log_dups_size":0,"ondisk_log_size":4664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305659904,"num_objects":4664,"num_object_clones":0,"num_object_copies":13992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4664,"num_objects_unfound":0,"num_objects_dirty":4664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4664,"num_write_kb":298496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":4664}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":262,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.204693+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:56:21.204693+0000","last_peered":"2023-09-23T22:56:21.204693+0000","last_clean":"2023-09-23T22:56:21.204693+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:56:21.204693+0000","last_undegraded":"2023-09-23T22:56:21.204693+0000","last_fullsized":"2023-09-23T22:56:21.204693+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":210,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.193881+0000","last_change":"2023-09-23T22:55:53.199112+0000","last_active":"2023-09-23T22:56:20.193881+0000","last_peered":"2023-09-23T22:56:20.193881+0000","last_clean":"2023-09-23T22:39:27.713294+0000","last_became_active":"2023-09-23T22:55:53.199112+0000","last_became_peered":"2023-09-23T22:55:53.199112+0000","last_unstale":"2023-09-23T22:56:20.193881+0000","last_undegraded":"2023-09-23T22:56:20.193881+0000","last_fullsized":"2023-09-23T22:55:53.196998+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:23.074066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":261,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.354928+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:56:20.354928+0000","last_peered":"2023-09-23T22:56:20.354928+0000","last_clean":"2023-09-23T22:56:20.354928+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:56:20.354928+0000","last_undegraded":"2023-09-23T22:56:20.354928+0000","last_fullsized":"2023-09-23T22:56:20.354928+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"330'4695","reported_seq":4728,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:21.603163+0000","last_change":"2023-09-23T22:56:21.416510+0000","last_active":"2023-09-23T22:56:21.603163+0000","last_peered":"2023-09-23T22:56:21.603163+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:56:21.402493+0000","last_became_peered":"2023-09-23T22:56:21.402493+0000","last_unstale":"2023-09-23T22:56:21.603163+0000","last_undegraded":"2023-09-23T22:56:21.254596+0000","last_fullsized":"2023-09-23T22:56:21.199812+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307691520,"num_objects":4695,"num_object_clones":0,"num_object_copies":14085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4688,"num_objects_misplaced":4695,"num_objects_unfound":0,"num_objects_dirty":4695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4695,"num_write_kb":300480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":7}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":89,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.202791+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:56:21.202791+0000","last_peered":"2023-09-23T22:56:21.202791+0000","last_clean":"2023-09-23T22:56:21.202791+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:56:21.202791+0000","last_undegraded":"2023-09-23T22:56:21.202791+0000","last_fullsized":"2023-09-23T22:56:21.202791+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":29,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.372777+0000","last_change":"2023-09-23T22:56:20.365359+0000","last_active":"2023-09-23T22:56:20.372777+0000","last_peered":"2023-09-23T22:56:20.372777+0000","last_clean":"2023-09-23T22:56:20.372777+0000","last_became_active":"2023-09-23T22:56:20.364964+0000","last_became_peered":"2023-09-23T22:56:20.364964+0000","last_unstale":"2023-09-23T22:56:20.372777+0000","last_undegraded":"2023-09-23T22:56:20.372777+0000","last_fullsized":"2023-09-23T22:56:20.372777+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":309,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.198441+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:56:21.198441+0000","last_peered":"2023-09-23T22:56:21.198441+0000","last_clean":"2023-09-23T22:56:21.198441+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:56:21.198441+0000","last_undegraded":"2023-09-23T22:56:21.198441+0000","last_fullsized":"2023-09-23T22:56:21.198441+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.f","version":"330'4602","reported_seq":4632,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:21.622057+0000","last_change":"2023-09-23T22:56:21.417472+0000","last_active":"2023-09-23T22:56:21.622057+0000","last_peered":"2023-09-23T22:56:21.622057+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:56:21.411008+0000","last_became_peered":"2023-09-23T22:56:21.411008+0000","last_unstale":"2023-09-23T22:56:21.622057+0000","last_undegraded":"2023-09-23T22:56:21.263090+0000","last_fullsized":"2023-09-23T22:56:21.200679+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4602,"log_dups_size":0,"ondisk_log_size":4602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301596672,"num_objects":4602,"num_object_clones":0,"num_object_copies":13806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4596,"num_objects_misplaced":4602,"num_objects_unfound":0,"num_objects_dirty":4602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4602,"num_write_kb":294528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":6}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.265404+0000","last_change":"2023-09-23T22:56:21.265404+0000","last_active":"2023-09-23T22:56:21.265404+0000","last_peered":"2023-09-23T22:56:21.265404+0000","last_clean":"2023-09-23T22:56:21.265404+0000","last_became_active":"2023-09-23T22:56:21.264984+0000","last_became_peered":"2023-09-23T22:56:21.264984+0000","last_unstale":"2023-09-23T22:56:21.265404+0000","last_undegraded":"2023-09-23T22:56:21.265404+0000","last_fullsized":"2023-09-23T22:56:21.265404+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":229,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T22:56:21.202356+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:56:21.202356+0000","last_peered":"2023-09-23T22:56:21.202356+0000","last_clean":"2023-09-23T22:56:21.202356+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:56:21.202356+0000","last_undegraded":"2023-09-23T22:56:21.202356+0000","last_fullsized":"2023-09-23T22:56:21.202356+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":289,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.365607+0000","last_change":"2023-09-23T22:56:20.364523+0000","last_active":"2023-09-23T22:56:20.365607+0000","last_peered":"2023-09-23T22:56:20.365607+0000","last_clean":"2023-09-23T22:56:20.365607+0000","last_became_active":"2023-09-23T22:56:20.363967+0000","last_became_peered":"2023-09-23T22:56:20.363967+0000","last_unstale":"2023-09-23T22:56:20.365607+0000","last_undegraded":"2023-09-23T22:56:20.365607+0000","last_fullsized":"2023-09-23T22:56:20.365607+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":264,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.355257+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:56:20.355257+0000","last_peered":"2023-09-23T22:56:20.355257+0000","last_clean":"2023-09-23T22:56:20.355257+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:56:20.355257+0000","last_undegraded":"2023-09-23T22:56:20.355257+0000","last_fullsized":"2023-09-23T22:56:20.355257+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":59,"reported_epoch":330,"state":"active+undersized","last_fresh":"2023-09-23T22:56:21.202201+0000","last_change":"2023-09-23T22:55:53.199764+0000","last_active":"2023-09-23T22:56:21.202201+0000","last_peered":"2023-09-23T22:56:21.202201+0000","last_clean":"2023-09-23T22:51:24.616891+0000","last_became_active":"2023-09-23T22:55:53.199764+0000","last_became_peered":"2023-09-23T22:55:53.199764+0000","last_unstale":"2023-09-23T22:56:21.202201+0000","last_undegraded":"2023-09-23T22:56:21.202201+0000","last_fullsized":"2023-09-23T22:55:53.195893+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:11.261891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":139,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.204001+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:56:20.204001+0000","last_peered":"2023-09-23T22:56:20.204001+0000","last_clean":"2023-09-23T22:56:20.204001+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:56:20.204001+0000","last_undegraded":"2023-09-23T22:56:20.204001+0000","last_fullsized":"2023-09-23T22:56:20.204001+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":344,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.200214+0000","last_change":"2023-09-23T22:55:53.200257+0000","last_active":"2023-09-23T22:56:20.200214+0000","last_peered":"2023-09-23T22:56:20.200214+0000","last_clean":"2023-09-23T22:55:51.197568+0000","last_became_active":"2023-09-23T22:55:53.200257+0000","last_became_peered":"2023-09-23T22:55:53.200257+0000","last_unstale":"2023-09-23T22:56:20.200214+0000","last_undegraded":"2023-09-23T22:56:20.200214+0000","last_fullsized":"2023-09-23T22:55:53.197355+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":153,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T22:56:20.290478+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:56:20.290478+0000","last_peered":"2023-09-23T22:56:20.290478+0000","last_clean":"2023-09-23T22:56:20.290478+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:56:20.290478+0000","last_undegraded":"2023-09-23T22:56:20.290478+0000","last_fullsized":"2023-09-23T22:56:20.290478+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":216,"reported_epoch":329,"state":"active+undersized","last_fresh":"2023-09-23T22:56:20.201410+0000","last_change":"2023-09-23T22:55:53.198913+0000","last_active":"2023-09-23T22:56:20.201410+0000","last_peered":"2023-09-23T22:56:20.201410+0000","last_clean":"2023-09-23T22:55:51.197306+0000","last_became_active":"2023-09-23T22:55:53.198913+0000","last_became_peered":"2023-09-23T22:55:53.198913+0000","last_unstale":"2023-09-23T22:56:20.201410+0000","last_undegraded":"2023-09-23T22:56:20.201410+0000","last_fullsized":"2023-09-23T22:55:53.197257+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:27.766495+0000","last_change":"2023-09-23T22:56:21.210926+0000","last_active":"2023-09-23T22:56:27.766495+0000","last_peered":"2023-09-23T22:56:27.766495+0000","last_clean":"2023-09-23T22:56:27.766495+0000","last_became_active":"2023-09-23T22:56:21.209449+0000","last_became_peered":"2023-09-23T22:56:21.209449+0000","last_unstale":"2023-09-23T22:56:27.766495+0000","last_undegraded":"2023-09-23T22:56:27.766495+0000","last_fullsized":"2023-09-23T22:56:27.766495+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:55.027845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":258,"reported_epoch":330,"state":"active+undersized","last_fresh":"2023-09-23T22:56:21.201778+0000","last_change":"2023-09-23T22:55:53.200092+0000","last_active":"2023-09-23T22:56:21.201778+0000","last_peered":"2023-09-23T22:56:21.201778+0000","last_clean":"2023-09-23T22:44:57.970339+0000","last_became_active":"2023-09-23T22:55:53.200092+0000","last_became_peered":"2023-09-23T22:55:53.200092+0000","last_unstale":"2023-09-23T22:56:21.201778+0000","last_undegraded":"2023-09-23T22:56:21.201778+0000","last_fullsized":"2023-09-23T22:55:53.195380+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:09.913132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":284,"reported_epoch":330,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:21.202675+0000","last_change":"2023-09-23T22:55:53.200660+0000","last_active":"2023-09-23T22:56:21.202675+0000","last_peered":"2023-09-23T22:56:21.202675+0000","last_clean":"2023-09-23T22:55:51.195937+0000","last_became_active":"2023-09-23T22:55:53.200660+0000","last_became_peered":"2023-09-23T22:55:53.200660+0000","last_unstale":"2023-09-23T22:56:21.202675+0000","last_undegraded":"2023-09-23T22:55:53.196609+0000","last_fullsized":"2023-09-23T22:55:53.196470+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:07.622461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":305,"reported_epoch":330,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:21.262507+0000","last_change":"2023-09-23T22:56:21.262507+0000","last_active":"2023-09-23T22:56:21.262507+0000","last_peered":"2023-09-23T22:56:21.262507+0000","last_clean":"2023-09-23T22:56:21.262507+0000","last_became_active":"2023-09-23T22:56:21.262143+0000","last_became_peered":"2023-09-23T22:56:21.262143+0000","last_unstale":"2023-09-23T22:56:21.262507+0000","last_undegraded":"2023-09-23T22:56:21.262507+0000","last_fullsized":"2023-09-23T22:56:21.262507+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":88,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4842192896,"num_objects":73886,"num_object_clones":0,"num_object_copies":221658,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":41646,"num_objects_misplaced":27127,"num_objects_unfound":0,"num_objects_dirty":73886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73886,"num_write_kb":4728704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":582,"num_bytes_recovered":38141952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13496090624,"data_stored":13496090624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73886,"ondisk_log_size":73886,"up":39,"acting":38,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":12,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":15028,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":80,"acting":89,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430561,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626789,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253099,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2091228,"kb_used_data":1475860,"kb_used_omap":9,"kb_used_meta":615350,"kb_avail":91653924,"statfs":{"total":95995035648,"available":93853618176,"internally_reserved":0,"allocated":1511280640,"data_stored":1510130811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":630119401},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:30 2023","interfaces":[{"interface":"back","average":{"1min":15.630000000000001,"5min":6.4039999999999999,"15min":6.1970000000000001},"min":{"1min":0.26200000000000001,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":129.49299999999999,"5min":129.49299999999999,"15min":129.49299999999999},"last":1.7470000000000001},{"interface":"front","average":{"1min":14.69,"5min":5.8090000000000002,"15min":5.9800000000000004},"min":{"1min":0.32100000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":129.80000000000001,"5min":129.80000000000001,"15min":129.80000000000001},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:55:30 2023","interfaces":[{"interface":"back","average":{"1min":9.8800000000000008,"5min":4.593,"15min":3.8439999999999999},"min":{"1min":0.246,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":111.452,"5min":111.452,"15min":111.452},"last":0.29799999999999999},{"interface":"front","average":{"1min":10.167,"5min":4.4580000000000002,"15min":3.9420000000000002},"min":{"1min":0.36199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":108.22799999999999,"5min":108.22799999999999,"15min":108.22799999999999},"last":1.329}]},{"osd":3,"last update":"Sat Sep 23 22:55:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.9650000000000001,"15min":2.7170000000000001},"min":{"1min":0.224,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.997999999999999,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.495},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.998,"15min":3.181},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.4050000000000002,"5min":10.845000000000001,"15min":89.861999999999995},"last":0.68999999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:55:37 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.9910000000000001,"15min":2.452},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":6.9790000000000001,"5min":11.388,"15min":78.948999999999998},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.714,"5min":2.4260000000000002,"15min":2.5390000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":20.788,"5min":20.788,"15min":78.963999999999999},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:55:37 2023","interfaces":[{"interface":"back","average":{"1min":3.6579999999999999,"5min":2.2669999999999999,"15min":2.0859999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":32.188000000000002,"5min":32.188000000000002,"15min":84.254999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":3.149,"5min":2.141,"15min":2.3769999999999998},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.28000000000000003},"max":{"1min":33.033000000000001,"5min":33.033000000000001,"15min":87.515000000000001},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:55:37 2023","interfaces":[{"interface":"back","average":{"1min":3.4820000000000002,"5min":2.6200000000000001,"15min":2.4460000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":20.742000000000001,"5min":20.742000000000001,"15min":79.311000000000007},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.3809999999999998,"15min":2.056},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.313},"max":{"1min":20.614000000000001,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:55:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.8520000000000001,"15min":2.6379999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.33400000000000002},"max":{"1min":2.875,"5min":9.9329999999999998,"15min":159.018},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.2370000000000001,"5min":2.4329999999999998,"15min":2.8799999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.371},"max":{"1min":20.902999999999999,"5min":20.902999999999999,"15min":158.96000000000001},"last":0.64200000000000002}]}]},{"osd":0,"up_from":219,"seq":940597837947,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3374356,"kb_used_data":2970956,"kb_used_omap":31,"kb_used_meta":403360,"kb_avail":90370796,"statfs":{"total":95995035648,"available":92539695104,"internally_reserved":0,"allocated":3042258944,"data_stored":3041061056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":413041586},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,14,2],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":3.1739999999999999,"15min":3.5910000000000002},"min":{"1min":0.215,"5min":0.19,"15min":0.19},"max":{"1min":13.066000000000001,"5min":40.180999999999997,"15min":40.180999999999997},"last":0.25},{"interface":"front","average":{"1min":2.4609999999999999,"5min":3.2250000000000001,"15min":3.5760000000000001},"min":{"1min":0.218,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":15.423999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":5.101,"15min":3.6499999999999999},"min":{"1min":0.251,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.683,"5min":111.508,"15min":111.508},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.0059999999999998,"5min":5.2539999999999996,"15min":3.3199999999999998},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.579000000000001,"5min":111.532,"15min":111.532},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":3.8140000000000001,"15min":3.0699999999999998},"min":{"1min":0.28100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":8.5289999999999999,"5min":126.73,"15min":126.73},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.8929999999999998,"15min":2.149},"min":{"1min":0.28899999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.9550000000000001,"5min":121.483,"15min":121.483},"last":0.29399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.024,"15min":2.1440000000000001},"min":{"1min":0.35899999999999999,"5min":0.33000000000000002,"15min":0.29999999999999999},"max":{"1min":13.161,"5min":13.161,"15min":18.670999999999999},"last":0.377},{"interface":"front","average":{"1min":2.3780000000000001,"5min":2.077,"15min":2.0779999999999998},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.255000000000001,"5min":10.255000000000001,"15min":20.009},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.595,"15min":1.9319999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.318},"max":{"1min":5.8659999999999997,"5min":13.851000000000001,"15min":61.756999999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.4299999999999999,"5min":2.2610000000000001,"15min":2.3399999999999999},"min":{"1min":0.39500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.194000000000001,"5min":13.194000000000001,"15min":61.777000000000001},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.7549999999999999,"15min":1.9410000000000001},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.5940000000000003,"5min":13.206,"15min":59.146999999999998},"last":0.443},{"interface":"front","average":{"1min":1.599,"5min":2.4830000000000001,"15min":2.4489999999999998},"min":{"1min":0.42399999999999999,"5min":0.375,"15min":0.375},"max":{"1min":13.125,"5min":13.125,"15min":59.164000000000001},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":2.3090000000000002,"15min":2.1150000000000002},"min":{"1min":0.44700000000000001,"5min":0.42799999999999999,"15min":0.41199999999999998},"max":{"1min":13.223000000000001,"5min":13.223000000000001,"15min":18.640000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.8090000000000002,"5min":1.95,"15min":1.7090000000000001},"min":{"1min":0.433,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":10.624000000000001,"5min":13.289,"15min":21.954999999999998},"last":0.495}]}]},{"osd":2,"up_from":131,"seq":562640715953,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1443208,"kb_used_data":981312,"kb_used_omap":81,"kb_used_meta":461806,"kb_avail":92301944,"statfs":{"total":95995035648,"available":94517190656,"internally_reserved":0,"allocated":1004863488,"data_stored":1003735035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":472889562},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:40 2023","interfaces":[{"interface":"back","average":{"1min":5.1609999999999996,"5min":6.0199999999999996,"15min":5.7270000000000003},"min":{"1min":0.311,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":49.741,"5min":114.483,"15min":132.79499999999999},"last":1.4650000000000001},{"interface":"front","average":{"1min":3.9390000000000001,"5min":5.6820000000000004,"15min":6.0030000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":49.645000000000003,"5min":113.873,"15min":132.58099999999999},"last":1.516}]},{"osd":1,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6970000000000001,"15min":3.1190000000000002},"min":{"1min":0.24099999999999999,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":19.504999999999999,"5min":77.004999999999995,"15min":88.316000000000003},"last":1.4850000000000001},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1160000000000001,"15min":3.4279999999999999},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":19.523,"5min":76.956000000000003,"15min":88.436999999999998},"last":1.0569999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":3.5609999999999999,"15min":4.4880000000000004},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":19.579999999999998,"5min":112.895,"15min":112.895},"last":1.3520000000000001},{"interface":"front","average":{"1min":3.137,"5min":3.5449999999999999,"15min":3.7599999999999998},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":19.542000000000002,"5min":113.995,"15min":113.995},"last":1.502}]},{"osd":4,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":5.5880000000000001,"5min":3.2669999999999999,"15min":3.1400000000000001},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.253},"max":{"1min":48.286999999999999,"5min":69.113,"15min":108.878},"last":3.2040000000000002},{"interface":"front","average":{"1min":4.9829999999999997,"5min":2.9889999999999999,"15min":3.6259999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":48.259999999999998,"5min":69.028999999999996,"15min":108.93000000000001},"last":1.452}]},{"osd":5,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.798,"15min":3.069},"min":{"1min":0.45000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":19.652000000000001,"5min":22.646000000000001,"15min":97.322999999999993},"last":1.5860000000000001},{"interface":"front","average":{"1min":2.113,"5min":2.1970000000000001,"15min":3.0710000000000002},"min":{"1min":0.47799999999999998,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":19.614999999999998,"5min":22.501000000000001,"15min":97.301000000000002},"last":1.1180000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":5.1059999999999999,"5min":3.2639999999999998,"15min":3.633},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":50.073999999999998,"5min":59.655000000000001,"15min":98.597999999999999},"last":1.98},{"interface":"front","average":{"1min":6.0739999999999998,"5min":3.2709999999999999,"15min":3.0499999999999998},"min":{"1min":0.42999999999999999,"5min":0.39000000000000001,"15min":0.313},"max":{"1min":49.469000000000001,"5min":59.612000000000002,"15min":98.509},"last":1.6040000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.919,"5min":2.0960000000000001,"15min":1.881},"min":{"1min":0.495,"5min":0.40300000000000002,"15min":0.33900000000000002},"max":{"1min":19.756,"5min":19.756,"15min":19.756},"last":1.5429999999999999},{"interface":"front","average":{"1min":2.3980000000000001,"5min":1.6439999999999999,"15min":2.4500000000000002},"min":{"1min":0.45800000000000002,"5min":0.371,"15min":0.37},"max":{"1min":19.765999999999998,"5min":19.765999999999998,"15min":88.584000000000003},"last":1.4199999999999999}]}]},{"osd":3,"up_from":0,"seq":403726926046,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019245,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3848416,"kb_used_data":3258712,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":89896736,"statfs":{"total":95995035648,"available":92054257664,"internally_reserved":0,"allocated":3336921088,"data_stored":3336346830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:24 2023","interfaces":[{"interface":"back","average":{"1min":26.692,"5min":9.3520000000000003,"15min":7.1840000000000002},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":159.62200000000001,"5min":159.62200000000001,"15min":159.62200000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":26.629000000000001,"5min":9.5440000000000005,"15min":7.7160000000000002},"min":{"1min":0.33100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":159.51400000000001,"5min":159.51400000000001,"15min":159.51400000000001},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:55:47 2023","interfaces":[{"interface":"back","average":{"1min":6.1849999999999996,"5min":5.7119999999999997,"15min":3.399},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":98.846999999999994,"5min":132.304,"15min":132.304},"last":0.28599999999999998},{"interface":"front","average":{"1min":6.1509999999999998,"5min":5.8239999999999998,"15min":3.2589999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":98.938999999999993,"5min":132.453,"15min":132.453},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:55:40 2023","interfaces":[{"interface":"back","average":{"1min":11.44,"5min":4.0789999999999997,"15min":3.7349999999999999},"min":{"1min":0.33800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":137.922,"5min":137.922,"15min":137.922},"last":0.46999999999999997},{"interface":"front","average":{"1min":11.497,"5min":4.1219999999999999,"15min":3.734},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":140.994,"5min":140.994,"15min":140.994},"last":0.433}]},{"osd":3,"last update":"Sat Sep 23 22:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":3.8100000000000001,"15min":4.5830000000000002},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":5.1509999999999998,"5min":164.453,"15min":164.453},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":3.9119999999999999,"15min":4.6779999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.27200000000000002},"max":{"1min":3.048,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.46400000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.0529999999999999,"15min":1.012},"min":{"1min":0.29399999999999998,"5min":0.28999999999999998,"15min":0.25900000000000001},"max":{"1min":3.0960000000000001,"5min":10.321999999999999,"15min":43.189999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.0129999999999999,"15min":1.0149999999999999},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.248},"max":{"1min":3.4119999999999999,"5min":6.4349999999999996,"15min":42.450000000000003},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.0329999999999999,"15min":0.97399999999999998},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7480000000000002,"5min":7.9139999999999997,"15min":27.056000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.0489999999999999,"15min":0.98799999999999999},"min":{"1min":0.373,"5min":0.29699999999999999,"15min":0.28199999999999997},"max":{"1min":3.0030000000000001,"5min":10.423999999999999,"15min":26.940999999999999},"last":0.435}]},{"osd":7,"last update":"Sat Sep 23 22:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.048,"15min":0.91400000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":1.9299999999999999,"5min":10.278,"15min":12.311},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.88400000000000001,"15min":0.83099999999999996},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":2.9580000000000002,"5min":4.077,"15min":8.6739999999999995},"last":0.45800000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823017,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2660484,"kb_used_data":2074876,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91084668,"statfs":{"total":95995035648,"available":93270700032,"internally_reserved":0,"allocated":2124673024,"data_stored":2124122482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:56:25 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":6.6219999999999999,"15min":5.1529999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":24.164999999999999,"5min":185.62,"15min":185.62},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.4740000000000002,"5min":6.6420000000000003,"15min":5.1449999999999996},"min":{"1min":0.29199999999999998,"5min":0.248,"15min":0.248},"max":{"1min":24.244,"5min":185.499,"15min":185.499},"last":0.373}]},{"osd":1,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":3.964,"15min":2.952},"min":{"1min":0.28799999999999998,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":3.5600000000000001,"5min":110.36499999999999,"15min":149.203},"last":0.40000000000000002},{"interface":"front","average":{"1min":2.4940000000000002,"5min":4.4850000000000003,"15min":3.327},"min":{"1min":0.43099999999999999,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":9.2050000000000001,"5min":110.89,"15min":149.554},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":4.992,"15min":5.04},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5309999999999997,"5min":189.126,"15min":189.126},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.4710000000000001,"5min":4.6379999999999999,"15min":4.8920000000000003},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":6.2530000000000001,"5min":188.74799999999999,"15min":188.74799999999999},"last":1.3149999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":3.1419999999999999,"15min":5.7699999999999996},"min":{"1min":0.40300000000000002,"5min":0.316,"15min":0.25700000000000001},"max":{"1min":6.415,"5min":89.995000000000005,"15min":257.05000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.972,"5min":3.105,"15min":5.6150000000000002},"min":{"1min":0.40200000000000002,"5min":0.33700000000000002,"15min":0.29699999999999999},"max":{"1min":6.3869999999999996,"5min":90.117999999999995,"15min":257.35000000000002},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:56:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.03,"15min":0.83799999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.24099999999999999},"max":{"1min":4.0460000000000003,"5min":6.3949999999999996,"15min":8.1110000000000007},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.647,"5min":1.3380000000000001,"15min":0.91500000000000004},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":8.6519999999999992,"5min":8.6519999999999992,"15min":10.535},"last":0.33700000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.2689999999999999,"15min":1.665},"min":{"1min":0.33100000000000002,"5min":0.307,"15min":0.26500000000000001},"max":{"1min":4.3360000000000003,"5min":8.0830000000000002,"15min":140.07499999999999},"last":0.434},{"interface":"front","average":{"1min":1.127,"5min":1.1739999999999999,"15min":1.675},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":4.1609999999999996,"5min":10.314,"15min":139.739},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.282,"15min":0.96399999999999997},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.32100000000000001},"max":{"1min":1.629,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.1279999999999999,"15min":0.89400000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32100000000000001},"max":{"1min":4.0110000000000001,"5min":6.7149999999999999,"15min":8.0459999999999994},"last":0.46400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3069378560,"data_stored":3069378560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1507065856,"data_stored":1507065856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1040777216,"data_stored":1040777216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3334209536,"data_stored":3334209536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2121728000,"data_stored":2121728000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2422931456,"data_stored":2422931456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:56:32.187 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T22:56:32.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:31 smithi151 ceph-mon[159210]: pgmap v970: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+remapped, 1 activating+degraded+remapped, 20 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 40 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 19 MiB/s wr, 302 op/s; 41650/221730 objects degraded (18.784%); 27141/221730 objects misplaced (12.241%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:56:32.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:31 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:56:32.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:32.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:32.345 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:32.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:32.384 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:32.446 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:32.609 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:32.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:32.718 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:32.791 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:32.876 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:32 smithi052 ceph-mon[180979]: from='client.55897 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:56:32.876 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:32 smithi052 ceph-mon[187743]: from='client.55897 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:56:32.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:56:32.951431+0000 min lat: 0.00121105 max lat: 11.5254 avg lat: 0.0127332 2023-09-23T22:56:32.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:56:32.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 60 16 75417 75401 78.5076 53 0.0142203 0.0127332 2023-09-23T22:56:32.991 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:56:33.147 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:33.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:32 smithi151 ceph-mon[159210]: from='client.55897 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:56:33.257 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:33.327 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:33.409 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:33.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:33.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:33.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:56:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:56:33] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-09-23T22:56:33.515 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:33.542 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:33.822 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:33.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:33 smithi052 ceph-mon[187743]: pgmap v971: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 6.7 MiB/s wr, 107 op/s; 45647/222501 objects degraded (20.515%); 22989/222501 objects misplaced (10.332%); 5.4 MiB/s, 0 keys/s, 86 objects/s recovering 2023-09-23T22:56:33.827 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:33 smithi052 ceph-mon[180979]: pgmap v971: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 6.7 MiB/s wr, 107 op/s; 45647/222501 objects degraded (20.515%); 22989/222501 objects misplaced (10.332%); 5.4 MiB/s, 0 keys/s, 86 objects/s recovering 2023-09-23T22:56:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:33.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:33.928 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:33.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 61 16 76149 76133 77.9707 45.75 0.0155439 0.0128199 2023-09-23T22:56:33.968 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:33 smithi151 ceph-mon[159210]: pgmap v971: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 6.7 MiB/s wr, 107 op/s; 45647/222501 objects degraded (20.515%); 22989/222501 objects misplaced (10.332%); 5.4 MiB/s, 0 keys/s, 86 objects/s recovering 2023-09-23T22:56:34.457 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:34.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:34.573 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:34.592 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:34.663 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:34.773 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:34.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:34.891 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:34.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 62 16 76997 76981 77.568 53 0.0135 0.0128853 2023-09-23T22:56:34.960 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:35.006 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:35.071 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:35.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:35.183 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:35.232 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:35.291 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:56:35.291 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:56:35.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:56:35.291 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:140973 8.f 8:f33d72b0:::benchmark_data_smithi052_337357_object38522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 456.49383956700001, 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.329060238, 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140973 2023-09-23T22:56:35.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795101+0000", 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795109+0000", 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6429999999999995e-06 2023-09-23T22:56:35.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795122+0000", 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3022e-05 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795130+0000", 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7400000000000004e-06 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810865+0000", 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015735702000000001 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for rw locks", 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810975+0000", 2023-09-23T22:56:35.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010935200000000001 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067039+0000", 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.256064085 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067161+0000", 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121952 2023-09-23T22:56:35.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067334+0000", 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017327 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.083778+0000", 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016443607999999998 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.084031+0000", 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025363099999999998 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120168+0000", 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036136433000000003 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120220+0000", 2023-09-23T22:56:35.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2271000000000001e-05 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.124163+0000", 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039430070000000001 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5823 9.c 9:36ec13c5:::benchmark_data_smithi052_402718_object5822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:56:35.301 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 313.67116239000001, 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22852289200000001, 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5823 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.302 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617779+0000", 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617800+0000", 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0721999999999999e-05 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617807+0000", 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.878e-06 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617814+0000", 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8519999999999998e-06 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618047+0000", 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023353299999999999 2023-09-23T22:56:35.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618201+0000", 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015408199999999999 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618341+0000", 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139765 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619519+0000", 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011783220000000001 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619626+0000", 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107017 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846195+0000", 2023-09-23T22:56:35.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22656818400000001 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846251+0000", 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6786000000000001e-05 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846303+0000", 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1248999999999997e-05 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.308 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5845 9.c 9:32588725:::benchmark_data_smithi052_402718_object5844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 313.664198967, 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.221823086, 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.309 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.310 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5845 2023-09-23T22:56:35.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.310 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624742+0000", 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624765+0000", 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2909e-05 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624775+0000", 2023-09-23T22:56:35.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.023e-05 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624783+0000", 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624865+0000", 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1631999999999994e-05 2023-09-23T22:56:35.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625069+0000", 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000203685 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625237+0000", 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168426 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625970+0000", 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00073263699999999996 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.626858+0000", 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088813800000000001 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846472+0000", 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21961366099999999 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846523+0000", 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0986000000000003e-05 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846566+0000", 2023-09-23T22:56:35.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3708000000000002e-05 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5878 9.c 9:34f7d49b:::benchmark_data_smithi052_402718_object5877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 313.65003082700002, 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.208484483, 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.317 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5878 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638910+0000", 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638928+0000", 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7866000000000001e-05 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638941+0000", 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2618e-05 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638947+0000", 2023-09-23T22:56:35.320 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5639999999999996e-06 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638986+0000", 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9031e-05 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639111+0000", 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124955 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639216+0000", 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105696 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639724+0000", 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00050770100000000005 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.640023+0000", 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029943599999999998 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847306+0000", 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20728270800000001 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847359+0000", 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3319999999999998e-05 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847396+0000", 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6473999999999997e-05 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.324 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13481 9.8 9:1f01482c:::benchmark_data_smithi052_402718_object13480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 309.95293903800001, 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.322004134, 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13481 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336002+0000", 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336029+0000", 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6545000000000001e-05 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336044+0000", 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5444999999999999e-05 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336063+0000", 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8729e-05 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336119+0000", 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6112000000000001e-05 2023-09-23T22:56:35.328 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336242+0000", 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012232500000000001 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336383+0000", 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014170900000000001 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337199+0000", 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00081607499999999998 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337730+0000", 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00053117300000000002 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657923+0000", 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32019289400000001 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657967+0000", 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3399999999999998e-05 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.658007+0000", 2023-09-23T22:56:35.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0534999999999997e-05 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13489 9.c 9:33ad037a:::benchmark_data_smithi052_402718_object13488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 309.95186435300002, 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32053898800000002, 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.332 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13489 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337077+0000", 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337098+0000", 2023-09-23T22:56:35.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1191000000000001e-05 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337107+0000", 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2179999999999999e-06 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337112+0000", 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T22:56:35.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337159+0000", 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6615e-05 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337308+0000", 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014894100000000001 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337456+0000", 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148066 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338621+0000", 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011655420000000001 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.339454+0000", 2023-09-23T22:56:35.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083201599999999998 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657516+0000", 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.318062489 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657570+0000", 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4024000000000003e-05 2023-09-23T22:56:35.338 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657617+0000", 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6826000000000002e-05 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59534 9.a 9:56ae8077:::benchmark_data_smithi052_402718_object59533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:56:35.339 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 266.19463995400002, 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20214791800000001, 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59534 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.340 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094300+0000", 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:56:35.341 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094331+0000", 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0930999999999999e-05 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094353+0000", 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2340999999999998e-05 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.342 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094362+0000", 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2519999999999999e-06 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094415+0000", 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2997000000000001e-05 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.343 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094637+0000", 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000222208 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094844+0000", 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000207631 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097382+0000", 2023-09-23T22:56:35.344 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025371360000000002 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097400+0000", 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8714000000000001e-05 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.098411+0000", 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001011131 2023-09-23T22:56:35.345 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294679+0000", 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19626732399999999 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294738+0000", 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8801000000000002e-05 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.346 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296450+0000", 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001712618 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59542 9.a 9:51ad1323:::benchmark_data_smithi052_402718_object59541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:56:35.347 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 266.18968576399999, 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19894724899999999, 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59542 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.348 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099254+0000", 2023-09-23T22:56:35.349 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099282+0000", 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7971999999999999e-05 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099301+0000", 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8593e-05 2023-09-23T22:56:35.350 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099311+0000", 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0074999999999999e-05 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099389+0000", 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7689999999999996e-05 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.351 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099595+0000", 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020635099999999999 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099809+0000", 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021421 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.352 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.102469+0000", 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002660314 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104681+0000", 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022113889999999998 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104831+0000", 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015015699999999999 2023-09-23T22:56:35.353 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296512+0000", 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19168089399999999 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296580+0000", 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8103999999999997e-05 2023-09-23T22:56:35.354 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298204+0000", 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016236810000000001 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98500 9.a 9:5e200aba:::benchmark_data_smithi052_402718_object8859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:56:35.355 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 212.321446313, 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22489546499999999, 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98500 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.356 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967494+0000", 2023-09-23T22:56:35.357 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967502+0000", 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6190000000000002e-06 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967516+0000", 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4651000000000001e-05 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.358 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967525+0000", 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6279999999999994e-06 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968657+0000", 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011323209999999999 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.359 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968810+0000", 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015326000000000001 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968949+0000", 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138429 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.974764+0000", 2023-09-23T22:56:35.360 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058149839999999996 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.975363+0000", 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059912599999999996 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977959+0000", 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025958399999999999 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.361 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190744+0000", 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.212785264 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190803+0000", 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8777000000000003e-05 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.362 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.192391+0000", 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015884969999999999 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110895 9.a 9:5225ae0d:::benchmark_data_smithi052_402718_object21254:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 194.22536334899999, 2023-09-23T22:56:35.363 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25354543899999998, 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110895 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:56:35.364 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063577+0000", 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.365 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063584+0000", 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1860000000000003e-06 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063611+0000", 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6279e-05 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.366 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063621+0000", 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8179999999999999e-06 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063692+0000", 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1106000000000007e-05 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.367 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063841+0000", 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014895899999999999 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063967+0000", 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012629000000000001 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.071444+0000", 2023-09-23T22:56:35.368 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074768090000000001 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072687+0000", 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00124325 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072764+0000", 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7496999999999993e-05 2023-09-23T22:56:35.369 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315667+0000", 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24290224499999999 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315734+0000", 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.370 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.317124+0000", 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001390575 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110898 9.a 9:538cf70b:::benchmark_data_smithi052_402718_object21257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 194.22529625300001, 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25922483400000002, 2023-09-23T22:56:35.371 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110898 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:56:35.372 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063645+0000", 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:35.373 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.374 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.374 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:56:35.374 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063650+0000", 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6210000000000003e-06 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063660+0000", 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8379999999999997e-06 2023-09-23T22:56:35.375 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063667+0000", 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9879999999999998e-06 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067524+0000", 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003857624 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.376 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067678+0000", 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015363900000000001 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067821+0000", 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143096 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:56:35.377 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072775+0000", 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049535079999999997 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.073473+0000", 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069833799999999995 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.074443+0000", 2023-09-23T22:56:35.378 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097013000000000004 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321598+0000", 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24715498 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321672+0000", 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4229999999999999e-05 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:56:35.379 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.322871+0000", 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001198483 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:56:35.380 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:56:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:35.381 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:35.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:35.482 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:35.581 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:35.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:35.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:35.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: pgmap v972: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 MiB/s wr, 181 op/s; 45890/224727 objects degraded (20.420%); 23104/224727 objects misplaced (10.281%); 5.4 MiB/s, 0 keys/s, 86 objects/s recovering 2023-09-23T22:56:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T22:56:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 45890/224727 objects degraded (20.420%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:35.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/828787443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: pgmap v972: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 MiB/s wr, 181 op/s; 45890/224727 objects degraded (20.420%); 23104/224727 objects misplaced (10.281%); 5.4 MiB/s, 0 keys/s, 86 objects/s recovering 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 45890/224727 objects degraded (20.420%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/828787443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:35.825 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:35.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:35.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 63 16 77740 77724 77.074 46.4375 0.0170986 0.012969 2023-09-23T22:56:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: pgmap v972: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 MiB/s wr, 181 op/s; 45890/224727 objects degraded (20.420%); 23104/224727 objects misplaced (10.281%); 5.4 MiB/s, 0 keys/s, 86 objects/s recovering 2023-09-23T22:56:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:35.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T22:56:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 45890/224727 objects degraded (20.420%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/828787443' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:35.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:56:36.205 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:56:36.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:36.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:56:36.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[187743]: osdmap e332: 8 total, 7 up, 5 in 2023-09-23T22:56:36.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[187743]: failed to encode map e332 with expected crc 2023-09-23T22:56:36.915 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:36.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:56:36.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[180979]: osdmap e332: 8 total, 7 up, 5 in 2023-09-23T22:56:36.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:36 smithi052 ceph-mon[180979]: failed to encode map e332 with expected crc 2023-09-23T22:56:36.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 64 16 78436 78420 76.5495 43.5 0.027947 0.0130586 2023-09-23T22:56:36.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:36 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:36.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:56:36.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:36 smithi151 ceph-mon[159210]: osdmap e332: 8 total, 7 up, 5 in 2023-09-23T22:56:36.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:36 smithi151 ceph-mon[159210]: failed to encode map e332 with expected crc 2023-09-23T22:56:36.954 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:56:37.272 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:37.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:37.379 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:37.464 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:37.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:37 smithi052 ceph-mon[180979]: failed to encode map e332 with expected crc 2023-09-23T22:56:37.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:37 smithi052 ceph-mon[180979]: failed to encode map e332 with expected crc 2023-09-23T22:56:37.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:37 smithi052 ceph-mon[180979]: pgmap v974: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 15 MiB/s wr, 235 op/s; 46139/225474 objects degraded (20.463%); 22993/225474 objects misplaced (10.198%); 2.9 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T22:56:37.538 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:37 smithi052 ceph-mon[187743]: failed to encode map e332 with expected crc 2023-09-23T22:56:37.538 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:37 smithi052 ceph-mon[187743]: failed to encode map e332 with expected crc 2023-09-23T22:56:37.538 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:37 smithi052 ceph-mon[187743]: pgmap v974: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 15 MiB/s wr, 235 op/s; 46139/225474 objects degraded (20.463%); 22993/225474 objects misplaced (10.198%); 2.9 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T22:56:37.608 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:37.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:37.707 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:56:37.721 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:37.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:37 smithi151 ceph-mon[159210]: failed to encode map e332 with expected crc 2023-09-23T22:56:37.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:37 smithi151 ceph-mon[159210]: failed to encode map e332 with expected crc 2023-09-23T22:56:37.748 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:37 smithi151 ceph-mon[159210]: pgmap v974: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 15 MiB/s wr, 235 op/s; 46139/225474 objects degraded (20.463%); 22993/225474 objects misplaced (10.198%); 2.9 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T22:56:37.753 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:37.924 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:37.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:37.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 65 16 79176 79160 76.0834 46.25 0.01567 0.0131385 2023-09-23T22:56:38.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:38.130 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:38.315 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:38.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:38.355 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:38.381 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:38.409 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:38.425 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:38.450 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:38.477 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:38.487 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:38.506 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:38.538 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.539 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:56:38.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:38.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:38.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/911457593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:38.726 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/911457593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:38.727 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/911457593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:38.727 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:38.782 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:38.852 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:38.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:38.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 66 16 79863 79847 75.5813 42.9375 0.0075991 0.0132021 2023-09-23T22:56:38.956 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:39.034 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:39.157 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:39.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:39.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:39.305 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:56:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:39.323 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:39.327 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:39.447 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:39.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:39.554 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:39.557 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:39 smithi052 ceph-mon[187743]: pgmap v975: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 27 MiB/s wr, 425 op/s; 46610/232080 objects degraded (20.084%); 23713/232080 objects misplaced (10.218%); 2.6 MiB/s, 0 keys/s, 40 objects/s recovering 2023-09-23T22:56:39.557 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/265933947' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:56:39.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:39 smithi052 ceph-mon[180979]: pgmap v975: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 27 MiB/s wr, 425 op/s; 46610/232080 objects degraded (20.084%); 23713/232080 objects misplaced (10.218%); 2.6 MiB/s, 0 keys/s, 40 objects/s recovering 2023-09-23T22:56:39.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/265933947' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:56:39.560 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 10 pgp_num to 16 2023-09-23T22:56:39.586 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:39 smithi151 ceph-mon[159210]: pgmap v975: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 27 MiB/s wr, 425 op/s; 46610/232080 objects degraded (20.084%); 23713/232080 objects misplaced (10.218%); 2.6 MiB/s, 0 keys/s, 40 objects/s recovering 2023-09-23T22:56:39.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/265933947' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:56:39.745 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:39.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:39.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:39.851 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:39.877 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:39.919 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:39.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:39.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:39.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 67 16 80511 80495 75.0578 40.5 0.135076 0.0133135 2023-09-23T22:56:40.022 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:40.126 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:40.245 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:40.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:40.342 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:40.352 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:40.406 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:40.501 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:40.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:40.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:40.679 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:40.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/265933947' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: osdmap e333: 8 total, 7 up, 5 in 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 46610/232080 objects degraded (20.084%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/265933947' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: osdmap e333: 8 total, 7 up, 5 in 2023-09-23T22:56:40.773 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 46610/232080 objects degraded (20.084%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:40.778 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:40.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:40.884 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:40.927 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:40.951 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 68 16 81315 81299 74.693 50.25 0.0144438 0.013382 2023-09-23T22:56:40.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/265933947' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:56:40.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: osdmap e333: 8 total, 7 up, 5 in 2023-09-23T22:56:40.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: failed to encode map e333 with expected crc 2023-09-23T22:56:40.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 46610/232080 objects degraded (20.084%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:41.015 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:41.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:41.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:41.126 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:41.153 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:41.198 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:41.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:41.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:41.324 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:56:41.344 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:41.574 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:41.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:41.694 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:41.759 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:41.873 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:41.878 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:41 smithi151 ceph-mon[159210]: pgmap v977: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 36 MiB/s wr, 575 op/s; 47087/236328 objects degraded (19.924%); 23948/236328 objects misplaced (10.133%); 896 KiB/s, 13 objects/s recovering 2023-09-23T22:56:41.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:41 smithi052 ceph-mon[180979]: pgmap v977: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 36 MiB/s wr, 575 op/s; 47087/236328 objects degraded (19.924%); 23948/236328 objects misplaced (10.133%); 896 KiB/s, 13 objects/s recovering 2023-09-23T22:56:41.878 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:41 smithi052 ceph-mon[187743]: pgmap v977: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 36 MiB/s wr, 575 op/s; 47087/236328 objects degraded (19.924%); 23948/236328 objects misplaced (10.133%); 896 KiB/s, 13 objects/s recovering 2023-09-23T22:56:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:41.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:41.953 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 69 16 82025 82009 74.2518 44.375 0.0182746 0.0134617 2023-09-23T22:56:41.980 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:42.046 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:42.138 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:42.157 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:42.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:42.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:42.321 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:42.447 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:42.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:42.554 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:42.578 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/344514237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:42.609 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:56:42.626 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:42.859 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:42.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/344514237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:42.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/344514237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:42.953 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 70 16 82833 82817 73.9125 50.5 0.0283195 0.0135241 2023-09-23T22:56:42.965 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:43.016 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:56:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:56:43] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.33.4" 2023-09-23T22:56:43.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[187743]: pgmap v978: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s; 47751/243042 objects degraded (19.647%); 24523/243042 objects misplaced (10.090%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:56:43.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:56:43.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[187743]: failed to encode map e334 with expected crc 2023-09-23T22:56:43.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/344514237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:43.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[187743]: osdmap e334: 8 total, 7 up, 5 in 2023-09-23T22:56:43.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[180979]: pgmap v978: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s; 47751/243042 objects degraded (19.647%); 24523/243042 objects misplaced (10.090%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:56:43.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:56:43.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[180979]: failed to encode map e334 with expected crc 2023-09-23T22:56:43.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/344514237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:43.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:43 smithi052 ceph-mon[180979]: osdmap e334: 8 total, 7 up, 5 in 2023-09-23T22:56:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:43 smithi151 ceph-mon[159210]: pgmap v978: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s; 47751/243042 objects degraded (19.647%); 24523/243042 objects misplaced (10.090%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:56:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:43 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:56:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:43 smithi151 ceph-mon[159210]: failed to encode map e334 with expected crc 2023-09-23T22:56:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/344514237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:43 smithi151 ceph-mon[159210]: osdmap e334: 8 total, 7 up, 5 in 2023-09-23T22:56:43.954 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 71 16 83644 83628 73.5855 50.6875 0.00973443 0.0135837 2023-09-23T22:56:44.000 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:44.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:44.106 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:44.163 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:44.339 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:44.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:44.451 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:44.496 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:44.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:44.576 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:44.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:44.592 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:44.622 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:44.657 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:44.675 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:84191 10.b e334/329 10:d5c7d4cc:::benchmark_data_smithi052_461816_object84190:head v 334'5353, mlcod=0'0)", 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.666568+0000", 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0076021279999999997, 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076261159999999996, 2023-09-23T22:56:44.676 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666568+0000", 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666568+0000", 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666568+0000", 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8482000000000001e-05 2023-09-23T22:56:44.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666616+0000", 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4489999999999996e-06 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666623+0000", 2023-09-23T22:56:44.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9470000000000002e-06 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666628+0000", 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103935 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666732+0000", 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1101e-05 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666753+0000", 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018453100000000001 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:84192 10.b e334/329 10:d6114ce9:::benchmark_data_smithi052_461816_object84191:head v 334'5354, mlcod=0'0)", 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.666916+0000", 2023-09-23T22:56:44.681 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0072543829999999997, 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073215110000000002, 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666916+0000", 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.682 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666916+0000", 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666915+0000", 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6263e-05 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666972+0000", 2023-09-23T22:56:44.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8570000000000001e-06 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666980+0000", 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1240000000000004e-06 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.666986+0000", 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056880300000000002 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672674+0000", 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8672e-05 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672692+0000", 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0057763090000000003 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:84195 10.3 10:cf30bbc6:::benchmark_data_smithi052_461816_object84194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e334)", 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.668818+0000", 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.005352496, 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054601279999999999, 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:56:44.686 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 84195 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668818+0000", 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668818+0000", 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:56:44.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668816+0000", 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6644999999999996e-05 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668883+0000", 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2019e-05 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668895+0000", 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4239999999999998e-06 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668901+0000", 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039479759999999997 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672849+0000", 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010577999999999999 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672955+0000", 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000135029 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.673090+0000", 2023-09-23T22:56:44.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0042721699999999996 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:84194 10.4 e334/317 10:2e53cf61:::benchmark_data_smithi052_461816_object84193:head v 334'5241, mlcod=334'5239)", 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.668844+0000", 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0053261159999999997, 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054832939999999997, 2023-09-23T22:56:44.691 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668844+0000", 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668844+0000", 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668844+0000", 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6700000000000003e-05 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668900+0000", 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5764e-05 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668916+0000", 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1499999999999998e-06 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668921+0000", 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011603300000000001 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669037+0000", 2023-09-23T22:56:44.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1413999999999999e-05 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669059+0000", 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021449699999999999 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:84196 10.f 10:fdf7163b:::benchmark_data_smithi052_461816_object84195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e334)", 2023-09-23T22:56:44.695 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.668908+0000", 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0052618939999999996, 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054579520000000003, 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 84196 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668908+0000", 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668908+0000", 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668908+0000", 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4037e-05 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668942+0000", 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6889999999999998e-06 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668949+0000", 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3259999999999997e-06 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668953+0000", 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003379244 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672332+0000", 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017353099999999999 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672506+0000", 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019399599999999999 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672700+0000", 2023-09-23T22:56:44.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037912369999999998 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:84197 10.1 10:8b6bf43a:::benchmark_data_smithi052_461816_object84196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e334)", 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.668968+0000", 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0052027230000000002, 2023-09-23T22:56:44.702 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054445029999999998, 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 84197 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.703 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668968+0000", 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668968+0000", 2023-09-23T22:56:44.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.668967+0000", 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6803e-05 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669004+0000", 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3570000000000003e-06 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669010+0000", 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9559999999999999e-06 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669014+0000", 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1923999999999996e-05 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669106+0000", 2023-09-23T22:56:44.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013893799999999999 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669245+0000", 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000158324 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:56:44.708 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669403+0000", 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00043578099999999999 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:84198 10.f 10.78ca096f (undecoded) ondisk+write+known_if_redirected e334)", 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.669026+0000", 2023-09-23T22:56:44.709 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0051446180000000001, 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054276159999999997, 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "queued for pg", 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 84198 2023-09-23T22:56:44.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669026+0000", 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669026+0000", 2023-09-23T22:56:44.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669025+0000", 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1749999999999999e-05 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669057+0000", 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0249999999999999e-06 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669063+0000", 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7479999999999999e-06 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.669067+0000", 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1023999999999999e-05 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.713 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:84199 10.3 10.927a13c3 (undecoded) ondisk+write+known_if_redirected e334)", 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.670409+0000", 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0037611099999999998, 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040773160000000001, 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "queued for pg", 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 84199 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.714 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.670409+0000", 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.670409+0000", 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:56:44.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.670407+0000", 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9979000000000003e-05 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.716 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.670467+0000", 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1793999999999999e-05 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.670479+0000", 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3499999999999999e-06 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.670487+0000", 2023-09-23T22:56:44.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7375999999999999e-05 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:84200 10.4 e334/317 10:27c440d3:::benchmark_data_smithi052_461816_object84199:head v 334'5242, mlcod=334'5239)", 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:44.672191+0000", 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.001979115, 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023282200000000002, 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:44.718 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672191+0000", 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672191+0000", 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672191+0000", 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7872999999999999e-05 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672219+0000", 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5219999999999998e-06 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672224+0000", 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0939999999999999e-06 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672227+0000", 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4662e-05 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672272+0000", 2023-09-23T22:56:44.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.313e-05 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:44.672295+0000", 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103935 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:56:44.722 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 9 2023-09-23T22:56:44.723 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:56:44.723 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:44.724 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:44.724 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4245534335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:44.728 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:44.757 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:44.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.758 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:44.758 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.758 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.758 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.758 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.758 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:56:44.767 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:44.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4245534335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:44.950 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4245534335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:44.954 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 72 16 84429 84413 73.2449 49.0625 0.0126764 0.0136474 2023-09-23T22:56:44.991 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:44.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:44.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:45.096 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:45.151 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:45.209 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:45.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:45.276 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5] out_osds: [7, 3, 6] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:56:45.276 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:56:45.276 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T22:56:45.276 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:56:45.310 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:45.378 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:45.470 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:45.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:45.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:45.573 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:45.586 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:45.604 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:45.652 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:45.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:45.696 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:45.759 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:45.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:45 smithi151 ceph-mon[159210]: pgmap v980: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 56 MiB/s wr, 903 op/s; 48198/247161 objects degraded (19.501%); 24747/247161 objects misplaced (10.013%); 880 KiB/s, 13 objects/s recovering 2023-09-23T22:56:45.830 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 48198/247161 objects degraded (19.501%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:45.837 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:45.934 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:45 smithi052 ceph-mon[187743]: pgmap v980: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 56 MiB/s wr, 903 op/s; 48198/247161 objects degraded (19.501%); 24747/247161 objects misplaced (10.013%); 880 KiB/s, 13 objects/s recovering 2023-09-23T22:56:45.934 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 48198/247161 objects degraded (19.501%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:45.935 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:45 smithi052 ceph-mon[180979]: pgmap v980: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 56 MiB/s wr, 903 op/s; 48198/247161 objects degraded (19.501%); 24747/247161 objects misplaced (10.013%); 880 KiB/s, 13 objects/s recovering 2023-09-23T22:56:45.935 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 48198/247161 objects degraded (19.501%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:45.942 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:56:45.954 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 73 16 84958 84942 72.6946 33.0625 0.00425308 0.0137492 2023-09-23T22:56:46.097 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:46.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:46.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:46.115 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:56:46.212 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:46.271 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:46.814 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:46.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:46.897 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:56:46.918 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:46.954 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 74 16 85662 85646 72.3069 44 0.0116618 0.0138239 2023-09-23T22:56:46.981 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:47.080 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:47.088 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:47.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:47.192 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:47.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:47.425 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:47.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:47.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:47.536 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:47.587 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:56:47.589 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:47.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:47 smithi151 ceph-mon[159210]: pgmap v981: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s; 48423/247836 objects degraded (19.538%); 24643/247836 objects misplaced (9.943%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:56:47.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:47.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:47 smithi052 ceph-mon[187743]: pgmap v981: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s; 48423/247836 objects degraded (19.538%); 24643/247836 objects misplaced (9.943%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:56:47.705 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:47 smithi052 ceph-mon[180979]: pgmap v981: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s; 48423/247836 objects degraded (19.538%); 24643/247836 objects misplaced (9.943%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:56:47.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:47.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:47.801 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:56:47.807 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:47.813 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:56:47.866 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:47.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:47.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:47.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 75 16 86405 86389 71.9597 46.4375 0.00780382 0.0138909 2023-09-23T22:56:48.038 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:48.114 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:48.209 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:48.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:48.316 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:48.348 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:48.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:48 smithi052 ceph-mon[187743]: from='client.46067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:56:48.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:48 smithi052 ceph-mon[180979]: from='client.46067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:56:48.873 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:48.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:48.908 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:48 smithi151 ceph-mon[159210]: from='client.46067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:56:48.956 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 76 16 87169 87153 71.6412 47.75 0.0113113 0.0139514 2023-09-23T22:56:48.977 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:48.997 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":981,"stamp":"2023-09-23T22:56:46.483586+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5413083040,"num_objects":82612,"num_object_clones":0,"num_object_copies":247836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":48423,"num_objects_misplaced":24643,"num_objects_unfound":0,"num_objects_dirty":82612,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":82848,"num_write_kb":5288077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1235,"num_bytes_recovered":79051478,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82182,"ondisk_log_size":82182,"up":221,"acting":240,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":257,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":14860116,"kb_used_data":12151660,"kb_used_omap":121,"kb_used_meta":2708294,"kb_avail":453865644,"statfs":{"total":479975178240,"available":464758419456,"internally_reserved":0,"allocated":12443299840,"data_stored":12438628053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124811,"internal_metadata":2773293173},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":344195072,"num_objects":5252,"num_object_clones":0,"num_object_copies":15756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1813,"num_objects_misplaced":930,"num_objects_unfound":0,"num_objects_dirty":5252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5252,"num_write_kb":336128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":214,"num_bytes_recovered":14024704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003013"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":336,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.244537+0000","last_change":"2023-09-23T22:55:53.200247+0000","last_active":"2023-09-23T22:56:36.244537+0000","last_peered":"2023-09-23T22:56:36.244537+0000","last_clean":"2023-09-23T22:55:51.197224+0000","last_became_active":"2023-09-23T22:55:53.200247+0000","last_became_peered":"2023-09-23T22:55:53.200247+0000","last_unstale":"2023-09-23T22:56:36.244537+0000","last_undegraded":"2023-09-23T22:56:36.244537+0000","last_fullsized":"2023-09-23T22:55:53.196948+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":138,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.246291+0000","last_change":"2023-09-23T22:56:20.211032+0000","last_active":"2023-09-23T22:56:36.246291+0000","last_peered":"2023-09-23T22:56:36.246291+0000","last_clean":"2023-09-23T22:56:36.246291+0000","last_became_active":"2023-09-23T22:56:20.210310+0000","last_became_peered":"2023-09-23T22:56:20.210310+0000","last_unstale":"2023-09-23T22:56:36.246291+0000","last_undegraded":"2023-09-23T22:56:36.246291+0000","last_fullsized":"2023-09-23T22:56:36.246291+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.250888+0000","last_change":"2023-09-23T22:56:21.208064+0000","last_active":"2023-09-23T22:56:36.250888+0000","last_peered":"2023-09-23T22:56:36.250888+0000","last_clean":"2023-09-23T22:56:36.250888+0000","last_became_active":"2023-09-23T22:56:21.207143+0000","last_became_peered":"2023-09-23T22:56:21.207143+0000","last_unstale":"2023-09-23T22:56:36.250888+0000","last_undegraded":"2023-09-23T22:56:36.250888+0000","last_fullsized":"2023-09-23T22:56:36.250888+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":314,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552473+0000","last_change":"2023-09-23T22:51:34.423159+0000","last_active":"2023-09-23T22:56:39.552473+0000","last_peered":"2023-09-23T22:56:39.552473+0000","last_clean":"2023-09-23T22:56:39.552473+0000","last_became_active":"2023-09-23T22:51:34.422861+0000","last_became_peered":"2023-09-23T22:51:34.422861+0000","last_unstale":"2023-09-23T22:56:39.552473+0000","last_undegraded":"2023-09-23T22:56:39.552473+0000","last_fullsized":"2023-09-23T22:56:39.552473+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:37.832383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":324,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.551368+0000","last_change":"2023-09-23T22:55:53.198868+0000","last_active":"2023-09-23T22:56:39.551368+0000","last_peered":"2023-09-23T22:56:39.551368+0000","last_clean":"2023-09-23T22:55:51.195765+0000","last_became_active":"2023-09-23T22:55:53.198868+0000","last_became_peered":"2023-09-23T22:55:53.198868+0000","last_unstale":"2023-09-23T22:56:39.551368+0000","last_undegraded":"2023-09-23T22:56:39.551368+0000","last_fullsized":"2023-09-23T22:55:53.195551+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:50:53.182763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":16,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.551719+0000","last_change":"2023-09-23T22:56:20.358021+0000","last_active":"2023-09-23T22:56:39.551719+0000","last_peered":"2023-09-23T22:56:39.551719+0000","last_clean":"2023-09-23T22:56:39.551719+0000","last_became_active":"2023-09-23T22:56:20.357519+0000","last_became_peered":"2023-09-23T22:56:20.357519+0000","last_unstale":"2023-09-23T22:56:39.551719+0000","last_undegraded":"2023-09-23T22:56:39.551719+0000","last_fullsized":"2023-09-23T22:56:39.551719+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:01.542689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":254,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.551894+0000","last_change":"2023-09-23T22:56:20.362169+0000","last_active":"2023-09-23T22:56:39.551894+0000","last_peered":"2023-09-23T22:56:39.551894+0000","last_clean":"2023-09-23T22:56:39.551894+0000","last_became_active":"2023-09-23T22:56:20.361766+0000","last_became_peered":"2023-09-23T22:56:20.361766+0000","last_unstale":"2023-09-23T22:56:39.551894+0000","last_undegraded":"2023-09-23T22:56:39.551894+0000","last_fullsized":"2023-09-23T22:56:39.551894+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:59.224393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.252510+0000","last_change":"2023-09-23T22:56:21.291040+0000","last_active":"2023-09-23T22:56:36.252510+0000","last_peered":"2023-09-23T22:56:36.252510+0000","last_clean":"2023-09-23T22:56:36.252510+0000","last_became_active":"2023-09-23T22:56:21.290100+0000","last_became_peered":"2023-09-23T22:56:21.290100+0000","last_unstale":"2023-09-23T22:56:36.252510+0000","last_undegraded":"2023-09-23T22:56:36.252510+0000","last_fullsized":"2023-09-23T22:56:36.252510+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":221,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.656017+0000","last_change":"2023-09-23T22:55:53.199541+0000","last_active":"2023-09-23T22:56:39.656017+0000","last_peered":"2023-09-23T22:56:39.656017+0000","last_clean":"2023-09-23T22:44:51.837412+0000","last_became_active":"2023-09-23T22:55:53.199541+0000","last_became_peered":"2023-09-23T22:55:53.199541+0000","last_unstale":"2023-09-23T22:56:39.656017+0000","last_undegraded":"2023-09-23T22:56:39.656017+0000","last_fullsized":"2023-09-23T22:55:53.197701+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.551548+0000","last_change":"2023-09-23T22:56:21.215512+0000","last_active":"2023-09-23T22:56:39.551548+0000","last_peered":"2023-09-23T22:56:39.551548+0000","last_clean":"2023-09-23T22:56:39.551548+0000","last_became_active":"2023-09-23T22:56:21.215127+0000","last_became_peered":"2023-09-23T22:56:21.215127+0000","last_unstale":"2023-09-23T22:56:39.551548+0000","last_undegraded":"2023-09-23T22:56:39.551548+0000","last_fullsized":"2023-09-23T22:56:39.551548+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:33.348767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":101,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.245911+0000","last_change":"2023-09-23T22:56:21.227390+0000","last_active":"2023-09-23T22:56:36.245911+0000","last_peered":"2023-09-23T22:56:36.245911+0000","last_clean":"2023-09-23T22:56:36.245911+0000","last_became_active":"2023-09-23T22:56:21.208490+0000","last_became_peered":"2023-09-23T22:56:21.208490+0000","last_unstale":"2023-09-23T22:56:36.245911+0000","last_undegraded":"2023-09-23T22:56:36.245911+0000","last_fullsized":"2023-09-23T22:56:36.245911+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":17,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.552171+0000","last_change":"2023-09-23T22:56:21.209792+0000","last_active":"2023-09-23T22:56:39.552171+0000","last_peered":"2023-09-23T22:56:39.552171+0000","last_clean":"2023-09-23T22:56:39.552171+0000","last_became_active":"2023-09-23T22:56:21.209180+0000","last_became_peered":"2023-09-23T22:56:21.209180+0000","last_unstale":"2023-09-23T22:56:39.552171+0000","last_undegraded":"2023-09-23T22:56:39.552171+0000","last_fullsized":"2023-09-23T22:56:39.552171+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:45.886895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":378,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.250018+0000","last_change":"2023-09-23T22:56:21.470686+0000","last_active":"2023-09-23T22:56:36.250018+0000","last_peered":"2023-09-23T22:56:36.250018+0000","last_clean":"2023-09-23T22:56:36.250018+0000","last_became_active":"2023-09-23T22:56:21.261869+0000","last_became_peered":"2023-09-23T22:56:21.261869+0000","last_unstale":"2023-09-23T22:56:36.250018+0000","last_undegraded":"2023-09-23T22:56:36.250018+0000","last_fullsized":"2023-09-23T22:56:36.250018+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":307,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.247159+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:56:36.247159+0000","last_peered":"2023-09-23T22:56:36.247159+0000","last_clean":"2023-09-23T22:56:36.247159+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:56:36.247159+0000","last_undegraded":"2023-09-23T22:56:36.247159+0000","last_fullsized":"2023-09-23T22:56:36.247159+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":213,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.247257+0000","last_change":"2023-09-23T22:56:31.306047+0000","last_active":"2023-09-23T22:56:36.247257+0000","last_peered":"2023-09-23T22:56:36.247257+0000","last_clean":"2023-09-23T22:56:36.247257+0000","last_became_active":"2023-09-23T22:56:28.565424+0000","last_became_peered":"2023-09-23T22:56:28.565424+0000","last_unstale":"2023-09-23T22:56:36.247257+0000","last_undegraded":"2023-09-23T22:56:36.247257+0000","last_fullsized":"2023-09-23T22:56:36.247257+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":224,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.248601+0000","last_change":"2023-09-23T22:55:53.198779+0000","last_active":"2023-09-23T22:56:36.248601+0000","last_peered":"2023-09-23T22:56:36.248601+0000","last_clean":"2023-09-23T22:44:51.858616+0000","last_became_active":"2023-09-23T22:55:53.198779+0000","last_became_peered":"2023-09-23T22:55:53.198779+0000","last_unstale":"2023-09-23T22:56:36.248601+0000","last_undegraded":"2023-09-23T22:56:36.248601+0000","last_fullsized":"2023-09-23T22:55:53.196667+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:46.378185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.552542+0000","last_change":"2023-09-23T22:55:53.200228+0000","last_active":"2023-09-23T22:56:39.552542+0000","last_peered":"2023-09-23T22:56:39.552542+0000","last_clean":"2023-09-23T22:55:51.196266+0000","last_became_active":"2023-09-23T22:55:53.200228+0000","last_became_peered":"2023-09-23T22:55:53.200228+0000","last_unstale":"2023-09-23T22:56:39.552542+0000","last_undegraded":"2023-09-23T22:56:39.552542+0000","last_fullsized":"2023-09-23T22:55:53.195451+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:23.575604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":202,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.553039+0000","last_change":"2023-09-23T22:56:20.364317+0000","last_active":"2023-09-23T22:56:39.553039+0000","last_peered":"2023-09-23T22:56:39.553039+0000","last_clean":"2023-09-23T22:56:39.553039+0000","last_became_active":"2023-09-23T22:56:20.363957+0000","last_became_peered":"2023-09-23T22:56:20.363957+0000","last_unstale":"2023-09-23T22:56:39.553039+0000","last_undegraded":"2023-09-23T22:56:39.553039+0000","last_fullsized":"2023-09-23T22:56:39.553039+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:34.748723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":193,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.251085+0000","last_change":"2023-09-23T22:56:21.209748+0000","last_active":"2023-09-23T22:56:36.251085+0000","last_peered":"2023-09-23T22:56:36.251085+0000","last_clean":"2023-09-23T22:56:36.251085+0000","last_became_active":"2023-09-23T22:56:21.206594+0000","last_became_peered":"2023-09-23T22:56:21.206594+0000","last_unstale":"2023-09-23T22:56:36.251085+0000","last_undegraded":"2023-09-23T22:56:36.251085+0000","last_fullsized":"2023-09-23T22:56:36.251085+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":318,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.552569+0000","last_change":"2023-09-23T22:56:21.215509+0000","last_active":"2023-09-23T22:56:39.552569+0000","last_peered":"2023-09-23T22:56:39.552569+0000","last_clean":"2023-09-23T22:56:39.552569+0000","last_became_active":"2023-09-23T22:56:21.215057+0000","last_became_peered":"2023-09-23T22:56:21.215057+0000","last_unstale":"2023-09-23T22:56:39.552569+0000","last_undegraded":"2023-09-23T22:56:39.552569+0000","last_fullsized":"2023-09-23T22:56:39.552569+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:10.120897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":29,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.551600+0000","last_change":"2023-09-23T22:56:21.213637+0000","last_active":"2023-09-23T22:56:39.551600+0000","last_peered":"2023-09-23T22:56:39.551600+0000","last_clean":"2023-09-23T22:56:39.551600+0000","last_became_active":"2023-09-23T22:56:21.212846+0000","last_became_peered":"2023-09-23T22:56:21.212846+0000","last_unstale":"2023-09-23T22:56:39.551600+0000","last_undegraded":"2023-09-23T22:56:39.551600+0000","last_fullsized":"2023-09-23T22:56:39.551600+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:46.186837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.551578+0000","last_change":"2023-09-23T22:56:21.209659+0000","last_active":"2023-09-23T22:56:39.551578+0000","last_peered":"2023-09-23T22:56:39.551578+0000","last_clean":"2023-09-23T22:56:39.551578+0000","last_became_active":"2023-09-23T22:56:21.209233+0000","last_became_peered":"2023-09-23T22:56:21.209233+0000","last_unstale":"2023-09-23T22:56:39.551578+0000","last_undegraded":"2023-09-23T22:56:39.551578+0000","last_fullsized":"2023-09-23T22:56:39.551578+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:14.844115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.6","version":"334'5223","reported_seq":5249,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T22:56:43.548678+0000","last_change":"2023-09-23T22:55:23.669240+0000","last_active":"2023-09-23T22:56:43.548678+0000","last_peered":"2023-09-23T22:56:43.548678+0000","last_clean":"2023-09-23T22:56:43.548678+0000","last_became_active":"2023-09-23T22:55:23.668982+0000","last_became_peered":"2023-09-23T22:55:23.668982+0000","last_unstale":"2023-09-23T22:56:43.548678+0000","last_undegraded":"2023-09-23T22:56:43.548678+0000","last_fullsized":"2023-09-23T22:56:43.548678+0000","mapping_epoch":317,"log_start":"321'100","ondisk_log_start":"321'100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5123,"log_dups_size":0,"ondisk_log_size":5123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:40.611676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342294528,"num_objects":5223,"num_object_clones":0,"num_object_copies":15669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5223,"num_write_kb":334272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":202,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.253814+0000","last_change":"2023-09-23T22:46:14.249508+0000","last_active":"2023-09-23T22:56:36.253814+0000","last_peered":"2023-09-23T22:56:36.253814+0000","last_clean":"2023-09-23T22:56:36.253814+0000","last_became_active":"2023-09-23T22:46:14.249097+0000","last_became_peered":"2023-09-23T22:46:14.249097+0000","last_unstale":"2023-09-23T22:56:36.253814+0000","last_undegraded":"2023-09-23T22:56:36.253814+0000","last_fullsized":"2023-09-23T22:56:36.253814+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":184,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.252759+0000","last_change":"2023-09-23T22:56:20.210137+0000","last_active":"2023-09-23T22:56:36.252759+0000","last_peered":"2023-09-23T22:56:36.252759+0000","last_clean":"2023-09-23T22:56:36.252759+0000","last_became_active":"2023-09-23T22:56:20.209865+0000","last_became_peered":"2023-09-23T22:56:20.209865+0000","last_unstale":"2023-09-23T22:56:36.252759+0000","last_undegraded":"2023-09-23T22:56:36.252759+0000","last_fullsized":"2023-09-23T22:56:36.252759+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.7","version":"334'5234","reported_seq":6365,"reported_epoch":334,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:46.045732+0000","last_change":"2023-09-23T22:56:31.315968+0000","last_active":"2023-09-23T22:56:46.045732+0000","last_peered":"2023-09-23T22:56:46.045732+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:56:21.421308+0000","last_became_peered":"2023-09-23T22:56:21.421308+0000","last_unstale":"2023-09-23T22:56:46.045732+0000","last_undegraded":"2023-09-23T22:56:21.233163+0000","last_fullsized":"2023-09-23T22:56:21.205823+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343015424,"num_objects":5234,"num_object_clones":0,"num_object_copies":15702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5234,"num_objects_misplaced":3966,"num_objects_unfound":0,"num_objects_dirty":5234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5234,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":35848192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1268}],"blocked_by":[],"up_primary":4,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":212,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.249561+0000","last_change":"2023-09-23T22:46:14.249435+0000","last_active":"2023-09-23T22:56:36.249561+0000","last_peered":"2023-09-23T22:56:36.249561+0000","last_clean":"2023-09-23T22:56:36.249561+0000","last_became_active":"2023-09-23T22:46:14.248837+0000","last_became_peered":"2023-09-23T22:46:14.248837+0000","last_unstale":"2023-09-23T22:56:36.249561+0000","last_undegraded":"2023-09-23T22:56:36.249561+0000","last_fullsized":"2023-09-23T22:56:36.249561+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":331,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.251006+0000","last_change":"2023-09-23T22:56:20.364338+0000","last_active":"2023-09-23T22:56:36.251006+0000","last_peered":"2023-09-23T22:56:36.251006+0000","last_clean":"2023-09-23T22:56:36.251006+0000","last_became_active":"2023-09-23T22:56:20.363441+0000","last_became_peered":"2023-09-23T22:56:20.363441+0000","last_unstale":"2023-09-23T22:56:36.251006+0000","last_undegraded":"2023-09-23T22:56:36.251006+0000","last_fullsized":"2023-09-23T22:56:36.251006+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"334'5184","reported_seq":5210,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T22:56:43.539701+0000","last_change":"2023-09-23T22:55:24.310230+0000","last_active":"2023-09-23T22:56:43.539701+0000","last_peered":"2023-09-23T22:56:43.539701+0000","last_clean":"2023-09-23T22:56:43.539701+0000","last_became_active":"2023-09-23T22:55:24.309921+0000","last_became_peered":"2023-09-23T22:55:24.309921+0000","last_unstale":"2023-09-23T22:56:43.539701+0000","last_undegraded":"2023-09-23T22:56:43.539701+0000","last_fullsized":"2023-09-23T22:56:43.539701+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5184,"log_dups_size":0,"ondisk_log_size":5184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:46.749922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339738624,"num_objects":5184,"num_object_clones":0,"num_object_copies":15552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5184,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":101,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.245291+0000","last_change":"2023-09-23T22:56:20.418726+0000","last_active":"2023-09-23T22:56:36.245291+0000","last_peered":"2023-09-23T22:56:36.245291+0000","last_clean":"2023-09-23T22:56:36.245291+0000","last_became_active":"2023-09-23T22:56:20.363404+0000","last_became_peered":"2023-09-23T22:56:20.363404+0000","last_unstale":"2023-09-23T22:56:36.245291+0000","last_undegraded":"2023-09-23T22:56:36.245291+0000","last_fullsized":"2023-09-23T22:56:36.245291+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":16,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552432+0000","last_change":"2023-09-23T22:56:20.363294+0000","last_active":"2023-09-23T22:56:39.552432+0000","last_peered":"2023-09-23T22:56:39.552432+0000","last_clean":"2023-09-23T22:56:39.552432+0000","last_became_active":"2023-09-23T22:56:20.362304+0000","last_became_peered":"2023-09-23T22:56:20.362304+0000","last_unstale":"2023-09-23T22:56:39.552432+0000","last_undegraded":"2023-09-23T22:56:39.552432+0000","last_fullsized":"2023-09-23T22:56:39.552432+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.994131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.5","version":"334'5213","reported_seq":5244,"reported_epoch":334,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:43.531758+0000","last_change":"2023-09-23T22:55:53.200485+0000","last_active":"2023-09-23T22:56:43.531758+0000","last_peered":"2023-09-23T22:56:43.531758+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:55:53.200485+0000","last_became_peered":"2023-09-23T22:55:53.200485+0000","last_unstale":"2023-09-23T22:56:43.531758+0000","last_undegraded":"2023-09-23T22:55:53.196004+0000","last_fullsized":"2023-09-23T22:55:53.195781+0000","mapping_epoch":324,"log_start":"321'100","ondisk_log_start":"321'100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5113,"log_dups_size":0,"ondisk_log_size":5113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:39.647676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341639168,"num_objects":5213,"num_object_clones":0,"num_object_copies":15639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5213,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5213,"num_write_kb":333632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":5213}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.551688+0000","last_change":"2023-09-23T22:56:20.358092+0000","last_active":"2023-09-23T22:56:39.551688+0000","last_peered":"2023-09-23T22:56:39.551688+0000","last_clean":"2023-09-23T22:56:39.551688+0000","last_became_active":"2023-09-23T22:56:20.357773+0000","last_became_peered":"2023-09-23T22:56:20.357773+0000","last_unstale":"2023-09-23T22:56:39.551688+0000","last_undegraded":"2023-09-23T22:56:39.551688+0000","last_fullsized":"2023-09-23T22:56:39.551688+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:14.698016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":229,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552431+0000","last_change":"2023-09-23T22:46:14.246779+0000","last_active":"2023-09-23T22:56:39.552431+0000","last_peered":"2023-09-23T22:56:39.552431+0000","last_clean":"2023-09-23T22:56:39.552431+0000","last_became_active":"2023-09-23T22:46:14.246478+0000","last_became_peered":"2023-09-23T22:46:14.246478+0000","last_unstale":"2023-09-23T22:56:39.552431+0000","last_undegraded":"2023-09-23T22:56:39.552431+0000","last_fullsized":"2023-09-23T22:56:39.552431+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:10.555567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"333'5091","reported_seq":5115,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:41.708011+0000","last_change":"2023-09-23T22:55:24.311230+0000","last_active":"2023-09-23T22:56:41.708011+0000","last_peered":"2023-09-23T22:56:41.708011+0000","last_clean":"2023-09-23T22:56:41.708011+0000","last_became_active":"2023-09-23T22:55:24.309796+0000","last_became_peered":"2023-09-23T22:55:24.309796+0000","last_unstale":"2023-09-23T22:56:41.708011+0000","last_undegraded":"2023-09-23T22:56:41.708011+0000","last_fullsized":"2023-09-23T22:56:41.708011+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333643776,"num_objects":5091,"num_object_clones":0,"num_object_copies":15273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5091,"num_write_kb":325824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":101,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552354+0000","last_change":"2023-09-23T22:51:34.442044+0000","last_active":"2023-09-23T22:56:39.552354+0000","last_peered":"2023-09-23T22:56:39.552354+0000","last_clean":"2023-09-23T22:56:39.552354+0000","last_became_active":"2023-09-23T22:51:34.430295+0000","last_became_peered":"2023-09-23T22:51:34.430295+0000","last_unstale":"2023-09-23T22:56:39.552354+0000","last_undegraded":"2023-09-23T22:56:39.552354+0000","last_fullsized":"2023-09-23T22:56:39.552354+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:42.872882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":316,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.249884+0000","last_change":"2023-09-23T22:56:21.262144+0000","last_active":"2023-09-23T22:56:36.249884+0000","last_peered":"2023-09-23T22:56:36.249884+0000","last_clean":"2023-09-23T22:56:36.249884+0000","last_became_active":"2023-09-23T22:56:21.261482+0000","last_became_peered":"2023-09-23T22:56:21.261482+0000","last_unstale":"2023-09-23T22:56:36.249884+0000","last_undegraded":"2023-09-23T22:56:36.249884+0000","last_fullsized":"2023-09-23T22:56:36.249884+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"333'5147","reported_seq":5176,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:41.720994+0000","last_change":"2023-09-23T22:55:53.199646+0000","last_active":"2023-09-23T22:56:41.720994+0000","last_peered":"2023-09-23T22:56:41.720994+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:55:53.199646+0000","last_became_peered":"2023-09-23T22:55:53.199646+0000","last_unstale":"2023-09-23T22:56:41.720994+0000","last_undegraded":"2023-09-23T22:55:53.196179+0000","last_fullsized":"2023-09-23T22:55:53.196049+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5147,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":5147}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.244844+0000","last_change":"2023-09-23T22:56:20.208394+0000","last_active":"2023-09-23T22:56:36.244844+0000","last_peered":"2023-09-23T22:56:36.244844+0000","last_clean":"2023-09-23T22:56:36.244844+0000","last_became_active":"2023-09-23T22:56:20.208065+0000","last_became_peered":"2023-09-23T22:56:20.208065+0000","last_unstale":"2023-09-23T22:56:36.244844+0000","last_undegraded":"2023-09-23T22:56:36.244844+0000","last_fullsized":"2023-09-23T22:56:36.244844+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":334,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.245658+0000","last_change":"2023-09-23T22:55:53.198961+0000","last_active":"2023-09-23T22:56:36.245658+0000","last_peered":"2023-09-23T22:56:36.245658+0000","last_clean":"2023-09-23T22:55:51.198609+0000","last_became_active":"2023-09-23T22:55:53.198961+0000","last_became_peered":"2023-09-23T22:55:53.198961+0000","last_unstale":"2023-09-23T22:56:36.245658+0000","last_undegraded":"2023-09-23T22:56:36.245658+0000","last_fullsized":"2023-09-23T22:55:53.196914+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"333'5086","reported_seq":6434,"reported_epoch":333,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:41.734471+0000","last_change":"2023-09-23T22:56:28.569171+0000","last_active":"2023-09-23T22:56:41.734471+0000","last_peered":"2023-09-23T22:56:41.734471+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:56:21.371406+0000","last_became_peered":"2023-09-23T22:56:21.371406+0000","last_unstale":"2023-09-23T22:56:41.734471+0000","last_undegraded":"2023-09-23T22:56:28.567437+0000","last_fullsized":"2023-09-23T22:56:21.199197+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3939,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5086,"num_write_kb":325504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":657,"num_bytes_recovered":43057152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4","objects":1147},{"shards":"1,4,6","objects":3939}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":347,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.250701+0000","last_change":"2023-09-23T22:55:53.202300+0000","last_active":"2023-09-23T22:56:36.250701+0000","last_peered":"2023-09-23T22:56:36.250701+0000","last_clean":"2023-09-23T22:55:51.197959+0000","last_became_active":"2023-09-23T22:55:53.202300+0000","last_became_peered":"2023-09-23T22:55:53.202300+0000","last_unstale":"2023-09-23T22:56:36.250701+0000","last_undegraded":"2023-09-23T22:56:36.250701+0000","last_fullsized":"2023-09-23T22:55:53.197851+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":325,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.249667+0000","last_change":"2023-09-23T22:56:20.210960+0000","last_active":"2023-09-23T22:56:36.249667+0000","last_peered":"2023-09-23T22:56:36.249667+0000","last_clean":"2023-09-23T22:56:36.249667+0000","last_became_active":"2023-09-23T22:56:20.210745+0000","last_became_peered":"2023-09-23T22:56:20.210745+0000","last_unstale":"2023-09-23T22:56:36.249667+0000","last_undegraded":"2023-09-23T22:56:36.249667+0000","last_fullsized":"2023-09-23T22:56:36.249667+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"333'4999","reported_seq":5023,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:41.715678+0000","last_change":"2023-09-23T22:55:24.310261+0000","last_active":"2023-09-23T22:56:41.715678+0000","last_peered":"2023-09-23T22:56:41.715678+0000","last_clean":"2023-09-23T22:56:41.715678+0000","last_became_active":"2023-09-23T22:55:24.309692+0000","last_became_peered":"2023-09-23T22:55:24.309692+0000","last_unstale":"2023-09-23T22:56:41.715678+0000","last_undegraded":"2023-09-23T22:56:41.715678+0000","last_fullsized":"2023-09-23T22:56:41.715678+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327614464,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4999,"num_write_kb":319936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":261,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.252670+0000","last_change":"2023-09-23T22:51:34.415484+0000","last_active":"2023-09-23T22:56:36.252670+0000","last_peered":"2023-09-23T22:56:36.252670+0000","last_clean":"2023-09-23T22:56:36.252670+0000","last_became_active":"2023-09-23T22:51:34.415118+0000","last_became_peered":"2023-09-23T22:51:34.415118+0000","last_unstale":"2023-09-23T22:56:36.252670+0000","last_undegraded":"2023-09-23T22:56:36.252670+0000","last_fullsized":"2023-09-23T22:56:36.252670+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.254057+0000","last_change":"2023-09-23T22:55:53.200618+0000","last_active":"2023-09-23T22:56:36.254057+0000","last_peered":"2023-09-23T22:56:36.254057+0000","last_clean":"2023-09-23T22:51:24.617408+0000","last_became_active":"2023-09-23T22:55:53.200618+0000","last_became_peered":"2023-09-23T22:55:53.200618+0000","last_unstale":"2023-09-23T22:56:36.254057+0000","last_undegraded":"2023-09-23T22:56:36.254057+0000","last_fullsized":"2023-09-23T22:55:53.196370+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"333'5091","reported_seq":5122,"reported_epoch":333,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:41.700632+0000","last_change":"2023-09-23T22:56:21.423950+0000","last_active":"2023-09-23T22:56:41.700632+0000","last_peered":"2023-09-23T22:56:41.700632+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:56:21.407987+0000","last_became_peered":"2023-09-23T22:56:21.407987+0000","last_unstale":"2023-09-23T22:56:41.700632+0000","last_undegraded":"2023-09-23T22:56:21.255394+0000","last_fullsized":"2023-09-23T22:56:21.199911+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333643776,"num_objects":5091,"num_object_clones":0,"num_object_copies":15273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4567,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5091,"num_write_kb":325824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4","objects":524},{"shards":"0,4,6","objects":4567}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":379,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.552177+0000","last_change":"2023-09-23T22:55:53.203502+0000","last_active":"2023-09-23T22:56:39.552177+0000","last_peered":"2023-09-23T22:56:39.552177+0000","last_clean":"2023-09-23T22:55:51.196240+0000","last_became_active":"2023-09-23T22:55:53.203502+0000","last_became_peered":"2023-09-23T22:55:53.203502+0000","last_unstale":"2023-09-23T22:56:39.552177+0000","last_undegraded":"2023-09-23T22:56:39.552177+0000","last_fullsized":"2023-09-23T22:55:53.196522+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:59.555018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":224,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.653909+0000","last_change":"2023-09-23T22:56:21.214721+0000","last_active":"2023-09-23T22:56:39.653909+0000","last_peered":"2023-09-23T22:56:39.653909+0000","last_clean":"2023-09-23T22:56:39.653909+0000","last_became_active":"2023-09-23T22:56:21.211329+0000","last_became_peered":"2023-09-23T22:56:21.211329+0000","last_unstale":"2023-09-23T22:56:39.653909+0000","last_undegraded":"2023-09-23T22:56:39.653909+0000","last_fullsized":"2023-09-23T22:56:39.653909+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,6],"avail_no_missing":["5","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":164,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.244436+0000","last_change":"2023-09-23T22:46:44.023850+0000","last_active":"2023-09-23T22:56:36.244436+0000","last_peered":"2023-09-23T22:56:36.244436+0000","last_clean":"2023-09-23T22:56:36.244436+0000","last_became_active":"2023-09-23T22:46:44.023443+0000","last_became_peered":"2023-09-23T22:46:44.023443+0000","last_unstale":"2023-09-23T22:56:36.244436+0000","last_undegraded":"2023-09-23T22:56:36.244436+0000","last_fullsized":"2023-09-23T22:56:36.244436+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.a","version":"334'5212","reported_seq":5250,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:43.535334+0000","last_change":"2023-09-23T22:56:21.236419+0000","last_active":"2023-09-23T22:56:43.535334+0000","last_peered":"2023-09-23T22:56:43.535334+0000","last_clean":"2023-09-23T22:56:43.535334+0000","last_became_active":"2023-09-23T22:56:21.215225+0000","last_became_peered":"2023-09-23T22:56:21.215225+0000","last_unstale":"2023-09-23T22:56:43.535334+0000","last_undegraded":"2023-09-23T22:56:43.535334+0000","last_fullsized":"2023-09-23T22:56:43.535334+0000","mapping_epoch":329,"log_start":"321'100","ondisk_log_start":"321'100","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5112,"log_dups_size":0,"ondisk_log_size":5112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:15.354698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341573632,"num_objects":5212,"num_object_clones":0,"num_object_copies":15636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5212,"num_objects_unfound":0,"num_objects_dirty":5212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5212,"num_write_kb":333568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":5212}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552808+0000","last_change":"2023-09-23T22:56:20.363703+0000","last_active":"2023-09-23T22:56:39.552808+0000","last_peered":"2023-09-23T22:56:39.552808+0000","last_clean":"2023-09-23T22:56:39.552808+0000","last_became_active":"2023-09-23T22:56:20.363272+0000","last_became_peered":"2023-09-23T22:56:20.363272+0000","last_unstale":"2023-09-23T22:56:39.552808+0000","last_undegraded":"2023-09-23T22:56:39.552808+0000","last_fullsized":"2023-09-23T22:56:39.552808+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:06.482210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":338,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.247298+0000","last_change":"2023-09-23T22:56:20.363589+0000","last_active":"2023-09-23T22:56:36.247298+0000","last_peered":"2023-09-23T22:56:36.247298+0000","last_clean":"2023-09-23T22:56:36.247298+0000","last_became_active":"2023-09-23T22:56:20.363246+0000","last_became_peered":"2023-09-23T22:56:20.363246+0000","last_unstale":"2023-09-23T22:56:36.247298+0000","last_undegraded":"2023-09-23T22:56:36.247298+0000","last_fullsized":"2023-09-23T22:56:36.247298+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":16,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.247787+0000","last_change":"2023-09-23T22:56:21.211430+0000","last_active":"2023-09-23T22:56:36.247787+0000","last_peered":"2023-09-23T22:56:36.247787+0000","last_clean":"2023-09-23T22:56:36.247787+0000","last_became_active":"2023-09-23T22:56:21.211069+0000","last_became_peered":"2023-09-23T22:56:21.211069+0000","last_unstale":"2023-09-23T22:56:36.247787+0000","last_undegraded":"2023-09-23T22:56:36.247787+0000","last_fullsized":"2023-09-23T22:56:36.247787+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:11:40.687951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"333'5101","reported_seq":5131,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:41.660921+0000","last_change":"2023-09-23T22:55:53.199706+0000","last_active":"2023-09-23T22:56:41.660921+0000","last_peered":"2023-09-23T22:56:41.660921+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:55:53.199706+0000","last_became_peered":"2023-09-23T22:55:53.199706+0000","last_unstale":"2023-09-23T22:56:41.660921+0000","last_undegraded":"2023-09-23T22:55:53.196616+0000","last_fullsized":"2023-09-23T22:55:53.196454+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5101,"log_dups_size":0,"ondisk_log_size":5101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299136,"num_objects":5101,"num_object_clones":0,"num_object_copies":15303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5101,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5101,"num_write_kb":326464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":5101}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":280,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.551797+0000","last_change":"2023-09-23T22:55:53.199820+0000","last_active":"2023-09-23T22:56:39.551797+0000","last_peered":"2023-09-23T22:56:39.551797+0000","last_clean":"2023-09-23T22:51:24.616651+0000","last_became_active":"2023-09-23T22:55:53.199820+0000","last_became_peered":"2023-09-23T22:55:53.199820+0000","last_unstale":"2023-09-23T22:56:39.551797+0000","last_undegraded":"2023-09-23T22:56:39.551797+0000","last_fullsized":"2023-09-23T22:55:53.195330+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:52.779172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":17,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.551244+0000","last_change":"2023-09-23T22:56:21.212434+0000","last_active":"2023-09-23T22:56:39.551244+0000","last_peered":"2023-09-23T22:56:39.551244+0000","last_clean":"2023-09-23T22:56:39.551244+0000","last_became_active":"2023-09-23T22:56:21.209765+0000","last_became_peered":"2023-09-23T22:56:21.209765+0000","last_unstale":"2023-09-23T22:56:39.551244+0000","last_undegraded":"2023-09-23T22:56:39.551244+0000","last_fullsized":"2023-09-23T22:56:39.551244+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:35.292030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":342,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.551800+0000","last_change":"2023-09-23T22:46:44.030841+0000","last_active":"2023-09-23T22:56:39.551800+0000","last_peered":"2023-09-23T22:56:39.551800+0000","last_clean":"2023-09-23T22:56:39.551800+0000","last_became_active":"2023-09-23T22:46:44.030543+0000","last_became_peered":"2023-09-23T22:46:44.030543+0000","last_unstale":"2023-09-23T22:56:39.551800+0000","last_undegraded":"2023-09-23T22:56:39.551800+0000","last_fullsized":"2023-09-23T22:56:39.551800+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:02.395742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"334'5299","reported_seq":5333,"reported_epoch":334,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:43.537352+0000","last_change":"2023-09-23T22:56:31.158156+0000","last_active":"2023-09-23T22:56:43.537352+0000","last_peered":"2023-09-23T22:56:43.537352+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:56:31.124755+0000","last_became_peered":"2023-09-23T22:56:31.124755+0000","last_unstale":"2023-09-23T22:56:43.537352+0000","last_undegraded":"2023-09-23T22:56:28.518043+0000","last_fullsized":"2023-09-23T22:56:21.199804+0000","mapping_epoch":329,"log_start":"321'200","ondisk_log_start":"321'200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5099,"log_dups_size":0,"ondisk_log_size":5099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:01.685913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347275264,"num_objects":5299,"num_object_clones":0,"num_object_copies":15897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4679,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5299,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1","objects":620},{"shards":"0,1,6","objects":4679}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.552530+0000","last_change":"2023-09-23T22:56:31.114543+0000","last_active":"2023-09-23T22:56:39.552530+0000","last_peered":"2023-09-23T22:56:39.552530+0000","last_clean":"2023-09-23T22:56:39.552530+0000","last_became_active":"2023-09-23T22:56:28.564908+0000","last_became_peered":"2023-09-23T22:56:28.564908+0000","last_unstale":"2023-09-23T22:56:39.552530+0000","last_undegraded":"2023-09-23T22:56:39.552530+0000","last_fullsized":"2023-09-23T22:56:39.552530+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:28.175175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":434,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.551411+0000","last_change":"2023-09-23T22:56:21.213296+0000","last_active":"2023-09-23T22:56:39.551411+0000","last_peered":"2023-09-23T22:56:39.551411+0000","last_clean":"2023-09-23T22:56:39.551411+0000","last_became_active":"2023-09-23T22:56:21.209805+0000","last_became_peered":"2023-09-23T22:56:21.209805+0000","last_unstale":"2023-09-23T22:56:39.551411+0000","last_undegraded":"2023-09-23T22:56:39.551411+0000","last_fullsized":"2023-09-23T22:56:39.551411+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:15.073184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":32,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.244366+0000","last_change":"2023-09-23T22:56:20.210570+0000","last_active":"2023-09-23T22:56:36.244366+0000","last_peered":"2023-09-23T22:56:36.244366+0000","last_clean":"2023-09-23T22:56:36.244366+0000","last_became_active":"2023-09-23T22:56:20.210257+0000","last_became_peered":"2023-09-23T22:56:20.210257+0000","last_unstale":"2023-09-23T22:56:36.244366+0000","last_undegraded":"2023-09-23T22:56:36.244366+0000","last_fullsized":"2023-09-23T22:56:36.244366+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":219,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.246061+0000","last_change":"2023-09-23T22:56:20.363700+0000","last_active":"2023-09-23T22:56:36.246061+0000","last_peered":"2023-09-23T22:56:36.246061+0000","last_clean":"2023-09-23T22:56:36.246061+0000","last_became_active":"2023-09-23T22:56:20.363347+0000","last_became_peered":"2023-09-23T22:56:20.363347+0000","last_unstale":"2023-09-23T22:56:36.246061+0000","last_undegraded":"2023-09-23T22:56:36.246061+0000","last_fullsized":"2023-09-23T22:56:36.246061+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"334'5257","reported_seq":5287,"reported_epoch":334,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:43.567394+0000","last_change":"2023-09-23T22:55:53.200491+0000","last_active":"2023-09-23T22:56:43.567394+0000","last_peered":"2023-09-23T22:56:43.567394+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:55:53.200491+0000","last_became_peered":"2023-09-23T22:55:53.200491+0000","last_unstale":"2023-09-23T22:56:43.567394+0000","last_undegraded":"2023-09-23T22:55:53.195494+0000","last_fullsized":"2023-09-23T22:55:53.195356+0000","mapping_epoch":324,"log_start":"321'100","ondisk_log_start":"321'100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5157,"log_dups_size":0,"ondisk_log_size":5157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:47.108767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344522752,"num_objects":5257,"num_object_clones":0,"num_object_copies":15771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5257,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5257,"num_write_kb":336448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":5257}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":311,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.551496+0000","last_change":"2023-09-23T22:51:34.423279+0000","last_active":"2023-09-23T22:56:39.551496+0000","last_peered":"2023-09-23T22:56:39.551496+0000","last_clean":"2023-09-23T22:56:39.551496+0000","last_became_active":"2023-09-23T22:51:34.422897+0000","last_became_peered":"2023-09-23T22:51:34.422897+0000","last_unstale":"2023-09-23T22:56:39.551496+0000","last_undegraded":"2023-09-23T22:56:39.551496+0000","last_fullsized":"2023-09-23T22:56:39.551496+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:19.989772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":283,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.551697+0000","last_change":"2023-09-23T22:51:34.418376+0000","last_active":"2023-09-23T22:56:39.551697+0000","last_peered":"2023-09-23T22:56:39.551697+0000","last_clean":"2023-09-23T22:56:39.551697+0000","last_became_active":"2023-09-23T22:51:34.417928+0000","last_became_peered":"2023-09-23T22:51:34.417928+0000","last_unstale":"2023-09-23T22:56:39.551697+0000","last_undegraded":"2023-09-23T22:56:39.551697+0000","last_fullsized":"2023-09-23T22:56:39.551697+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:39.248489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.552030+0000","last_change":"2023-09-23T22:55:53.198950+0000","last_active":"2023-09-23T22:56:39.552030+0000","last_peered":"2023-09-23T22:56:39.552030+0000","last_clean":"2023-09-23T22:55:51.196259+0000","last_became_active":"2023-09-23T22:55:53.198950+0000","last_became_peered":"2023-09-23T22:55:53.198950+0000","last_unstale":"2023-09-23T22:56:39.552030+0000","last_undegraded":"2023-09-23T22:56:39.552030+0000","last_fullsized":"2023-09-23T22:55:53.195857+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:23.246246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"333'5131","reported_seq":5163,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:41.705749+0000","last_change":"2023-09-23T22:56:21.229053+0000","last_active":"2023-09-23T22:56:41.705749+0000","last_peered":"2023-09-23T22:56:41.705749+0000","last_clean":"2023-09-23T22:56:41.705749+0000","last_became_active":"2023-09-23T22:56:21.207374+0000","last_became_peered":"2023-09-23T22:56:21.207374+0000","last_unstale":"2023-09-23T22:56:41.705749+0000","last_undegraded":"2023-09-23T22:56:41.705749+0000","last_fullsized":"2023-09-23T22:56:41.705749+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5131,"log_dups_size":0,"ondisk_log_size":5131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336265216,"num_objects":5131,"num_object_clones":0,"num_object_copies":15393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5131,"num_objects_unfound":0,"num_objects_dirty":5131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5131,"num_write_kb":328384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":5131}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":266,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.553173+0000","last_change":"2023-09-23T22:46:14.252190+0000","last_active":"2023-09-23T22:56:39.553173+0000","last_peered":"2023-09-23T22:56:39.553173+0000","last_clean":"2023-09-23T22:56:39.553173+0000","last_became_active":"2023-09-23T22:46:14.251911+0000","last_became_peered":"2023-09-23T22:46:14.251911+0000","last_unstale":"2023-09-23T22:56:39.553173+0000","last_undegraded":"2023-09-23T22:56:39.553173+0000","last_fullsized":"2023-09-23T22:56:39.553173+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:20.950923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":213,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.248746+0000","last_change":"2023-09-23T22:55:53.199112+0000","last_active":"2023-09-23T22:56:36.248746+0000","last_peered":"2023-09-23T22:56:36.248746+0000","last_clean":"2023-09-23T22:39:27.713294+0000","last_became_active":"2023-09-23T22:55:53.199112+0000","last_became_peered":"2023-09-23T22:55:53.199112+0000","last_unstale":"2023-09-23T22:56:36.248746+0000","last_undegraded":"2023-09-23T22:56:36.248746+0000","last_fullsized":"2023-09-23T22:55:53.196998+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:23.074066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":266,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552171+0000","last_change":"2023-09-23T22:46:14.245365+0000","last_active":"2023-09-23T22:56:39.552171+0000","last_peered":"2023-09-23T22:56:39.552171+0000","last_clean":"2023-09-23T22:56:39.552171+0000","last_became_active":"2023-09-23T22:46:14.245064+0000","last_became_peered":"2023-09-23T22:46:14.245064+0000","last_unstale":"2023-09-23T22:56:39.552171+0000","last_undegraded":"2023-09-23T22:56:39.552171+0000","last_fullsized":"2023-09-23T22:56:39.552171+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:40.034482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"333'5189","reported_seq":5225,"reported_epoch":333,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:41.661223+0000","last_change":"2023-09-23T22:56:21.416510+0000","last_active":"2023-09-23T22:56:41.661223+0000","last_peered":"2023-09-23T22:56:41.661223+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:56:21.402493+0000","last_became_peered":"2023-09-23T22:56:21.402493+0000","last_unstale":"2023-09-23T22:56:41.661223+0000","last_undegraded":"2023-09-23T22:56:21.254596+0000","last_fullsized":"2023-09-23T22:56:21.199812+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5189,"log_dups_size":0,"ondisk_log_size":5189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340066304,"num_objects":5189,"num_object_clones":0,"num_object_copies":15567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4688,"num_objects_misplaced":5189,"num_objects_unfound":0,"num_objects_dirty":5189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5189,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":501}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":92,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552250+0000","last_change":"2023-09-23T22:51:34.430421+0000","last_active":"2023-09-23T22:56:39.552250+0000","last_peered":"2023-09-23T22:56:39.552250+0000","last_clean":"2023-09-23T22:56:39.552250+0000","last_became_active":"2023-09-23T22:51:34.416846+0000","last_became_peered":"2023-09-23T22:51:34.416846+0000","last_unstale":"2023-09-23T22:56:39.552250+0000","last_undegraded":"2023-09-23T22:56:39.552250+0000","last_fullsized":"2023-09-23T22:56:39.552250+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:59.574799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.250178+0000","last_change":"2023-09-23T22:56:20.365359+0000","last_active":"2023-09-23T22:56:36.250178+0000","last_peered":"2023-09-23T22:56:36.250178+0000","last_clean":"2023-09-23T22:56:36.250178+0000","last_became_active":"2023-09-23T22:56:20.364964+0000","last_became_peered":"2023-09-23T22:56:20.364964+0000","last_unstale":"2023-09-23T22:56:36.250178+0000","last_undegraded":"2023-09-23T22:56:36.250178+0000","last_fullsized":"2023-09-23T22:56:36.250178+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":312,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552980+0000","last_change":"2023-09-23T22:51:34.421103+0000","last_active":"2023-09-23T22:56:39.552980+0000","last_peered":"2023-09-23T22:56:39.552980+0000","last_clean":"2023-09-23T22:56:39.552980+0000","last_became_active":"2023-09-23T22:51:34.420756+0000","last_became_peered":"2023-09-23T22:51:34.420756+0000","last_unstale":"2023-09-23T22:56:39.552980+0000","last_undegraded":"2023-09-23T22:56:39.552980+0000","last_fullsized":"2023-09-23T22:56:39.552980+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:09.289828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.f","version":"333'5131","reported_seq":5164,"reported_epoch":333,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:56:41.700705+0000","last_change":"2023-09-23T22:56:21.417472+0000","last_active":"2023-09-23T22:56:41.700705+0000","last_peered":"2023-09-23T22:56:41.700705+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:56:21.411008+0000","last_became_peered":"2023-09-23T22:56:21.411008+0000","last_unstale":"2023-09-23T22:56:41.700705+0000","last_undegraded":"2023-09-23T22:56:21.263090+0000","last_fullsized":"2023-09-23T22:56:21.200679+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5131,"log_dups_size":0,"ondisk_log_size":5131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336265216,"num_objects":5131,"num_object_clones":0,"num_object_copies":15393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4596,"num_objects_misplaced":5131,"num_objects_unfound":0,"num_objects_dirty":5131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5131,"num_write_kb":328384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":535}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.251155+0000","last_change":"2023-09-23T22:56:21.265404+0000","last_active":"2023-09-23T22:56:36.251155+0000","last_peered":"2023-09-23T22:56:36.251155+0000","last_clean":"2023-09-23T22:56:36.251155+0000","last_became_active":"2023-09-23T22:56:21.264984+0000","last_became_peered":"2023-09-23T22:56:21.264984+0000","last_unstale":"2023-09-23T22:56:36.251155+0000","last_undegraded":"2023-09-23T22:56:36.251155+0000","last_fullsized":"2023-09-23T22:56:36.251155+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":232,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552378+0000","last_change":"2023-09-23T22:46:14.250773+0000","last_active":"2023-09-23T22:56:39.552378+0000","last_peered":"2023-09-23T22:56:39.552378+0000","last_clean":"2023-09-23T22:56:39.552378+0000","last_became_active":"2023-09-23T22:46:14.249879+0000","last_became_peered":"2023-09-23T22:46:14.249879+0000","last_unstale":"2023-09-23T22:56:39.552378+0000","last_undegraded":"2023-09-23T22:56:39.552378+0000","last_fullsized":"2023-09-23T22:56:39.552378+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:11.293195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":292,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.250810+0000","last_change":"2023-09-23T22:56:20.364523+0000","last_active":"2023-09-23T22:56:36.250810+0000","last_peered":"2023-09-23T22:56:36.250810+0000","last_clean":"2023-09-23T22:56:36.250810+0000","last_became_active":"2023-09-23T22:56:20.363967+0000","last_became_peered":"2023-09-23T22:56:20.363967+0000","last_unstale":"2023-09-23T22:56:36.250810+0000","last_undegraded":"2023-09-23T22:56:36.250810+0000","last_fullsized":"2023-09-23T22:56:36.250810+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":268,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T22:56:39.552260+0000","last_change":"2023-09-23T22:46:14.246665+0000","last_active":"2023-09-23T22:56:39.552260+0000","last_peered":"2023-09-23T22:56:39.552260+0000","last_clean":"2023-09-23T22:56:39.552260+0000","last_became_active":"2023-09-23T22:46:14.246018+0000","last_became_peered":"2023-09-23T22:46:14.246018+0000","last_unstale":"2023-09-23T22:56:39.552260+0000","last_undegraded":"2023-09-23T22:56:39.552260+0000","last_fullsized":"2023-09-23T22:56:39.552260+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:29.044235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":62,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.551909+0000","last_change":"2023-09-23T22:55:53.199764+0000","last_active":"2023-09-23T22:56:39.551909+0000","last_peered":"2023-09-23T22:56:39.551909+0000","last_clean":"2023-09-23T22:51:24.616891+0000","last_became_active":"2023-09-23T22:55:53.199764+0000","last_became_peered":"2023-09-23T22:55:53.199764+0000","last_unstale":"2023-09-23T22:56:39.551909+0000","last_undegraded":"2023-09-23T22:56:39.551909+0000","last_fullsized":"2023-09-23T22:55:53.195893+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:11.261891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":142,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.249800+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:56:36.249800+0000","last_peered":"2023-09-23T22:56:36.249800+0000","last_clean":"2023-09-23T22:56:36.249800+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:56:36.249800+0000","last_undegraded":"2023-09-23T22:56:36.249800+0000","last_fullsized":"2023-09-23T22:56:36.249800+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":347,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.247542+0000","last_change":"2023-09-23T22:55:53.200257+0000","last_active":"2023-09-23T22:56:36.247542+0000","last_peered":"2023-09-23T22:56:36.247542+0000","last_clean":"2023-09-23T22:55:51.197568+0000","last_became_active":"2023-09-23T22:55:53.200257+0000","last_became_peered":"2023-09-23T22:55:53.200257+0000","last_unstale":"2023-09-23T22:56:36.247542+0000","last_undegraded":"2023-09-23T22:56:36.247542+0000","last_fullsized":"2023-09-23T22:55:53.197355+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":156,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T22:56:36.245804+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:56:36.245804+0000","last_peered":"2023-09-23T22:56:36.245804+0000","last_clean":"2023-09-23T22:56:36.245804+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:56:36.245804+0000","last_undegraded":"2023-09-23T22:56:36.245804+0000","last_fullsized":"2023-09-23T22:56:36.245804+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":219,"reported_epoch":332,"state":"active+undersized","last_fresh":"2023-09-23T22:56:36.244702+0000","last_change":"2023-09-23T22:55:53.198913+0000","last_active":"2023-09-23T22:56:36.244702+0000","last_peered":"2023-09-23T22:56:36.244702+0000","last_clean":"2023-09-23T22:55:51.197306+0000","last_became_active":"2023-09-23T22:55:53.198913+0000","last_became_peered":"2023-09-23T22:55:53.198913+0000","last_unstale":"2023-09-23T22:56:36.244702+0000","last_undegraded":"2023-09-23T22:56:36.244702+0000","last_fullsized":"2023-09-23T22:55:53.197257+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":17,"reported_epoch":333,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:39.551761+0000","last_change":"2023-09-23T22:56:21.210926+0000","last_active":"2023-09-23T22:56:39.551761+0000","last_peered":"2023-09-23T22:56:39.551761+0000","last_clean":"2023-09-23T22:56:39.551761+0000","last_became_active":"2023-09-23T22:56:21.209449+0000","last_became_peered":"2023-09-23T22:56:21.209449+0000","last_unstale":"2023-09-23T22:56:39.551761+0000","last_undegraded":"2023-09-23T22:56:39.551761+0000","last_fullsized":"2023-09-23T22:56:39.551761+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:55.027845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,6],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":261,"reported_epoch":333,"state":"active+undersized","last_fresh":"2023-09-23T22:56:39.551943+0000","last_change":"2023-09-23T22:55:53.200092+0000","last_active":"2023-09-23T22:56:39.551943+0000","last_peered":"2023-09-23T22:56:39.551943+0000","last_clean":"2023-09-23T22:44:57.970339+0000","last_became_active":"2023-09-23T22:55:53.200092+0000","last_became_peered":"2023-09-23T22:55:53.200092+0000","last_unstale":"2023-09-23T22:56:39.551943+0000","last_undegraded":"2023-09-23T22:56:39.551943+0000","last_fullsized":"2023-09-23T22:55:53.195380+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:09.913132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":288,"reported_epoch":333,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:56:39.552015+0000","last_change":"2023-09-23T22:55:53.200660+0000","last_active":"2023-09-23T22:56:39.552015+0000","last_peered":"2023-09-23T22:56:39.552015+0000","last_clean":"2023-09-23T22:55:51.195937+0000","last_became_active":"2023-09-23T22:55:53.200660+0000","last_became_peered":"2023-09-23T22:55:53.200660+0000","last_unstale":"2023-09-23T22:56:39.552015+0000","last_undegraded":"2023-09-23T22:55:53.196609+0000","last_fullsized":"2023-09-23T22:55:53.196470+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:07.622461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":308,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:56:36.252848+0000","last_change":"2023-09-23T22:56:21.262507+0000","last_active":"2023-09-23T22:56:36.252848+0000","last_peered":"2023-09-23T22:56:36.252848+0000","last_clean":"2023-09-23T22:56:36.252848+0000","last_became_active":"2023-09-23T22:56:21.262143+0000","last_became_peered":"2023-09-23T22:56:21.262143+0000","last_unstale":"2023-09-23T22:56:36.252848+0000","last_undegraded":"2023-09-23T22:56:36.252848+0000","last_fullsized":"2023-09-23T22:56:36.252848+0000","mapping_epoch":329,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,6],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":88,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5412487168,"num_objects":82588,"num_object_clones":0,"num_object_copies":247764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":48421,"num_objects_misplaced":24629,"num_objects_unfound":0,"num_objects_dirty":82588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82588,"num_write_kb":5285632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1206,"num_bytes_recovered":79036416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15207694336,"data_stored":15207694336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81988,"ondisk_log_size":81988,"up":39,"acting":38,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":12,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":15062,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7549,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":80,"acting":89,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430565,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626793,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253102,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2345996,"kb_used_data":1700292,"kb_used_omap":9,"kb_used_meta":645686,"kb_avail":91399156,"statfs":{"total":95995035648,"available":93592735744,"internally_reserved":0,"allocated":1741099008,"data_stored":1739927922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":661183465},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:56:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":6.1699999999999999,"15min":5.9660000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":2.8140000000000001,"5min":129.49299999999999,"15min":129.49299999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.72599999999999998,"5min":5.7290000000000001,"15min":5.7279999999999998},"min":{"1min":0.25600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.0759999999999996,"5min":129.80000000000001,"15min":129.80000000000001},"last":0.435}]},{"osd":2,"last update":"Sat Sep 23 22:55:30 2023","interfaces":[{"interface":"back","average":{"1min":9.8800000000000008,"5min":4.593,"15min":3.8439999999999999},"min":{"1min":0.246,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":111.452,"5min":111.452,"15min":111.452},"last":0.29799999999999999},{"interface":"front","average":{"1min":10.167,"5min":4.4580000000000002,"15min":3.9420000000000002},"min":{"1min":0.36199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":108.22799999999999,"5min":108.22799999999999,"15min":108.22799999999999},"last":1.329}]},{"osd":3,"last update":"Sat Sep 23 22:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":1.772,"15min":2.7189999999999999},"min":{"1min":0.23400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.141,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.8440000000000001,"15min":3.2149999999999999},"min":{"1min":0.217,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.133,"5min":10.845000000000001,"15min":89.861999999999995},"last":1.2030000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.992,"15min":2.238},"min":{"1min":0.33100000000000002,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":4.6589999999999998,"5min":11.388,"15min":78.948999999999998},"last":0.497},{"interface":"front","average":{"1min":1.008,"5min":2.415,"15min":2.29},"min":{"1min":0.32200000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3820000000000001,"5min":20.788,"15min":78.963999999999999},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":2.4049999999999998,"15min":2.0510000000000002},"min":{"1min":0.38600000000000001,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":14.317,"5min":32.188000000000002,"15min":84.254999999999995},"last":14.317},{"interface":"front","average":{"1min":1.4830000000000001,"5min":2.1789999999999998,"15min":2.3359999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":8.0500000000000007,"5min":33.033000000000001,"15min":87.515000000000001},"last":3.472}]},{"osd":6,"last update":"Sat Sep 23 22:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":2.6240000000000001,"15min":2.4870000000000001},"min":{"1min":0.435,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.6680000000000001,"5min":20.742000000000001,"15min":79.311000000000007},"last":4.202},{"interface":"front","average":{"1min":1.6659999999999999,"5min":2.484,"15min":2.1240000000000001},"min":{"1min":0.36899999999999999,"5min":0.34999999999999998,"15min":0.313},"max":{"1min":8.5370000000000008,"5min":20.614000000000001,"15min":79.328999999999994},"last":8.0790000000000006}]},{"osd":7,"last update":"Sat Sep 23 22:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.8080000000000001,"15min":2.0299999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.33400000000000002},"max":{"1min":4.0970000000000004,"5min":8.702,"15min":29.539000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.3420000000000001,"5min":2.4319999999999999,"15min":2.2770000000000001},"min":{"1min":0.436,"5min":0.38800000000000001,"15min":0.371},"max":{"1min":5.3959999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":5.3959999999999999}]}]},{"osd":0,"up_from":219,"seq":940597837950,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3806152,"kb_used_data":3380608,"kb_used_omap":31,"kb_used_meta":425504,"kb_avail":89939000,"statfs":{"total":95995035648,"available":92097536000,"internally_reserved":0,"allocated":3461742592,"data_stored":3460519385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31822,"internal_metadata":435717042},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:56:40 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":2.867,"15min":3.3690000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.19},"max":{"1min":1.5349999999999999,"5min":40.180999999999997,"15min":40.180999999999997},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.76700000000000002,"5min":2.9590000000000001,"15min":3.3900000000000001},"min":{"1min":0.219,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":2.7269999999999999,"5min":40.137999999999998,"15min":40.137999999999998},"last":0.219}]},{"osd":2,"last update":"Sat Sep 23 22:55:39 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":5.101,"15min":3.6499999999999999},"min":{"1min":0.251,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.683,"5min":111.508,"15min":111.508},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.0059999999999998,"5min":5.2539999999999996,"15min":3.3199999999999998},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.579000000000001,"5min":111.532,"15min":111.532},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:56:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.085,"15min":2.9900000000000002},"min":{"1min":0.26600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.7200000000000002,"5min":9.2759999999999998,"15min":126.73},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":1.3640000000000001,"15min":2.109},"min":{"1min":0.28699999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.347,"5min":7.8419999999999996,"15min":121.483},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.149,"15min":2.1459999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.8029999999999999,"5min":13.161,"15min":18.670999999999999},"last":7.8029999999999999},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.052,"15min":2.0800000000000001},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.9729999999999999,"5min":10.255000000000001,"15min":20.009},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:56:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":1.4139999999999999,"15min":1.8939999999999999},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.318},"max":{"1min":2.2989999999999999,"5min":13.851000000000001,"15min":61.756999999999998},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":2.1619999999999999,"15min":2.2909999999999999},"min":{"1min":0.38100000000000001,"5min":0.35799999999999998,"15min":0.33500000000000002},"max":{"1min":2.3359999999999999,"5min":13.194000000000001,"15min":61.777000000000001},"last":1.034}]},{"osd":6,"last update":"Sat Sep 23 22:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.7070000000000001,"15min":1.9450000000000001},"min":{"1min":0.42099999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6230000000000002,"5min":12.683,"15min":59.146999999999998},"last":1.333},{"interface":"front","average":{"1min":1.276,"5min":2.46,"15min":2.4220000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":3.8069999999999999,"5min":13.125,"15min":59.164000000000001},"last":1.3480000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:56:40 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":2.2120000000000002,"15min":2.0760000000000001},"min":{"1min":0.47299999999999998,"5min":0.438,"15min":0.41199999999999998},"max":{"1min":4.0510000000000002,"5min":13.223000000000001,"15min":18.640000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.7789999999999999,"15min":1.6950000000000001},"min":{"1min":0.43099999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.0869999999999997,"5min":10.624000000000001,"15min":21.954999999999998},"last":0.45400000000000001}]}]},{"osd":2,"up_from":131,"seq":562640715953,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1443208,"kb_used_data":981312,"kb_used_omap":81,"kb_used_meta":461806,"kb_avail":92301944,"statfs":{"total":95995035648,"available":94517190656,"internally_reserved":0,"allocated":1004863488,"data_stored":1003735035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":472889562},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:55:40 2023","interfaces":[{"interface":"back","average":{"1min":5.1609999999999996,"5min":6.0199999999999996,"15min":5.7270000000000003},"min":{"1min":0.311,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":49.741,"5min":114.483,"15min":132.79499999999999},"last":1.4650000000000001},{"interface":"front","average":{"1min":3.9390000000000001,"5min":5.6820000000000004,"15min":6.0030000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":49.645000000000003,"5min":113.873,"15min":132.58099999999999},"last":1.516}]},{"osd":1,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6970000000000001,"15min":3.1190000000000002},"min":{"1min":0.24099999999999999,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":19.504999999999999,"5min":77.004999999999995,"15min":88.316000000000003},"last":1.4850000000000001},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1160000000000001,"15min":3.4279999999999999},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":19.523,"5min":76.956000000000003,"15min":88.436999999999998},"last":1.0569999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":3.5609999999999999,"15min":4.4880000000000004},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":19.579999999999998,"5min":112.895,"15min":112.895},"last":1.3520000000000001},{"interface":"front","average":{"1min":3.137,"5min":3.5449999999999999,"15min":3.7599999999999998},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":19.542000000000002,"5min":113.995,"15min":113.995},"last":1.502}]},{"osd":4,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":5.5880000000000001,"5min":3.2669999999999999,"15min":3.1400000000000001},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.253},"max":{"1min":48.286999999999999,"5min":69.113,"15min":108.878},"last":3.2040000000000002},{"interface":"front","average":{"1min":4.9829999999999997,"5min":2.9889999999999999,"15min":3.6259999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":48.259999999999998,"5min":69.028999999999996,"15min":108.93000000000001},"last":1.452}]},{"osd":5,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.798,"15min":3.069},"min":{"1min":0.45000000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":19.652000000000001,"5min":22.646000000000001,"15min":97.322999999999993},"last":1.5860000000000001},{"interface":"front","average":{"1min":2.113,"5min":2.1970000000000001,"15min":3.0710000000000002},"min":{"1min":0.47799999999999998,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":19.614999999999998,"5min":22.501000000000001,"15min":97.301000000000002},"last":1.1180000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":5.1059999999999999,"5min":3.2639999999999998,"15min":3.633},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.32700000000000001},"max":{"1min":50.073999999999998,"5min":59.655000000000001,"15min":98.597999999999999},"last":1.98},{"interface":"front","average":{"1min":6.0739999999999998,"5min":3.2709999999999999,"15min":3.0499999999999998},"min":{"1min":0.42999999999999999,"5min":0.39000000000000001,"15min":0.313},"max":{"1min":49.469000000000001,"5min":59.612000000000002,"15min":98.509},"last":1.6040000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:55:45 2023","interfaces":[{"interface":"back","average":{"1min":2.919,"5min":2.0960000000000001,"15min":1.881},"min":{"1min":0.495,"5min":0.40300000000000002,"15min":0.33900000000000002},"max":{"1min":19.756,"5min":19.756,"15min":19.756},"last":1.5429999999999999},{"interface":"front","average":{"1min":2.3980000000000001,"5min":1.6439999999999999,"15min":2.4500000000000002},"min":{"1min":0.45800000000000002,"5min":0.371,"15min":0.37},"max":{"1min":19.765999999999998,"5min":19.765999999999998,"15min":88.584000000000003},"last":1.4199999999999999}]}]},{"osd":3,"up_from":0,"seq":403726926050,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019248,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4265796,"kb_used_data":3676092,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":89479356,"statfs":{"total":95995035648,"available":91626860544,"internally_reserved":0,"allocated":3764318208,"data_stored":3763733561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":9.1389999999999993,"15min":6.6989999999999998},"min":{"1min":0.25800000000000001,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":2.5619999999999998,"5min":159.62200000000001,"15min":159.62200000000001},"last":1.655},{"interface":"front","average":{"1min":4.9820000000000002,"5min":10.222,"15min":7.4800000000000004},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.217},"max":{"1min":89.790999999999997,"5min":159.51400000000001,"15min":159.51400000000001},"last":8.9329999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:55:47 2023","interfaces":[{"interface":"back","average":{"1min":6.1849999999999996,"5min":5.7119999999999997,"15min":3.399},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":98.846999999999994,"5min":132.304,"15min":132.304},"last":0.31900000000000001},{"interface":"front","average":{"1min":6.1509999999999998,"5min":5.8239999999999998,"15min":3.2589999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":98.938999999999993,"5min":132.453,"15min":132.453},"last":4.9139999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:55:40 2023","interfaces":[{"interface":"back","average":{"1min":11.44,"5min":4.0789999999999997,"15min":3.7349999999999999},"min":{"1min":0.33800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":137.922,"5min":137.922,"15min":137.922},"last":0.46999999999999997},{"interface":"front","average":{"1min":11.497,"5min":4.1219999999999999,"15min":3.734},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":140.994,"5min":140.994,"15min":140.994},"last":0.433}]},{"osd":3,"last update":"Sat Sep 23 22:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":3.8100000000000001,"15min":4.5830000000000002},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":5.1509999999999998,"5min":164.453,"15min":164.453},"last":1.631},{"interface":"front","average":{"1min":0.63600000000000001,"5min":3.9119999999999999,"15min":4.6779999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.27200000000000002},"max":{"1min":3.048,"5min":167.05199999999999,"15min":167.05199999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":1.075,"15min":0.89500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26100000000000001},"max":{"1min":4.3129999999999997,"5min":10.321999999999999,"15min":12.238},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.121,"15min":0.90900000000000003},"min":{"1min":0.29299999999999998,"5min":0.28799999999999998,"15min":0.248},"max":{"1min":5.5279999999999996,"5min":6.4349999999999996,"15min":9.5190000000000001},"last":5.6849999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.0329999999999999,"15min":0.97399999999999998},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7480000000000002,"5min":7.9139999999999997,"15min":27.056000000000001},"last":5.617},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.0489999999999999,"15min":0.98799999999999999},"min":{"1min":0.373,"5min":0.29699999999999999,"15min":0.28199999999999997},"max":{"1min":3.0030000000000001,"5min":10.423999999999999,"15min":26.940999999999999},"last":0.34599999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.048,"15min":0.91400000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":1.9299999999999999,"5min":10.278,"15min":12.311},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.88400000000000001,"15min":0.83099999999999996},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":2.9580000000000002,"5min":4.077,"15min":8.6739999999999995},"last":1.5940000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823021,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2998964,"kb_used_data":2413356,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90746188,"statfs":{"total":95995035648,"available":92924096512,"internally_reserved":0,"allocated":2471276544,"data_stored":2470712150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:56:25 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":6.6219999999999999,"15min":5.1529999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":24.164999999999999,"5min":185.62,"15min":185.62},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.4740000000000002,"5min":6.6420000000000003,"15min":5.1449999999999996},"min":{"1min":0.29199999999999998,"5min":0.248,"15min":0.248},"max":{"1min":24.244,"5min":185.499,"15min":185.499},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:56:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.9369999999999998,"15min":2.96},"min":{"1min":0.31900000000000001,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":4.9020000000000001,"5min":110.36499999999999,"15min":149.203},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.4930000000000001,"5min":4.4219999999999997,"15min":3.3860000000000001},"min":{"1min":0.39200000000000002,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":4.8520000000000003,"5min":110.89,"15min":149.554},"last":0.42899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":4.992,"15min":5.04},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5309999999999997,"5min":189.126,"15min":189.126},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.4710000000000001,"5min":4.6379999999999999,"15min":4.8920000000000003},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":6.2530000000000001,"5min":188.74799999999999,"15min":188.74799999999999},"last":1.3149999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:56:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.8559999999999999,"15min":5.8140000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":4.5069999999999997,"5min":89.995000000000005,"15min":257.05000000000001},"last":0.378},{"interface":"front","average":{"1min":0.98499999999999999,"5min":2.7040000000000002,"15min":5.6289999999999996},"min":{"1min":0.35899999999999999,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":4.7619999999999996,"5min":90.117999999999995,"15min":257.35000000000002},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:56:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.03,"15min":0.83799999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.24099999999999999},"max":{"1min":4.0460000000000003,"5min":6.3949999999999996,"15min":8.1110000000000007},"last":115.06699999999999},{"interface":"front","average":{"1min":1.647,"5min":1.3380000000000001,"15min":0.91500000000000004},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":8.6519999999999992,"5min":8.6519999999999992,"15min":10.535},"last":114.908}]},{"osd":6,"last update":"Sat Sep 23 22:56:33 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.341,"15min":1.1699999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.26500000000000001},"max":{"1min":4.077,"5min":7.649,"15min":48.360999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.872,"5min":1.22,"15min":1.1499999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.3179999999999996,"5min":10.314,"15min":48.292999999999999},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.282,"15min":0.96399999999999997},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.32100000000000001},"max":{"1min":1.629,"5min":10.367000000000001,"15min":10.367000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.1279999999999999,"15min":0.89400000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32100000000000001},"max":{"1min":4.0110000000000001,"5min":6.7149999999999999,"15min":8.0459999999999994},"last":0.33400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2974,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3494510592,"data_stored":3494510592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1758068736,"data_stored":1758068736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1040777216,"data_stored":1040777216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3781361664,"data_stored":3781361664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2483748864,"data_stored":2483748864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2649227264,"data_stored":2649227264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:56:49.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-09-23T22:56:49.049 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:49.275 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:49.295 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:49.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:49.399 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:49.467 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:49.587 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:49.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:49.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:49 smithi151 ceph-mon[159210]: pgmap v982: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 659 op/s; 48877/254010 objects degraded (19.242%); 25300/254010 objects misplaced (9.960%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:56:49.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:49.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:49.645 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:56:49.645 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:56:49.646 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:56:49.646 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:56:49.646 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:56:49.695 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:49.784 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:49.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:49 smithi052 ceph-mon[187743]: pgmap v982: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 659 op/s; 48877/254010 objects degraded (19.242%); 25300/254010 objects misplaced (9.960%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:56:49.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:49.860 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:49.860 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:49 smithi052 ceph-mon[180979]: pgmap v982: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 659 op/s; 48877/254010 objects degraded (19.242%); 25300/254010 objects misplaced (9.960%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:56:49.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:56:49.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:56:49.877 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:49.930 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:49.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:49.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:49.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 77 16 87899 87883 71.3034 45.625 0.0101394 0.0140192 2023-09-23T22:56:50.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:50.096 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:50.582 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:50.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:50.651 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:56:50.692 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:50.748 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:50.835 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4165022629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:56:50.835 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 48877/254010 objects degraded (19.242%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:50.836 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4165022629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:56:50.836 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 48877/254010 objects degraded (19.242%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4165022629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:56:50.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 48877/254010 objects degraded (19.242%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:56:50.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 78 16 88481 88465 70.8557 36.375 0.016517 0.0141075 2023-09-23T22:56:50.991 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:56:51.379 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:51.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:51.484 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:51.524 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:51.640 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: pgmap v983: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 49330/257991 objects degraded (19.121%); 25501/257991 objects misplaced (9.884%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:51.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:51.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4165022629' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:56:51.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: osdmap e335: 8 total, 7 up, 5 in 2023-09-23T22:56:51.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.641 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: pgmap v983: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 49330/257991 objects degraded (19.121%); 25501/257991 objects misplaced (9.884%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:51.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:51.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4165022629' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:56:51.644 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: osdmap e335: 8 total, 7 up, 5 in 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.645 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[187743]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.712 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:56:51.808 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:51.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:51.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:51.913 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: pgmap v983: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 45 MiB/s wr, 721 op/s; 49330/257991 objects degraded (19.121%); 25501/257991 objects misplaced (9.884%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4165022629' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: osdmap e335: 8 total, 7 up, 5 in 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:51 smithi151 ceph-mon[159210]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 79 16 89169 89153 70.5031 43 0.0160629 0.014178 2023-09-23T22:56:51.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:51 smithi052 ceph-mon[180979]: failed to encode map e335 with expected crc 2023-09-23T22:56:51.963 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:52.025 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:52.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:52.054 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:52.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:52.158 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:52.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:52.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:52.247 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:52.247 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:52.248 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:52.248 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:52.248 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:52.248 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:52.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.248 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.249 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:52.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.249 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:56:52.306 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:52.368 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:52.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2903604425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:52.901 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2903604425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:52.901 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2903604425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:52.901 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:52.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:52.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T22:56:52.957543+0000 min lat: 0.00121105 max lat: 11.5254 avg lat: 0.0142636 2023-09-23T22:56:52.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:56:52.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 80 16 89756 89740 70.0805 36.6875 0.00537157 0.0142636 2023-09-23T22:56:53.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:53.053 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:53.137 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:53.142 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:53.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:53.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:53.247 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:53.289 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:53.361 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:53.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:53.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:53.450 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:56:53.468 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:53.554 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:53.667 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:53 smithi052 ceph-mon[180979]: pgmap v985: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 45 MiB/s wr, 727 op/s; 49969/264669 objects degraded (18.880%); 26081/264669 objects misplaced (9.854%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:53.668 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:56:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:56:53] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.33.4" 2023-09-23T22:56:53.668 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:53 smithi052 ceph-mon[187743]: pgmap v985: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 45 MiB/s wr, 727 op/s; 49969/264669 objects degraded (18.880%); 26081/264669 objects misplaced (9.854%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:53.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:53 smithi151 ceph-mon[159210]: pgmap v985: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 45 MiB/s wr, 727 op/s; 49969/264669 objects degraded (18.880%); 26081/264669 objects misplaced (9.854%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:53.800 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:53.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:53.911 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:53.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 81 16 90528 90512 69.8108 48.25 0.0279122 0.0143191 2023-09-23T22:56:53.961 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:53.976 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T22:56:54.003 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:54.133 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:56:54.133 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:56:54.257 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:54.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:54.361 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:54.400 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:54.473 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:54.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:54.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:54.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:54.617 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:54.694 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:54.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:54.797 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:54.834 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:54.874 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4056781815' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:56:54.874 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:54 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:56:54.874 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4056781815' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:56:54.874 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:54 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:56:54.882 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:54.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T22:56:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4056781815' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:56:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:54 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T22:56:54.957 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 82 16 91117 91101 69.4084 36.8125 0.0255252 0.0143957 2023-09-23T22:56:54.986 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:55.029 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:55.143 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:56:55.161 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:55.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:55.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:55.271 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:55.324 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:56:55.386 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:55.586 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:55.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:55.699 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:56:55.751 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:55.887 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:55 smithi052 ceph-mon[187743]: pgmap v986: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 50370/268404 objects degraded (18.766%); 26297/268404 objects misplaced (9.798%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:55.887 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 50370/268404 objects degraded (18.766%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:56:55.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:55 smithi052 ceph-mon[180979]: pgmap v986: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 50370/268404 objects degraded (18.766%); 26297/268404 objects misplaced (9.798%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:55.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 50370/268404 objects degraded (18.766%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:56:55.918 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:56:55.925 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:56:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:55.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:56:55.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:55 smithi151 ceph-mon[159210]: pgmap v986: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 50370/268404 objects degraded (18.766%); 26297/268404 objects misplaced (9.798%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:55.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 50370/268404 objects degraded (18.766%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:56:55.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 83 16 91869 91853 69.1385 47 0.0400478 0.0144585 2023-09-23T22:56:56.032 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:56.127 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:56.134 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:56:56.257 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:56.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:56.362 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:56:56.462 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:56.588 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:56:56.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:56.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:56:56.690 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:56.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1959643888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:56.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:56 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:56.698 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:56:56.734 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:56.919 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:56.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:56.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 84 16 92505 92489 68.7887 39.75 0.00448634 0.0145311 2023-09-23T22:56:56.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1959643888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:56.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:56 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1959643888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:56 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:56:57.037 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:57.095 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:57.415 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:57.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:57.519 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:57.609 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:57.698 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[180979]: pgmap v987: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 50594/269076 objects degraded (18.803%); 26192/269076 objects misplaced (9.734%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:57.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:57.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[180979]: failed to encode map e336 with expected crc 2023-09-23T22:56:57.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:57.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[180979]: osdmap e336: 8 total, 7 up, 5 in 2023-09-23T22:56:57.947 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:57.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:57 smithi151 ceph-mon[159210]: pgmap v987: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 50594/269076 objects degraded (18.803%); 26192/269076 objects misplaced (9.734%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:57.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:57 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:57.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:57 smithi151 ceph-mon[159210]: failed to encode map e336 with expected crc 2023-09-23T22:56:57.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:57 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:57.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:57 smithi151 ceph-mon[159210]: osdmap e336: 8 total, 7 up, 5 in 2023-09-23T22:56:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:57.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:57.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 85 16 93311 93295 68.5721 50.375 0.0320845 0.0145777 2023-09-23T22:56:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[187743]: pgmap v987: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 50594/269076 objects degraded (18.803%); 26192/269076 objects misplaced (9.734%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:56:57.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:56:57.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[187743]: failed to encode map e336 with expected crc 2023-09-23T22:56:57.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:56:57.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:57 smithi052 ceph-mon[187743]: osdmap e336: 8 total, 7 up, 5 in 2023-09-23T22:56:58.057 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:56:58.100 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:58.178 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:56:58.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:58.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:56:58.183 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:56:58.208 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:56:58.236 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:56:58.274 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:56:58.283 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:58.301 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:56:58.340 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:56:58.364 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:56:58.364 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.364 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.365 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:56:58.365 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.365 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.365 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.365 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.365 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:56:58.384 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:58.556 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:58.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:93786 10.8 e336/324 10:193f7009:::benchmark_data_smithi052_461816_object93785:head v 336'5865, mlcod=336'5863)", 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.628354+0000", 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0014710579999999999, 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00150227, 2023-09-23T22:56:58.638 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628354+0000", 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628354+0000", 2023-09-23T22:56:58.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628354+0000", 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5036000000000001e-05 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628389+0000", 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3580000000000003e-06 2023-09-23T22:56:58.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628397+0000", 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4720000000000003e-06 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628403+0000", 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000224773 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628627+0000", 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.931e-05 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628647+0000", 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00029256799999999999 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:93790 10.8 e336/324)", 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.629154+0000", 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.00067150800000000002, 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00077276500000000002, 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "queued for pg", 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629154+0000", 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629154+0000", 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629153+0000", 2023-09-23T22:56:58.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4189000000000005e-05 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629217+0000", 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1144999999999999e-05 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629228+0000", 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6400000000000002e-06 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629234+0000", 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0094999999999998e-05 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:93794 10.a e336/329)", 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.629262+0000", 2023-09-23T22:56:58.646 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.00056350900000000002, 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071318499999999997, 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "queued for pg", 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629262+0000", 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.647 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629262+0000", 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629261+0000", 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8826000000000002e-05 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.648 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629320+0000", 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8240000000000002e-06 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629327+0000", 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3390000000000003e-06 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.629331+0000", 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9484000000000001e-05 2023-09-23T22:56:58.649 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:93783 10.7 10:e4e77049:::benchmark_data_smithi052_461816_object93782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e336)", 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.620643+0000", 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0091821149999999994, 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0093933790000000003, 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:56:58.650 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 93783 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.651 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620643+0000", 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620643+0000", 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620641+0000", 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3058000000000001e-05 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620674+0000", 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5381999999999999e-05 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620690+0000", 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7259999999999996e-06 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620697+0000", 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7939000000000002e-05 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620754+0000", 2023-09-23T22:56:58.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000160851 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.620915+0000", 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000198455 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:56:58.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.621114+0000", 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047067399999999997 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:93784 10.9 e336/324 10:966bd9f6:::benchmark_data_smithi052_461816_object93783:head v 336'5858, mlcod=336'5853)", 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.623622+0000", 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0062037070000000001, 2023-09-23T22:56:58.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064973039999999998, 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.623622+0000", 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.623622+0000", 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.623620+0000", 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0722999999999997e-05 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.623651+0000", 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1099e-05 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.623662+0000", 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2790000000000004e-06 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.623668+0000", 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097231200000000002 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.624641+0000", 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8349999999999999e-05 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.624659+0000", 2023-09-23T22:56:58.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001037622 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop_reply(client.45953.0:93783 10.7 e336/329)", 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.627979+0000", 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0018463920000000001, 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002201803, 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.661 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "queued for pg", 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.627979+0000", 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.627979+0000", 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:56:58.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.627978+0000", 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0550000000000002e-06 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.627985+0000", 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6699000000000001e-05 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628002+0000", 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5290000000000003e-06 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628009+0000", 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0227e-05 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.664 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:93788 10.6 e336/317 10:645cbabf:::benchmark_data_smithi052_461816_object93787:head v 336'5868, mlcod=336'5866)", 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:58.628295+0000", 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.001529732, 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019370050000000001, 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628295+0000", 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628295+0000", 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:56:58.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628295+0000", 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4199000000000002e-05 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628329+0000", 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2569999999999996e-06 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628338+0000", 2023-09-23T22:56:58.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4490000000000001e-06 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628343+0000", 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4718000000000001e-05 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628387+0000", 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7262000000000001e-05 2023-09-23T22:56:58.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:58.628415+0000", 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000119194 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 7 2023-09-23T22:56:58.669 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:56:58.673 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:56:58.772 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:58.875 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/473179125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:58.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 86 16 94045 94029 68.3082 45.875 0.0215141 0.0146343 2023-09-23T22:56:58.963 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:56:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:58.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:56:58.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/473179125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:58.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/473179125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:56:59.068 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:56:59.077 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:56:59.108 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:59.164 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:59.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:59.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:59.270 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:59.311 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:59.352 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:56:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:59.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:56:59.455 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:56:59.481 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:59.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:56:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:59.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:56:59.677 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:56:59.714 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:56:59.740 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:56:59.815 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:56:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:56:59.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:56:59.893 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5] out_osds: [7, 3, 6] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:56:59.893 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:56:59.893 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:56:59.893 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.731190 2023-09-23T22:56:59.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.7311899720090196 2023-09-23T22:56:59.924 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:56:59 smithi151 ceph-mon[159210]: pgmap v989: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 44 MiB/s wr, 703 op/s; 51047/274878 objects degraded (18.571%); 26804/274878 objects misplaced (9.751%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:56:59.926 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:56:59.951 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:56:59.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 87 16 94654 94638 67.9606 38.0625 0.0515956 0.014708 2023-09-23T22:56:59.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:56:59 smithi052 ceph-mon[187743]: pgmap v989: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 44 MiB/s wr, 703 op/s; 51047/274878 objects degraded (18.571%); 26804/274878 objects misplaced (9.751%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:56:59.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:56:59 smithi052 ceph-mon[180979]: pgmap v989: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 44 MiB/s wr, 703 op/s; 51047/274878 objects degraded (18.571%); 26804/274878 objects misplaced (9.751%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:57:00.005 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:00.199 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:00.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:00.299 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:00.315 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:00.408 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:00.528 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:00.543 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:00.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:00.652 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:00.727 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:00.811 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:00.825 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 51047/274878 objects degraded (18.571%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:00.825 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 51047/274878 objects degraded (18.571%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:00.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:00.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 51047/274878 objects degraded (18.571%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:00.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:00.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:00.947 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:00.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 88 16 95349 95333 67.682 43.4375 0.0160265 0.01477 2023-09-23T22:57:01.013 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:01.078 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:57:01.398 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:01.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:01.504 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:01.604 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:01.712 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:01.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:01.819 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:01.865 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:01 smithi151 ceph-mon[159210]: pgmap v990: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 44 MiB/s wr, 701 op/s; 51476/278718 objects degraded (18.469%); 27034/278718 objects misplaced (9.699%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:57:01.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 89 16 96068 96052 67.4265 44.9375 0.0190291 0.0148254 2023-09-23T22:57:01.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:01 smithi052 ceph-mon[187743]: pgmap v990: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 44 MiB/s wr, 701 op/s; 51476/278718 objects degraded (18.469%); 27034/278718 objects misplaced (9.699%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:57:01.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:01 smithi052 ceph-mon[180979]: pgmap v990: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 44 MiB/s wr, 701 op/s; 51476/278718 objects degraded (18.469%); 27034/278718 objects misplaced (9.699%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T22:57:02.451 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:02.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:02.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:02.560 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:02.623 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:02.708 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:02.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:02.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:02.791 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:02.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:02.891 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:02.958 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total time run: 90.0258 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total writes made: 96798 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Write size: 65536 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object size: 65536 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Bandwidth (MB/sec): 67.2015 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Bandwidth: 45.4981 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max bandwidth (MB/sec): 217.812 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average IOPS: 1075 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev IOPS: 727.981 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max IOPS: 3485 2023-09-23T22:57:02.959 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min IOPS: 0 2023-09-23T22:57:02.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average Latency(s): 0.014875 2023-09-23T22:57:02.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Latency(s): 0.130846 2023-09-23T22:57:02.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max latency(s): 11.5254 2023-09-23T22:57:02.960 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min latency(s): 0.00121105 2023-09-23T22:57:02.963 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:57:03.039 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:03.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:03.155 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:03.243 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:03.366 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:57:03.366 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:57:03.366 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:57:03.366 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:57:03.366 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:57:03.388 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:03.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:03.493 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:03.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:57:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:57:03] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.33.4" 2023-09-23T22:57:03.596 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:03.672 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:03.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:03.737 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.5 primary-affinity to 0.73119 (8479192) 2023-09-23T22:57:03.737 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:57:03.776 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:03.804 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:03.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:03.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:03.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:03.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:03 smithi151 ceph-mon[159210]: pgmap v991: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 43 MiB/s wr, 687 op/s; 52145/285303 objects degraded (18.277%); 27560/285303 objects misplaced (9.660%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:57:03.954 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4190652510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:03.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:03 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:03.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1100759284' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]: dispatch 2023-09-23T22:57:03.955 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:03 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]: dispatch 2023-09-23T22:57:03.973 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[187743]: pgmap v991: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 43 MiB/s wr, 687 op/s; 52145/285303 objects degraded (18.277%); 27560/285303 objects misplaced (9.660%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:57:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4190652510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1100759284' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]: dispatch 2023-09-23T22:57:03.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]: dispatch 2023-09-23T22:57:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[180979]: pgmap v991: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 43 MiB/s wr, 687 op/s; 52145/285303 objects degraded (18.277%); 27560/285303 objects misplaced (9.660%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:57:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4190652510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1100759284' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]: dispatch 2023-09-23T22:57:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:03 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]: dispatch 2023-09-23T22:57:04.008 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:04.059 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:04.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:04.164 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:04.189 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:04.233 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:04.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:04.336 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:04.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:04.682 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:57:04.942 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:04.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: failed to encode map e337 with expected crc 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]': finished 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: osdmap e337: 8 total, 7 up, 5 in 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:04.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: failed to encode map e337 with expected crc 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]': finished 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: osdmap e337: 8 total, 7 up, 5 in 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: failed to encode map e337 with expected crc 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:57:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7311899720090196}]': finished 2023-09-23T22:57:04.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: osdmap e337: 8 total, 7 up, 5 in 2023-09-23T22:57:04.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:04.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:05.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:05.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:05.311 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:05.614 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:05.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:05.718 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:05.788 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:05.875 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:05.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:05.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: pgmap v993: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 843 op/s; 52450/288135 objects degraded (18.203%); 27727/288135 objects misplaced (9.623%); 864 KiB/s, 13 objects/s recovering 2023-09-23T22:57:05.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.889 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: osdmap e338: 8 total, 7 up, 5 in 2023-09-23T22:57:05.889 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.889 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.889 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 52450/288135 objects degraded (18.203%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:05.983 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: pgmap v993: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 843 op/s; 52450/288135 objects degraded (18.203%); 27727/288135 objects misplaced (9.623%); 864 KiB/s, 13 objects/s recovering 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: osdmap e338: 8 total, 7 up, 5 in 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 52450/288135 objects degraded (18.203%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: pgmap v993: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 843 op/s; 52450/288135 objects degraded (18.203%); 27727/288135 objects misplaced (9.623%); 864 KiB/s, 13 objects/s recovering 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: osdmap e338: 8 total, 7 up, 5 in 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: failed to encode map e338 with expected crc 2023-09-23T22:57:05.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 52450/288135 objects degraded (18.203%), 11 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:06.041 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:06.116 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:06.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:06.146 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:57:06.177 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:06.205 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:06.222 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:06.232 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:06.261 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:06.265 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:06.290 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:06.332 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.333 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:57:06.348 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:06.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:06.456 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:06.539 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:06.674 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:06.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:06.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:06.779 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:06.802 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:06 smithi151 ceph-mon[159210]: failed to encode map e339 with expected crc 2023-09-23T22:57:06.803 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:06 smithi151 ceph-mon[159210]: osdmap e339: 8 total, 7 up, 5 in 2023-09-23T22:57:06.803 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:06 smithi151 ceph-mon[159210]: failed to encode map e339 with expected crc 2023-09-23T22:57:06.803 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:06 smithi151 ceph-mon[159210]: failed to encode map e339 with expected crc 2023-09-23T22:57:06.803 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1365549039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:06.805 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:06.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:06.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:06.952 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:06.991 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:07.230 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[180979]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.230 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[180979]: osdmap e339: 8 total, 7 up, 5 in 2023-09-23T22:57:07.231 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[180979]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.231 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[180979]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.231 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1365549039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:07.236 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[187743]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.236 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[187743]: osdmap e339: 8 total, 7 up, 5 in 2023-09-23T22:57:07.236 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[187743]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.236 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[187743]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.237 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1365549039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:07.269 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:07.269 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:07.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:07.374 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:07.459 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:07.601 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:07.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:07.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:07.717 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:07.749 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:57:07.761 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:07.856 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:07.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:07.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:07.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:07 smithi151 ceph-mon[159210]: failed to encode map e339 with expected crc 2023-09-23T22:57:07.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:07 smithi151 ceph-mon[159210]: pgmap v996: 89 pgs: 22 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 36 MiB/s wr, 641 op/s; 50088/287970 objects degraded (17.393%); 29975/287970 objects misplaced (10.409%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:57:07.962 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:07.995 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:07.995 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:08.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:07 smithi052 ceph-mon[187743]: failed to encode map e339 with expected crc 2023-09-23T22:57:08.008 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:07 smithi052 ceph-mon[187743]: pgmap v996: 89 pgs: 22 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 36 MiB/s wr, 641 op/s; 50088/287970 objects degraded (17.393%); 29975/287970 objects misplaced (10.409%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:57:08.009 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:07 smithi052 ceph-mon[180979]: failed to encode map e339 with expected crc 2023-09-23T22:57:08.010 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:07 smithi052 ceph-mon[180979]: pgmap v996: 89 pgs: 22 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 36 MiB/s wr, 641 op/s; 50088/287970 objects degraded (17.393%); 29975/287970 objects misplaced (10.409%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T22:57:08.017 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:08.095 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:08.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:08.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:08.200 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:08.268 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:08.297 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:08.402 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:08.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:08.509 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:08.566 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:08.609 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:08.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:08.712 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:08.741 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:08.939 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:08.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:09.044 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:09.106 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:09.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:09.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:09.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:09.270 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:57:09.317 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:57:09.346 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:09.423 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:09.593 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:09.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:09.712 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:09.793 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:09.941 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:09.965 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:09.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:10.044 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:10.078 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:10.099 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:09 smithi052 ceph-mon[187743]: pgmap v997: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 170 B/s rd, 18 MiB/s wr, 525 op/s; 51275/286176 objects degraded (17.917%); 28415/286176 objects misplaced (9.929%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:57:10.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:09 smithi052 ceph-mon[180979]: pgmap v997: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 170 B/s rd, 18 MiB/s wr, 525 op/s; 51275/286176 objects degraded (17.917%); 28415/286176 objects misplaced (9.929%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:57:10.138 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:09 smithi151 ceph-mon[159210]: pgmap v997: 89 pgs: 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 170 B/s rd, 18 MiB/s wr, 525 op/s; 51275/286176 objects degraded (17.917%); 28415/286176 objects misplaced (9.929%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:57:10.155 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:10.282 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:10.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:10.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:10.388 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:10.471 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:10.473 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5] out_osds: [7, 3, 6] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:57:10.473 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:57:10.473 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:57:10.473 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:57:10.474 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:57:10.474 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:57:10.782 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:10.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:10.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:11.049 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 51275/286176 objects degraded (17.917%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:11.049 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1268248314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:11.050 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 51275/286176 objects degraded (17.917%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:11.050 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1268248314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:11.052 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:11.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 51275/286176 objects degraded (17.917%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:11.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1268248314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:11.218 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:57:11.489 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:11.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:11.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:11.594 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:11.669 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:11.935 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:11 smithi151 ceph-mon[159210]: pgmap v998: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 151 B/s rd, 5.4 MiB/s wr, 427 op/s; 46265/282954 objects degraded (16.351%); 32677/282954 objects misplaced (11.549%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:57:11.935 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:11 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:57:11.936 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:11 smithi151 ceph-mon[159210]: failed to encode map e340 with expected crc 2023-09-23T22:57:11.936 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1268248314' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:11.936 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:11 smithi151 ceph-mon[159210]: osdmap e340: 8 total, 7 up, 5 in 2023-09-23T22:57:11.957 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:11.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:12.066 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:12.087 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[180979]: pgmap v998: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 151 B/s rd, 5.4 MiB/s wr, 427 op/s; 46265/282954 objects degraded (16.351%); 32677/282954 objects misplaced (11.549%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:57:12.088 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:57:12.088 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[180979]: failed to encode map e340 with expected crc 2023-09-23T22:57:12.088 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1268248314' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:12.088 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[180979]: osdmap e340: 8 total, 7 up, 5 in 2023-09-23T22:57:12.088 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[187743]: pgmap v998: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 151 B/s rd, 5.4 MiB/s wr, 427 op/s; 46265/282954 objects degraded (16.351%); 32677/282954 objects misplaced (11.549%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:57:12.089 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:57:12.089 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[187743]: failed to encode map e340 with expected crc 2023-09-23T22:57:12.089 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1268248314' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:12.089 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:11 smithi052 ceph-mon[187743]: osdmap e340: 8 total, 7 up, 5 in 2023-09-23T22:57:12.172 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:12.358 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:12.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:12.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:12.471 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:12.571 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:12.683 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:12.699 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:12.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:12.737 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:12.737 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:12.803 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:12.844 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:57:12.878 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:12.894 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:12.911 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:12.952 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:12.976 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:13.012 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:13.036 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:13.036 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.037 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:57:13.181 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:13.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:13.285 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:13.337 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:13.394 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:13.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:13.498 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:13.555 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:57:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:57:13] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.33.4" 2023-09-23T22:57:13.811 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:13.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:13 smithi151 ceph-mon[159210]: pgmap v1000: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 132 B/s rd, 4.7 MiB/s wr, 640 op/s; 45646/276720 objects degraded (16.495%); 31959/276720 objects misplaced (11.549%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:57:13.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/738941688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:13.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:13.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:13.919 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:13.970 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:57:13.999 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:14.116 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:14.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:14.230 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:14.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:13 smithi052 ceph-mon[180979]: pgmap v1000: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 132 B/s rd, 4.7 MiB/s wr, 640 op/s; 45646/276720 objects degraded (16.495%); 31959/276720 objects misplaced (11.549%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:57:14.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/738941688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:13 smithi052 ceph-mon[187743]: pgmap v1000: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 132 B/s rd, 4.7 MiB/s wr, 640 op/s; 45646/276720 objects degraded (16.495%); 31959/276720 objects misplaced (11.549%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:57:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/738941688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:14.277 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:14.483 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:57:14.630 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:14.855 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:14.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:14.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:14.971 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:15.048 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:15.255 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:15.373 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:57:15.374 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "" 2023-09-23T22:57:15.374 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:57:15.725 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:15.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:15.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:15.832 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:15.867 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:16.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:15 smithi151 ceph-mon[159210]: pgmap v1001: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 117 B/s rd, 4.1 MiB/s wr, 715 op/s; 45287/272865 objects degraded (16.597%); 31526/272865 objects misplaced (11.554%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:57:16.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 45287/272865 objects degraded (16.597%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:16.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:15 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:16.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:15 smithi052 ceph-mon[187743]: pgmap v1001: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 117 B/s rd, 4.1 MiB/s wr, 715 op/s; 45287/272865 objects degraded (16.597%); 31526/272865 objects misplaced (11.554%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:57:16.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 45287/272865 objects degraded (16.597%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:16.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:15 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:16.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:15 smithi052 ceph-mon[180979]: pgmap v1001: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 117 B/s rd, 4.1 MiB/s wr, 715 op/s; 45287/272865 objects degraded (16.597%); 31526/272865 objects misplaced (11.554%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:57:16.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 45287/272865 objects degraded (16.597%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:16.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:15 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:57:16.270 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:16.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:16.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:16.374 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:16.816 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:16.911 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:16.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:16.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:57:17.023 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:17.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:16 smithi151 ceph-mon[159210]: pgmap v1002: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 102 B/s rd, 2.8 MiB/s wr, 629 op/s; 45096/271767 objects degraded (16.594%); 31351/271767 objects misplaced (11.536%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:57:17.115 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:17.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:16 smithi052 ceph-mon[180979]: pgmap v1002: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 102 B/s rd, 2.8 MiB/s wr, 629 op/s; 45096/271767 objects degraded (16.594%); 31351/271767 objects misplaced (11.536%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:57:17.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:16 smithi052 ceph-mon[187743]: pgmap v1002: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 102 B/s rd, 2.8 MiB/s wr, 629 op/s; 45096/271767 objects degraded (16.594%); 31351/271767 objects misplaced (11.536%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:57:17.277 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:17.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:17.398 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:17.463 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:17.537 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:57:17.537 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:57:17.537 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:57:17.538 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:57:17.538 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:57:17.567 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:17.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:17.643 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:17.678 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:17.735 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:18.053 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:18.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:18.158 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:18.424 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:57:18.700 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4208045186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:57:18.702 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4208045186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:57:18.714 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4208045186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:57:18.851 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:57:18.880 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:19.003 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:19.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:19.115 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:19.163 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4208045186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: osdmap e341: 8 total, 7 up, 5 in 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.445 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[180979]: pgmap v1004: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 44691/266847 objects degraded (16.748%); 30842/266847 objects misplaced (11.558%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4208045186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: osdmap e341: 8 total, 7 up, 5 in 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:19 smithi151 ceph-mon[159210]: pgmap v1004: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 44691/266847 objects degraded (16.748%); 30842/266847 objects misplaced (11.558%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:57:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4208045186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:57:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: osdmap e341: 8 total, 7 up, 5 in 2023-09-23T22:57:19.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: failed to encode map e341 with expected crc 2023-09-23T22:57:19.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:19 smithi052 ceph-mon[187743]: pgmap v1004: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 670 op/s; 44691/266847 objects degraded (16.748%); 30842/266847 objects misplaced (11.558%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T22:57:19.847 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:20.157 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:20.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:20.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:20.267 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:20.493 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:20.575 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:57:20.607 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:20.639 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:20.679 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:20.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:20.707 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 44691/266847 objects degraded (16.748%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:20.726 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:20.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 44691/266847 objects degraded (16.748%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:20.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 44691/266847 objects degraded (16.748%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:20.777 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:20.809 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:20.809 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:57:20.809 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:57:20.809 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:20.809 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:57:20.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:57:20.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:57:20.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:57:20.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:57:21.094 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:21.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:21.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:21.216 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:21.263 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:21.475 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5] out_osds: [7, 3, 6] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:57:21.476 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:57:21.476 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T22:57:21.476 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T22:57:21.508 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:21 smithi052 ceph-mon[187743]: pgmap v1005: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 44314/263133 objects degraded (16.841%); 30442/263133 objects misplaced (11.569%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T22:57:21.508 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3721199770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:21.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:21 smithi052 ceph-mon[180979]: pgmap v1005: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 44314/263133 objects degraded (16.841%); 30442/263133 objects misplaced (11.569%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T22:57:21.508 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3721199770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:21.760 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:21.857 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:21 smithi151 ceph-mon[159210]: pgmap v1005: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.4 GiB data, 16 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 44314/263133 objects degraded (16.841%); 30442/263133 objects misplaced (11.569%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T22:57:21.857 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3721199770' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:21.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:21.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:21.860 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:21.860 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:21.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:22.060 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:22.129 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:57:22.142 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:22.553 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:22.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:22.667 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:22.764 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:22.920 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:22.920 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:22.935 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:22.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:23.046 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:23.164 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:57:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:57:23] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.33.4" 2023-09-23T22:57:23.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:23 smithi052 ceph-mon[187743]: pgmap v1006: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 657 op/s; 43684/256989 objects degraded (16.998%); 29692/256989 objects misplaced (11.554%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:57:23.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:23 smithi052 ceph-mon[180979]: pgmap v1006: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 657 op/s; 43684/256989 objects degraded (16.998%); 29692/256989 objects misplaced (11.554%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:57:23.802 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:23.861 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:57:23.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:23 smithi151 ceph-mon[159210]: pgmap v1006: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 657 op/s; 43684/256989 objects degraded (16.998%); 29692/256989 objects misplaced (11.554%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T22:57:24.071 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:24.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:24.177 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:24.232 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:57:24.262 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:24.415 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:24.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:24.519 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:24.614 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:24.716 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:24.727 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:24.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:24.846 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:24.939 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:24.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:24 smithi052 conmon[180954]: 2023-09-23T22:57:24.780+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e341 definitely_dead 0 2023-09-23T22:57:25.135 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:25.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:25.247 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:25.268 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:25.320 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:25.530 INFO:teuthology.orchestra.run.smithi052.stderr:marked in osd.3. 2023-09-23T22:57:25.584 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:25.587 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:25.587 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:25.587 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[180979]: pgmap v1007: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 43388/253686 objects degraded (17.103%); 29318/253686 objects misplaced (11.557%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:57:25.587 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1183452174' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:57:25.587 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:57:25.587 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 43684/256989 objects degraded (16.998%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:25.588 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:25.588 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:25.588 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[187743]: pgmap v1007: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 43388/253686 objects degraded (17.103%); 29318/253686 objects misplaced (11.557%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:57:25.588 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1183452174' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:57:25.588 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:57:25.588 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 43684/256989 objects degraded (16.998%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:25.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:25.690 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:25.721 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:25 smithi151 ceph-mon[159210]: pgmap v1007: 89 pgs: 22 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 43388/253686 objects degraded (17.103%); 29318/253686 objects misplaced (11.557%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T22:57:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1183452174' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:57:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:25 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:57:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 43684/256989 objects degraded (16.998%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:26.534 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:26.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:26.619 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T22:57:26.650 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:26.679 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:26 smithi151 ceph-mon[159210]: failed to encode map e342 with expected crc 2023-09-23T22:57:26.679 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:57:26.679 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:26 smithi151 ceph-mon[159210]: osdmap e342: 8 total, 7 up, 6 in 2023-09-23T22:57:26.682 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:26.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:26 smithi052 ceph-mon[187743]: failed to encode map e342 with expected crc 2023-09-23T22:57:26.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:57:26.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:26 smithi052 ceph-mon[187743]: osdmap e342: 8 total, 7 up, 6 in 2023-09-23T22:57:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:26 smithi052 ceph-mon[180979]: failed to encode map e342 with expected crc 2023-09-23T22:57:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:57:26.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:26 smithi052 ceph-mon[180979]: osdmap e342: 8 total, 7 up, 6 in 2023-09-23T22:57:26.930 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:26.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:27.034 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:27.062 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:27.270 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:27.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:27.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:27.374 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:27.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:27.501 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:27.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:27.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:27.520 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:57:27.606 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:27.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: osdmap e343: 8 total, 7 up, 6 in 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: pgmap v1010: 89 pgs: 12 peering, 3 remapped+peering, 6 unknown, 20 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 9 active+undersized, 30 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34250/236547 objects degraded (14.479%); 24034/236547 objects misplaced (10.160%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 341'7020 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2117534659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:27.671 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:27.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:27 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: osdmap e343: 8 total, 7 up, 6 in 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: pgmap v1010: 89 pgs: 12 peering, 3 remapped+peering, 6 unknown, 20 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 9 active+undersized, 30 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34250/236547 objects degraded (14.479%); 24034/236547 objects misplaced (10.160%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 341'7020 2023-09-23T22:57:27.750 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2117534659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: osdmap e343: 8 total, 7 up, 6 in 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: failed to encode map e343 with expected crc 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: pgmap v1010: 89 pgs: 12 peering, 3 remapped+peering, 6 unknown, 20 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 9 active+undersized, 30 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34250/236547 objects degraded (14.479%); 24034/236547 objects misplaced (10.160%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T22:57:27.751 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 341'7020 2023-09-23T22:57:27.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2117534659' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:27.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:27.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:27.752 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:27 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:27.752 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:28.057 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:28.057 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:28.204 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:28.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:28.311 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:28.363 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:28.413 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:28.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:28.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:28.518 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:28.604 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: osdmap e344: 8 total, 7 up, 6 in 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6929 2023-09-23T22:57:28.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6938 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6826 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6969 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 341'6948 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 341'6925 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:28.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:28.815 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:28.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:28.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:28.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:28.826 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: osdmap e344: 8 total, 7 up, 6 in 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6929 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6938 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6826 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6969 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 341'6948 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 341'6925 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:28.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: osdmap e344: 8 total, 7 up, 6 in 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: 10.4 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6929 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: 10.5 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6938 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: failed to encode map e344 with expected crc 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: 10.7 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6826 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 341'6969 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 341'6948 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: 10.f starting backfill to osd.3 from (0'0,0'0] MAX to 341'6925 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:57:28.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:28.921 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:29.004 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:29.196 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:57:29.230 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:29.258 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:29.302 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:29.316 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:29.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:29.335 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:29.366 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.402 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.403 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:57:29.427 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:29.477 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:29.491 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:57:29.606 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:29.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:29.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:29 smithi151 ceph-mon[159210]: pgmap v1012: 89 pgs: 15 peering, 6 remapped+peering, 5 unknown, 16 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 7 active+undersized, 32 active+clean; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 428 op/s; 23967/249273 objects degraded (9.615%); 19473/249273 objects misplaced (7.812%); 2.4 MiB/s, 39 objects/s recovering 2023-09-23T22:57:29.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4237528409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:29.710 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:29.739 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:29 smithi052 ceph-mon[187743]: pgmap v1012: 89 pgs: 15 peering, 6 remapped+peering, 5 unknown, 16 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 7 active+undersized, 32 active+clean; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 428 op/s; 23967/249273 objects degraded (9.615%); 19473/249273 objects misplaced (7.812%); 2.4 MiB/s, 39 objects/s recovering 2023-09-23T22:57:29.739 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4237528409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:29.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:29 smithi052 ceph-mon[180979]: pgmap v1012: 89 pgs: 15 peering, 6 remapped+peering, 5 unknown, 16 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 7 active+undersized, 32 active+clean; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 428 op/s; 23967/249273 objects degraded (9.615%); 19473/249273 objects misplaced (7.812%); 2.4 MiB/s, 39 objects/s recovering 2023-09-23T22:57:29.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4237528409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:29.743 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:29.789 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:29.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:29.893 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:29.931 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:30.255 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:30.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:30.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:30.265 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:30.283 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:30.361 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:30.421 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:30.595 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:30.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:30.689 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 23967/249273 objects degraded (9.615%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:30.708 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:30.745 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:30.802 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:30.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:30.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 23967/249273 objects degraded (9.615%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:30.811 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 23967/249273 objects degraded (9.615%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:30.852 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:57:30.908 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:30.985 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:31.174 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:31.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:31.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:31.286 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:57:31.333 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:31.412 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:31.416 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:57:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:31.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:57:31.527 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:31.570 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:31.620 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [7, 6] dead_osds: [2] live_osds: [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:57:31.621 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:57:31.621 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [1, 3, 4, 5, 6, 7, 0] 2023-09-23T22:57:31.621 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T22:57:31.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:57:31.662 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:31 smithi052 ceph-mon[187743]: pgmap v1013: 89 pgs: 15 peering, 6 remapped+peering, 5 unknown, 15 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 33 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 27274/247407 objects degraded (11.024%); 25841/247407 objects misplaced (10.445%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:57:31.665 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:31 smithi052 ceph-mon[180979]: pgmap v1013: 89 pgs: 15 peering, 6 remapped+peering, 5 unknown, 15 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 33 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 27274/247407 objects degraded (11.024%); 25841/247407 objects misplaced (10.445%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:57:31.709 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:31.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:31.818 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:31.885 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:31 smithi151 ceph-mon[159210]: pgmap v1013: 89 pgs: 15 peering, 6 remapped+peering, 5 unknown, 15 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 33 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 27274/247407 objects degraded (11.024%); 25841/247407 objects misplaced (10.445%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:57:31.953 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:31.960 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:31 smithi052 systemd[1]: Stopping Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:31.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:32.062 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:32.143 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:32.266 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:57:32.475 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:32.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:32.578 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:32 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:57:32.579 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:32.668 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:32.861 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:32.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:32.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:32 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:57:32.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:32.885 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:32 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:57:32.974 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:33.020 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:33.074 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:33.189 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:32 smithi052 conmon[332594]: 2023-09-23T22:57:32.883+0000 7f2fd7030700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:57:33.189 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:32 smithi052 conmon[332594]: 2023-09-23T22:57:32.883+0000 7f2fd7030700 -1 osd.0 344 *** Got signal Terminated *** 2023-09-23T22:57:33.189 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:32 smithi052 conmon[332594]: 2023-09-23T22:57:32.883+0000 7f2fd7030700 -1 osd.0 344 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:57:33.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:33.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:33.352 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:33.365 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:57:33.365 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:57:33.365 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:57:33.365 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:57:33.365 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:57:33.410 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:33.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:57:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:57:33] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.33.4" 2023-09-23T22:57:33.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:33 smithi151 ceph-mon[159210]: pgmap v1014: 89 pgs: 14 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 42444/243426 objects degraded (17.436%); 43840/243426 objects misplaced (18.010%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T22:57:33.598 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:33 smithi151 ceph-mon[159210]: osd.0 marked itself down and dead 2023-09-23T22:57:33.670 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:33.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:33.774 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:33.802 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:33.847 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:33.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:33.951 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:33.978 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:33.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:33 smithi052 ceph-mon[187743]: pgmap v1014: 89 pgs: 14 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 42444/243426 objects degraded (17.436%); 43840/243426 objects misplaced (18.010%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T22:57:33.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:33 smithi052 ceph-mon[187743]: osd.0 marked itself down and dead 2023-09-23T22:57:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:33 smithi052 ceph-mon[180979]: pgmap v1014: 89 pgs: 14 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 6 active+undersized, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 42444/243426 objects degraded (17.436%); 43840/243426 objects misplaced (18.010%); 8.0 MiB/s, 129 objects/s recovering 2023-09-23T22:57:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:33 smithi052 ceph-mon[180979]: osd.0 marked itself down and dead 2023-09-23T22:57:34.021 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:34.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:34.125 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:34.152 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:34.195 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:34.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:34.299 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:34.325 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:34.375 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:34.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:34.479 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:34.516 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:34.625 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:57:34.655 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:34 smithi052 bash[491891]: db01d2d31a60c927d426b4187f1616f5bf6d3b7c5306687c710e5f5a4f0bfe23 2023-09-23T22:57:34.655 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:57:34.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: osdmap e345: 8 total, 6 up, 6 in 2023-09-23T22:57:34.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1444828676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:34.656 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: osdmap e345: 8 total, 6 up, 6 in 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1444828676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:34.657 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:34.675 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:34.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:34.779 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:34.807 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:57:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: osdmap e345: 8 total, 6 up, 6 in 2023-09-23T22:57:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: failed to encode map e345 with expected crc 2023-09-23T22:57:34.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1444828676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:34.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:57:34.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:35.064 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:35.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:35.118 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:57:35.167 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:35.195 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:35.245 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:35.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:35.295 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:35.314 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:35.333 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:35.349 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:35.352 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:35.370 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:35.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:35.388 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:35.388 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:57:35.388 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:57:35.388 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:35.388 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:57:35.388 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:57:35.389 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:57:35.389 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:57:35.389 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:57:35.432 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:35.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:35.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:35.520 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:57:35.538 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:35.573 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: pgmap v1016: 89 pgs: 1 stale+active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 5 stale+active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 stale+active+clean, 2 stale+active+undersized, 1 stale+active+undersized+degraded+remapped+backfill_wait, 2 stale+active+undersized+degraded, 9 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 1.24k op/s; 42444/243426 objects degraded (17.436%); 43840/243426 objects misplaced (18.010%); 7.0 MiB/s, 113 objects/s recovering 2023-09-23T22:57:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:57:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: osdmap e346: 8 total, 6 up, 6 in 2023-09-23T22:57:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: 10.d continuing backfill to osd.3 from (322'1500,344'7206] MIN to 344'7206 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: 10.4 continuing backfill to osd.3 from (322'1800,344'7124] MIN to 344'7124 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: 10.5 continuing backfill to osd.3 from (322'1800,344'7155] 10:a0377578:::benchmark_data_smithi052_461816_object50540:head to 344'7155 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 42444/243426 objects degraded (17.436%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:35.695 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2889821331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: pgmap v1016: 89 pgs: 1 stale+active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 5 stale+active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 stale+active+clean, 2 stale+active+undersized, 1 stale+active+undersized+degraded+remapped+backfill_wait, 2 stale+active+undersized+degraded, 9 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 1.24k op/s; 42444/243426 objects degraded (17.436%); 43840/243426 objects misplaced (18.010%); 7.0 MiB/s, 113 objects/s recovering 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: osdmap e346: 8 total, 6 up, 6 in 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: 10.d continuing backfill to osd.3 from (322'1500,344'7206] MIN to 344'7206 2023-09-23T22:57:35.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: 10.4 continuing backfill to osd.3 from (322'1800,344'7124] MIN to 344'7124 2023-09-23T22:57:35.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: 10.5 continuing backfill to osd.3 from (322'1800,344'7155] 10:a0377578:::benchmark_data_smithi052_461816_object50540:head to 344'7155 2023-09-23T22:57:35.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 42444/243426 objects degraded (17.436%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:35.697 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2889821331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: pgmap v1016: 89 pgs: 1 stale+active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 5 stale+active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 stale+active+clean, 2 stale+active+undersized, 1 stale+active+undersized+degraded+remapped+backfill_wait, 2 stale+active+undersized+degraded, 9 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 1.24k op/s; 42444/243426 objects degraded (17.436%); 43840/243426 objects misplaced (18.010%); 7.0 MiB/s, 113 objects/s recovering 2023-09-23T22:57:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: osdmap e346: 8 total, 6 up, 6 in 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: failed to encode map e346 with expected crc 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: 10.d continuing backfill to osd.3 from (322'1500,344'7206] MIN to 344'7206 2023-09-23T22:57:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: 10.4 continuing backfill to osd.3 from (322'1800,344'7124] MIN to 344'7124 2023-09-23T22:57:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: 10.5 continuing backfill to osd.3 from (322'1800,344'7155] 10:a0377578:::benchmark_data_smithi052_461816_object50540:head to 344'7155 2023-09-23T22:57:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 42444/243426 objects degraded (17.436%), 11 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2889821331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:35.727 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:35.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:35.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:35.831 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:35.882 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:35.952 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:35.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:36.056 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:36.085 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:36.207 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:36.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:36.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:36.267 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:36.311 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:36.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:36.393 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:36.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:36.500 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:36.529 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:36.575 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:36.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:36.679 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:36.719 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:36.868 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:36.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[180979]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[180979]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[180979]: osdmap e347: 8 total, 6 up, 6 in 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[180979]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[180979]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[180979]: 10.c starting backfill to osd.3 from (322'1400,344'7137] MIN to 344'7142 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[187743]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[187743]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[187743]: osdmap e347: 8 total, 6 up, 6 in 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[187743]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[187743]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:36 smithi052 ceph-mon[187743]: 10.c starting backfill to osd.3 from (322'1400,344'7137] MIN to 344'7142 2023-09-23T22:57:36.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:36.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:36 smithi151 ceph-mon[159210]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:36 smithi151 ceph-mon[159210]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:36 smithi151 ceph-mon[159210]: osdmap e347: 8 total, 6 up, 6 in 2023-09-23T22:57:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:36 smithi151 ceph-mon[159210]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:36 smithi151 ceph-mon[159210]: failed to encode map e347 with expected crc 2023-09-23T22:57:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:36 smithi151 ceph-mon[159210]: 10.c starting backfill to osd.3 from (322'1400,344'7137] MIN to 344'7142 2023-09-23T22:57:36.973 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:37.002 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:37.050 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:37.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:37.154 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:37.185 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:37.334 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:37.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:37.412 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:37.412 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:37.437 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:37.465 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:37.511 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:37.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:37.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:37.613 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:37.646 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:37.694 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:37.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:37 smithi052 ceph-mon[180979]: pgmap v1019: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 stale+active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 stale+active+undersized, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 45307/241230 objects degraded (18.782%); 56323/241230 objects misplaced (23.348%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:57:37.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:37 smithi052 ceph-mon[187743]: pgmap v1019: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 stale+active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 stale+active+undersized, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 45307/241230 objects degraded (18.782%); 56323/241230 objects misplaced (23.348%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:57:37.789 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T22:57:37.797 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:37.832 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:57:37.832 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T22:57:37.900 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:37.946 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:37.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:37 smithi151 ceph-mon[159210]: pgmap v1019: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 stale+active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 stale+active+undersized, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 45307/241230 objects degraded (18.782%); 56323/241230 objects misplaced (23.348%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:57:37.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:37.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:38.017 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:57:38.053 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:38.062 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:57:38.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:38.142 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:38.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:38.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:38.250 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:38.268 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:57:38.277 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:38.321 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:38.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:38.430 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:38.460 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:38.499 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:38.516 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:38.531 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:38.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:38.637 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:38.668 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:38.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:38.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:38.874 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:38.904 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:38.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:38 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:57:38.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:38 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:57:38.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:38 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:57:39.505 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:39.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:39.609 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:39.638 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:39.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:39 smithi052 ceph-mon[187743]: pgmap v1020: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 45307/241230 objects degraded (18.782%); 56325/241230 objects misplaced (23.349%); 1.5 MiB/s, 22 objects/s recovering 2023-09-23T22:57:39.706 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:39 smithi052 ceph-mon[180979]: pgmap v1020: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 45307/241230 objects degraded (18.782%); 56325/241230 objects misplaced (23.349%); 1.5 MiB/s, 22 objects/s recovering 2023-09-23T22:57:39.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:39 smithi151 ceph-mon[159210]: pgmap v1020: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 45307/241230 objects degraded (18.782%); 56325/241230 objects misplaced (23.349%); 1.5 MiB/s, 22 objects/s recovering 2023-09-23T22:57:40.028 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:40.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:40.129 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:40.131 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:40.158 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:40.207 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:40.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:40.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:40.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:40.338 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:40.385 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:40.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:40.490 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:40.515 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:40.560 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:40.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:40.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:40.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:40.693 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:40.743 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:40.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:40.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:40.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:40.871 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 45307/241230 objects degraded (18.782%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:40.874 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:40.918 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:40.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:40.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:40.969 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:40.969 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:40.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 45307/241230 objects degraded (18.782%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:40.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 45307/241230 objects degraded (18.782%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:41.026 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:41.053 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:41.099 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:41.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:41.204 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:41.230 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:41.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:41.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:41.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:41.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:41.411 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:41.460 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:41.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:41.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:41.567 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:41.593 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:41.646 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:41.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:41.683 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:57:41.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:41 smithi052 ceph-mon[187743]: pgmap v1021: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 45307/241230 objects degraded (18.782%); 56325/241230 objects misplaced (23.349%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:57:41.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3783154091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:41.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:41.708 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:41 smithi052 ceph-mon[180979]: pgmap v1021: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 45307/241230 objects degraded (18.782%); 56325/241230 objects misplaced (23.349%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:57:41.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3783154091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:41.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:41.753 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:41.794 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:41 smithi151 ceph-mon[159210]: pgmap v1021: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 45307/241230 objects degraded (18.782%); 56325/241230 objects misplaced (23.349%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:57:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3783154091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:42.221 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:57:42.350 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:42.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:42.454 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:42.480 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:42.526 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:42.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:42.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:42.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:42.707 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:42.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:57:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[180979]: failed to encode map e348 with expected crc 2023-09-23T22:57:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:42.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[180979]: osdmap e348: 8 total, 6 up, 6 in 2023-09-23T22:57:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:57:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[187743]: failed to encode map e348 with expected crc 2023-09-23T22:57:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:42.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:42 smithi052 ceph-mon[187743]: osdmap e348: 8 total, 6 up, 6 in 2023-09-23T22:57:42.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:42.833 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [7, 6] dead_osds: [2, 0] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:57:42.833 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:57:42.833 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:57:42.833 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.610458 2023-09-23T22:57:42.833 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0.6104579423016018 2023-09-23T22:57:42.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:42 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:57:42.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:42 smithi151 ceph-mon[159210]: failed to encode map e348 with expected crc 2023-09-23T22:57:42.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:42.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:42 smithi151 ceph-mon[159210]: osdmap e348: 8 total, 6 up, 6 in 2023-09-23T22:57:42.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:42.849 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:42.883 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:42.892 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:42.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:42.904 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:43.000 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:43.028 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:43.086 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:43.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:43.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:43.191 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:43.217 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:43.260 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:43.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:43.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:43.330 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:43.367 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:43.399 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:43.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:57:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:57:43] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.33.4" 2023-09-23T22:57:43.495 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:43.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:43.599 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:43.625 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:43.669 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:43.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:43.776 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:43.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:43 smithi052 ceph-mon[187743]: pgmap v1023: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 45191/241230 objects degraded (18.734%); 56325/241230 objects misplaced (23.349%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:57:43.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2471076691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:43.802 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:43 smithi052 ceph-mon[180979]: pgmap v1023: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 45191/241230 objects degraded (18.734%); 56325/241230 objects misplaced (23.349%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:57:43.802 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2471076691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:43.805 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:43 smithi151 ceph-mon[159210]: pgmap v1023: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 45191/241230 objects degraded (18.734%); 56325/241230 objects misplaced (23.349%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:57:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2471076691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:44.003 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:44.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:44.108 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:44.138 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:44.288 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:44.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:44.391 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:44.420 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:44.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:57:44.617 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:44.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:44.721 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:44.750 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:44.901 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:44.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:44.962 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:57:44.962 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:57:44.962 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:57:44.962 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:57:44.962 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:57:45.006 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:45.046 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:45.725 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:57:45.794 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:45 smithi052 ceph-mon[187743]: pgmap v1024: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 45191/241230 objects degraded (18.734%); 56325/241230 objects misplaced (23.349%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:57:45.794 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 45191/241230 objects degraded (18.734%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:45.794 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:45 smithi052 ceph-mon[180979]: pgmap v1024: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 45191/241230 objects degraded (18.734%); 56325/241230 objects misplaced (23.349%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:57:45.794 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 45191/241230 objects degraded (18.734%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:45.795 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:45.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:45.820 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:45.900 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:45.928 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:45.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:45 smithi151 ceph-mon[159210]: pgmap v1024: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 45191/241230 objects degraded (18.734%); 56325/241230 objects misplaced (23.349%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:57:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 45191/241230 objects degraded (18.734%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:46.029 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:46.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:46.135 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:46.161 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:46.208 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:46.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:46.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:46.314 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:46.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:46.386 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:46.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:46.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:46.520 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:46.563 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:46.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:46.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:46.668 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:46.692 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.0 primary-affinity to 0.610458 (8400062) 2023-09-23T22:57:46.699 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:46.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1959962201' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]: dispatch 2023-09-23T22:57:46.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:46 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]: dispatch 2023-09-23T22:57:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1959962201' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]: dispatch 2023-09-23T22:57:46.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:46 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]: dispatch 2023-09-23T22:57:46.758 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:46.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:46.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:46.864 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:46.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1959962201' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]: dispatch 2023-09-23T22:57:46.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:46 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]: dispatch 2023-09-23T22:57:46.891 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:46.936 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:46.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:46.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:46.971 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:47.017 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:47.037 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:47.043 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:47.055 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:47.070 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:47.075 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:47.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:57:47.075 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:57:47.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:47.075 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:57:47.075 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:57:47.076 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:57:47.076 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:57:47.076 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:57:47.116 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:47.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:47.222 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:47.256 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:47.306 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:47.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:47.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:47.442 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:47.608 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:47.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:47.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[187743]: pgmap v1025: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 45076/241230 objects degraded (18.686%); 56325/241230 objects misplaced (23.349%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:57:47.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[187743]: failed to encode map e349 with expected crc 2023-09-23T22:57:47.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]': finished 2023-09-23T22:57:47.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[187743]: osdmap e349: 8 total, 6 up, 6 in 2023-09-23T22:57:47.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[180979]: pgmap v1025: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 45076/241230 objects degraded (18.686%); 56325/241230 objects misplaced (23.349%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:57:47.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[180979]: failed to encode map e349 with expected crc 2023-09-23T22:57:47.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]': finished 2023-09-23T22:57:47.690 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:47 smithi052 ceph-mon[180979]: osdmap e349: 8 total, 6 up, 6 in 2023-09-23T22:57:47.712 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:47.739 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:47.783 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:47.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:47.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:47.891 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:47.917 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:47 smithi151 ceph-mon[159210]: pgmap v1025: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 31 active+clean, 3 active+recovery_wait+degraded+remapped, 21 active+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 stale+active+clean, 2 undersized+peered, 2 undersized+degraded+peered, 6 active+clean+remapped, 1 stale+active+remapped+backfill_wait; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 45076/241230 objects degraded (18.686%); 56325/241230 objects misplaced (23.349%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T22:57:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:47 smithi151 ceph-mon[159210]: failed to encode map e349 with expected crc 2023-09-23T22:57:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6104579423016018}]': finished 2023-09-23T22:57:47.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:47 smithi151 ceph-mon[159210]: osdmap e349: 8 total, 6 up, 6 in 2023-09-23T22:57:47.962 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:47.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:48.067 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:48.098 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:48.290 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:48.298 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:48.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:48.401 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:48.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:48.531 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:48.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:48.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:48.635 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:48.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:48.693 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:48 smithi052 ceph-mon[187743]: failed to encode map e349 with expected crc 2023-09-23T22:57:48.694 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:48 smithi052 ceph-mon[187743]: failed to encode map e349 with expected crc 2023-09-23T22:57:48.694 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1209107610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:57:48.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:48 smithi052 ceph-mon[180979]: failed to encode map e349 with expected crc 2023-09-23T22:57:48.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:48 smithi052 ceph-mon[180979]: failed to encode map e349 with expected crc 2023-09-23T22:57:48.694 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1209107610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:57:48.698 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:57:48.710 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:48.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:48.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:48.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:48.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:48.912 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:48.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:48 smithi151 ceph-mon[159210]: failed to encode map e349 with expected crc 2023-09-23T22:57:48.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:48 smithi151 ceph-mon[159210]: failed to encode map e349 with expected crc 2023-09-23T22:57:48.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1209107610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:57:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:48.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:49.018 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:49.045 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:49.090 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:49.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:49.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:49.197 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:49.224 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:49.273 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:49.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:49.380 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:49.410 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:49.459 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:49.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:49.564 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:49.592 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:49.691 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:49.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:49.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:49.796 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:49.822 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:49.866 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:49.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:49.908 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:57:49.908 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:57:49.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:57:49.908 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:57:49.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110891 9.3 9:c27b0003:::benchmark_data_smithi052_402718_object21250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 268.850057488, 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25767365199999998, 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110891 2023-09-23T22:57:49.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055633+0000", 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055637+0000", 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.126e-06 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055645+0000", 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9850000000000006e-06 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055650+0000", 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4990000000000002e-06 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055758+0000", 2023-09-23T22:57:49.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010737 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055921+0000", 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163437 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.056098+0000", 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000176563 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.059728+0000", 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036304570000000001 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.062797+0000", 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00306851 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312384+0000", 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.249587273 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312438+0000", 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4354999999999997e-05 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313307+0000", 2023-09-23T22:57:49.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00086921899999999996 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110899 9.3 9:c9f9c7b0:::benchmark_data_smithi052_402718_object21258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 268.84239417800001, 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29968218299999999, 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.916 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110899 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063299+0000", 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5469999999999999e-06 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063306+0000", 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6889999999999998e-06 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063312+0000", 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6489999999999996e-06 2023-09-23T22:57:49.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066300+0000", 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029886909999999999 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066461+0000", 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016098600000000001 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:57:49.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066620+0000", 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015845099999999999 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312307+0000", 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24568741599999999 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312349+0000", 2023-09-23T22:57:49.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1686e-05 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362127+0000", 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049778356000000003 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362181+0000", 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3421999999999998e-05 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362979+0000", 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079871 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110900 9.3 9:cfcfdec5:::benchmark_data_smithi052_402718_object21259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:57:49.923 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:57:49.924 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 268.84237410100002, 2023-09-23T22:57:49.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.300308513, 2023-09-23T22:57:49.924 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.924 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110900 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063319+0000", 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.525e-06 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063325+0000", 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5999999999999997e-06 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063329+0000", 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5170000000000001e-06 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069300+0000", 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059708189999999996 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069454+0000", 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015409899999999999 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069610+0000", 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155332 2023-09-23T22:57:49.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312314+0000", 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24270393900000001 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313359+0000", 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010457520000000001 2023-09-23T22:57:49.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363082+0000", 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049722865999999998 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363133+0000", 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0757999999999998e-05 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363626+0000", 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000492659 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110905 9.3 9:cdd24a36:::benchmark_data_smithi052_402718_object21264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 268.84148946200003, 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29997646300000003, 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.932 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110905 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064201+0000", 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064206+0000", 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7419999999999997e-06 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064217+0000", 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.094e-05 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064224+0000", 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2529999999999998e-06 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072314+0000", 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0080899820000000008 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072465+0000", 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015051600000000001 2023-09-23T22:57:49.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.309128+0000", 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23666314399999999 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.314036+0000", 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049078869999999997 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.320379+0000", 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00634324 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363720+0000", 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.043340956999999999 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363771+0000", 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1734999999999998e-05 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.364178+0000", 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040673700000000001 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44617 10.2 e325/317 10:42f612af:::benchmark_data_smithi052_461816_object44616:head v 325'2757, mlcod=325'2755)", 2023-09-23T22:57:49.939 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.230235+0000", 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 113.67545663, 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.097530256999999995, 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230234+0000", 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230284+0000", 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0538999999999998e-05 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230294+0000", 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2580000000000001e-06 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230301+0000", 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0979999999999996e-06 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230348+0000", 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6904999999999999e-05 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230370+0000", 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2036999999999999e-05 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.231896+0000", 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015261840000000001 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327765+0000", 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095868996999999997 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.944 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44626 10.2 e325/317 10:4fdb9bab:::benchmark_data_smithi052_461816_object44625:head v 325'2758, mlcod=325'2755)", 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.234607+0000", 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 113.67108465699999, 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096288367, 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234606+0000", 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234684+0000", 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8349999999999994e-05 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327708+0000", 2023-09-23T22:57:49.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.093023764999999994 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327737+0000", 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9223000000000001e-05 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327860+0000", 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122982 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327916+0000", 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5405999999999998e-05 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330864+0000", 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002948492 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330895+0000", 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0898999999999997e-05 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44632 10.b e325/317 10:d9603e28:::benchmark_data_smithi052_461816_object44631:head v 325'2853, mlcod=325'2851)", 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.237428+0000", 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 113.668263079, 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096280273999999999, 2023-09-23T22:57:49.950 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237427+0000", 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237481+0000", 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3566000000000002e-05 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327715+0000", 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.090234522999999997 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327736+0000", 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0746000000000001e-05 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327869+0000", 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013274200000000001 2023-09-23T22:57:49.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327926+0000", 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7645999999999997e-05 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333664+0000", 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057375569999999999 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333708+0000", 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4566e-05 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71910 10.b e327/317 10:da00891c:::benchmark_data_smithi052_461816_object71909:head v 327'4562, mlcod=327'4560)", 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.838808+0000", 2023-09-23T22:57:49.955 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 92.066883492000002, 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070775292000000004, 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838807+0000", 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838881+0000", 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3671999999999995e-05 2023-09-23T22:57:49.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838927+0000", 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6146999999999997e-05 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838934+0000", 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3769999999999998e-06 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838979+0000", 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.494e-05 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838998+0000", 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8782000000000001e-05 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909541+0000", 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070543146000000001 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909583+0000", 2023-09-23T22:57:49.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1967e-05 2023-09-23T22:57:49.960 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.960 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.960 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71913 10.1 e327/317 10:868eb1a2:::benchmark_data_smithi052_461816_object71912:head v 327'4350, mlcod=327'4347)", 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.840731+0000", 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 92.064960163999999, 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.066217576, 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840730+0000", 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840777+0000", 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6655e-05 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T22:57:49.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061592056999999999 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902407+0000", 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7666000000000002e-05 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902538+0000", 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000131793 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902555+0000", 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6888000000000002e-05 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906899+0000", 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043436869999999997 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906949+0000", 2023-09-23T22:57:49.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9629999999999997e-05 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71918 10.b e327/317 10:d9e3a463:::benchmark_data_smithi052_461816_object71917:head v 327'4563, mlcod=327'4560)", 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.844519+0000", 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 92.061172165000002, 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074161447000000005, 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.966 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844518+0000", 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844600+0000", 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2000000000000001e-05 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.057768742999999997 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902406+0000", 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7252999999999997e-05 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.904983+0000", 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025770709999999998 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.905005+0000", 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2028e-05 2023-09-23T22:57:49.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918640+0000", 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01363467 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918681+0000", 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0522000000000003e-05 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.970 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104103 10.c e340/329 10:3a122c6e:::benchmark_data_smithi052_461816_object7302:head v 340'6509, mlcod=0'0)", 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.450864+0000", 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.454827180000002, 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083293366999999993, 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450864+0000", 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450864+0000", 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450862+0000", 2023-09-23T22:57:49.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450879+0000", 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7056000000000001e-05 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450910+0000", 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0960000000000002e-05 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450929+0000", 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.853e-05 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450997+0000", 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8225000000000005e-05 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451014+0000", 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6988999999999999e-05 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534114+0000", 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083099808999999997 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534157+0000", 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3625000000000003e-05 2023-09-23T22:57:49.975 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104104 10.c e340/329 10:3017b5bb:::benchmark_data_smithi052_461816_object7303:head v 340'6510, mlcod=0'0)", 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.451247+0000", 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.454444520000003, 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.088953937999999996, 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451247+0000", 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451247+0000", 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451246+0000", 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:49.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451251+0000", 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3050000000000003e-06 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451258+0000", 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9550000000000003e-06 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451263+0000", 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1240000000000004e-06 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457336+0000", 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060733820000000004 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457354+0000", 2023-09-23T22:57:49.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7779999999999999e-05 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540161+0000", 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.082807243000000003 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540201+0000", 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9532e-05 2023-09-23T22:57:49.980 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104107 10.4 e340/317 10:2627ee68:::benchmark_data_smithi052_461816_object7306:head v 340'6525, mlcod=340'6521)", 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.451379+0000", 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.454312561999998, 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.082812793999999995, 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.981 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451379+0000", 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451379+0000", 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451378+0000", 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451384+0000", 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4760000000000003e-06 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451394+0000", 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9410000000000002e-06 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451400+0000", 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.015e-06 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457323+0000", 2023-09-23T22:57:49.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059230339999999998 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457338+0000", 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4263e-05 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534163+0000", 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.076825685000000005 2023-09-23T22:57:49.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534191+0000", 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8124999999999999e-05 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104106 10.c e340/329 10:3b7c91e7:::benchmark_data_smithi052_461816_object7305:head v 340'6511, mlcod=0'0)", 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.457021+0000", 2023-09-23T22:57:49.986 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.448669907999999, 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083208802999999998, 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457021+0000", 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457021+0000", 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457020+0000", 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457029+0000", 2023-09-23T22:57:49.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2579999999999995e-06 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457043+0000", 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4239e-05 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457063+0000", 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9656999999999998e-05 2023-09-23T22:57:49.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.463087+0000", 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060247390000000003 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.463117+0000", 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9611000000000002e-05 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540206+0000", 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.077089550000000007 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540230+0000", 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3666999999999999e-05 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.991 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104110 10.c e340/329 10:3405d415:::benchmark_data_smithi052_461816_object7309:head v 340'6512, mlcod=0'0)", 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.460318+0000", 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.445373625000002, 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.079936283999999996, 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460318+0000", 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460318+0000", 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460316+0000", 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:49.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460327+0000", 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0478e-05 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460337+0000", 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5440000000000004e-06 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460342+0000", 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2249999999999999e-06 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.520173+0000", 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.059831670000000003 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:49.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.520196+0000", 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2121e-05 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540232+0000", 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020036824000000002 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:49.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540254+0000", 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1539000000000002e-05 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104853 10.c e340/329 10:3b1f5b6f:::benchmark_data_smithi052_461816_object8052:head v 340'6559, mlcod=0'0)", 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924581+0000", 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 33.981109967000002, 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080972484999999997, 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:49.997 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T22:57:49.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924585+0000", 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0080000000000004e-06 2023-09-23T22:57:49.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924591+0000", 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.942e-06 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924595+0000", 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4329999999999997e-06 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924640+0000", 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4134999999999997e-05 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924664+0000", 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4360999999999999e-05 2023-09-23T22:57:50.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:16.005514+0000", 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080849657000000005 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:16.005554+0000", 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0201999999999999e-05 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:50.002 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104855 10.4 e340/317 10:2c6443b8:::benchmark_data_smithi052_461816_object8054:head v 340'6568, mlcod=340'6566)", 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924876+0000", 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 33.980815405999998, 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.075044514000000007, 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924876+0000", 2023-09-23T22:57:50.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924876+0000", 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924875+0000", 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924881+0000", 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.57e-06 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924890+0000", 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1330000000000005e-06 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:50.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924896+0000", 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924934+0000", 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7849000000000003e-05 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924952+0000", 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8175000000000001e-05 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.999878+0000", 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074926648999999998 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.999920+0000", 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1924e-05 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:50.007 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104856 10.2 e340/317 10:4d1d554a:::benchmark_data_smithi052_461816_object8055:head v 340'6546, mlcod=340'6544)", 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924973+0000", 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 33.980718084000003, 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070607910999999995, 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T22:57:50.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:57:50.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924978+0000", 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9599999999999999e-06 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924986+0000", 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4589999999999998e-06 2023-09-23T22:57:50.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924992+0000", 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5219999999999998e-06 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.986453+0000", 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061461233999999997 2023-09-23T22:57:50.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.986477+0000", 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3586000000000001e-05 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.995543+0000", 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090668009999999993 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.995581+0000", 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7732999999999999e-05 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113501 10.4 e344/343 10:2471dee2:::benchmark_data_smithi052_461816_object16700:head v 344'7088, mlcod=344'7086)", 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.300805+0000", 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.604885993, 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.114329005, 2023-09-23T22:57:50.013 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300805+0000", 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300805+0000", 2023-09-23T22:57:50.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300804+0000", 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300813+0000", 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.312e-06 2023-09-23T22:57:50.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300829+0000", 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5485000000000002e-05 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300839+0000", 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0029e-05 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300894+0000", 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5551999999999999e-05 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405126+0000", 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.104231453 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.415005+0000", 2023-09-23T22:57:50.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098791069999999998 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.415134+0000", 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012936600000000001 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113493 10.c e344/343 10:3a2fa64b:::benchmark_data_smithi052_461816_object16692:head v 344'7088, mlcod=0'0)", 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.302714+0000", 2023-09-23T22:57:50.018 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.602977682999999, 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.115107467, 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302714+0000", 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:57:50.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302714+0000", 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302713+0000", 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302720+0000", 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9609999999999999e-06 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405115+0000", 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10239490699999999 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405128+0000", 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3213e-05 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405167+0000", 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8720000000000002e-05 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405189+0000", 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2401000000000001e-05 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:57:50.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.417788+0000", 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012598971 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.417821+0000", 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3126000000000001e-05 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:57:50.023 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:57:50.024 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: pgmap v1027: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 undersized+degraded+remapped+backfilling+peered, 9 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 77109/238197 objects degraded (32.372%); 59602/238197 objects misplaced (25.022%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: failed to encode map e350 with expected crc 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: failed to encode map e350 with expected crc 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1209107610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: osdmap e350: 8 total, 6 up, 6 in 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: pgmap v1027: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 undersized+degraded+remapped+backfilling+peered, 9 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 77109/238197 objects degraded (32.372%); 59602/238197 objects misplaced (25.022%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:50.025 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: failed to encode map e350 with expected crc 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: failed to encode map e350 with expected crc 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1209107610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: osdmap e350: 8 total, 6 up, 6 in 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: pgmap v1027: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 undersized+degraded+remapped+backfilling+peered, 9 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 77109/238197 objects degraded (32.372%); 59602/238197 objects misplaced (25.022%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: failed to encode map e350 with expected crc 2023-09-23T22:57:50.026 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: failed to encode map e350 with expected crc 2023-09-23T22:57:50.027 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1209107610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:57:50.027 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: osdmap e350: 8 total, 6 up, 6 in 2023-09-23T22:57:50.027 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:57:50.027 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:57:50.055 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:50.204 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:50.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:50.291 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:57:50.308 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:50.334 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:50.376 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:50.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:50.482 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:50.513 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:50.593 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:57:50.612 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:50.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:50.670 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:50.718 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:50.744 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:50.786 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:50.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:50.874 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 77109/238197 objects degraded (32.372%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:50.874 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 77109/238197 objects degraded (32.372%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:50.891 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:50.917 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:50.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 77109/238197 objects degraded (32.372%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:50.959 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:50.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:51.066 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:51.097 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:51.295 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:51.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:51.399 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:51.424 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:51.467 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:51.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:51.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:51.516 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:57:51.573 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:51.601 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:51.669 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:51.699 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:51.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:51.803 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:51.828 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:51.870 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:51.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:51.883 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:51 smithi052 ceph-mon[187743]: pgmap v1029: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 undersized+degraded+remapped+backfilling+peered, 9 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 77109/238197 objects degraded (32.372%); 59602/238197 objects misplaced (25.022%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:57:51.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:51 smithi052 ceph-mon[180979]: pgmap v1029: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 undersized+degraded+remapped+backfilling+peered, 9 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 77109/238197 objects degraded (32.372%); 59602/238197 objects misplaced (25.022%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:57:51.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:51 smithi151 ceph-mon[159210]: pgmap v1029: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 undersized+degraded+remapped+backfilling+peered, 9 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 77109/238197 objects degraded (32.372%); 59602/238197 objects misplaced (25.022%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:57:51.976 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:52.002 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:52.043 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:52.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:52.148 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:52.173 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:52.215 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:52.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:52.274 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:57:52.299 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:57:52.319 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:57:52.321 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:52.344 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:57:52.346 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:52.364 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:57:52.382 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:57:52.393 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:52.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:52.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:52.431 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:52.496 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:52.527 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:52.624 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [7, 6] dead_osds: [2, 0] live_osds: [1, 3, 4, 5, 6, 7] 2023-09-23T22:57:52.624 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-09-23T22:57:52.624 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-09-23T22:57:52.625 DEBUG:teuthology.orchestra.run.smithi052:osd.0> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T22:57:52.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl start ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:57:52.628 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:52.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:52.657 INFO:journalctl@ceph.osd.0.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:57:52.731 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:52.756 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:52.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:52.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3910711977' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:52.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3910711977' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:52.995 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:52 smithi052 systemd[1]: Starting Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:57:53.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:53.040 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3910711977' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:57:53.371 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:53.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:53.475 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:53.507 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:57:53.587 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:57:53.621 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:57:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:57:53] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T22:57:53.860 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:53.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:53.873 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:53 smithi052 ceph-mon[187743]: pgmap v1030: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 76992/238191 objects degraded (32.324%); 59600/238191 objects misplaced (25.022%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:57:53.873 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:53 smithi052 ceph-mon[180979]: pgmap v1030: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 76992/238191 objects degraded (32.324%); 59600/238191 objects misplaced (25.022%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:57:53.965 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:53.988 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:53 smithi151 ceph-mon[159210]: pgmap v1030: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 76992/238191 objects degraded (32.324%); 59600/238191 objects misplaced (25.022%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:57:53.991 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:54.033 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:54.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:54.139 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:54.163 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:54.211 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:54.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:54.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:54.316 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:54.348 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:54.397 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:54.397 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:54.498 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:54.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:54.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:54.603 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:54.628 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:54.671 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:54.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:54.778 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:54.803 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:54.845 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:54.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:54.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:54.951 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:54.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:55.238 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:55.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:55.344 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:55.347 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:57:55.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:55.411 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:55.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:55.516 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:55.549 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:55.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:57:55.657 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:55.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:55.744 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:55.762 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:55.787 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:55.833 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:55.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:55.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:55.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[180979]: pgmap v1031: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 76992/238191 objects degraded (32.324%); 59600/238191 objects misplaced (25.022%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:57:55.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 76992/238191 objects degraded (32.324%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:55.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/418188027' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:55.942 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:55.943 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[187743]: pgmap v1031: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 76992/238191 objects degraded (32.324%); 59600/238191 objects misplaced (25.022%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:57:55.943 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 76992/238191 objects degraded (32.324%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:55.943 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/418188027' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:55.943 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:55 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:55.978 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:56.029 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:56.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:56.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:56.132 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:56.156 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:55 smithi151 ceph-mon[159210]: pgmap v1031: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 76992/238191 objects degraded (32.324%); 59600/238191 objects misplaced (25.022%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:57:56.156 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 76992/238191 objects degraded (32.324%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:57:56.156 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/418188027' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:56.156 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:55 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:57:56.158 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:56.201 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:56.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:56.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:56.224 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:57:56.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:57:56.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:57:56.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T22:57:56.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:57:56.243 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:57:56.244 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T22:57:56.244 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:57:56.244 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:55 smithi052 bash[499125]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T22:57:56.306 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:56.331 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:56.377 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:56.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:56.482 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:56.513 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:56.769 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:56.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:56.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:56.770 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[187743]: osdmap e351: 8 total, 6 up, 6 in 2023-09-23T22:57:56.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[187743]: failed to encode map e351 with expected crc 2023-09-23T22:57:56.771 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:56.771 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:56.771 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[180979]: osdmap e351: 8 total, 6 up, 6 in 2023-09-23T22:57:56.771 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:56 smithi052 ceph-mon[180979]: failed to encode map e351 with expected crc 2023-09-23T22:57:56.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:56.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:56.873 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:56.897 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:56.938 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:56.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:57.043 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:57:57.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:56 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:57:57.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:56 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:57:57.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:56 smithi151 ceph-mon[159210]: osdmap e351: 8 total, 6 up, 6 in 2023-09-23T22:57:57.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:56 smithi151 ceph-mon[159210]: failed to encode map e351 with expected crc 2023-09-23T22:57:57.069 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:57.115 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:57:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:57.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:57:57.220 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:57.246 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:57.288 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:57.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:57.393 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:57.422 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:57.609 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:57.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:57.679 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:57.713 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:57.739 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:57.782 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:57.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:57.889 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:57.914 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:57.956 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:57.964 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:57 smithi151 ceph-mon[159210]: failed to encode map e351 with expected crc 2023-09-23T22:57:57.964 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:57 smithi151 ceph-mon[159210]: pgmap v1033: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76876/238188 objects degraded (32.275%); 59598/238188 objects misplaced (25.021%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:57:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:57.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:57 smithi052 ceph-mon[187743]: failed to encode map e351 with expected crc 2023-09-23T22:57:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:57 smithi052 ceph-mon[187743]: pgmap v1033: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76876/238188 objects degraded (32.275%); 59598/238188 objects misplaced (25.021%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:57:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:57 smithi052 ceph-mon[180979]: failed to encode map e351 with expected crc 2023-09-23T22:57:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:57 smithi052 ceph-mon[180979]: pgmap v1033: 89 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait+laggy, 2 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76876/238188 objects degraded (32.275%); 59598/238188 objects misplaced (25.021%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:57:58.067 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:57:58.092 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:58.134 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:57:58.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:58.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:57:58.239 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:57:58.279 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:58.377 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:57:58.483 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:57:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:58.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:57:58.587 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:58.612 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:58.654 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:58.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:58.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:58.663 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:57:58.663 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:57:58.687 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:58 smithi052 ceph-osd[500529]: -- 172.21.15.52:0/4113058525 <== mon.0 v2:172.21.15.52:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a27422c4e0 con 0x55a2741ff000 2023-09-23T22:57:58.687 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:58 smithi052 bash[500360]: e622ae7bfc72d210da7cb0bf8f045803ef675690320eece65db595fce2686381 2023-09-23T22:57:58.687 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:58 smithi052 systemd[1]: Started Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:57:58.762 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:57:58.788 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:58.829 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:57:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:58.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:57:58.934 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:58.959 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:58.977 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:57:58 smithi052 conmon[500475]: 2023-09-23T22:57:58.881+0000 7f03a73d9640 -1 Falling back to public interface 2023-09-23T22:57:59.008 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:59.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:59.112 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:57:59.142 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:59.435 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:57:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:59.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:57:59.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:59.565 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:59.607 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:59.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:59.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:59.711 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:57:59 smithi052 ceph-mon[187743]: pgmap v1034: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76871/238170 objects degraded (32.276%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:57:59.711 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:57:59 smithi052 ceph-mon[180979]: pgmap v1034: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76871/238170 objects degraded (32.276%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:57:59.711 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:57:59.714 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:59.739 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:59.781 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:59.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:57:59.809 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:57:59.871 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:57:59.887 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:57:59.910 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:57:59 smithi151 ceph-mon[159210]: pgmap v1034: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76871/238170 objects degraded (32.276%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:57:59.913 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:57:59.954 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:57:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:57:59.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:00.059 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:00.084 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:00.125 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:00.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:00.198 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:00.224 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:00.231 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:00.249 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:00.256 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:00.270 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:00.288 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:00.303 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:00.306 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:58:00.307 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:58:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:00.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:00.409 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:00.435 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:00.478 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:00.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:00.583 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:00.614 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:00.761 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:00.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:00.834 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1789708441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:00.835 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 76871/238170 objects degraded (32.276%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:00.835 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1789708441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:00.835 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 76871/238170 objects degraded (32.276%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:00.865 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:00.892 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:00.934 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:00.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:01.040 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:01.070 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:01.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1789708441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 76871/238170 objects degraded (32.276%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:01.291 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:01.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:01.395 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:01.410 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:58:01.411 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:58:01.421 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:01.438 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:01.464 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:01.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:01.489 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:01.569 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:01.606 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:01.820 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:01.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:01.873 INFO:teuthology.orchestra.run.smithi052.stderr:no valid command found; 10 closest matches: 2023-09-23T22:58:01.873 INFO:teuthology.orchestra.run.smithi052.stderr:0 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:1 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:2 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:abort 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:assert 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs debug_inject_read_zeros 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs files list 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:bluefs stats 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:bluestore bluefs device info [] 2023-09-23T22:58:01.874 INFO:teuthology.orchestra.run.smithi052.stderr:config diff 2023-09-23T22:58:01.875 INFO:teuthology.orchestra.run.smithi052.stderr:admin_socket: invalid command 2023-09-23T22:58:01.891 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:01 smithi052 ceph-mon[187743]: pgmap v1035: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76871/238170 objects degraded (32.276%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:01.891 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:01 smithi052 ceph-mon[180979]: pgmap v1035: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76871/238170 objects degraded (32.276%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:01.925 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:01.951 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:01.994 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:01.998 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:01 smithi151 ceph-mon[159210]: pgmap v1035: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76871/238170 objects degraded (32.276%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:01.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:02.100 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:02.125 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:02.168 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:02.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:02.275 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:02.307 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:02.557 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:02.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:02.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:02.694 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:03.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:58:03.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:58:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:58:03] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T22:58:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:58:03.597 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2023-09-23T22:58:03.599 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:03.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:03.705 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:03.731 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:03.773 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:03.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:03.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:03 smithi052 ceph-mon[180979]: pgmap v1036: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76755/238170 objects degraded (32.227%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:03.850 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:03 smithi052 ceph-mon[187743]: pgmap v1036: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76755/238170 objects degraded (32.227%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:03.879 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:03.904 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:03.946 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:03.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:04.052 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:58:04.052 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T22:58:04.053 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T22:58:04.053 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:58:04.053 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.053 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45206.0:140973 8.f 8:f33d72b0:::benchmark_data_smithi052_337357_object38522:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-09-23T22:58:04.053 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:58:04.053 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 545.25405915299996, 2023-09-23T22:58:04.054 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.329060238, 2023-09-23T22:58:04.054 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.054 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.054 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.054 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45206", 2023-09-23T22:58:04.054 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/869084036", 2023-09-23T22:58:04.055 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140973 2023-09-23T22:58:04.055 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.055 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.055 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.055 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.055 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:58:04.056 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.056 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.056 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.056 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.056 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795103+0000", 2023-09-23T22:58:04.056 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.057 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.057 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.057 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.057 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795101+0000", 2023-09-23T22:58:04.057 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:04.058 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.058 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.058 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.058 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795109+0000", 2023-09-23T22:58:04.058 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6429999999999995e-06 2023-09-23T22:58:04.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.059 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.059 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.059 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795122+0000", 2023-09-23T22:58:04.059 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3022e-05 2023-09-23T22:58:04.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.060 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.060 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.060 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.795130+0000", 2023-09-23T22:58:04.060 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7400000000000004e-06 2023-09-23T22:58:04.060 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.060 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.061 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.061 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810865+0000", 2023-09-23T22:58:04.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015735702000000001 2023-09-23T22:58:04.061 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.061 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.061 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for rw locks", 2023-09-23T22:58:04.062 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:58.810975+0000", 2023-09-23T22:58:04.062 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010935200000000001 2023-09-23T22:58:04.062 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.062 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.062 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.062 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067039+0000", 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.256064085 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067161+0000", 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121952 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.067334+0000", 2023-09-23T22:58:04.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017327 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.083778+0000", 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016443607999999998 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.084031+0000", 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025363099999999998 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120168+0000", 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036136433000000003 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.120220+0000", 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2271000000000001e-05 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:48:59.124163+0000", 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039430070000000001 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5823 9.c 9:36ec13c5:::benchmark_data_smithi052_402718_object5822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 402.43138197600001, 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22852289200000001, 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.066 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5823 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617780+0000", 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617779+0000", 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617800+0000", 2023-09-23T22:58:04.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0721999999999999e-05 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617807+0000", 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.878e-06 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.617814+0000", 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8519999999999998e-06 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.069 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618047+0000", 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023353299999999999 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618201+0000", 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015408199999999999 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.618341+0000", 2023-09-23T22:58:04.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139765 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619519+0000", 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011783220000000001 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.619626+0000", 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107017 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846195+0000", 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22656818400000001 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846251+0000", 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6786000000000001e-05 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846303+0000", 2023-09-23T22:58:04.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1248999999999997e-05 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5845 9.c 9:32588725:::benchmark_data_smithi052_402718_object5844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 402.42441855300001, 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.221823086, 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.073 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5845 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624743+0000", 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624742+0000", 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624765+0000", 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2909e-05 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624775+0000", 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.023e-05 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624783+0000", 2023-09-23T22:58:04.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.624865+0000", 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1631999999999994e-05 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625069+0000", 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000203685 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625237+0000", 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168426 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.625970+0000", 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00073263699999999996 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.626858+0000", 2023-09-23T22:58:04.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088813800000000001 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846472+0000", 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21961366099999999 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846523+0000", 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0986000000000003e-05 2023-09-23T22:58:04.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.846566+0000", 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3708000000000002e-05 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:5878 9.c 9:34f7d49b:::benchmark_data_smithi052_402718_object5877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:58:04.080 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 402.41025041299997, 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.208484483, 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 5878 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638911+0000", 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638910+0000", 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638928+0000", 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7866000000000001e-05 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638941+0000", 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2618e-05 2023-09-23T22:58:04.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638947+0000", 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5639999999999996e-06 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.638986+0000", 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9031e-05 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639111+0000", 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000124955 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639216+0000", 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105696 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.639724+0000", 2023-09-23T22:58:04.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00050770100000000005 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.640023+0000", 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029943599999999998 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847306+0000", 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20728270800000001 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847359+0000", 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3319999999999998e-05 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:21.847396+0000", 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6473999999999997e-05 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13481 9.8 9:1f01482c:::benchmark_data_smithi052_402718_object13480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 398.71315862400002, 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.322004134, 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13481 2023-09-23T22:58:04.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336003+0000", 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336002+0000", 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336029+0000", 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6545000000000001e-05 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336044+0000", 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5444999999999999e-05 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336063+0000", 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8729e-05 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336119+0000", 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6112000000000001e-05 2023-09-23T22:58:04.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336242+0000", 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012232500000000001 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.336383+0000", 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014170900000000001 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337199+0000", 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00081607499999999998 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337730+0000", 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00053117300000000002 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657923+0000", 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32019289400000001 2023-09-23T22:58:04.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657967+0000", 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3399999999999998e-05 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.658007+0000", 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0534999999999997e-05 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.094 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:13489 9.c 9:33ad037a:::benchmark_data_smithi052_402718_object13488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 398.71208393900002, 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.32053898800000002, 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.095 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 13489 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337078+0000", 2023-09-23T22:58:04.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337077+0000", 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337098+0000", 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1191000000000001e-05 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337107+0000", 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2179999999999999e-06 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337112+0000", 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337159+0000", 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6615e-05 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337308+0000", 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014894100000000001 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.337456+0000", 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148066 2023-09-23T22:58:04.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.338621+0000", 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011655420000000001 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.339454+0000", 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083201599999999998 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657516+0000", 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.318062489 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657570+0000", 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4024000000000003e-05 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:51:25.657617+0000", 2023-09-23T22:58:04.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6826000000000002e-05 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59534 9.a 9:56ae8077:::benchmark_data_smithi052_402718_object59533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 354.95485953999997, 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20214791800000001, 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.102 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59534 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094300+0000", 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094331+0000", 2023-09-23T22:58:04.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0930999999999999e-05 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094353+0000", 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2340999999999998e-05 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094362+0000", 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2519999999999999e-06 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.105 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094415+0000", 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2997000000000001e-05 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094637+0000", 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000222208 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094844+0000", 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000207631 2023-09-23T22:58:04.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097382+0000", 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025371360000000002 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097400+0000", 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8714000000000001e-05 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.098411+0000", 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001011131 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294679+0000", 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19626732399999999 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294738+0000", 2023-09-23T22:58:04.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8801000000000002e-05 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296450+0000", 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001712618 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59542 9.a 9:51ad1323:::benchmark_data_smithi052_402718_object59541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T22:58:04.109 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 354.94990534999999, 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19894724899999999, 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59542 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099254+0000", 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099282+0000", 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7971999999999999e-05 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099301+0000", 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8593e-05 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099311+0000", 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0074999999999999e-05 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099389+0000", 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7689999999999996e-05 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099595+0000", 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020635099999999999 2023-09-23T22:58:04.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099809+0000", 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021421 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.102469+0000", 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002660314 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104681+0000", 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022113889999999998 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104831+0000", 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015015699999999999 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296512+0000", 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19168089399999999 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296580+0000", 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8103999999999997e-05 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298204+0000", 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016236810000000001 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.116 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98500 9.a 9:5e200aba:::benchmark_data_smithi052_402718_object8859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 301.08166589899997, 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22489546499999999, 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.117 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98500 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967494+0000", 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967502+0000", 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6190000000000002e-06 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967516+0000", 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4651000000000001e-05 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967525+0000", 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6279999999999994e-06 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968657+0000", 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011323209999999999 2023-09-23T22:58:04.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968810+0000", 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015326000000000001 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968949+0000", 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138429 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.974764+0000", 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058149839999999996 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.975363+0000", 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059912599999999996 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977959+0000", 2023-09-23T22:58:04.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025958399999999999 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190744+0000", 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.212785264 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190803+0000", 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8777000000000003e-05 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.192391+0000", 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015884969999999999 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110895 9.a 9:5225ae0d:::benchmark_data_smithi052_402718_object21254:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 282.98558293500002, 2023-09-23T22:58:04.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25354543899999998, 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110895 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063577+0000", 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063584+0000", 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1860000000000003e-06 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063611+0000", 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6279e-05 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063621+0000", 2023-09-23T22:58:04.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8179999999999999e-06 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063692+0000", 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1106000000000007e-05 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063841+0000", 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014895899999999999 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063967+0000", 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012629000000000001 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.071444+0000", 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074768090000000001 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072687+0000", 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00124325 2023-09-23T22:58:04.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072764+0000", 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7496999999999993e-05 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315667+0000", 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24290224499999999 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315734+0000", 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.317124+0000", 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001390575 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110898 9.a 9:538cf70b:::benchmark_data_smithi052_402718_object21257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 282.98551583900002, 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25922483400000002, 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110898 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.132 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063645+0000", 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063650+0000", 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6210000000000003e-06 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063660+0000", 2023-09-23T22:58:04.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8379999999999997e-06 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063667+0000", 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9879999999999998e-06 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067524+0000", 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003857624 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067678+0000", 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015363900000000001 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067821+0000", 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143096 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072775+0000", 2023-09-23T22:58:04.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049535079999999997 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.073473+0000", 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069833799999999995 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.074443+0000", 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097013000000000004 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321598+0000", 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24715498 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321672+0000", 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4229999999999999e-05 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.322871+0000", 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001198483 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113539 10.4 e344/343 10:2b438690:::benchmark_data_smithi052_461816_object16738:head v 344'7089, mlcod=344'7087)", 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.420345+0000", 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.628816726, 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012921103999999999, 2023-09-23T22:58:04.139 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420345+0000", 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420345+0000", 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420345+0000", 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420350+0000", 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4979999999999997e-06 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420359+0000", 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5050000000000007e-06 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420364+0000", 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.507e-06 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420448+0000", 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4258999999999994e-05 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.420471+0000", 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3132000000000001e-05 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.433228+0000", 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012756656999999999 2023-09-23T22:58:04.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.433266+0000", 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8019000000000003e-05 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113711 10.5 e344/343 10:a41688cc:::benchmark_data_smithi052_461816_object16910:head v 344'7120, mlcod=344'7118)", 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.759236+0000", 2023-09-23T22:58:04.144 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.289925370999999, 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013040774, 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.759236+0000", 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.759236+0000", 2023-09-23T22:58:04.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.759236+0000", 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.759240+0000", 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5540000000000002e-06 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.759245+0000", 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2390000000000002e-06 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.759249+0000", 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0239999999999999e-06 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.762082+0000", 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028328210000000001 2023-09-23T22:58:04.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.762101+0000", 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8833000000000001e-05 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.772219+0000", 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010118494 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.772277+0000", 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8028000000000001e-05 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113715 10.4 e344/343 10:27be032a:::benchmark_data_smithi052_461816_object16914:head v 344'7100, mlcod=344'7096)", 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.784549+0000", 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.264612938999999, 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013676176999999999, 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.149 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.784549+0000", 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.784549+0000", 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.784548+0000", 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.784557+0000", 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4229999999999994e-06 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.784576+0000", 2023-09-23T22:58:04.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8519e-05 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.784584+0000", 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6209999999999993e-06 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.790997+0000", 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0064125559999999998 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.791021+0000", 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3638999999999998e-05 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.798161+0000", 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071400760000000004 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.798225+0000", 2023-09-23T22:58:04.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4364999999999998e-05 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113718 10.f e344/343 10:f7ba7a2d:::benchmark_data_smithi052_461816_object16917:head v 344'7098, mlcod=0'0)", 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.801170+0000", 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.247991710000001, 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017963037000000001, 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.154 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801170+0000", 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801170+0000", 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801169+0000", 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801179+0000", 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8689999999999999e-06 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801196+0000", 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6954999999999999e-05 2023-09-23T22:58:04.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801206+0000", 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0044000000000001e-05 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801266+0000", 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0256e-05 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801291+0000", 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5378999999999999e-05 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.819085+0000", 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017794152000000001 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.819133+0000", 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7685999999999997e-05 2023-09-23T22:58:04.158 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113723 10.c e344/343 10:379ab819:::benchmark_data_smithi052_461816_object16922:head v 344'7110, mlcod=0'0)", 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.803520+0000", 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.245642086, 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014268787, 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803520+0000", 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803520+0000", 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803518+0000", 2023-09-23T22:58:04.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803528+0000", 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.781e-06 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803549+0000", 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0758999999999999e-05 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803558+0000", 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6500000000000002e-06 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803652+0000", 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4458999999999998e-05 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803679+0000", 2023-09-23T22:58:04.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.656e-05 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817736+0000", 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014057246000000001 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817788+0000", 2023-09-23T22:58:04.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2654000000000001e-05 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113730 10.5 e344/343 10:afcc18d7:::benchmark_data_smithi052_461816_object16929:head v 344'7125, mlcod=344'7119)", 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.803701+0000", 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.245460831999999, 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014089620000000001, 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.164 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803701+0000", 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803701+0000", 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803700+0000", 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803708+0000", 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.537e-06 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803718+0000", 2023-09-23T22:58:04.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.083e-05 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803725+0000", 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7970000000000003e-06 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.807471+0000", 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037456780000000001 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.807493+0000", 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2413999999999999e-05 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817756+0000", 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010262699 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817791+0000", 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4584e-05 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113731 10.5 e344/343 10:a90ca0ca:::benchmark_data_smithi052_461816_object16930:head v 344'7126, mlcod=344'7119)", 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.803740+0000", 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.245421852, 2023-09-23T22:58:04.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014073373, 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803740+0000", 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803740+0000", 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803740+0000", 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803743+0000", 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.331e-06 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803750+0000", 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2280000000000001e-06 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803755+0000", 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3789999999999999e-06 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.812206+0000", 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0084509489999999993 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.812235+0000", 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9298e-05 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817793+0000", 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055584880000000003 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817813+0000", 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9959999999999999e-05 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113733 10.4 e344/343 10:2cc8a87a:::benchmark_data_smithi052_461816_object16932:head v 344'7101, mlcod=344'7099)", 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.803759+0000", 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.245403019000001, 2023-09-23T22:58:04.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014056569999999999, 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803759+0000", 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803759+0000", 2023-09-23T22:58:04.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803759+0000", 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803761+0000", 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8040000000000002e-06 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803766+0000", 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9590000000000003e-06 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.803770+0000", 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7359999999999999e-06 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.805966+0000", 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021961849999999998 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.805987+0000", 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0655000000000001e-05 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817794+0000", 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011806952000000001 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.817815+0000", 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1458e-05 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113883 10.c e344/343 10:3b417e4e:::benchmark_data_smithi052_461816_object17082:head v 344'7117, mlcod=0'0)", 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:33.063351+0000", 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 30.985810775000001, 2023-09-23T22:58:04.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013542904999999999, 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063351+0000", 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063351+0000", 2023-09-23T22:58:04.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063350+0000", 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063359+0000", 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9940000000000006e-06 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063381+0000", 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2565000000000002e-05 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063392+0000", 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0411000000000001e-05 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:58:04.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063453+0000", 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0995000000000001e-05 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.063477+0000", 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.429e-05 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.076835+0000", 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013358195 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:58:04.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:33.076894+0000", 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8560000000000002e-05 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:58:04.184 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:58:04.184 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:04.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:03 smithi151 ceph-mon[159210]: pgmap v1036: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76755/238170 objects degraded (32.227%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:04.210 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:04.234 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:04.257 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:04.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:04.364 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:04.397 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:04.505 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:04.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:04.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:04.639 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:04.717 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:58:04 smithi052 conmon[500475]: 2023-09-23T22:58:04.501+0000 7f03a73d9640 -1 osd.0 344 log_to_monitors true 2023-09-23T22:58:04.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:04.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:58:04.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[180979]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:58:04.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/489274821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:04.758 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:58:04.993 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 22:58:04 smithi052 conmon[500475]: 2023-09-23T22:58:04.737+0000 7f0398846700 -1 osd.0 344 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:58:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:58:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[187743]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:58:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/489274821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:04 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:58:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:04 smithi151 ceph-mon[159210]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:58:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/489274821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:05.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:58:05.833 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:05.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:05.936 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:05.962 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:05.989 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: pgmap v1037: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76755/238170 objects degraded (32.227%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: failed to encode map e352 with expected crc 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/489274821' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: osdmap e352: 8 total, 6 up, 6 in 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: from='osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:05.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 76755/238170 objects degraded (32.227%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: pgmap v1037: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76755/238170 objects degraded (32.227%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: failed to encode map e352 with expected crc 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/489274821' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: osdmap e352: 8 total, 6 up, 6 in 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: from='osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:05.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:05.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 76755/238170 objects degraded (32.227%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:06.006 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:06.009 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: pgmap v1037: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized+remapped, 22 active+clean, 5 active+recovery_wait+degraded+remapped, 30 active+undersized, 10 active+undersized+degraded, 2 undersized+peered, 2 undersized+degraded+peered, 4 active+clean+remapped; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 76755/238170 objects degraded (32.227%); 59594/238170 objects misplaced (25.022%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: failed to encode map e352 with expected crc 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/489274821' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: osdmap e352: 8 total, 6 up, 6 in 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: from='osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:06.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 76755/238170 objects degraded (32.227%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:06.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:06.112 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:06.140 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:06.190 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:06.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:06.245 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:06.296 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:06.322 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:06.399 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:06.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:06.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:06.503 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:06.534 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:06.584 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:06.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:06.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:06.687 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:06.715 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:06.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: failed to encode map e352 with expected crc 2023-09-23T22:58:06.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:58:06.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: failed to encode map e353 with expected crc 2023-09-23T22:58:06.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510] boot 2023-09-23T22:58:06.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: osdmap e353: 8 total, 7 up, 6 in 2023-09-23T22:58:06.891 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:58:06.891 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: failed to encode map e353 with expected crc 2023-09-23T22:58:06.891 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:06 smithi151 ceph-mon[159210]: osdmap e354: 8 total, 7 up, 6 in 2023-09-23T22:58:06.945 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:06.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:06.950 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:58:06.969 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:06.988 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: failed to encode map e352 with expected crc 2023-09-23T22:58:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:58:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: failed to encode map e353 with expected crc 2023-09-23T22:58:06.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510] boot 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: osdmap e353: 8 total, 7 up, 6 in 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: failed to encode map e353 with expected crc 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[187743]: osdmap e354: 8 total, 7 up, 6 in 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: failed to encode map e352 with expected crc 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: failed to encode map e353 with expected crc 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: osd.0 [v2:172.21.15.52:6802/1932666510,v1:172.21.15.52:6803/1932666510] boot 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: osdmap e353: 8 total, 7 up, 6 in 2023-09-23T22:58:06.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:58:06.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: failed to encode map e353 with expected crc 2023-09-23T22:58:06.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:06 smithi052 ceph-mon[180979]: osdmap e354: 8 total, 7 up, 6 in 2023-09-23T22:58:07.007 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:07.026 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:07.044 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:07.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:07.063 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:07.063 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.063 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.063 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:58:07.064 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.064 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.064 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.064 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.064 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:58:07.081 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:07.287 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:07.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:07.390 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:07.417 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:07.461 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:07.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:07.564 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:07.589 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:07.634 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:07.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:07.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:58:07.704 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:58:07.738 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:07.786 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:58:07.789 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:07.965 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:07.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:07.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: pgmap v1040: 89 pgs: 1 active+recovering+degraded+remapped, 13 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 20 active+undersized, 8 active+undersized+degraded, 5 remapped+peering, 2 undersized+peered, 1 undersized+degraded+peered, 3 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 68384/238170 objects degraded (28.712%); 46691/238170 objects misplaced (19.604%); 991 KiB/s, 15 objects/s recovering 2023-09-23T22:58:07.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: 10.5 continuing backfill to osd.3 from (322'1800,344'7155] 10:a29216c4:::benchmark_data_smithi052_461816_object66347:head to 344'7155 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1419372208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[180979]: osdmap e355: 8 total, 7 up, 6 in 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: pgmap v1040: 89 pgs: 1 active+recovering+degraded+remapped, 13 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 20 active+undersized, 8 active+undersized+degraded, 5 remapped+peering, 2 undersized+peered, 1 undersized+degraded+peered, 3 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 68384/238170 objects degraded (28.712%); 46691/238170 objects misplaced (19.604%); 991 KiB/s, 15 objects/s recovering 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: 10.5 continuing backfill to osd.3 from (322'1800,344'7155] 10:a29216c4:::benchmark_data_smithi052_461816_object66347:head to 344'7155 2023-09-23T22:58:07.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: failed to encode map e354 with expected crc 2023-09-23T22:58:07.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1419372208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:07.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:07 smithi052 ceph-mon[187743]: osdmap e355: 8 total, 7 up, 6 in 2023-09-23T22:58:08.073 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: pgmap v1040: 89 pgs: 1 active+recovering+degraded+remapped, 13 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+undersized+remapped, 22 active+clean, 4 active+recovery_wait+degraded+remapped, 20 active+undersized, 8 active+undersized+degraded, 5 remapped+peering, 2 undersized+peered, 1 undersized+degraded+peered, 3 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 68384/238170 objects degraded (28.712%); 46691/238170 objects misplaced (19.604%); 991 KiB/s, 15 objects/s recovering 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: 10.5 continuing backfill to osd.3 from (322'1800,344'7155] 10:a29216c4:::benchmark_data_smithi052_461816_object66347:head to 344'7155 2023-09-23T22:58:08.127 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.128 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.128 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: failed to encode map e354 with expected crc 2023-09-23T22:58:08.128 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1419372208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:08.128 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:07 smithi151 ceph-mon[159210]: osdmap e355: 8 total, 7 up, 6 in 2023-09-23T22:58:08.130 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:08.208 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:08.239 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:08.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:08.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:08.347 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:08.394 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:08.445 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:08.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:08.479 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:58:08.479 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:58:08.550 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:08.586 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:08.597 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:58:08.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:08.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:08.966 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:08.996 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:09.042 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:09.044 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.044 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.044 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.044 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: 10.4 continuing backfill to osd.3 from (322'1800,349'7126] MIN to 349'7126 2023-09-23T22:58:09.045 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.045 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.045 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: 10.d continuing backfill to osd.3 from (322'1500,347'7208] MIN to 347'7208 2023-09-23T22:58:09.045 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:08 smithi151 ceph-mon[159210]: 10.c continuing backfill to osd.3 from (322'1400,347'7145] MIN to 347'7145 2023-09-23T22:58:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:09.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:09.146 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:09.175 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:09.228 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:09.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:09.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:09.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: 10.4 continuing backfill to osd.3 from (322'1800,349'7126] MIN to 349'7126 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: 10.d continuing backfill to osd.3 from (322'1500,347'7208] MIN to 347'7208 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[187743]: 10.c continuing backfill to osd.3 from (322'1400,347'7145] MIN to 347'7145 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: 10.4 continuing backfill to osd.3 from (322'1800,349'7126] MIN to 349'7126 2023-09-23T22:58:09.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: 10.d continuing backfill to osd.3 from (322'1500,347'7208] MIN to 347'7208 2023-09-23T22:58:09.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:08 smithi052 ceph-mon[180979]: 10.c continuing backfill to osd.3 from (322'1400,347'7145] MIN to 347'7145 2023-09-23T22:58:09.333 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:09.364 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:09.420 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:09.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:09.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:09.506 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:09.525 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:09.557 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:09.558 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:58:09.607 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:09.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:09.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:09.705 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:58:09.711 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:09.777 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:09.828 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:09.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:09.881 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.881 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[187743]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.881 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[187743]: pgmap v1043: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 14 peering, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+remapped, 29 active+clean, 2 active+recovery_wait+degraded+remapped, 16 active+undersized, 4 active+undersized+degraded, 10 remapped+peering, 1 undersized+peered, 1 undersized+degraded+peered, 3 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 44001/238125 objects degraded (18.478%); 36333/238125 objects misplaced (15.258%); 192 KiB/s, 1 objects/s recovering 2023-09-23T22:58:09.881 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[187743]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:58:09.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[180979]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[180979]: pgmap v1043: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 14 peering, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+remapped, 29 active+clean, 2 active+recovery_wait+degraded+remapped, 16 active+undersized, 4 active+undersized+degraded, 10 remapped+peering, 1 undersized+peered, 1 undersized+degraded+peered, 3 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 44001/238125 objects degraded (18.478%); 36333/238125 objects misplaced (15.258%); 192 KiB/s, 1 objects/s recovering 2023-09-23T22:58:09.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:09 smithi052 ceph-mon[180979]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:58:09.935 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:09 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:09 smithi151 ceph-mon[159210]: failed to encode map e355 with expected crc 2023-09-23T22:58:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:09 smithi151 ceph-mon[159210]: pgmap v1043: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 14 peering, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+remapped, 29 active+clean, 2 active+recovery_wait+degraded+remapped, 16 active+undersized, 4 active+undersized+degraded, 10 remapped+peering, 1 undersized+peered, 1 undersized+degraded+peered, 3 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 44001/238125 objects degraded (18.478%); 36333/238125 objects misplaced (15.258%); 192 KiB/s, 1 objects/s recovering 2023-09-23T22:58:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:09 smithi151 ceph-mon[159210]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T22:58:09.962 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:09.965 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:10.013 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:10.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:10.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:10.119 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:10.148 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:10.246 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:10.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:10.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:10.350 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:10.383 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:10.484 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:10.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:10.589 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:10.617 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:10.649 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:10.664 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:10.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:10.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:10.760 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:58:10.769 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:10.796 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:10.844 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:10.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/169145340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:10.845 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:10.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 44001/238125 objects degraded (18.478%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:10.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:10.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:10.947 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:10.974 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/169145340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:10.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:10.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 44001/238125 objects degraded (18.478%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:10.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/169145340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:10.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:10.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 44001/238125 objects degraded (18.478%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:10.978 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:11.379 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:11.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:11.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:11.483 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:11.506 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:58:11.506 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:58:11.513 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:11.771 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:58:11.771 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: pgmap v1044: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 14 peering, 2 active+undersized+degraded+remapped+backfill_wait, 39 active+clean, 3 active+recovery_wait+degraded+remapped, 10 active+undersized, 2 active+undersized+degraded, 9 remapped+peering, 6 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 12 op/s; 33968/237951 objects degraded (14.275%); 36227/237951 objects misplaced (15.225%) 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: failed to encode map e356 with expected crc 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: osdmap e356: 8 total, 7 up, 6 in 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: failed to encode map e356 with expected crc 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: failed to encode map e356 with expected crc 2023-09-23T22:58:11.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:11 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T22:58:12.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: pgmap v1044: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 14 peering, 2 active+undersized+degraded+remapped+backfill_wait, 39 active+clean, 3 active+recovery_wait+degraded+remapped, 10 active+undersized, 2 active+undersized+degraded, 9 remapped+peering, 6 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 12 op/s; 33968/237951 objects degraded (14.275%); 36227/237951 objects misplaced (15.225%) 2023-09-23T22:58:12.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:58:12.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: osdmap e356: 8 total, 7 up, 6 in 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: pgmap v1044: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 14 peering, 2 active+undersized+degraded+remapped+backfill_wait, 39 active+clean, 3 active+recovery_wait+degraded+remapped, 10 active+undersized, 2 active+undersized+degraded, 9 remapped+peering, 6 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 12 op/s; 33968/237951 objects degraded (14.275%); 36227/237951 objects misplaced (15.225%) 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: osdmap e356: 8 total, 7 up, 6 in 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:11 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T22:58:12.625 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:58:12.744 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T22:58:12.744 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T22:58:12.744 DEBUG:teuthology.orchestra.run.smithi052:osd.2> sudo journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T22:58:12.746 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl start ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:58:12.768 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:12.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:12.779 INFO:journalctl@ceph.osd.2.smithi052.stdout:-- Logs begin at Sat 2023-09-23 22:05:46 UTC. -- 2023-09-23T22:58:12.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:12.901 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:12 smithi151 ceph-mon[159210]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.903 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:12.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:12 smithi052 ceph-mon[180979]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:12 smithi052 ceph-mon[187743]: failed to encode map e356 with expected crc 2023-09-23T22:58:12.917 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:12 smithi052 systemd[1]: Starting Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T22:58:12.953 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:12.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:13.027 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:13.057 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:13.086 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:13.121 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:58:13.149 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:13.176 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:13.194 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:13.195 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:13.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:13.217 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:13.297 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:13.326 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:13.526 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:13.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:13.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:13.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:13.661 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:58:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:58:13] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.33.4" 2023-09-23T22:58:13.665 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:13.717 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:13.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:13.821 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:13.849 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:13.899 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:13.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:13.912 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:13 smithi052 ceph-mon[187743]: failed to encode map e356 with expected crc 2023-09-23T22:58:13.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:13 smithi052 ceph-mon[187743]: pgmap v1046: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 1 remapped+peering, 14 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 34 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:13.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/722901119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:13.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:13 smithi052 ceph-mon[180979]: failed to encode map e356 with expected crc 2023-09-23T22:58:13.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:13 smithi052 ceph-mon[180979]: pgmap v1046: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 1 remapped+peering, 14 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 34 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:13.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/722901119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:13 smithi151 ceph-mon[159210]: failed to encode map e356 with expected crc 2023-09-23T22:58:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:13 smithi151 ceph-mon[159210]: pgmap v1046: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 1 remapped+peering, 14 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 34 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/722901119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:14.003 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:14.033 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:14.243 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:58:14.243 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:58:14.243 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T22:58:14.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:58:14.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:58:14.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T22:58:14.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:58:14.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:13 smithi052 bash[505090]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T22:58:14.284 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:14.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:14.390 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:14.420 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:14.523 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:14.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:14.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:14.662 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:14.686 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:58:14.718 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126042 9.c 9:3b764d1a:::benchmark_data_smithi052_402718_object36401:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.696860+0000", 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 271.01703064399999, 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0697906479999999, 2023-09-23T22:58:14.719 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126042 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.696860+0000", 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.696860+0000", 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.696857+0000", 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0558e-05 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.696868+0000", 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8151999999999998e-05 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.696886+0000", 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4490000000000002e-06 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.696893+0000", 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012567899999999999 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.697019+0000", 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4100000000000001e-05 2023-09-23T22:58:14.723 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.697033+0000", 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.03345208 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.730485+0000", 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016528199999999999 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.730650+0000", 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012936600000000001 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.730780+0000", 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013961458 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.744741+0000", 2023-09-23T22:58:14.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021285643999999999 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766027+0000", 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00051712700000000004 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766544+0000", 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0371000000000002e-05 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766604+0000", 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5766999999999997e-05 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766650+0000", 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0697906479999999 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126055 9.c 9:3e32f4c3:::benchmark_data_smithi052_402718_object36414:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.709224+0000", 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 271.00466579699997, 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.066071446, 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.728 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126055 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709224+0000", 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709224+0000", 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:14.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709222+0000", 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1085e-05 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709234+0000", 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1687000000000001e-05 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709255+0000", 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2020000000000004e-06 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709263+0000", 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8474999999999997e-05 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709332+0000", 2023-09-23T22:58:14.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3339000000000001e-05 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.709345+0000", 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.024080514 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.733426+0000", 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012804799999999999 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.733554+0000", 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015865599999999999 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.733712+0000", 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011039478 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.744752+0000", 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021325862000000001 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766078+0000", 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0091590040000000001 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775237+0000", 2023-09-23T22:58:14.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1708000000000002e-05 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775269+0000", 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7324000000000001e-05 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775296+0000", 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.066071446 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.735 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126076 9.c 9:3cc55d78:::benchmark_data_smithi052_402718_object36435:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.745261+0000", 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.96862871899998, 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0301869960000001, 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.736 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126076 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745261+0000", 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745261+0000", 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745261+0000", 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5620000000000003e-06 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745266+0000", 2023-09-23T22:58:14.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3441999999999999e-05 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745280+0000", 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6790000000000002e-06 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745285+0000", 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2929999999999995e-05 2023-09-23T22:58:14.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745348+0000", 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2096000000000001e-05 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.745360+0000", 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99066963900000005 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.736030+0000", 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0205999999999997e-05 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.736110+0000", 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011112600000000001 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.736221+0000", 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012754806 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.748976+0000", 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.01715299 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766129+0000", 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0092210179999999992 2023-09-23T22:58:14.742 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775350+0000", 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2889000000000001e-05 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775403+0000", 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5439999999999999e-05 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.743 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775448+0000", 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0301869960000001 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126083 9.c 9:3c95c16c:::benchmark_data_smithi052_402718_object36442:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.751299+0000", 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.96259149299999, 2023-09-23T22:58:14.744 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0242578980000001, 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126083 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751299+0000", 2023-09-23T22:58:14.745 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751299+0000", 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751298+0000", 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4510000000000001e-06 2023-09-23T22:58:14.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751302+0000", 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2860000000000002e-06 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751309+0000", 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2320000000000003e-06 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751313+0000", 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00055019200000000002 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751863+0000", 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7600000000000001e-05 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.751881+0000", 2023-09-23T22:58:14.748 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.98635421099999998 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.738235+0000", 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012297799999999999 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.738358+0000", 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014939700000000001 2023-09-23T22:58:14.749 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.738508+0000", 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010478345999999999 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.748986+0000", 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017193028999999999 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766179+0000", 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0093256580000000006 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775505+0000", 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0369e-05 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.751 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775535+0000", 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1668e-05 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775557+0000", 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0242578980000001 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.752 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126086 9.c 9:3a681989:::benchmark_data_smithi052_402718_object36445:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.760293+0000", 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.95359711399999, 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.015415567, 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126086 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.753 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760293+0000", 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760293+0000", 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760292+0000", 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4150000000000002e-06 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760299+0000", 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3302000000000001e-05 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760312+0000", 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5219999999999998e-06 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760318+0000", 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1307e-05 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760389+0000", 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1603000000000001e-05 2023-09-23T22:58:14.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760401+0000", 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97994868700000004 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.740349+0000", 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116453 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.740466+0000", 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015730400000000001 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.740623+0000", 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0083719720000000001 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.748995+0000", 2023-09-23T22:58:14.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017230638999999999 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766226+0000", 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0093888919999999994 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775615+0000", 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9805999999999997e-05 2023-09-23T22:58:14.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775664+0000", 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4166999999999999e-05 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775709+0000", 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.015415567 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126089 9.c 9:3b5e0f20:::benchmark_data_smithi052_402718_object36448:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.760903+0000", 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.95298703600002, 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.014892095, 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126089 2023-09-23T22:58:14.761 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760903+0000", 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760903+0000", 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760902+0000", 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.48e-06 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760908+0000", 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4905e-05 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760923+0000", 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4300000000000003e-06 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760929+0000", 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3025999999999997e-05 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760982+0000", 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0431000000000001e-05 2023-09-23T22:58:14.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.760993+0000", 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.98144755100000003 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.742440+0000", 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000126351 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.742566+0000", 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012288699999999999 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.742689+0000", 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010500311 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.753190+0000", 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013133537000000001 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766323+0000", 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0094223659999999997 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775746+0000", 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7355000000000002e-05 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775773+0000", 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2345e-05 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775795+0000", 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.014892095 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126091 9.c 9:39218863:::benchmark_data_smithi052_402718_object36450:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.762144+0000", 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.95174576699998, 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.013774247, 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.769 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126091 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762144+0000", 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762144+0000", 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762144+0000", 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.843e-06 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762149+0000", 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3733999999999999e-05 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762162+0000", 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9009999999999997e-06 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762169+0000", 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6208000000000002e-05 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762215+0000", 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.541e-06 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.762224+0000", 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.98257571099999996 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.744800+0000", 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4742999999999998e-05 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.744884+0000", 2023-09-23T22:58:14.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0080000000000005e-05 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.744974+0000", 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0082239089999999997 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.753198+0000", 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013170075 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766368+0000", 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0094617139999999995 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775830+0000", 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6081000000000001e-05 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775876+0000", 2023-09-23T22:58:14.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2466999999999998e-05 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.775919+0000", 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.013774247 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126096 9.c 9:3a258c32:::benchmark_data_smithi052_402718_object36455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.772786+0000", 2023-09-23T22:58:14.777 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.941104575, 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.013601336, 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126096 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772786+0000", 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772786+0000", 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772784+0000", 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0530000000000001e-06 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772791+0000", 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2221999999999999e-05 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772804+0000", 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.81e-06 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772810+0000", 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7929000000000001e-05 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772858+0000", 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7889999999999995e-06 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772866+0000", 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97400909599999996 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.746875+0000", 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120142 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.746995+0000", 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000149927 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.747145+0000", 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010230737 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.757376+0000", 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0090122819999999999 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766388+0000", 2023-09-23T22:58:14.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019935234999999999 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786323+0000", 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6412000000000003e-05 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786360+0000", 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7047e-05 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786387+0000", 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.013601336 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126098 9.c 9:37f594de:::benchmark_data_smithi052_402718_object36457:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.772851+0000", 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.94103871599998, 2023-09-23T22:58:14.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.013693379, 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126098 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772851+0000", 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772851+0000", 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772851+0000", 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3620000000000002e-06 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772854+0000", 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6029999999999999e-06 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772862+0000", 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.34e-06 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.772866+0000", 2023-09-23T22:58:14.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000454621 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.773321+0000", 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4089999999999992e-06 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.773330+0000", 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97571376300000001 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.749044+0000", 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012997700000000001 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.749174+0000", 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7718000000000003e-05 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.749262+0000", 2023-09-23T22:58:14.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081175399999999995 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.757379+0000", 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0090491740000000001 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766429+0000", 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020017288000000001 2023-09-23T22:58:14.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786446+0000", 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4404999999999998e-05 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786500+0000", 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4654000000000003e-05 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786545+0000", 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.013693379 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45593.0:126118 9.c 9:3442434b:::benchmark_data_smithi052_402718_object36477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:53:43.808163+0000", 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 270.90572691099999, 2023-09-23T22:58:14.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97847682199999997, 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45593", 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 126118 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808163+0000", 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808163+0000", 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808161+0000", 2023-09-23T22:58:14.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2739999999999996e-06 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808170+0000", 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5923999999999999e-05 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808186+0000", 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5190000000000003e-06 2023-09-23T22:58:14.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808194+0000", 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1310999999999994e-05 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808285+0000", 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0154e-05 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:43.808295+0000", 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94283011400000005 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.751125+0000", 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117411 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.751243+0000", 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000152242 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.751395+0000", 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010281557 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.761677+0000", 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004770781 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.766447+0000", 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020140228999999999 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786588+0000", 2023-09-23T22:58:14.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.463e-05 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786612+0000", 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7763000000000002e-05 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:53:44.786640+0000", 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.97847682199999997 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.801 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73725 10.d 10:bb4c55d8:::benchmark_data_smithi052_461816_object73724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e328)", 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:19.736394+0000", 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 114.977496659, 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4793088839999999, 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.802 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73725 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736394+0000", 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736394+0000", 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736393+0000", 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6150999999999997e-05 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736459+0000", 2023-09-23T22:58:14.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0729e-05 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736470+0000", 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0130000000000001e-06 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736475+0000", 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7129000000000002e-05 2023-09-23T22:58:14.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736522+0000", 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0084e-05 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736532+0000", 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.46529093500000002 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201823+0000", 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9469999999999998e-06 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201833+0000", 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0108204620000001 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.212653+0000", 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030491759999999998 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.215702+0000", 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4793088839999999 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73736 10.d 10:b6d6833f:::benchmark_data_smithi052_461816_object73735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e328)", 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:19.736489+0000", 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 114.97740167400001, 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4793379980000001, 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.809 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73736 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736489+0000", 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736489+0000", 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736488+0000", 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2418000000000001e-05 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736510+0000", 2023-09-23T22:58:14.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3950000000000003e-06 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736515+0000", 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2629999999999999e-06 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736518+0000", 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00044128700000000001 2023-09-23T22:58:14.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736959+0000", 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4780000000000002e-06 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736964+0000", 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.46491047499999999 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201874+0000", 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6979999999999993e-06 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201884+0000", 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0138582380000001 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.215742+0000", 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4228999999999996e-05 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.215827+0000", 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4793379980000001 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73743 10.d 10:be204043:::benchmark_data_smithi052_461816_object73742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e328)", 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:19.736652+0000", 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 114.977237823, 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5049905890000002, 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73743 2023-09-23T22:58:14.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736652+0000", 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736652+0000", 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:14.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736652+0000", 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1970000000000001e-05 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736684+0000", 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5359999999999998e-06 2023-09-23T22:58:14.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736690+0000", 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2559999999999998e-06 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.736694+0000", 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074487000000000002 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.737439+0000", 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3309999999999993e-06 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.737448+0000", 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.46445961099999999 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201908+0000", 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1139999999999995e-06 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201917+0000", 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0139382180000001 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.215855+0000", 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001784264 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.217639+0000", 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010095659999999999 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.218649+0000", 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010574297 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.229223+0000", 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2660000000000001e-06 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.229231+0000", 2023-09-23T22:58:14.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045338469999999997 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.233765+0000", 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0664000000000001e-05 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.233826+0000", 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0078171820000000006 2023-09-23T22:58:14.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.241643+0000", 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5049905890000002 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73746 10.c 10:30645706:::benchmark_data_smithi052_461816_object73745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T22:58:14.825 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:19.746370+0000", 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 114.96751985, 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6820375379999999, 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73746 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746370+0000", 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746370+0000", 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746369+0000", 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9074999999999998e-05 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746438+0000", 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1987999999999999e-05 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746450+0000", 2023-09-23T22:58:14.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4880000000000004e-06 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746457+0000", 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7731999999999995e-05 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746525+0000", 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0366e-05 2023-09-23T22:58:14.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746535+0000", 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.45580134 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202336+0000", 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6330000000000002e-06 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202345+0000", 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2162695610000001 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.418614+0000", 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000172926 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.418787+0000", 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00023521600000000001 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.419023+0000", 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072438829999999996 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.426266+0000", 2023-09-23T22:58:14.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6487999999999999e-05 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.426283+0000", 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6404999999999997e-05 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.426369+0000", 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6461000000000002e-05 2023-09-23T22:58:14.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.426406+0000", 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020020860000000001 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.428408+0000", 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6820375379999999 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.834 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73747 10.d 10:bec037a8:::benchmark_data_smithi052_461816_object73746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:19.746473+0000", 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 114.96741765100001, 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.500590622, 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.835 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73747 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746473+0000", 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746473+0000", 2023-09-23T22:58:14.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746472+0000", 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6863999999999998e-05 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746509+0000", 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2989999999999999e-06 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746516+0000", 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.437e-06 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746520+0000", 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00035065900000000002 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746871+0000", 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9289999999999998e-06 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.746879+0000", 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.45507138600000002 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201950+0000", 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3383999999999999e-05 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.201964+0000", 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0220461169999999 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.224010+0000", 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000111469 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.224121+0000", 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000166985 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,6", 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.224288+0000", 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049489219999999997 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.229237+0000", 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000168627 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.229406+0000", 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017520765000000001 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.246927+0000", 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0686000000000003e-05 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.246967+0000", 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5920000000000003e-05 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.247063+0000", 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.500590622 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73752 10.e 10:77adb5ed:::benchmark_data_smithi052_461816_object73751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e328)", 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:19.751627+0000", 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 114.96226351200001, 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.67203283, 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.844 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73752 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751627+0000", 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751627+0000", 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751626+0000", 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2148999999999998e-05 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751678+0000", 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2159e-05 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751690+0000", 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1410000000000004e-06 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751695+0000", 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6091999999999999e-05 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751731+0000", 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7820000000000001e-06 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:19.751736+0000", 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.45294525299999999 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.204681+0000", 2023-09-23T22:58:14.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7810000000000005e-06 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.204689+0000", 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2085753269999999 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.413264+0000", 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017770199999999999 2023-09-23T22:58:14.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.413442+0000", 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00024711299999999998 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.413689+0000", 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005783959 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.419473+0000", 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7159e-05 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.419490+0000", 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001685073 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.421175+0000", 2023-09-23T22:58:14.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5864e-05 2023-09-23T22:58:14.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.421221+0000", 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002438188 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.423660+0000", 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.67203283 2023-09-23T22:58:14.853 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114197 10.6 10:6cbf7a9f:::benchmark_data_smithi052_461816_object17396:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.638868+0000", 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 41.075022240999999, 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0609476529999999, 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.854 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114197 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638868+0000", 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638868+0000", 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638867+0000", 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6000000000000003e-06 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638873+0000", 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2157999999999999e-05 2023-09-23T22:58:14.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638885+0000", 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6440000000000003e-06 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638892+0000", 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4397000000000001e-05 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638946+0000", 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4919999999999993e-06 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638955+0000", 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0465119249999999 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.685467+0000", 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024042859999999998 2023-09-23T22:58:14.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.687871+0000", 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155284 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.688026+0000", 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.009768288 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.697795+0000", 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018617110000000001 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699656+0000", 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4583999999999998e-05 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699711+0000", 2023-09-23T22:58:14.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000104646 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699816+0000", 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0609476529999999 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114199 10.6 10:63a78c60:::benchmark_data_smithi052_461816_object17398:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.638909+0000", 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 41.074980815000004, 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0610776770000001, 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114199 2023-09-23T22:58:14.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638909+0000", 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638909+0000", 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:14.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638909+0000", 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.937e-06 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638912+0000", 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9689999999999999e-06 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638918+0000", 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.126e-06 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638922+0000", 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045948000000000001 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.639381+0000", 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6140000000000002e-06 2023-09-23T22:58:14.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.639386+0000", 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.05181372 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691200+0000", 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000126706 2023-09-23T22:58:14.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691326+0000", 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000119423 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691446+0000", 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0063587169999999998 2023-09-23T22:58:14.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.697805+0000", 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020814200000000001 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699886+0000", 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3377999999999997e-05 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699929+0000", 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7663e-05 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699987+0000", 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0610776770000001 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.869 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114201 10.6 10:63900912:::benchmark_data_smithi052_461816_object17400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.646195+0000", 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 41.067694807000002, 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.053975222, 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.870 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114201 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646195+0000", 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646195+0000", 2023-09-23T22:58:14.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646194+0000", 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8939999999999993e-06 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646202+0000", 2023-09-23T22:58:14.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4620000000000002e-06 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646211+0000", 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8049999999999997e-06 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646216+0000", 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1202999999999997e-05 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646267+0000", 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0254999999999999e-05 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646277+0000", 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.048210514 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.694488+0000", 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117799 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.694606+0000", 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012992200000000001 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.694736+0000", 2023-09-23T22:58:14.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031115409999999998 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.697847+0000", 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021845380000000002 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.700032+0000", 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7054e-05 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.700079+0000", 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1739999999999999e-05 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.700171+0000", 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.053975222 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.877 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114204 10.d 10:b3b88576:::benchmark_data_smithi052_461816_object17403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.664220+0000", 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 41.049669747999999, 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0153031850000001, 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114204 2023-09-23T22:58:14.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664220+0000", 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664220+0000", 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664219+0000", 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2110000000000001e-06 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664226+0000", 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4001e-05 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664240+0000", 2023-09-23T22:58:14.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6459999999999999e-06 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664249+0000", 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3594000000000002e-05 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664302+0000", 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1223000000000001e-05 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664313+0000", 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0007254290000001 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.665039+0000", 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014699799999999999 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:14.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.665186+0000", 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018358900000000001 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.665369+0000", 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033414080000000001 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.668711+0000", 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004222188 2023-09-23T22:58:14.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.672933+0000", 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0063869929999999997 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.679320+0000", 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4951999999999997e-05 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.679385+0000", 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013865500000000001 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.679524+0000", 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0153031850000001 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.885 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:14.886 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:14.886 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:14.887 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:14.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:14.988 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:15.018 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:15.065 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:15.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:15.168 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:15.194 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:15.235 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:58:15.247 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:15.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:15.300 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:15.350 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:15.378 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:15.493 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:15 smithi052 ceph-osd[505824]: -- 172.21.15.52:0/3836314234 <== mon.2 v2:172.21.15.52:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55772779e4e0 con 0x557727771400 2023-09-23T22:58:15.493 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:15 smithi052 bash[505680]: 82bbd993ad69cf5a61e29cf428fd01fdd972046411462527ade6c91dfd82cdc5 2023-09-23T22:58:15.493 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:15 smithi052 systemd[1]: Started Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T22:58:15.587 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:15.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:15.641 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:58:15.691 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:15.720 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:15.770 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:15.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:15.863 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:15 smithi052 conmon[505773]: 2023-09-23T22:58:15.744+0000 7f9e167bb640 -1 Falling back to public interface 2023-09-23T22:58:15.864 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:15 smithi052 ceph-mon[180979]: pgmap v1047: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:15.864 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 41028/237549 objects degraded (17.271%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:15.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:15 smithi052 ceph-mon[187743]: pgmap v1047: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:15.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 41028/237549 objects degraded (17.271%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:15.878 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:15.910 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:15 smithi151 ceph-mon[159210]: pgmap v1047: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 41028/237549 objects degraded (17.271%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:15.956 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:15.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:16.032 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:16.061 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:16.091 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:16.187 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:16.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:16.290 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:16.321 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:16.321 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:16.451 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:16.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:16.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:16.555 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:16.585 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:16.635 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:16.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:16.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:16.777 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:16.826 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:16.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:16.930 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:16.960 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:17.459 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:17.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:17.563 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:17.595 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:17.645 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:17.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:17.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:17.689 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:17.689 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:58:17.689 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T22:58:17.689 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:17.748 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:17.768 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:58:17.784 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:17.885 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:17.886 INFO:teuthology.orchestra.run.smithi052.stderr:no valid command found; 10 closest matches: 2023-09-23T22:58:17.886 INFO:teuthology.orchestra.run.smithi052.stderr:0 2023-09-23T22:58:17.886 INFO:teuthology.orchestra.run.smithi052.stderr:1 2023-09-23T22:58:17.886 INFO:teuthology.orchestra.run.smithi052.stderr:2 2023-09-23T22:58:17.886 INFO:teuthology.orchestra.run.smithi052.stderr:abort 2023-09-23T22:58:17.886 INFO:teuthology.orchestra.run.smithi052.stderr:assert 2023-09-23T22:58:17.887 INFO:teuthology.orchestra.run.smithi052.stderr:config diff 2023-09-23T22:58:17.887 INFO:teuthology.orchestra.run.smithi052.stderr:config diff get 2023-09-23T22:58:17.887 INFO:teuthology.orchestra.run.smithi052.stderr:config get 2023-09-23T22:58:17.887 INFO:teuthology.orchestra.run.smithi052.stderr:config help [] 2023-09-23T22:58:17.887 INFO:teuthology.orchestra.run.smithi052.stderr:config set ... 2023-09-23T22:58:17.887 INFO:teuthology.orchestra.run.smithi052.stderr:admin_socket: invalid command 2023-09-23T22:58:17.887 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:17 smithi151 ceph-mon[159210]: pgmap v1048: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 21 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:17.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1907862744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:17.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:17.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:17.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:17 smithi052 ceph-mon[187743]: pgmap v1048: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 21 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:17.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1907862744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:17.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:17.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:17 smithi052 ceph-mon[180979]: pgmap v1048: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 21 op/s; 41028/237549 objects degraded (17.271%); 47559/237549 objects misplaced (20.021%) 2023-09-23T22:58:17.948 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1907862744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:17.948 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:17.989 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:18.016 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:18.061 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:18.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:18.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:18.164 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:18.197 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:18.537 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:58:18.564 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:18.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:18.668 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:18.694 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:18.749 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:18.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:18.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:18 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:18.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:18 smithi151 ceph-mon[159210]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:18 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:58:18.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:18 smithi151 ceph-mon[159210]: osdmap e357: 8 total, 7 up, 6 in 2023-09-23T22:58:18.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:18 smithi151 ceph-mon[159210]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.774 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:18 smithi151 ceph-mon[159210]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.855 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:18.886 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:18.930 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:18.985 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:18.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:18.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[187743]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[187743]: osdmap e357: 8 total, 7 up, 6 in 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[187743]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[187743]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[180979]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:58:18.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[180979]: osdmap e357: 8 total, 7 up, 6 in 2023-09-23T22:58:18.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[180979]: failed to encode map e357 with expected crc 2023-09-23T22:58:18.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:18 smithi052 ceph-mon[180979]: failed to encode map e357 with expected crc 2023-09-23T22:58:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:58:19.029 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-2, ['dump_ops_in_flight'] 2023-09-23T22:58:19.088 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:19.119 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:19.173 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:19.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:19.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:19.277 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:19.305 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:19.353 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:19.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:19.457 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:19.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:19.537 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:19.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:19.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:19.640 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:19.669 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:19.713 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:19.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:19.817 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:19.844 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:19.890 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:19.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:19.993 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:20.021 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:20.068 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:20.070 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:19 smithi052 ceph-mon[180979]: pgmap v1050: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 16 op/s; 41028/237549 objects degraded (17.271%); 47443/237549 objects misplaced (19.972%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:58:20.070 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:20.070 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:20.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:20.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:19 smithi052 ceph-mon[187743]: pgmap v1050: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 16 op/s; 41028/237549 objects degraded (17.271%); 47443/237549 objects misplaced (19.972%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:58:20.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:20.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:19 smithi151 ceph-mon[159210]: pgmap v1050: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 16 op/s; 41028/237549 objects degraded (17.271%); 47443/237549 objects misplaced (19.972%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:58:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:20.098 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:20.173 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:20.201 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:20.252 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:20.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:20.355 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:20.387 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:20.490 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:20.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:20.559 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:20.560 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:58:20.560 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:58:20.560 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:58:20.560 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:20.586 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:58:20.594 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:20.657 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:20.703 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:20.705 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:20 smithi052 conmon[505773]: 2023-09-23T22:58:20.433+0000 7f9e167bb640 -1 osd.2 323 log_to_monitors true 2023-09-23T22:58:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:20.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:20.807 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:20.836 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:20.885 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:20.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:20.949 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:20.989 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:21.016 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:21.296 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:21.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:21.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:21 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:58:21.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:21 smithi151 ceph-mon[159210]: pgmap v1051: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 13 op/s; 41028/237549 objects degraded (17.271%); 47443/237549 objects misplaced (19.972%); 1.7 MiB/s, 28 objects/s recovering 2023-09-23T22:58:21.356 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:58:21.401 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:21.428 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:21.482 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:21.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:21.494 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 22:58:21 smithi052 conmon[505773]: 2023-09-23T22:58:21.244+0000 7f9e03c20700 -1 osd.2 323 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:58:21.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:21 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:58:21.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:21 smithi052 ceph-mon[187743]: pgmap v1051: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 13 op/s; 41028/237549 objects degraded (17.271%); 47443/237549 objects misplaced (19.972%); 1.7 MiB/s, 28 objects/s recovering 2023-09-23T22:58:21.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:21 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:58:21.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:21 smithi052 ceph-mon[180979]: pgmap v1051: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 13 op/s; 41028/237549 objects degraded (17.271%); 47443/237549 objects misplaced (19.972%); 1.7 MiB/s, 28 objects/s recovering 2023-09-23T22:58:21.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:21.620 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:21.667 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:21.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:21.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:21.754 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:21.771 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:21.803 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:22.085 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:22.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:22.189 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:22.219 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:22.247 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:22 smithi151 ceph-mon[159210]: failed to encode map e358 with expected crc 2023-09-23T22:58:22.247 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:22 smithi151 ceph-mon[159210]: failed to encode map e358 with expected crc 2023-09-23T22:58:22.248 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:22 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:58:22.248 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:22 smithi151 ceph-mon[159210]: osdmap e358: 8 total, 7 up, 6 in 2023-09-23T22:58:22.248 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:22 smithi151 ceph-mon[159210]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:22.354 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[187743]: failed to encode map e358 with expected crc 2023-09-23T22:58:22.354 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[187743]: failed to encode map e358 with expected crc 2023-09-23T22:58:22.354 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:58:22.354 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[187743]: osdmap e358: 8 total, 7 up, 6 in 2023-09-23T22:58:22.354 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[187743]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:22.355 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[180979]: failed to encode map e358 with expected crc 2023-09-23T22:58:22.355 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[180979]: failed to encode map e358 with expected crc 2023-09-23T22:58:22.355 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:58:22.355 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[180979]: osdmap e358: 8 total, 7 up, 6 in 2023-09-23T22:58:22.355 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:22 smithi052 ceph-mon[180979]: from='osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi052", "root=default"]}]: dispatch 2023-09-23T22:58:22.390 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:22.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:22.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:22.522 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:22.571 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:22.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:22.675 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:22.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:22.758 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:22.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:22.862 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:22.889 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:23.102 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:23.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:23.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:23.209 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:23.255 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:58:23.271 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:23.356 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:23.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:23.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:23.464 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:23.492 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:23.537 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:23.539 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:58:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:58:23] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.33.4" 2023-09-23T22:58:23.539 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:58:23.539 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482] boot 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: osdmap e359: 8 total, 8 up, 6 in 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: pgmap v1054: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41028/237549 objects degraded (17.271%); 47326/237549 objects misplaced (19.923%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/363934485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:23.540 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[187743]: osdmap e360: 8 total, 8 up, 6 in 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482] boot 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: osdmap e359: 8 total, 8 up, 6 in 2023-09-23T22:58:23.541 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: pgmap v1054: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41028/237549 objects degraded (17.271%); 47326/237549 objects misplaced (19.923%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/363934485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:23.542 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:23 smithi052 ceph-mon[180979]: osdmap e360: 8 total, 8 up, 6 in 2023-09-23T22:58:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:23.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:23.597 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73740 10.7 10:e8cc0162:::benchmark_data_smithi052_461816_object73739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202451+0000", 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 123.391687488, 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2554468189999999, 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.598 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73740 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202451+0000", 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202451+0000", 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202449+0000", 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000173795 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202623+0000", 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2417e-05 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202635+0000", 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5763000000000001e-05 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202651+0000", 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8651000000000001e-05 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202749+0000", 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.145e-05 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202761+0000", 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2256656210000001 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.428426+0000", 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014574740000000001 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.429884+0000", 2023-09-23T22:58:23.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000133375 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.430017+0000", 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010129305999999999 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.440147+0000", 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011353296000000001 2023-09-23T22:58:23.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451500+0000", 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003660782 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.455161+0000", 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3852e-05 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.455205+0000", 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026931049999999999 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.457898+0000", 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2554468189999999 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.606 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73741 10.7 10:e8bfd85c:::benchmark_data_smithi052_461816_object73740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202698+0000", 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 123.391439893, 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2575494979999999, 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.607 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73741 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202698+0000", 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202698+0000", 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202698+0000", 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0708000000000003e-05 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202748+0000", 2023-09-23T22:58:23.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1382e-05 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202760+0000", 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0619999999999997e-06 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202766+0000", 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000388203 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203154+0000", 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.893e-06 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203164+0000", 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2287006620000001 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.431865+0000", 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000108812 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.431973+0000", 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000142452 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.432116+0000", 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011844664 2023-09-23T22:58:23.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.443960+0000", 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075977429999999997 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451558+0000", 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064268040000000004 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.457985+0000", 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0398000000000001e-05 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.458035+0000", 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022125040000000001 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.460248+0000", 2023-09-23T22:58:23.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2575494979999999 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73749 10.7 10:ec9af002:::benchmark_data_smithi052_461816_object73748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202791+0000", 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 123.391346842, 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.263480012, 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.615 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73749 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202791+0000", 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202791+0000", 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202791+0000", 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4153999999999999e-05 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202825+0000", 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0713000000000001e-05 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202836+0000", 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0301e-05 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202846+0000", 2023-09-23T22:58:23.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00077028199999999995 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203616+0000", 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9519999999999997e-06 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203623+0000", 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2305048009999999 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.434128+0000", 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012141099999999999 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.434249+0000", 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000146086 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.434396+0000", 2023-09-23T22:58:23.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095770060000000008 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.443973+0000", 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076323290000000002 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451605+0000", 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012022138999999999 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.463627+0000", 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2588000000000004e-05 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.463700+0000", 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0025718580000000002 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.623 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.466271+0000", 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.263480012 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73751 10.7 10:e00325f7:::benchmark_data_smithi052_461816_object73750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202864+0000", 2023-09-23T22:58:23.625 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 123.39127402299999, 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2747165890000001, 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73751 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.626 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202864+0000", 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202864+0000", 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202863+0000", 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6653000000000003e-05 2023-09-23T22:58:23.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202900+0000", 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0098000000000001e-05 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202910+0000", 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1009999999999997e-06 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202916+0000", 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010056469999999999 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203922+0000", 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0539999999999997e-06 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203929+0000", 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.232469193 2023-09-23T22:58:23.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.436398+0000", 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013066400000000001 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.436529+0000", 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012086 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.436650+0000", 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073337649999999999 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.443984+0000", 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076664180000000004 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451650+0000", 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022855486000000001 2023-09-23T22:58:23.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.474505+0000", 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1630000000000005e-05 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.474567+0000", 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030137919999999999 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.477581+0000", 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2747165890000001 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97333 10.7 10:e37b5792:::benchmark_data_smithi052_461816_object532:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.854177+0000", 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 79.739961112000003, 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94848450200000001, 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.633 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97333 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854177+0000", 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854177+0000", 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854175+0000", 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8010000000000004e-06 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.635 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854183+0000", 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5661000000000001e-05 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854199+0000", 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.114e-06 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854206+0000", 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6054000000000001e-05 2023-09-23T22:58:23.636 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854272+0000", 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3147e-05 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854285+0000", 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92368130199999998 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.637 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.777966+0000", 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013291099999999999 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.778099+0000", 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014655700000000001 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.778246+0000", 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012212475 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.790458+0000", 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099334079999999995 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800392+0000", 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7689999999999998e-05 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800449+0000", 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8202000000000003e-05 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800508+0000", 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021539710000000002 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.802662+0000", 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94848450200000001 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.640 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97340 10.9 10:92897f12:::benchmark_data_smithi052_461816_object539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.860204+0000", 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 79.733934081000001, 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93248454700000005, 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.641 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97340 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860204+0000", 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860204+0000", 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860202+0000", 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.465e-06 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860211+0000", 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.808e-05 2023-09-23T22:58:23.643 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860229+0000", 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860238+0000", 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0139000000000002e-05 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860288+0000", 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0019999999999995e-06 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860297+0000", 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90190687999999997 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.762204+0000", 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002049875 2023-09-23T22:58:23.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.764253+0000", 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0403999999999997e-05 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.764344+0000", 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0083045500000000008 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.772648+0000", 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019917872 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792566+0000", 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5818000000000001e-05 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792612+0000", 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6701999999999998e-05 2023-09-23T22:58:23.647 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792689+0000", 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93248454700000005 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97348 10.9 10:9297810a:::benchmark_data_smithi052_461816_object547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.871378+0000", 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 79.722760516999998, 2023-09-23T22:58:23.648 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92148298399999995, 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97348 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871378+0000", 2023-09-23T22:58:23.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871378+0000", 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871376+0000", 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8349999999999999e-06 2023-09-23T22:58:23.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871386+0000", 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5957999999999999e-05 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871401+0000", 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9549999999999992e-06 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871409+0000", 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000133244 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871543+0000", 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0589e-05 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871553+0000", 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.89562549300000005 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767179+0000", 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000122841 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767302+0000", 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013168399999999999 2023-09-23T22:58:23.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767433+0000", 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0078544730000000007 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.775288+0000", 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017446303999999999 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792734+0000", 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5383999999999998e-05 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792779+0000", 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1309000000000004e-05 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792861+0000", 2023-09-23T22:58:23.655 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92148298399999995 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97368 10.9 10:977fa808:::benchmark_data_smithi052_461816_object567:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.885379+0000", 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 79.708759420999996, 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.908468681, 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.656 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97368 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885379+0000", 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885379+0000", 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885377+0000", 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1136e-05 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885388+0000", 2023-09-23T22:58:23.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1736e-05 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885409+0000", 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9409999999999996e-06 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885418+0000", 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073186099999999999 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.886150+0000", 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9389999999999992e-06 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.886160+0000", 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88392771000000003 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770088+0000", 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010628699999999999 2023-09-23T22:58:23.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770194+0000", 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100766 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770295+0000", 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076737589999999996 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.661 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.777969+0000", 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015755661000000001 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793724+0000", 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0615000000000003e-05 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793765+0000", 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2626999999999993e-05 2023-09-23T22:58:23.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793848+0000", 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.908468681 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97377 10.7 10:e47c8c4a:::benchmark_data_smithi052_461816_object576:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.898849+0000", 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 79.695288860000005, 2023-09-23T22:58:23.663 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90806386800000005, 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97377 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898849+0000", 2023-09-23T22:58:23.664 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898849+0000", 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898847+0000", 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0633e-05 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898858+0000", 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8763e-05 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898877+0000", 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0770000000000008e-06 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898885+0000", 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2481e-05 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898947+0000", 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9250000000000007e-06 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898957+0000", 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88245452400000002 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781412+0000", 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013464799999999999 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781547+0000", 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017924 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:58:23.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781726+0000", 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016204418000000002 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.797930+0000", 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00481702 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.802747+0000", 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001920106 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.669 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.804667+0000", 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8881000000000001e-05 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.804726+0000", 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021871030000000001 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.670 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.806913+0000", 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90806386800000005 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114190 10.a 10:552a2e4c:::benchmark_data_smithi052_461816_object17389:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637641+0000", 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 49.956497061999997, 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.055871596, 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T22:58:23.671 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114190 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637641+0000", 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.672 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637641+0000", 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637640+0000", 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7300000000000002e-06 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637646+0000", 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1294999999999999e-05 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.673 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637657+0000", 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6370000000000004e-06 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637663+0000", 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3239000000000003e-05 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.674 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637716+0000", 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0421000000000001e-05 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637726+0000", 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0540947350000001 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691821+0000", 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00169184 2023-09-23T22:58:23.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.693513+0000", 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.055871596 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114194 10.8 10.1b16bda8 (undecoded) ondisk+retry+write+known_if_redirected e345)", 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637677+0000", 2023-09-23T22:58:23.676 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 49.956461152000003, 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058402421, 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114194 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T22:58:23.677 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9859999999999999e-06 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637680+0000", 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.246e-06 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637686+0000", 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0130000000000002e-06 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637689+0000", 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6356000000000001e-05 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637735+0000", 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637742+0000", 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058305386 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.696048+0000", 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1797000000000001e-05 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.696080+0000", 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058402421 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.681 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114198 10.5 10.b52bfca5 (undecoded) ondisk+retry+write+known_if_redirected e345)", 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637701+0000", 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 49.956437592, 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0533679819999999, 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114198 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.682 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637701+0000", 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637701+0000", 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637700+0000", 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6850000000000001e-06 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637703+0000", 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4110000000000004e-06 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637708+0000", 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.89e-06 2023-09-23T22:58:23.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637712+0000", 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0080000000000002e-05 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637762+0000", 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0999999999999993e-06 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637771+0000", 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0532533829999999 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691025+0000", 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3807000000000003e-05 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691069+0000", 2023-09-23T22:58:23.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0533679819999999 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114194 10.8 10:15bd68d8:::benchmark_data_smithi052_461816_object17393:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e352)", 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733139+0000", 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860999360000001, 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.051601287, 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T22:58:23.687 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114194 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733139+0000", 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733139+0000", 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999919 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733131+0000", 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6708999999999999e-05 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733148+0000", 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.889e-05 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733167+0000", 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9740000000000008e-06 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733176+0000", 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00030421400000000003 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733480+0000", 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0818999999999999e-05 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733491+0000", 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.047172094 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.780663+0000", 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040775170000000001 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.784740+0000", 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.051601287 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114211 10.8 10:14a263e5:::benchmark_data_smithi052_461816_object17410:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733194+0000", 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860944147000001, 2023-09-23T22:58:23.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083107408, 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114211 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9090000000000002e-06 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733197+0000", 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0261000000000001e-05 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733208+0000", 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6239999999999999e-06 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733213+0000", 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045623199999999999 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733670+0000", 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0780000000000001e-06 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733676+0000", 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.05109607 2023-09-23T22:58:23.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.784772+0000", 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00058614799999999999 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.785358+0000", 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015455199999999999 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.785512+0000", 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017875975999999998 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803388+0000", 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012748966 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816137+0000", 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0511999999999999e-05 2023-09-23T22:58:23.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816198+0000", 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103699 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816302+0000", 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083107408 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.699 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114213 10.8 10:18638f4e:::benchmark_data_smithi052_461816_object17412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733219+0000", 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860919328000001, 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0833211110000001, 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.700 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114213 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733219+0000", 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733219+0000", 2023-09-23T22:58:23.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733218+0000", 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.574e-06 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733222+0000", 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6910000000000003e-06 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733231+0000", 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2459999999999997e-06 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733235+0000", 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000902928 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.703 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734138+0000", 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3409999999999997e-06 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734145+0000", 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.054099326 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788245+0000", 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139839 2023-09-23T22:58:23.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788384+0000", 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013002399999999999 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788514+0000", 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014879720000000001 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803394+0000", 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012988685999999999 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816383+0000", 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0869999999999999e-05 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816434+0000", 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106334 2023-09-23T22:58:23.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816540+0000", 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0833211110000001 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114214 10.8 10:12e4921e:::benchmark_data_smithi052_461816_object17413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733241+0000", 2023-09-23T22:58:23.707 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860897632, 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083505803, 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114214 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.708 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733241+0000", 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733241+0000", 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733240+0000", 2023-09-23T22:58:23.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9979999999999999e-06 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733243+0000", 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3750000000000003e-06 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733252+0000", 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4340000000000002e-06 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733256+0000", 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001189924 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734446+0000", 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5579999999999997e-06 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734452+0000", 2023-09-23T22:58:23.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0568490829999999 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791302+0000", 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124091 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791426+0000", 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019142000000000001 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791617+0000", 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011782971999999999 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803400+0000", 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013200309 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816600+0000", 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7296999999999998e-05 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816648+0000", 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8789000000000006e-05 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816746+0000", 2023-09-23T22:58:23.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083505803 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114223 10.8 10:1be79ee9:::benchmark_data_smithi052_461816_object17422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733260+0000", 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860877951999999, 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0836756359999999, 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.715 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114223 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.716 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9550000000000001e-06 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733263+0000", 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9780000000000005e-06 2023-09-23T22:58:23.717 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733271+0000", 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4569999999999998e-06 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733275+0000", 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001796401 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.718 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735072+0000", 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7419999999999997e-06 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735079+0000", 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0592689369999999 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794348+0000", 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129245 2023-09-23T22:58:23.719 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794478+0000", 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016843500000000001 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794646+0000", 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021672981000000001 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.720 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816319+0000", 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047669500000000002 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816796+0000", 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5318000000000002e-05 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816841+0000", 2023-09-23T22:58:23.721 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4890999999999997e-05 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816936+0000", 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0836756359999999 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.722 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114229 10.8 10:11c38b0a:::benchmark_data_smithi052_461816_object17428:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733280+0000", 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860858458999999, 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083835967, 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114229 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.723 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733280+0000", 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733280+0000", 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.724 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733279+0000", 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3809999999999999e-06 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733283+0000", 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2919999999999996e-06 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733291+0000", 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8060000000000002e-06 2023-09-23T22:58:23.725 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733296+0000", 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022410709999999999 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735537+0000", 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7049999999999994e-06 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.726 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735547+0000", 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0619144519999999 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797461+0000", 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124339 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.727 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797585+0000", 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013691500000000001 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797722+0000", 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018607162999999999 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816330+0000", 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00064679199999999998 2023-09-23T22:58:23.728 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816976+0000", 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5336000000000001e-05 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817022+0000", 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4147000000000005e-05 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.729 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817116+0000", 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083835967 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114239 10.8 10:1ab2d4f2:::benchmark_data_smithi052_461816_object17438:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733300+0000", 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860838130000001, 2023-09-23T22:58:23.730 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0839988570000001, 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114239 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T22:58:23.731 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.101e-06 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.732 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733303+0000", 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9850000000000003e-06 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733309+0000", 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0110000000000002e-06 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.733 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733313+0000", 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023753369999999999 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735688+0000", 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5669999999999995e-06 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735697+0000", 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.064783783 2023-09-23T22:58:23.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800481+0000", 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011374699999999999 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800594+0000", 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012472500000000001 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800719+0000", 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015619889 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816339+0000", 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00081903599999999998 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817158+0000", 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5899999999999998e-05 2023-09-23T22:58:23.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817204+0000", 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5204000000000005e-05 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817299+0000", 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0839988570000001 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114244 10.8 10:16611384:::benchmark_data_smithi052_461816_object17443:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733317+0000", 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 17.860821618999999, 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0845525629999999, 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114244 2023-09-23T22:58:23.738 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.739 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:23.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:23.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733317+0000", 2023-09-23T22:58:23.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:23.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733317+0000", 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733316+0000", 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.531e-06 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733319+0000", 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6490000000000008e-06 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733326+0000", 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0069999999999999e-06 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:23.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733330+0000", 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031344089999999999 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.736465+0000", 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1509999999999998e-06 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.736472+0000", 2023-09-23T22:58:23.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0669546030000001 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803427+0000", 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114894 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T22:58:23.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803542+0000", 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129417 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803671+0000", 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013664033000000001 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817335+0000", 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037598800000000001 2023-09-23T22:58:23.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817711+0000", 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1017999999999998e-05 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817772+0000", 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7250999999999994e-05 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T22:58:23.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817869+0000", 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0845525629999999 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:23.780 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:23.780 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:23.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:58:23.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: osd.2 [v2:172.21.15.52:6818/929264482,v1:172.21.15.52:6819/929264482] boot 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: osdmap e359: 8 total, 8 up, 6 in 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: failed to encode map e359 with expected crc 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: pgmap v1054: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 15 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41028/237549 objects degraded (17.271%); 47326/237549 objects misplaced (19.923%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/363934485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:23.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:23 smithi151 ceph-mon[159210]: osdmap e360: 8 total, 8 up, 6 in 2023-09-23T22:58:23.814 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:23.867 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:23.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:23.972 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:24.000 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:24.030 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 355'7174 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: 10.3 continuing backfill to osd.3 from (322'1500,355'7195] MIN to 355'7195 2023-09-23T22:58:24.266 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: 10.5 continuing backfill to osd.3 from (322'1800,355'7173] 10:a29216c4:::benchmark_data_smithi052_461816_object66347:head to 355'7173 2023-09-23T22:58:24.267 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[187743]: osdmap e361: 8 total, 8 up, 6 in 2023-09-23T22:58:24.324 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:24.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:24.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:24.431 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:24.485 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:24.521 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:24.522 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:58:24.522 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 355'7174 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: 10.3 continuing backfill to osd.3 from (322'1500,355'7195] MIN to 355'7195 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: 10.5 continuing backfill to osd.3 from (322'1800,355'7173] 10:a29216c4:::benchmark_data_smithi052_461816_object66347:head to 355'7173 2023-09-23T22:58:24.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:24 smithi151 ceph-mon[159210]: osdmap e361: 8 total, 8 up, 6 in 2023-09-23T22:58:24.535 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:24.537 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.537 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: failed to encode map e360 with expected crc 2023-09-23T22:58:24.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 355'7174 2023-09-23T22:58:24.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: 10.3 continuing backfill to osd.3 from (322'1500,355'7195] MIN to 355'7195 2023-09-23T22:58:24.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: 10.5 continuing backfill to osd.3 from (322'1800,355'7173] 10:a29216c4:::benchmark_data_smithi052_461816_object66347:head to 355'7173 2023-09-23T22:58:24.538 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:24 smithi052 ceph-mon[180979]: osdmap e361: 8 total, 8 up, 6 in 2023-09-23T22:58:24.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:24.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:24.640 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:24.667 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:24.764 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:24.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:24.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:24.868 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:24.898 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:24.953 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:25.046 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:25.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:25.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:25.150 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:25.185 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:25.241 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:25.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:25.346 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:25.374 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: 10.f continuing backfill to osd.3 from (322'1400,355'7132] MIN to 355'7132 2023-09-23T22:58:25.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: 10.7 continuing backfill to osd.3 from (321'400,355'7055] MIN to 355'7055 2023-09-23T22:58:25.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:25 smithi151 ceph-mon[159210]: pgmap v1057: 89 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 14 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41028/237549 objects degraded (17.271%); 47326/237549 objects misplaced (19.923%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:58:25.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.675 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: 10.f continuing backfill to osd.3 from (322'1400,355'7132] MIN to 355'7132 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: 10.7 continuing backfill to osd.3 from (321'400,355'7055] MIN to 355'7055 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[187743]: pgmap v1057: 89 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 14 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41028/237549 objects degraded (17.271%); 47326/237549 objects misplaced (19.923%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: failed to encode map e361 with expected crc 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: 10.f continuing backfill to osd.3 from (322'1400,355'7132] MIN to 355'7132 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: 10.7 continuing backfill to osd.3 from (321'400,355'7055] MIN to 355'7055 2023-09-23T22:58:25.677 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:25 smithi052 ceph-mon[180979]: pgmap v1057: 89 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 52 active+clean, 5 active+recovery_wait+degraded+remapped, 6 active+undersized, 3 active+undersized+degraded, 14 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41028/237549 objects degraded (17.271%); 47326/237549 objects misplaced (19.923%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:58:25.694 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:25.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:25.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:25.798 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:25.828 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:25.879 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:25.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:25.985 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:26.025 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:26.141 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:26.150 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:58:26.150 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:58:26.150 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T22:58:26.151 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.162 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.163 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:58:26.402 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:58:26.478 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:26.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:26.582 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:26.608 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:26.652 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:26.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:26.755 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:26.781 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:26.826 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:58:26.968 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:26.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:27.071 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:27.101 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:27.168 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:27.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:27.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:27.271 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:27.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:27 smithi151 ceph-mon[159210]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:27 smithi151 ceph-mon[159210]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:27 smithi151 ceph-mon[159210]: osdmap e362: 8 total, 8 up, 6 in 2023-09-23T22:58:27.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:27 smithi151 ceph-mon[159210]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:27 smithi151 ceph-mon[159210]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:27 smithi151 ceph-mon[159210]: pgmap v1059: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 68 active+clean, 3 active+recovery_wait+degraded+remapped, 3 active+undersized, 1 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 36904/237549 objects degraded (15.535%); 44291/237549 objects misplaced (18.645%); 565 KiB/s, 8 objects/s recovering 2023-09-23T22:58:27.304 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:58:27.410 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:58:27.449 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:27.452 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:27.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:27.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:27.464 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:27.464 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T22:58:27.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114254 10.b 10:d67f3bd7:::benchmark_data_smithi052_461816_object17453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:06.805895+0000", 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 20.657851618999999, 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 20.657879253000001, 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "delayed", 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114254 2023-09-23T22:58:27.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.805895+0000", 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.805895+0000", 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.805893+0000", 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2693e-05 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.805905+0000", 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6167e-05 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:27.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.805922+0000", 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4020000000000002e-06 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.805930+0000", 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018078409999999999 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.807738+0000", 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2974e-05 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.807751+0000", 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.447509116999999 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.255260+0000", 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.01521871 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for readable", 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.270479+0000", 2023-09-23T22:58:27.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.464583431000001 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114256 10.b 10:dfc198f9:::benchmark_data_smithi052_461816_object17455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:06.815613+0000", 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 20.648134029000001, 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 20.648221325000002, 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "delayed", 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:27.470 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114256 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815613+0000", 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815613+0000", 2023-09-23T22:58:27.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815610+0000", 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5291999999999999e-05 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815625+0000", 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5252000000000001e-05 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815650+0000", 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.6460000000000005e-06 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815660+0000", 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6154000000000003e-05 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815736+0000", 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4277000000000001e-05 2023-09-23T22:58:27.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.815751+0000", 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.456132511 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.271883+0000", 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034183460000000001 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for readable", 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.275301+0000", 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.459688538999998 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114263 10.b 10:d12570d8:::benchmark_data_smithi052_461816_object17462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:06.827840+0000", 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 20.635906446, 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 20.636044254000002, 2023-09-23T22:58:27.475 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "delayed", 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114263 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827840+0000", 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827840+0000", 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827838+0000", 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.28e-06 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827845+0000", 2023-09-23T22:58:27.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2649999999999996e-06 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827853+0000", 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.262e-06 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827857+0000", 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2712000000000005e-05 2023-09-23T22:58:27.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827920+0000", 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2418e-05 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.827932+0000", 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.448838344999999 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.276771+0000", 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034110680000000002 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for readable", 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.280182+0000", 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.452341254 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114283 10.b 10:d3d4d917:::benchmark_data_smithi052_461816_object17482:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-09-23T22:58:27.480 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:06.868863+0000", 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 20.594884050000001, 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 20.595071426000001, 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "delayed", 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114283 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868863+0000", 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868863+0000", 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868861+0000", 2023-09-23T22:58:27.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8249999999999999e-06 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868868+0000", 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4136e-05 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868882+0000", 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6400000000000002e-06 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868888+0000", 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3781999999999993e-05 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868952+0000", 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9881000000000002e-05 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.868972+0000", 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.412833478 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.281805+0000", 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055921210000000002 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for readable", 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:26.287397+0000", 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 19.41853454 2023-09-23T22:58:27.485 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 4 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:27.486 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:27.559 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:27.589 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:27.640 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:27.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:27.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[187743]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[187743]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[187743]: osdmap e362: 8 total, 8 up, 6 in 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[187743]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[187743]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[187743]: pgmap v1059: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 68 active+clean, 3 active+recovery_wait+degraded+remapped, 3 active+undersized, 1 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 36904/237549 objects degraded (15.535%); 44291/237549 objects misplaced (18.645%); 565 KiB/s, 8 objects/s recovering 2023-09-23T22:58:27.653 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[180979]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[180979]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[180979]: osdmap e362: 8 total, 8 up, 6 in 2023-09-23T22:58:27.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[180979]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[180979]: failed to encode map e362 with expected crc 2023-09-23T22:58:27.654 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:27 smithi052 ceph-mon[180979]: pgmap v1059: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 activating+remapped, 3 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 68 active+clean, 3 active+recovery_wait+degraded+remapped, 3 active+undersized, 1 active+clean+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 36904/237549 objects degraded (15.535%); 44291/237549 objects misplaced (18.645%); 565 KiB/s, 8 objects/s recovering 2023-09-23T22:58:27.744 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:27.773 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:27.821 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:27.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:27.924 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:27.953 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:28.466 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:58:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 36904/237549 objects degraded (15.535%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[180979]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[180979]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[180979]: osdmap e363: 8 total, 8 up, 6 in 2023-09-23T22:58:28.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[180979]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[180979]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 36904/237549 objects degraded (15.535%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[187743]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[187743]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[187743]: osdmap e363: 8 total, 8 up, 6 in 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[187743]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:28 smithi052 ceph-mon[187743]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:28 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 36904/237549 objects degraded (15.535%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:58:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:28 smithi151 ceph-mon[159210]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:28 smithi151 ceph-mon[159210]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:28 smithi151 ceph-mon[159210]: osdmap e363: 8 total, 8 up, 6 in 2023-09-23T22:58:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:28 smithi151 ceph-mon[159210]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:28 smithi151 ceph-mon[159210]: failed to encode map e363 with expected crc 2023-09-23T22:58:28.909 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:28.958 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:28.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:28.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:29.062 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:29.092 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:29.268 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:29.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:29.372 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:29.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:29.412 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:58:29.501 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:29.503 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:29.503 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:29 smithi151 ceph-mon[159210]: pgmap v1061: 89 pgs: 1 active+recovering+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 72 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41533/237549 objects degraded (17.484%); 38630/237549 objects misplaced (16.262%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:58:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:29.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:29.605 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:29.635 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:29.635 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:29 smithi052 ceph-mon[180979]: pgmap v1061: 89 pgs: 1 active+recovering+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 72 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41533/237549 objects degraded (17.484%); 38630/237549 objects misplaced (16.262%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:58:29.635 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:58:29.635 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:29 smithi052 ceph-mon[187743]: pgmap v1061: 89 pgs: 1 active+recovering+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 72 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41533/237549 objects degraded (17.484%); 38630/237549 objects misplaced (16.262%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:58:29.639 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:29.810 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:29.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:29.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:29.914 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:29.933 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:29.940 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:30.291 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:58:30.299 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:58:30.497 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:30.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:30.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:30.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4193092663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:30.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3306735123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:58:30.632 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:30.670 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4193092663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:30.670 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3306735123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:58:30.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4193092663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:30.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3306735123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:58:30.702 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:30.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:30.809 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:30.840 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:31.160 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:31.160 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:58:31.160 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:58:31.160 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:58:31.160 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:31.392 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:31.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:31.496 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:31.525 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4193092663' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3306735123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: osdmap e364: 8 total, 8 up, 6 in 2023-09-23T22:58:31.579 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: pgmap v1063: 89 pgs: 1 active+recovering+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 72 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41533/237549 objects degraded (17.484%); 38630/237549 objects misplaced (16.262%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:58:31.580 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[187743]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.580 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4193092663' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3306735123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: osdmap e364: 8 total, 8 up, 6 in 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: pgmap v1063: 89 pgs: 1 active+recovering+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 72 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41533/237549 objects degraded (17.484%); 38630/237549 objects misplaced (16.262%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:58:31.581 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:31 smithi052 ceph-mon[180979]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4193092663' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3306735123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: osdmap e364: 8 total, 8 up, 6 in 2023-09-23T22:58:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: pgmap v1063: 89 pgs: 1 active+recovering+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 72 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41533/237549 objects degraded (17.484%); 38630/237549 objects misplaced (16.262%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:58:31.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:31 smithi151 ceph-mon[159210]: failed to encode map e364 with expected crc 2023-09-23T22:58:31.793 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:58:32.267 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:58:32.281 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:32.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:32.386 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:32.427 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:32.469 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:32.579 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:32.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:32.684 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:32.716 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:32.816 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:32.820 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:58:32.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:32.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:32.827 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:32 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 41417/237549 objects degraded (17.435%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:32.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:32 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 41417/237549 objects degraded (17.435%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:32.847 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:32.851 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:32.870 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:58:32.891 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:32.912 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:32.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:32.944 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:32 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 41417/237549 objects degraded (17.435%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:32.953 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:32.964 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:32.985 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.986 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:58:32.987 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:58:33.078 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:33.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:33.182 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:33.211 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:33.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:58:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:58:33] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.33.4" 2023-09-23T22:58:33.564 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:33.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:33.581 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:58:33.667 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:33.696 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:33 smithi151 ceph-mon[159210]: pgmap v1064: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41417/237549 objects degraded (17.435%); 38514/237549 objects misplaced (16.213%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:58:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/624558793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:33.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:33 smithi052 ceph-mon[187743]: pgmap v1064: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41417/237549 objects degraded (17.435%); 38514/237549 objects misplaced (16.213%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:58:33.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/624558793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:33 smithi052 ceph-mon[180979]: pgmap v1064: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41417/237549 objects degraded (17.435%); 38514/237549 objects misplaced (16.213%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:58:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/624558793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:34.049 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:34.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:34.154 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:34.182 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:34.345 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:34.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:34.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:34.450 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:34.483 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:34.767 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:35.042 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:35.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:35.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:35.059 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:58:35.060 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:58:35.147 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:35.196 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:35.289 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:35.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:35.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:35.396 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:35.443 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:35.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:35.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:35.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[187743]: pgmap v1065: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41417/237549 objects degraded (17.435%); 38514/237549 objects misplaced (16.213%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:58:35.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/966414696' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:58:35.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:35.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[180979]: pgmap v1065: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41417/237549 objects degraded (17.435%); 38514/237549 objects misplaced (16.213%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:58:35.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.622 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/966414696' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:58:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:35.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:35.724 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:35.751 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:35 smithi151 ceph-mon[159210]: pgmap v1065: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 41417/237549 objects degraded (17.435%); 38514/237549 objects misplaced (16.213%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:58:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:35.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/966414696' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:58:36.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:36.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:36.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:36.124 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:58:36.189 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:58:36.222 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:36.272 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:36.340 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:36.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:36.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:36.409 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:58:36.446 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:36.477 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:36.551 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:36.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:36.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:36.656 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:36.691 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:36.694 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:36.907 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:36.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:36.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:36.961 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:58:37.011 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:37.041 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:37.216 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/966414696' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: osdmap e365: 8 total, 8 up, 6 in 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: pgmap v1067: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 41300/237393 objects degraded (17.397%); 38500/237393 objects misplaced (16.218%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:58:37.219 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:37.220 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:37.220 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/966414696' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:58:37.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: osdmap e365: 8 total, 8 up, 6 in 2023-09-23T22:58:37.220 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: pgmap v1067: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 41300/237393 objects degraded (17.397%); 38500/237393 objects misplaced (16.218%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:37.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:37.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:37.277 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/966414696' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: osdmap e365: 8 total, 8 up, 6 in 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: failed to encode map e365 with expected crc 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: pgmap v1067: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 41300/237393 objects degraded (17.397%); 38500/237393 objects misplaced (16.218%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:58:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:58:37.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:58:37.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:58:37.323 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:37.359 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:37.417 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:37.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:37.437 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:37.521 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:37.563 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:37.638 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:37.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:37.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:37.748 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:37.779 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:37.937 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:37.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:38.041 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:38.081 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:38.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:38 smithi052 ceph-mon[187743]: failed to encode map e365 with expected crc 2023-09-23T22:58:38.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4231125835' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:38.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:38.237 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:58:38.365 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:38 smithi151 ceph-mon[159210]: failed to encode map e365 with expected crc 2023-09-23T22:58:38.365 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4231125835' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:38.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:38.383 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:38.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:38.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:38 smithi052 ceph-mon[180979]: failed to encode map e365 with expected crc 2023-09-23T22:58:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4231125835' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:38.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:38.525 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:38.623 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:38.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:38.730 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:38.781 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:38.854 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T22:58:38.854 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:58:38.875 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:38.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:38.985 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:39.042 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:39.110 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:39.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:39.221 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:39.255 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:39 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:58:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:39 smithi151 ceph-mon[159210]: failed to encode map e366 with expected crc 2023-09-23T22:58:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:39 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:39 smithi151 ceph-mon[159210]: osdmap e366: 8 total, 8 up, 6 in 2023-09-23T22:58:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:39 smithi151 ceph-mon[159210]: failed to encode map e366 with expected crc 2023-09-23T22:58:39.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:39 smithi151 ceph-mon[159210]: pgmap v1069: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 41277/236853 objects degraded (17.427%); 38319/236853 objects misplaced (16.178%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[187743]: failed to encode map e366 with expected crc 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[187743]: osdmap e366: 8 total, 8 up, 6 in 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[187743]: failed to encode map e366 with expected crc 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[187743]: pgmap v1069: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 41277/236853 objects degraded (17.427%); 38319/236853 objects misplaced (16.178%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:58:39.474 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[180979]: failed to encode map e366 with expected crc 2023-09-23T22:58:39.475 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:39.475 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[180979]: osdmap e366: 8 total, 8 up, 6 in 2023-09-23T22:58:39.475 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[180979]: failed to encode map e366 with expected crc 2023-09-23T22:58:39.475 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:39 smithi052 ceph-mon[180979]: pgmap v1069: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 41277/236853 objects degraded (17.427%); 38319/236853 objects misplaced (16.178%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:58:39.752 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:39.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:39.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:39.857 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:39.897 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:39.972 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:58:40.122 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:40.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:40.168 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:58:40.190 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:40.216 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:58:40.228 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:40.255 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:40.272 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:40.292 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:40.322 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:40.344 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:40.348 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:40.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:40.372 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:40.373 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.373 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.373 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.374 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:58:40.395 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 41277/236853 objects degraded (17.427%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:40.396 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2954139537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:40.396 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 41277/236853 objects degraded (17.427%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:40.396 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2954139537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:40.450 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:40.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 41277/236853 objects degraded (17.427%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:40.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2954139537' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:40.489 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:40.497 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:40.577 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:40.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:40.689 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:40.717 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:40.779 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:40.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:40.888 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:40.922 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:41.159 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:41.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:41.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:41.187 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:58:41.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:41 smithi052 ceph-mon[180979]: pgmap v1070: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 41232/236007 objects degraded (17.471%); 38202/236007 objects misplaced (16.187%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:58:41.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3867736743' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:58:41.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:58:41.241 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:58:41.263 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:41.293 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:41 smithi151 ceph-mon[159210]: pgmap v1070: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 41232/236007 objects degraded (17.471%); 38202/236007 objects misplaced (16.187%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:58:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3867736743' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:58:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:58:41.452 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:41.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:41.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:41 smithi052 ceph-mon[187743]: pgmap v1070: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 41232/236007 objects degraded (17.471%); 38202/236007 objects misplaced (16.187%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:58:41.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3867736743' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:58:41.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:58:41.556 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:41.585 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:41.635 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:41.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:41.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:41.743 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:41.777 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:41.984 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:41.992 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:58:41.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:41.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:42.068 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:42.098 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:42.140 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:42.201 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:42.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:42.307 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:42.341 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[187743]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[187743]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[187743]: osdmap e367: 8 total, 8 up, 6 in 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[187743]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[180979]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[180979]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.342 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:58:42.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[180979]: osdmap e367: 8 total, 8 up, 6 in 2023-09-23T22:58:42.343 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:42 smithi052 ceph-mon[180979]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.345 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:42.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:42 smithi151 ceph-mon[159210]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:42 smithi151 ceph-mon[159210]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:58:42.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:42 smithi151 ceph-mon[159210]: osdmap e367: 8 total, 8 up, 6 in 2023-09-23T22:58:42.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:42 smithi151 ceph-mon[159210]: failed to encode map e367 with expected crc 2023-09-23T22:58:42.509 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:42.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:42.613 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:42.643 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:42.643 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T22:58:42.644 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T22:58:42.644 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T22:58:42.644 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:42.649 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:42.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:42.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:42.806 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:42.839 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:43.151 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:43.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:43.161 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:58:43.184 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:43.255 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:43.288 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:43.397 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:43.404 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:43 smithi052 ceph-mon[187743]: pgmap v1072: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 40992/234165 objects degraded (17.506%); 37877/234165 objects misplaced (16.175%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T22:58:43.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:43.405 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:58:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:58:43] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T22:58:43.405 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:43 smithi052 ceph-mon[180979]: pgmap v1072: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 40992/234165 objects degraded (17.506%); 37877/234165 objects misplaced (16.175%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T22:58:43.506 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:43.556 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:43.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:43 smithi151 ceph-mon[159210]: pgmap v1072: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 40992/234165 objects degraded (17.506%); 37877/234165 objects misplaced (16.175%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T22:58:43.735 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:58:43.790 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:43.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:43.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:43.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:43.951 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:44.026 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:44.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:44.134 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:44.164 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:44.215 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:44.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:44.254 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:58:44.267 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:58:44.319 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:44.383 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:44.440 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/931964003' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:44.440 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/931964003' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:44.509 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/931964003' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:58:44.519 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:44.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:44.625 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:44.656 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:44.716 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:44.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:44.820 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:44.850 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:44.910 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:44.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:45.013 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:45.052 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:45.135 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:45.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:45.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:45.266 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:45.296 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:45.392 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:45.392 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.392 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/931964003' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:58:45.392 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: pgmap v1074: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 40992/234165 objects degraded (17.506%); 37877/234165 objects misplaced (16.175%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 40992/234165 objects degraded (17.506%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/931964003' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.393 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.394 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: pgmap v1074: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 40992/234165 objects degraded (17.506%); 37877/234165 objects misplaced (16.175%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:58:45.394 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 40992/234165 objects degraded (17.506%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:45.481 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:45.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:45.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:45.586 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:45.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:45.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/931964003' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:58:45.622 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T22:58:45.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: failed to encode map e368 with expected crc 2023-09-23T22:58:45.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: pgmap v1074: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 40992/234165 objects degraded (17.506%); 37877/234165 objects misplaced (16.175%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:58:45.623 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 40992/234165 objects degraded (17.506%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:45.625 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:45.799 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:45.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:45.908 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:45.942 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:46.101 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:46.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:46.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:46.209 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:46.249 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:46.271 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:58:46.329 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T22:58:46.329 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T22:58:46.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T22:58:46.329 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [] 2023-09-23T22:58:46.329 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T22:58:46.359 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:46.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:46.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:46.469 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:46.503 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3777762323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:58:46.503 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:46 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:58:46.518 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:46.597 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3777762323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:58:46.597 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:46 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:58:46.598 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3777762323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:58:46.598 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:46 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:58:46.626 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:46.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:46.733 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:46.763 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:47.019 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:47.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:47.122 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:47.153 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:58:47.159 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:47.185 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:58:47.277 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[180979]: failed to encode map e369 with expected crc 2023-09-23T22:58:47.277 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:58:47.277 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[180979]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T22:58:47.277 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[180979]: failed to encode map e369 with expected crc 2023-09-23T22:58:47.277 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[180979]: pgmap v1076: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 40835/233037 objects degraded (17.523%); 37716/233037 objects misplaced (16.185%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:58:47.422 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:47.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:47.425 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:47 smithi151 ceph-mon[159210]: failed to encode map e369 with expected crc 2023-09-23T22:58:47.425 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:58:47.425 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:47 smithi151 ceph-mon[159210]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T22:58:47.425 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:47 smithi151 ceph-mon[159210]: failed to encode map e369 with expected crc 2023-09-23T22:58:47.425 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:47 smithi151 ceph-mon[159210]: pgmap v1076: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 40835/233037 objects degraded (17.523%); 37716/233037 objects misplaced (16.185%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:58:47.499 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:58:47.525 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:47.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:58:47.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[187743]: failed to encode map e369 with expected crc 2023-09-23T22:58:47.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:58:47.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[187743]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T22:58:47.572 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[187743]: failed to encode map e369 with expected crc 2023-09-23T22:58:47.573 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:47 smithi052 ceph-mon[187743]: pgmap v1076: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 40835/233037 objects degraded (17.523%); 37716/233037 objects misplaced (16.185%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:58:47.616 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:47.619 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:47.653 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:58:47.704 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:47.739 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:47.766 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:47.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:47.777 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:47.807 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:47.838 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:47.839 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.841 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.842 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:58:47.875 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:47.910 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:47.947 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:48.046 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:48.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:48.159 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:48.187 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:48.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2975576778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:48.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2975576778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:48.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2975576778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:48.760 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:48.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:48.865 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:48.916 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:48.993 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:48.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:49.100 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:49.133 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:49.186 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:49.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:49.290 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:49.316 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:49 smithi052 ceph-mon[187743]: pgmap v1077: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 40738/231630 objects degraded (17.588%); 37457/231630 objects misplaced (16.171%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:58:49.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:49 smithi052 ceph-mon[180979]: pgmap v1077: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 40738/231630 objects degraded (17.588%); 37457/231630 objects misplaced (16.171%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:58:49.319 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:49 smithi151 ceph-mon[159210]: pgmap v1077: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 40738/231630 objects degraded (17.588%); 37457/231630 objects misplaced (16.171%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:58:49.818 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:49.830 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:49.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:49.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:49.935 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:49.977 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:50.055 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:50.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:50.162 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:50.194 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:50.299 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:50.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:50.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:50.404 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:50.435 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:50.449 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:50.449 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 40738/231630 objects degraded (17.588%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:50.450 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:50.450 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 40738/231630 objects degraded (17.588%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:50.595 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:50.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:58:50.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 40738/231630 objects degraded (17.588%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:50.702 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:50.728 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:50.773 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:58:50.786 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:50.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:50.895 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:50.930 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:51.014 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:51.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:51.119 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:51.176 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:51.248 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:51.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:51.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:51.301 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:51 smithi052 ceph-mon[187743]: pgmap v1078: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 40684/230682 objects degraded (17.636%); 37318/230682 objects misplaced (16.177%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:58:51.301 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3653974543' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:51.301 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:51 smithi052 ceph-mon[180979]: pgmap v1078: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 40684/230682 objects degraded (17.636%); 37318/230682 objects misplaced (16.177%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:58:51.301 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3653974543' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:51.313 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:58:51.354 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:51.387 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:51 smithi151 ceph-mon[159210]: pgmap v1078: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 40684/230682 objects degraded (17.636%); 37318/230682 objects misplaced (16.177%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:58:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3653974543' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:58:52.194 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:52.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:52.300 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:52.334 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:52.399 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:52.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:52.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3653974543' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T22:58:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[180979]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3653974543' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T22:58:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:52 smithi052 ceph-mon[187743]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.511 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:52.550 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3653974543' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:58:52.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T22:58:52.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:52 smithi151 ceph-mon[159210]: failed to encode map e370 with expected crc 2023-09-23T22:58:52.704 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:52.764 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:52.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:52.811 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:58:52.811 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:58:52.869 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:52.898 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:52.960 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:52.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:53.067 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:53.107 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:53.318 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:58:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:58:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:58:53] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-09-23T22:58:53.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:53 smithi052 ceph-mon[187743]: pgmap v1080: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 241 op/s; 40439/228366 objects degraded (17.708%); 36907/228366 objects misplaced (16.161%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:58:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2474412095' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:58:53.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:53 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:58:53.496 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:53 smithi052 ceph-mon[180979]: pgmap v1080: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 241 op/s; 40439/228366 objects degraded (17.708%); 36907/228366 objects misplaced (16.161%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:58:53.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2474412095' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:58:53.497 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:53 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:58:53.666 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:53.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:53 smithi151 ceph-mon[159210]: pgmap v1080: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 241 op/s; 40439/228366 objects degraded (17.708%); 36907/228366 objects misplaced (16.161%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:58:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2474412095' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:58:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:53 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:58:53.771 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:53.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:54.235 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:58:54.313 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[180979]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.313 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:58:54.313 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[180979]: osdmap e371: 8 total, 8 up, 6 in 2023-09-23T22:58:54.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[180979]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[180979]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[180979]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:58:54.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[187743]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:58:54.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[187743]: osdmap e371: 8 total, 8 up, 6 in 2023-09-23T22:58:54.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[187743]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.576 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[187743]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.576 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:54 smithi052 ceph-mon[187743]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.599 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T22:58:54.651 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:54.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:54.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:54 smithi151 ceph-mon[159210]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:54 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:58:54.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:54 smithi151 ceph-mon[159210]: osdmap e371: 8 total, 8 up, 6 in 2023-09-23T22:58:54.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:54 smithi151 ceph-mon[159210]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:54 smithi151 ceph-mon[159210]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:54 smithi151 ceph-mon[159210]: failed to encode map e371 with expected crc 2023-09-23T22:58:54.758 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:54.797 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:54.856 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:54.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:54.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:54.923 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:54.961 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:58:54.993 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:55.053 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:58:55.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:55.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:58:55.159 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:55.192 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:55.279 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:55.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:55.297 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:58:55.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:55 smithi151 ceph-mon[159210]: pgmap v1082: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 40439/228366 objects degraded (17.708%); 36907/228366 objects misplaced (16.161%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:58:55.316 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 40439/228366 objects degraded (17.708%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:55.317 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3413849794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:55.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:55 smithi052 ceph-mon[187743]: pgmap v1082: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 40439/228366 objects degraded (17.708%); 36907/228366 objects misplaced (16.161%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:58:55.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 40439/228366 objects degraded (17.708%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:55.317 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3413849794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:55.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:55 smithi052 ceph-mon[180979]: pgmap v1082: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 40439/228366 objects degraded (17.708%); 36907/228366 objects misplaced (16.161%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:58:55.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 40439/228366 objects degraded (17.708%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:58:55.318 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3413849794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:58:55.324 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:58:55.350 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:58:55.378 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:58:55.387 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:55.408 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:58:55.440 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:58:55.445 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:55.462 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:58:55.488 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.491 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:58:55.588 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:55.588 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:58:55.604 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:55.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:55.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:55.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:55.740 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:56.200 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:56.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:56.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:56.305 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:56.338 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:56.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:56.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:56.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:56.526 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:58:56.554 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:56.600 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:58:56.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:56.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:58:56.707 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:58:56.739 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:56.854 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:58:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:56.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:58:56.958 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:58:56.990 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:57.086 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:58:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:57.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:58:57.195 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:57.240 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:57.554 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:57.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:57.661 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:57.692 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:57 smithi151 ceph-mon[159210]: pgmap v1083: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 40271/226908 objects degraded (17.748%); 36702/226908 objects misplaced (16.175%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:58:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:57 smithi052 ceph-mon[187743]: pgmap v1083: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 40271/226908 objects degraded (17.748%); 36702/226908 objects misplaced (16.175%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:58:57.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:57 smithi052 ceph-mon[180979]: pgmap v1083: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 196 op/s; 40271/226908 objects degraded (17.748%); 36702/226908 objects misplaced (16.175%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:58:58.088 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:58.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:58.192 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:58:58.222 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:58.434 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:58:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:58.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:58:58.540 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:58:58.568 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:58.609 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:58:58.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3001222569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:58 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3001222569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:58 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:58.815 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:58:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3001222569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:58 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:58:59.202 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:58:59.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:59.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:58:59.305 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:58:59.336 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:59.386 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:58:59.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:59.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:58:59.491 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:58:59.523 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[180979]: pgmap v1084: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 40161/225234 objects degraded (17.831%); 36404/225234 objects misplaced (16.163%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[180979]: failed to encode map e372 with expected crc 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[180979]: failed to encode map e372 with expected crc 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[180979]: osdmap e372: 8 total, 8 up, 6 in 2023-09-23T22:58:59.779 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[187743]: pgmap v1084: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 40161/225234 objects degraded (17.831%); 36404/225234 objects misplaced (16.163%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T22:58:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[187743]: failed to encode map e372 with expected crc 2023-09-23T22:58:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[187743]: failed to encode map e372 with expected crc 2023-09-23T22:58:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:58:59 smithi052 ceph-mon[187743]: osdmap e372: 8 total, 8 up, 6 in 2023-09-23T22:58:59.911 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:58:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:58:59.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:58:59.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:59 smithi151 ceph-mon[159210]: pgmap v1084: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 40161/225234 objects degraded (17.831%); 36404/225234 objects misplaced (16.163%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T22:58:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:59 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:58:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:59 smithi151 ceph-mon[159210]: failed to encode map e372 with expected crc 2023-09-23T22:58:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:59 smithi151 ceph-mon[159210]: failed to encode map e372 with expected crc 2023-09-23T22:58:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:58:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:58:59 smithi151 ceph-mon[159210]: osdmap e372: 8 total, 8 up, 6 in 2023-09-23T22:59:00.016 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:00.032 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:00.056 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:00.130 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:00.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:00.182 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:59:00.182 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:59:00.182 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:59:00.182 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:59:00.182 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:59:00.243 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:00.314 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:00.485 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:00.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:00.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:00.590 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:00.624 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 40161/225234 objects degraded (17.831%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:59:00.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3002689872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:59:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 40161/225234 objects degraded (17.831%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:59:00.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3002689872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:59:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 40161/225234 objects degraded (17.831%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:59:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3002689872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:59:01.207 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:01.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:01.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:01.338 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:01.378 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T22:59:01.395 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:01.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:01.503 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:01.539 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:01.544 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:59:01.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:01 smithi151 ceph-mon[159210]: pgmap v1086: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 167 op/s; 40108/224334 objects degraded (17.879%); 36294/224334 objects misplaced (16.179%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:59:01.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:01 smithi151 ceph-mon[159210]: failed to encode map e373 with expected crc 2023-09-23T22:59:01.602 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3002689872' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:59:01.603 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:01 smithi151 ceph-mon[159210]: osdmap e373: 8 total, 8 up, 6 in 2023-09-23T22:59:01.625 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:01.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:01.670 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:59:01.732 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:01.784 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:01.846 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:01.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:01.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:01.875 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[187743]: pgmap v1086: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 167 op/s; 40108/224334 objects degraded (17.879%); 36294/224334 objects misplaced (16.179%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:59:01.875 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[187743]: failed to encode map e373 with expected crc 2023-09-23T22:59:01.875 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3002689872' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:59:01.875 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[187743]: osdmap e373: 8 total, 8 up, 6 in 2023-09-23T22:59:01.875 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[180979]: pgmap v1086: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 167 op/s; 40108/224334 objects degraded (17.879%); 36294/224334 objects misplaced (16.179%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T22:59:01.876 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[180979]: failed to encode map e373 with expected crc 2023-09-23T22:59:01.876 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3002689872' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:59:01.876 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:01 smithi052 ceph-mon[180979]: osdmap e373: 8 total, 8 up, 6 in 2023-09-23T22:59:01.952 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:01.989 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:02.050 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:02.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:02.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:02.156 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:02.188 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:02.387 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:59:02.388 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:59:02.402 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:02.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:02.506 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:02.547 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:02.628 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:02.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:02.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:02.737 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:02.775 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:02.783 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:02.828 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:02.873 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:02.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:02.893 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:02.914 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:02.937 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:02.970 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:02.984 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:02.993 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:03.014 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:03.019 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:03.036 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:03.064 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:03.066 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.066 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.066 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.066 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.066 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.067 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.067 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.067 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:59:03.077 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:03.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:03.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:03.216 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:03.266 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:03.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:03.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:03.400 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:03.446 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:59:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:59:03] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-09-23T22:59:03.485 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:03.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:03.597 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:03.634 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:03.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:03 smithi052 ceph-mon[180979]: pgmap v1088: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 39691/222144 objects degraded (17.867%); 36066/222144 objects misplaced (16.235%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T22:59:03.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4121813817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:03.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:03 smithi052 ceph-mon[187743]: pgmap v1088: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 39691/222144 objects degraded (17.867%); 36066/222144 objects misplaced (16.235%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T22:59:03.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4121813817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:03 smithi151 ceph-mon[159210]: pgmap v1088: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 39691/222144 objects degraded (17.867%); 36066/222144 objects misplaced (16.235%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T22:59:03.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4121813817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:04.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:04.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:04.847 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:04.916 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:04.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:05.021 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:05.066 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:05.124 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:05.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:05.234 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:05.280 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:05.734 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:05 smithi052 ceph-mon[180979]: pgmap v1089: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 39691/222144 objects degraded (17.867%); 36066/222144 objects misplaced (16.235%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:59:05.734 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 39691/222144 objects degraded (17.867%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:59:05.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:05 smithi052 ceph-mon[187743]: pgmap v1089: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 39691/222144 objects degraded (17.867%); 36066/222144 objects misplaced (16.235%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:59:05.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 39691/222144 objects degraded (17.867%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:59:05.781 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:59:05.781 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:59:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:05 smithi151 ceph-mon[159210]: pgmap v1089: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 39691/222144 objects degraded (17.867%); 36066/222144 objects misplaced (16.235%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:59:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 39691/222144 objects degraded (17.867%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:59:06.045 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:06.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:06.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:06.150 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:06.190 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:06.260 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:06.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:06.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:06.365 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:06.396 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:06.648 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:59:06.671 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2405641843' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:06.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2405641843' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:06.878 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:59:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2405641843' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:07.014 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:59:07.043 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:07.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:07.148 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:07.185 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:07.563 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:07.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:07.667 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: pgmap v1090: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 39452/220728 objects degraded (17.874%); 35916/220728 objects misplaced (16.272%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:59:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:59:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.673 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2405641843' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: osdmap e374: 8 total, 8 up, 6 in 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[187743]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: pgmap v1090: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 39452/220728 objects degraded (17.874%); 35916/220728 objects misplaced (16.272%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2405641843' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: osdmap e374: 8 total, 8 up, 6 in 2023-09-23T22:59:07.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:07 smithi052 ceph-mon[180979]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.744 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:07.802 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:07.803 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:07.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:07.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:07.834 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:07.911 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:07.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: pgmap v1090: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 39452/220728 objects degraded (17.874%); 35916/220728 objects misplaced (16.272%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:59:07.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:59:07.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.941 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2405641843' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:07.941 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: osdmap e374: 8 total, 8 up, 6 in 2023-09-23T22:59:07.941 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:07 smithi151 ceph-mon[159210]: failed to encode map e374 with expected crc 2023-09-23T22:59:07.943 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:08.006 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:08.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:08.114 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:08.161 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:08.236 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:08.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:08.341 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:08.375 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:08.398 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:08.401 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:59:08.684 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:08.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:08.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:08.791 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:08.821 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:08.877 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:08 smithi052 ceph-mon[180979]: failed to encode map e374 with expected crc 2023-09-23T22:59:08.878 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:08 smithi052 ceph-mon[187743]: failed to encode map e374 with expected crc 2023-09-23T22:59:08.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:08 smithi151 ceph-mon[159210]: failed to encode map e374 with expected crc 2023-09-23T22:59:09.337 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:09.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:09.419 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1091,"stamp":"2023-09-23T22:59:06.631736+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4820834232,"num_objects":73576,"num_object_clones":0,"num_object_copies":220728,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":39452,"num_objects_misplaced":35916,"num_objects_unfound":0,"num_objects_dirty":73576,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":120306,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6805,"num_bytes_recovered":442448828,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95040,"ondisk_log_size":95040,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14237816,"kb_used_data":11385800,"kb_used_omap":137,"kb_used_meta":2851830,"kb_avail":548233096,"statfs":{"total":575970213888,"available":561390690304,"internally_reserved":0,"allocated":11659059200,"data_stored":11651828604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":2920274104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-98435072,"num_objects":-1502,"num_object_clones":0,"num_object_copies":-4506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-709,"num_objects_misplaced":-488,"num_objects_unfound":0,"num_objects_dirty":-1502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1502,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":488,"num_bytes_recovered":31981568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.034267"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":385,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.601827+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:58:58.601827+0000","last_peered":"2023-09-23T22:58:58.601827+0000","last_clean":"2023-09-23T22:58:58.601827+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:58:58.601827+0000","last_undegraded":"2023-09-23T22:58:58.601827+0000","last_fullsized":"2023-09-23T22:58:58.601827+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":41,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617318+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:58:58.617318+0000","last_peered":"2023-09-23T22:58:58.617318+0000","last_clean":"2023-09-23T22:58:58.617318+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:58:58.617318+0000","last_undegraded":"2023-09-23T22:58:58.617318+0000","last_fullsized":"2023-09-23T22:58:58.617318+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":40,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615382+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:58:58.615382+0000","last_peered":"2023-09-23T22:58:58.615382+0000","last_clean":"2023-09-23T22:58:58.615382+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:58:58.615382+0000","last_undegraded":"2023-09-23T22:58:58.615382+0000","last_fullsized":"2023-09-23T22:58:58.615382+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":368,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617152+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:58:58.617152+0000","last_peered":"2023-09-23T22:58:58.617152+0000","last_clean":"2023-09-23T22:58:58.617152+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:58:58.617152+0000","last_undegraded":"2023-09-23T22:58:58.617152+0000","last_fullsized":"2023-09-23T22:58:58.617152+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615022+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:58:58.615022+0000","last_peered":"2023-09-23T22:58:58.615022+0000","last_clean":"2023-09-23T22:58:58.615022+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:58:58.615022+0000","last_undegraded":"2023-09-23T22:58:58.615022+0000","last_fullsized":"2023-09-23T22:58:58.615022+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":42,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.604684+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T22:58:58.604684+0000","last_peered":"2023-09-23T22:58:58.604684+0000","last_clean":"2023-09-23T22:58:58.604684+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T22:58:58.604684+0000","last_undegraded":"2023-09-23T22:58:58.604684+0000","last_fullsized":"2023-09-23T22:58:58.604684+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":308,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.618913+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:58:58.618913+0000","last_peered":"2023-09-23T22:58:58.618913+0000","last_clean":"2023-09-23T22:58:58.618913+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:58:58.618913+0000","last_undegraded":"2023-09-23T22:58:58.618913+0000","last_fullsized":"2023-09-23T22:58:58.618913+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":50,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.614905+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:58:58.614905+0000","last_peered":"2023-09-23T22:58:58.614905+0000","last_clean":"2023-09-23T22:58:58.614905+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:58:58.614905+0000","last_undegraded":"2023-09-23T22:58:58.614905+0000","last_fullsized":"2023-09-23T22:58:58.614905+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.616093+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:58:58.616093+0000","last_peered":"2023-09-23T22:58:58.616093+0000","last_clean":"2023-09-23T22:58:58.616093+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:58:58.616093+0000","last_undegraded":"2023-09-23T22:58:58.616093+0000","last_fullsized":"2023-09-23T22:58:58.616093+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":55,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615264+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:58:58.615264+0000","last_peered":"2023-09-23T22:58:58.615264+0000","last_clean":"2023-09-23T22:58:58.615264+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:58:58.615264+0000","last_undegraded":"2023-09-23T22:58:58.615264+0000","last_fullsized":"2023-09-23T22:58:58.615264+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":61,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617226+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:58:58.617226+0000","last_peered":"2023-09-23T22:58:58.617226+0000","last_clean":"2023-09-23T22:58:58.617226+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:58:58.617226+0000","last_undegraded":"2023-09-23T22:58:58.617226+0000","last_fullsized":"2023-09-23T22:58:58.617226+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":117,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613188+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:58:58.613188+0000","last_peered":"2023-09-23T22:58:58.613188+0000","last_clean":"2023-09-23T22:58:58.613188+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:58:58.613188+0000","last_undegraded":"2023-09-23T22:58:58.613188+0000","last_fullsized":"2023-09-23T22:58:58.613188+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":444,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.609004+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:58:58.609004+0000","last_peered":"2023-09-23T22:58:58.609004+0000","last_clean":"2023-09-23T22:58:58.609004+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:58:58.609004+0000","last_undegraded":"2023-09-23T22:58:58.609004+0000","last_fullsized":"2023-09-23T22:58:58.609004+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":347,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608704+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:58:58.608704+0000","last_peered":"2023-09-23T22:58:58.608704+0000","last_clean":"2023-09-23T22:58:58.608704+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:58:58.608704+0000","last_undegraded":"2023-09-23T22:58:58.608704+0000","last_fullsized":"2023-09-23T22:58:58.608704+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":273,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608775+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:58:58.608775+0000","last_peered":"2023-09-23T22:58:58.608775+0000","last_clean":"2023-09-23T22:58:58.608775+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:58:58.608775+0000","last_undegraded":"2023-09-23T22:58:58.608775+0000","last_fullsized":"2023-09-23T22:58:58.608775+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":52,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.616853+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:58:58.616853+0000","last_peered":"2023-09-23T22:58:58.616853+0000","last_clean":"2023-09-23T22:58:58.616853+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:58:58.616853+0000","last_undegraded":"2023-09-23T22:58:58.616853+0000","last_fullsized":"2023-09-23T22:58:58.616853+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":361,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615156+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:58:58.615156+0000","last_peered":"2023-09-23T22:58:58.615156+0000","last_clean":"2023-09-23T22:58:58.615156+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:58:58.615156+0000","last_undegraded":"2023-09-23T22:58:58.615156+0000","last_fullsized":"2023-09-23T22:58:58.615156+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":273,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615596+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:58:58.615596+0000","last_peered":"2023-09-23T22:58:58.615596+0000","last_clean":"2023-09-23T22:58:58.615596+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:58:58.615596+0000","last_undegraded":"2023-09-23T22:58:58.615596+0000","last_fullsized":"2023-09-23T22:58:58.615596+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":258,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602004+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:58:58.602004+0000","last_peered":"2023-09-23T22:58:58.602004+0000","last_clean":"2023-09-23T22:58:58.602004+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:58:58.602004+0000","last_undegraded":"2023-09-23T22:58:58.602004+0000","last_fullsized":"2023-09-23T22:58:58.602004+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":385,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617242+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T22:58:58.617242+0000","last_peered":"2023-09-23T22:58:58.617242+0000","last_clean":"2023-09-23T22:58:58.617242+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T22:58:58.617242+0000","last_undegraded":"2023-09-23T22:58:58.617242+0000","last_fullsized":"2023-09-23T22:58:58.617242+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":90,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.606001+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:58:58.606001+0000","last_peered":"2023-09-23T22:58:58.606001+0000","last_clean":"2023-09-23T22:58:58.606001+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:58:58.606001+0000","last_undegraded":"2023-09-23T22:58:58.606001+0000","last_fullsized":"2023-09-23T22:58:58.606001+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617384+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:58:58.617384+0000","last_peered":"2023-09-23T22:58:58.617384+0000","last_clean":"2023-09-23T22:58:58.617384+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:58:58.617384+0000","last_undegraded":"2023-09-23T22:58:58.617384+0000","last_fullsized":"2023-09-23T22:58:58.617384+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"373'7509","reported_seq":7585,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:04.507859+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T22:59:04.507859+0000","last_peered":"2023-09-23T22:59:04.507859+0000","last_clean":"2023-09-23T22:59:04.507859+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T22:59:04.507859+0000","last_undegraded":"2023-09-23T22:59:04.507859+0000","last_fullsized":"2023-09-23T22:59:04.507859+0000","mapping_epoch":353,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5509,"log_dups_size":0,"ondisk_log_size":5509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299958272,"num_objects":4577,"num_object_clones":0,"num_object_copies":13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7509,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":257,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.603322+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:58:58.603322+0000","last_peered":"2023-09-23T22:58:58.603322+0000","last_clean":"2023-09-23T22:58:58.603322+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:58:58.603322+0000","last_undegraded":"2023-09-23T22:58:58.603322+0000","last_fullsized":"2023-09-23T22:58:58.603322+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615217+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:58:58.615217+0000","last_peered":"2023-09-23T22:58:58.615217+0000","last_clean":"2023-09-23T22:58:58.615217+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:58:58.615217+0000","last_undegraded":"2023-09-23T22:58:58.615217+0000","last_fullsized":"2023-09-23T22:58:58.615217+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"373'7428","reported_seq":9834,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:06.063333+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T22:59:06.063333+0000","last_peered":"2023-09-23T22:59:06.063333+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T22:59:06.063333+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296353792,"num_objects":4522,"num_object_clones":0,"num_object_copies":13566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4522,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":4522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7428,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":955}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":267,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602055+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:58:58.602055+0000","last_peered":"2023-09-23T22:58:58.602055+0000","last_clean":"2023-09-23T22:58:58.602055+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:58:58.602055+0000","last_undegraded":"2023-09-23T22:58:58.602055+0000","last_fullsized":"2023-09-23T22:58:58.602055+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":380,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.601957+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:58:58.601957+0000","last_peered":"2023-09-23T22:58:58.601957+0000","last_clean":"2023-09-23T22:58:58.601957+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:58:58.601957+0000","last_undegraded":"2023-09-23T22:58:58.601957+0000","last_fullsized":"2023-09-23T22:58:58.601957+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"373'7512","reported_seq":7582,"reported_epoch":373,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:04.634835+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T22:59:04.634835+0000","last_peered":"2023-09-23T22:59:04.634835+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T22:59:04.634835+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T22:59:04.634835+0000","mapping_epoch":354,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298844160,"num_objects":4560,"num_object_clones":0,"num_object_copies":13680,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":4560,"num_objects_unfound":0,"num_objects_dirty":4560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7512,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":4560}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":124,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615635+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:58:58.615635+0000","last_peered":"2023-09-23T22:58:58.615635+0000","last_clean":"2023-09-23T22:58:58.615635+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:58:58.615635+0000","last_undegraded":"2023-09-23T22:58:58.615635+0000","last_fullsized":"2023-09-23T22:58:58.615635+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":242,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615848+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:58:58.615848+0000","last_peered":"2023-09-23T22:58:58.615848+0000","last_clean":"2023-09-23T22:58:58.615848+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:58:58.615848+0000","last_undegraded":"2023-09-23T22:58:58.615848+0000","last_fullsized":"2023-09-23T22:58:58.615848+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"373'7549","reported_seq":7805,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:04.702999+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T22:59:04.702999+0000","last_peered":"2023-09-23T22:59:04.702999+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T22:59:04.702999+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5549,"log_dups_size":0,"ondisk_log_size":5549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303104000,"num_objects":4625,"num_object_clones":0,"num_object_copies":13875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3904,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":4625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7549,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":1015}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":41,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.616821+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:58:58.616821+0000","last_peered":"2023-09-23T22:58:58.616821+0000","last_clean":"2023-09-23T22:58:58.616821+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:58:58.616821+0000","last_undegraded":"2023-09-23T22:58:58.616821+0000","last_fullsized":"2023-09-23T22:58:58.616821+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":271,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.606027+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:58:58.606027+0000","last_peered":"2023-09-23T22:58:58.606027+0000","last_clean":"2023-09-23T22:58:58.606027+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:58:58.606027+0000","last_undegraded":"2023-09-23T22:58:58.606027+0000","last_fullsized":"2023-09-23T22:58:58.606027+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"373'7481","reported_seq":7559,"reported_epoch":373,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:01.750085+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T22:59:01.750085+0000","last_peered":"2023-09-23T22:59:01.750085+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T22:59:01.750085+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T22:59:01.750085+0000","mapping_epoch":353,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302317568,"num_objects":4613,"num_object_clones":0,"num_object_copies":13839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7481,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4613}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.616883+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:58:58.616883+0000","last_peered":"2023-09-23T22:58:58.616883+0000","last_clean":"2023-09-23T22:58:58.616883+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:58:58.616883+0000","last_undegraded":"2023-09-23T22:58:58.616883+0000","last_fullsized":"2023-09-23T22:58:58.616883+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":366,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608915+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:58:58.608915+0000","last_peered":"2023-09-23T22:58:58.608915+0000","last_clean":"2023-09-23T22:58:58.608915+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:58:58.608915+0000","last_undegraded":"2023-09-23T22:58:58.608915+0000","last_fullsized":"2023-09-23T22:58:58.608915+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"373'7527","reported_seq":7613,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:01.942685+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T22:59:01.942685+0000","last_peered":"2023-09-23T22:59:01.942685+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T22:59:01.942685+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4675,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7527,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":1056}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":156,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602312+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:58:58.602312+0000","last_peered":"2023-09-23T22:58:58.602312+0000","last_clean":"2023-09-23T22:58:58.602312+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:58:58.602312+0000","last_undegraded":"2023-09-23T22:58:58.602312+0000","last_fullsized":"2023-09-23T22:58:58.602312+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":383,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602108+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:58:58.602108+0000","last_peered":"2023-09-23T22:58:58.602108+0000","last_clean":"2023-09-23T22:58:58.602108+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:58:58.602108+0000","last_undegraded":"2023-09-23T22:58:58.602108+0000","last_fullsized":"2023-09-23T22:58:58.602108+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"373'7454","reported_seq":9922,"reported_epoch":373,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:02.021798+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T22:59:02.021798+0000","last_peered":"2023-09-23T22:59:02.021798+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T22:59:02.021798+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T22:59:02.021798+0000","mapping_epoch":354,"log_start":"322'1700","ondisk_log_start":"322'1700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5754,"log_dups_size":0,"ondisk_log_size":5754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301858816,"num_objects":4606,"num_object_clones":0,"num_object_copies":13818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3413,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7454,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1187,"num_bytes_recovered":77791232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":3172}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.601805+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:58:58.601805+0000","last_peered":"2023-09-23T22:58:58.601805+0000","last_clean":"2023-09-23T22:58:58.601805+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:58:58.601805+0000","last_undegraded":"2023-09-23T22:58:58.601805+0000","last_fullsized":"2023-09-23T22:58:58.601805+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":374,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602139+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:58:58.602139+0000","last_peered":"2023-09-23T22:58:58.602139+0000","last_clean":"2023-09-23T22:58:58.602139+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:58:58.602139+0000","last_undegraded":"2023-09-23T22:58:58.602139+0000","last_fullsized":"2023-09-23T22:58:58.602139+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"373'7372","reported_seq":7450,"reported_epoch":373,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:01.906477+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T22:59:01.906477+0000","last_peered":"2023-09-23T22:59:01.906477+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T22:59:01.906477+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T22:59:01.906477+0000","mapping_epoch":353,"log_start":"322'1700","ondisk_log_start":"322'1700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5672,"log_dups_size":0,"ondisk_log_size":5672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301203456,"num_objects":4596,"num_object_clones":0,"num_object_copies":13788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7372,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4596}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":41,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615156+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:58:58.615156+0000","last_peered":"2023-09-23T22:58:58.615156+0000","last_clean":"2023-09-23T22:58:58.615156+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:58:58.615156+0000","last_undegraded":"2023-09-23T22:58:58.615156+0000","last_fullsized":"2023-09-23T22:58:58.615156+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":275,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613685+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:58:58.613685+0000","last_peered":"2023-09-23T22:58:58.613685+0000","last_clean":"2023-09-23T22:58:58.613685+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:58:58.613685+0000","last_undegraded":"2023-09-23T22:58:58.613685+0000","last_fullsized":"2023-09-23T22:58:58.613685+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"373'7479","reported_seq":10491,"reported_epoch":373,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:59:02.076026+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T22:59:02.076026+0000","last_peered":"2023-09-23T22:59:02.076026+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T22:59:02.076026+0000","last_undegraded":"2023-09-23T22:59:02.076026+0000","last_fullsized":"2023-09-23T22:59:02.076026+0000","mapping_epoch":354,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5679,"log_dups_size":0,"ondisk_log_size":5679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2215,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7479,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1450,"num_bytes_recovered":94502912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2382},{"shards":"0,4,6","objects":2215}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":279,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613292+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:58:58.613292+0000","last_peered":"2023-09-23T22:58:58.613292+0000","last_clean":"2023-09-23T22:58:58.613292+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:58:58.613292+0000","last_undegraded":"2023-09-23T22:58:58.613292+0000","last_fullsized":"2023-09-23T22:58:58.613292+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":283,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615533+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:58:58.615533+0000","last_peered":"2023-09-23T22:58:58.615533+0000","last_clean":"2023-09-23T22:58:58.615533+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:58:58.615533+0000","last_undegraded":"2023-09-23T22:58:58.615533+0000","last_fullsized":"2023-09-23T22:58:58.615533+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":96,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615723+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:58:58.615723+0000","last_peered":"2023-09-23T22:58:58.615723+0000","last_clean":"2023-09-23T22:58:58.615723+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:58:58.615723+0000","last_undegraded":"2023-09-23T22:58:58.615723+0000","last_fullsized":"2023-09-23T22:58:58.615723+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"373'7566","reported_seq":7655,"reported_epoch":373,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:59:04.675893+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T22:59:04.675893+0000","last_peered":"2023-09-23T22:59:04.675893+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T22:59:04.675893+0000","last_undegraded":"2023-09-23T22:59:04.675893+0000","last_fullsized":"2023-09-23T22:59:04.675893+0000","mapping_epoch":359,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298713088,"num_objects":4558,"num_object_clones":0,"num_object_copies":13674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4558,"num_objects_unfound":0,"num_objects_dirty":4558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7566,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4558}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":42,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.604575+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T22:58:58.604575+0000","last_peered":"2023-09-23T22:58:58.604575+0000","last_clean":"2023-09-23T22:58:58.604575+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T22:58:58.604575+0000","last_undegraded":"2023-09-23T22:58:58.604575+0000","last_fullsized":"2023-09-23T22:58:58.604575+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":393,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608789+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:58:58.608789+0000","last_peered":"2023-09-23T22:58:58.608789+0000","last_clean":"2023-09-23T22:58:58.608789+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:58:58.608789+0000","last_undegraded":"2023-09-23T22:58:58.608789+0000","last_fullsized":"2023-09-23T22:58:58.608789+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":264,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613331+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:58:58.613331+0000","last_peered":"2023-09-23T22:58:58.613331+0000","last_clean":"2023-09-23T22:58:58.613331+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:58:58.613331+0000","last_undegraded":"2023-09-23T22:58:58.613331+0000","last_fullsized":"2023-09-23T22:58:58.613331+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"373'7555","reported_seq":9570,"reported_epoch":373,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:06.133743+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T22:59:06.133743+0000","last_peered":"2023-09-23T22:59:06.133743+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T22:59:06.133743+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295501824,"num_objects":4509,"num_object_clones":0,"num_object_copies":13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2606,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7555,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":965,"num_bytes_recovered":63242240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1903}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":339,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613030+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:58:58.613030+0000","last_peered":"2023-09-23T22:58:58.613030+0000","last_clean":"2023-09-23T22:58:58.613030+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:58:58.613030+0000","last_undegraded":"2023-09-23T22:58:58.613030+0000","last_fullsized":"2023-09-23T22:58:58.613030+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":216,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.612963+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:58:58.612963+0000","last_peered":"2023-09-23T22:58:58.612963+0000","last_clean":"2023-09-23T22:58:58.612963+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:58:58.612963+0000","last_undegraded":"2023-09-23T22:58:58.612963+0000","last_fullsized":"2023-09-23T22:58:58.612963+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":283,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.618987+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:58:58.618987+0000","last_peered":"2023-09-23T22:58:58.618987+0000","last_clean":"2023-09-23T22:58:58.618987+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:58:58.618987+0000","last_undegraded":"2023-09-23T22:58:58.618987+0000","last_fullsized":"2023-09-23T22:58:58.618987+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"373'7563","reported_seq":9438,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:01.575733+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T22:59:01.575733+0000","last_peered":"2023-09-23T22:59:01.575733+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T22:59:01.575733+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7563,"log_dups_size":0,"ondisk_log_size":7563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305463296,"num_objects":4661,"num_object_clones":0,"num_object_copies":13983,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7563,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":3228},{"shards":"0,6","objects":-320}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":356,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617563+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:58:58.617563+0000","last_peered":"2023-09-23T22:58:58.617563+0000","last_clean":"2023-09-23T22:58:58.617563+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:58:58.617563+0000","last_undegraded":"2023-09-23T22:58:58.617563+0000","last_fullsized":"2023-09-23T22:58:58.617563+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":468,"reported_epoch":372,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:58:58.608759+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T22:58:58.608759+0000","last_peered":"2023-09-23T22:58:58.608759+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T22:58:58.608759+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":41,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.616600+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:58:58.616600+0000","last_peered":"2023-09-23T22:58:58.616600+0000","last_clean":"2023-09-23T22:58:58.616600+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:58:58.616600+0000","last_undegraded":"2023-09-23T22:58:58.616600+0000","last_fullsized":"2023-09-23T22:58:58.616600+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602206+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:58:58.602206+0000","last_peered":"2023-09-23T22:58:58.602206+0000","last_clean":"2023-09-23T22:58:58.602206+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:58:58.602206+0000","last_undegraded":"2023-09-23T22:58:58.602206+0000","last_fullsized":"2023-09-23T22:58:58.602206+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"373'7537","reported_seq":7618,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:04.626967+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T22:59:04.626967+0000","last_peered":"2023-09-23T22:59:04.626967+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T22:59:04.626967+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5537,"log_dups_size":0,"ondisk_log_size":5537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297992192,"num_objects":4547,"num_object_clones":0,"num_object_copies":13641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7537,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":1040}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":362,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.606105+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T22:58:58.606105+0000","last_peered":"2023-09-23T22:58:58.606105+0000","last_clean":"2023-09-23T22:58:58.606105+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T22:58:58.606105+0000","last_undegraded":"2023-09-23T22:58:58.606105+0000","last_fullsized":"2023-09-23T22:58:58.606105+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":333,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.604401+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T22:58:58.604401+0000","last_peered":"2023-09-23T22:58:58.604401+0000","last_clean":"2023-09-23T22:58:58.604401+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T22:58:58.604401+0000","last_undegraded":"2023-09-23T22:58:58.604401+0000","last_fullsized":"2023-09-23T22:58:58.604401+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":293,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.605322+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:58:58.605322+0000","last_peered":"2023-09-23T22:58:58.605322+0000","last_clean":"2023-09-23T22:58:58.605322+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:58:58.605322+0000","last_undegraded":"2023-09-23T22:58:58.605322+0000","last_fullsized":"2023-09-23T22:58:58.605322+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"373'7548","reported_seq":7645,"reported_epoch":373,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:02.034869+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T22:59:02.034869+0000","last_peered":"2023-09-23T22:59:02.034869+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T22:59:02.034869+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T22:59:02.034869+0000","mapping_epoch":354,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070104,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":4656,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7548,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":4656}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":308,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.606168+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T22:58:58.606168+0000","last_peered":"2023-09-23T22:58:58.606168+0000","last_clean":"2023-09-23T22:58:58.606168+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T22:58:58.606168+0000","last_undegraded":"2023-09-23T22:58:58.606168+0000","last_fullsized":"2023-09-23T22:58:58.606168+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":216,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613367+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:58:58.613367+0000","last_peered":"2023-09-23T22:58:58.613367+0000","last_clean":"2023-09-23T22:58:58.613367+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:58:58.613367+0000","last_undegraded":"2023-09-23T22:58:58.613367+0000","last_fullsized":"2023-09-23T22:58:58.613367+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":186,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617654+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:58:58.617654+0000","last_peered":"2023-09-23T22:58:58.617654+0000","last_clean":"2023-09-23T22:58:58.617654+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:58:58.617654+0000","last_undegraded":"2023-09-23T22:58:58.617654+0000","last_fullsized":"2023-09-23T22:58:58.617654+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"373'7486","reported_seq":9225,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:02.080132+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T22:59:02.080132+0000","last_peered":"2023-09-23T22:59:02.080132+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T22:59:02.080132+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305397760,"num_objects":4660,"num_object_clones":0,"num_object_copies":13980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8528,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7486,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1079},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":65,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617496+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:58:58.617496+0000","last_peered":"2023-09-23T22:58:58.617496+0000","last_clean":"2023-09-23T22:58:58.617496+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:58:58.617496+0000","last_undegraded":"2023-09-23T22:58:58.617496+0000","last_fullsized":"2023-09-23T22:58:58.617496+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":81,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602648+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:58:58.602648+0000","last_peered":"2023-09-23T22:58:58.602648+0000","last_clean":"2023-09-23T22:58:58.602648+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:58:58.602648+0000","last_undegraded":"2023-09-23T22:58:58.602648+0000","last_fullsized":"2023-09-23T22:58:58.602648+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615074+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:58:58.615074+0000","last_peered":"2023-09-23T22:58:58.615074+0000","last_clean":"2023-09-23T22:58:58.615074+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:58:58.615074+0000","last_undegraded":"2023-09-23T22:58:58.615074+0000","last_fullsized":"2023-09-23T22:58:58.615074+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"373'7480","reported_seq":7987,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:01.780480+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T22:59:01.780480+0000","last_peered":"2023-09-23T22:59:01.780480+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T22:59:01.780480+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300810240,"num_objects":4590,"num_object_clones":0,"num_object_copies":13770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4590,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":4590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7480,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":1039}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":328,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.616472+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:58:58.616472+0000","last_peered":"2023-09-23T22:58:58.616472+0000","last_clean":"2023-09-23T22:58:58.616472+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:58:58.616472+0000","last_undegraded":"2023-09-23T22:58:58.616472+0000","last_fullsized":"2023-09-23T22:58:58.616472+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":275,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.605253+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:58:58.605253+0000","last_peered":"2023-09-23T22:58:58.605253+0000","last_clean":"2023-09-23T22:58:58.605253+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:58:58.605253+0000","last_undegraded":"2023-09-23T22:58:58.605253+0000","last_fullsized":"2023-09-23T22:58:58.605253+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":347,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.601856+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:58:58.601856+0000","last_peered":"2023-09-23T22:58:58.601856+0000","last_clean":"2023-09-23T22:58:58.601856+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:58:58.601856+0000","last_undegraded":"2023-09-23T22:58:58.601856+0000","last_fullsized":"2023-09-23T22:58:58.601856+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":310,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608432+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:58:58.608432+0000","last_peered":"2023-09-23T22:58:58.608432+0000","last_clean":"2023-09-23T22:58:58.608432+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:58:58.608432+0000","last_undegraded":"2023-09-23T22:58:58.608432+0000","last_fullsized":"2023-09-23T22:58:58.608432+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":120,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.613186+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:58:58.613186+0000","last_peered":"2023-09-23T22:58:58.613186+0000","last_clean":"2023-09-23T22:58:58.613186+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:58:58.613186+0000","last_undegraded":"2023-09-23T22:58:58.613186+0000","last_fullsized":"2023-09-23T22:58:58.613186+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":182,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608832+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:58:58.608832+0000","last_peered":"2023-09-23T22:58:58.608832+0000","last_clean":"2023-09-23T22:58:58.608832+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:58:58.608832+0000","last_undegraded":"2023-09-23T22:58:58.608832+0000","last_fullsized":"2023-09-23T22:58:58.608832+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":396,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.608864+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:58:58.608864+0000","last_peered":"2023-09-23T22:58:58.608864+0000","last_clean":"2023-09-23T22:58:58.608864+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:58:58.608864+0000","last_undegraded":"2023-09-23T22:58:58.608864+0000","last_fullsized":"2023-09-23T22:58:58.608864+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":196,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.602159+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:58:58.602159+0000","last_peered":"2023-09-23T22:58:58.602159+0000","last_clean":"2023-09-23T22:58:58.602159+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:58:58.602159+0000","last_undegraded":"2023-09-23T22:58:58.602159+0000","last_fullsized":"2023-09-23T22:58:58.602159+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":283,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.601833+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:58:58.601833+0000","last_peered":"2023-09-23T22:58:58.601833+0000","last_clean":"2023-09-23T22:58:58.601833+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:58:58.601833+0000","last_undegraded":"2023-09-23T22:58:58.601833+0000","last_fullsized":"2023-09-23T22:58:58.601833+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":65,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.617305+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:58:58.617305+0000","last_peered":"2023-09-23T22:58:58.617305+0000","last_clean":"2023-09-23T22:58:58.617305+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:58:58.617305+0000","last_undegraded":"2023-09-23T22:58:58.617305+0000","last_fullsized":"2023-09-23T22:58:58.617305+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":68,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.615501+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:58:58.615501+0000","last_peered":"2023-09-23T22:58:58.615501+0000","last_clean":"2023-09-23T22:58:58.615501+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:58:58.615501+0000","last_undegraded":"2023-09-23T22:58:58.615501+0000","last_fullsized":"2023-09-23T22:58:58.615501+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":314,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.605107+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T22:58:58.605107+0000","last_peered":"2023-09-23T22:58:58.605107+0000","last_clean":"2023-09-23T22:58:58.605107+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T22:58:58.605107+0000","last_undegraded":"2023-09-23T22:58:58.605107+0000","last_fullsized":"2023-09-23T22:58:58.605107+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":358,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:58:58.601660+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:58:58.601660+0000","last_peered":"2023-09-23T22:58:58.601660+0000","last_clean":"2023-09-23T22:58:58.601660+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:58:58.601660+0000","last_undegraded":"2023-09-23T22:58:58.601660+0000","last_fullsized":"2023-09-23T22:58:58.601660+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4820238360,"num_objects":73552,"num_object_clones":0,"num_object_copies":220656,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":39450,"num_objects_misplaced":35916,"num_objects_unfound":0,"num_objects_dirty":73552,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120046,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6770,"num_bytes_recovered":442433536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14043262976,"data_stored":14043250760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94846,"ondisk_log_size":94846,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430593,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626817,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253130,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2403452,"kb_used_data":1727348,"kb_used_omap":9,"kb_used_meta":676086,"kb_avail":91341700,"statfs":{"total":95995035648,"available":93533900800,"internally_reserved":0,"allocated":1768804352,"data_stored":1767420729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":692313065},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:57:31 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":5.7149999999999999,"15min":5.7949999999999999},"min":{"1min":0.20899999999999999,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":8.7859999999999996,"5min":129.49299999999999,"15min":129.49299999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.2150000000000001,"5min":5.3460000000000001,"15min":5.508},"min":{"1min":0.20399999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.0529999999999999,"5min":129.80000000000001,"15min":129.80000000000001},"last":0.629}]},{"osd":2,"last update":"Sat Sep 23 22:55:30 2023","interfaces":[{"interface":"back","average":{"1min":9.8800000000000008,"5min":4.593,"15min":3.8439999999999999},"min":{"1min":0.246,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":111.452,"5min":111.452,"15min":111.452},"last":0.504},{"interface":"front","average":{"1min":10.167,"5min":4.4580000000000002,"15min":3.9420000000000002},"min":{"1min":0.36199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":108.22799999999999,"5min":108.22799999999999,"15min":108.22799999999999},"last":0.99099999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":1.0629999999999999,"15min":2.3849999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63100000000000001,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.40200000000000002,"5min":1.147,"15min":2.8340000000000001},"min":{"1min":0.27400000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.624,"5min":8.0099999999999998,"15min":89.861999999999995},"last":0.96799999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5730000000000004,"5min":3.1190000000000002,"15min":2.766},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":151.036,"5min":151.036,"15min":151.036},"last":0.66100000000000003},{"interface":"front","average":{"1min":7.5510000000000002,"5min":3.2469999999999999,"15min":2.7709999999999999},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":151.06899999999999,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":3.4449999999999998,"15min":2.5329999999999999},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":0.95099999999999996,"5min":116.098,"15min":116.098},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":3.1200000000000001,"15min":2.7890000000000001},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":2.698,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.69299999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.8600000000000001,"15min":2.431},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.85499999999999998,"5min":20.742000000000001,"15min":79.311000000000007},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.9119999999999999,"15min":2.125},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":4.4219999999999997,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.3580000000000001,"15min":2.028},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":0.87,"5min":8.1400000000000006,"15min":29.539000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.6990000000000001,"15min":2.2120000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.45100000000000001}]}]},{"osd":0,"up_from":353,"seq":1516123455501,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3154420,"kb_used_data":3033452,"kb_used_omap":37,"kb_used_meta":120922,"kb_avail":90590732,"statfs":{"total":95995035648,"available":92764909568,"internally_reserved":0,"allocated":3106254848,"data_stored":3104762422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":123824290},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.262}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26500000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.2770000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.4969999999999999}]}]},{"osd":2,"up_from":359,"seq":1541893259274,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1088576,"kb_used_data":1001400,"kb_used_omap":83,"kb_used_meta":87084,"kb_avail":92656576,"statfs":{"total":95995035648,"available":94880333824,"internally_reserved":0,"allocated":1025433600,"data_stored":1023991049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":89174469},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0449999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0760000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.624}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926078,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528728,"kb_used_data":50320,"kb_used_omap":7,"kb_used_meta":478392,"kb_avail":93216424,"statfs":{"total":95995035648,"available":95453618176,"internally_reserved":0,"allocated":51527680,"data_stored":50057394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":489874024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:57:32 2023","interfaces":[{"interface":"back","average":{"1min":11.765000000000001,"5min":9.7530000000000001,"15min":5.25},"min":{"1min":0.23100000000000001,"5min":0.217,"15min":0.21099999999999999},"max":{"1min":93.506,"5min":216.51599999999999,"15min":216.51599999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":12.666,"5min":10.047000000000001,"15min":5.2969999999999997},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":93.864999999999995,"5min":216.84800000000001,"15min":216.84800000000001},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.097,"15min":2.786},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.60899999999999999,"5min":16.596,"15min":123.866},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.35499999999999998,"5min":1.179,"15min":2.4449999999999998},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":0.47499999999999998,"5min":16.649999999999999,"15min":122.265},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:55:36 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":2.431,"15min":3.1230000000000002},"min":{"1min":0.251,"5min":0.222,"15min":0.222},"max":{"1min":6.8920000000000003,"5min":38.292999999999999,"15min":125.446},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":2.544,"15min":3.7759999999999998},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.217},"max":{"1min":5.9269999999999996,"5min":38.225999999999999,"15min":182.483},"last":0.308}]},{"osd":4,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.3129999999999999,"15min":1.8640000000000001},"min":{"1min":0.30399999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":1.0700000000000001,"5min":7.8929999999999998,"15min":18.757000000000001},"last":2.4790000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.3129999999999999,"15min":1.629},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.27500000000000002},"max":{"1min":0.72199999999999998,"5min":7.8780000000000001,"15min":15.037000000000001},"last":2.4670000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.393,"15min":1.52},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.59399999999999997,"5min":10.260999999999999,"15min":12.127000000000001},"last":3.036},{"interface":"front","average":{"1min":0.46800000000000003,"5min":1.546,"15min":1.954},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75600000000000001,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7709999999999999,"15min":1.788},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":14.097,"15min":14.097},"last":2.5390000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.677,"15min":2.0390000000000001},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5,"5min":12.887,"15min":12.887},"last":2.4969999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.3480000000000001,"15min":2.2109999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79700000000000004,"5min":6.9950000000000001,"15min":91.757999999999996},"last":3.1040000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.238,"15min":2.2240000000000002},"min":{"1min":0.434,"5min":0.41699999999999998,"15min":0.33200000000000002},"max":{"1min":1.1479999999999999,"5min":7.1219999999999999,"15min":91.709999999999994},"last":3.1230000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019274,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4026856,"kb_used_data":3441248,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":89718296,"statfs":{"total":95995035648,"available":91871535104,"internally_reserved":0,"allocated":3523837952,"data_stored":3523103203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:57:27 2023","interfaces":[{"interface":"back","average":{"1min":12.035,"5min":9.827,"15min":6.9669999999999996},"min":{"1min":0.40400000000000003,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":106.324,"5min":159.62200000000001,"15min":159.62200000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":12.058999999999999,"5min":10.849,"15min":7.7149999999999999},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.217},"max":{"1min":106.086,"5min":159.51400000000001,"15min":159.51400000000001},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":3.7690000000000001,"15min":3.786},"min":{"1min":0.40000000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":1.966,"5min":98.846999999999994,"15min":132.304},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.71699999999999997,"5min":3.9740000000000002,"15min":3.6259999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":3.2269999999999999,"5min":98.938999999999993,"15min":132.453},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:55:40 2023","interfaces":[{"interface":"back","average":{"1min":11.44,"5min":4.0789999999999997,"15min":3.7349999999999999},"min":{"1min":0.33800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":137.922,"5min":137.922,"15min":137.922},"last":1.028},{"interface":"front","average":{"1min":11.497,"5min":4.1219999999999999,"15min":3.734},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":140.994,"5min":140.994,"15min":140.994},"last":1.097}]},{"osd":3,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.536,"15min":4.5099999999999998},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":3.399,"5min":8.5510000000000002,"15min":164.453},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.478,"15min":4.5720000000000001},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.99,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:58:27 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":3.0800000000000001,"15min":1.641},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":12.961,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.376},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.5019999999999998,"15min":1.7869999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.248},"max":{"1min":5.4619999999999997,"5min":151.905,"15min":151.905},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5820000000000001,"15min":1.1559999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":2.113,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.423,"15min":1.101},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0350000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:58:12 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.4299999999999999,"15min":1.0820000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.84,"5min":12.84,"15min":12.84},"last":0.499},{"interface":"front","average":{"1min":1.397,"5min":1.278,"15min":0.98399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8700000000000001,"5min":8.5190000000000001,"15min":8.6739999999999995},"last":10.587}]}]},{"osd":5,"up_from":36,"seq":154618823049,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3035784,"kb_used_data":2132032,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":90709368,"statfs":{"total":95995035648,"available":92886392832,"internally_reserved":0,"allocated":2183200768,"data_stored":2182493807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:57:26 2023","interfaces":[{"interface":"back","average":{"1min":14.138999999999999,"5min":8.0350000000000001,"15min":5.9749999999999996},"min":{"1min":0.29099999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":135.78800000000001,"5min":185.62,"15min":185.62},"last":0.43099999999999999},{"interface":"front","average":{"1min":14.194000000000001,"5min":8.0440000000000005,"15min":5.9809999999999999},"min":{"1min":0.27700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":135.17099999999999,"5min":185.499,"15min":185.499},"last":0.55400000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":2.9249999999999998,"15min":2.3330000000000002},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":3.2429999999999999,"5min":110.36499999999999,"15min":110.36499999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":3.4340000000000002,"15min":2.8410000000000002},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":10.353,"5min":110.89,"15min":110.89},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":4.992,"15min":5.04},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5309999999999997,"5min":189.126,"15min":189.126},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":4.6379999999999999,"15min":4.8920000000000003},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":6.2530000000000001,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.59499999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.6160000000000001,"15min":5.117},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":3.149,"5min":11.298999999999999,"15min":257.05000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.5469999999999999,"15min":4.9409999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.4940000000000002,"5min":12.185,"15min":257.35000000000002},"last":1.01}]},{"osd":4,"last update":"Sat Sep 23 22:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":2.7559999999999998,"15min":1.4650000000000001},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.603,"5min":115.06699999999999,"15min":115.06699999999999},"last":1.135},{"interface":"front","average":{"1min":0.56599999999999995,"5min":2.6909999999999998,"15min":1.4419999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":3.093,"5min":114.908,"15min":114.908},"last":0.625}]},{"osd":6,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.3300000000000001,"15min":1.2649999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.498,"5min":11.121,"15min":48.360999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.272,"15min":1.2709999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.7759999999999998,"5min":12.137,"15min":48.292999999999999},"last":1.0600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.24,"15min":1.012},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.4390000000000001,"5min":12.23,"15min":12.23},"last":1.036},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.2889999999999999,"15min":0.98899999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.3760000000000003,"5min":11.112,"15min":11.112},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3096580096,"data_stored":3096576024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1761083392,"data_stored":1761083392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1020198912,"data_stored":1020198912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":47185920,"data_stored":47185920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3519811584,"data_stored":3519807512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2178613248,"data_stored":2178613248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2419789824,"data_stored":2419785752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:59:09.425 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:59:09.442 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:09.472 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:09.525 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:09.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:09.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:09.630 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:09.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:09 smithi052 ceph-mon[187743]: from='client.46319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:09.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:09 smithi052 ceph-mon[187743]: pgmap v1092: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 39158/218790 objects degraded (17.898%); 35730/218790 objects misplaced (16.331%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:59:09.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:09 smithi052 ceph-mon[180979]: from='client.46319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:09.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:09 smithi052 ceph-mon[180979]: pgmap v1092: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 39158/218790 objects degraded (17.898%); 35730/218790 objects misplaced (16.331%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:59:09.663 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:09.673 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:59:09.919 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:09.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:09.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:09 smithi151 ceph-mon[159210]: from='client.46319 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:09 smithi151 ceph-mon[159210]: pgmap v1092: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 39158/218790 objects degraded (17.898%); 35730/218790 objects misplaced (16.331%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:59:10.024 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:10.061 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:10.461 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:10.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:10.567 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:10.624 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:10.760 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:10.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:10.864 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:10.871 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:10.876 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:10.912 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 39158/218790 objects degraded (17.898%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:10.913 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 39158/218790 objects degraded (17.898%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:10.915 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 39158/218790 objects degraded (17.898%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:10.978 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:10.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:10.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:10.992 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:59:10.992 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:59:11.083 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:11.109 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:11.167 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:11.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:11.273 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:11.306 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:11.361 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:11.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:11.467 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:11.507 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:11.564 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:11.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:11.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:11.650 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:11 smithi151 ceph-mon[159210]: pgmap v1093: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 39102/217788 objects degraded (17.954%); 35591/217788 objects misplaced (16.342%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:59:11.650 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3332440298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:11.673 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:11.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:11 smithi052 ceph-mon[187743]: pgmap v1093: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 39102/217788 objects degraded (17.954%); 35591/217788 objects misplaced (16.342%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:59:11.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3332440298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:11.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:11 smithi052 ceph-mon[180979]: pgmap v1093: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 39102/217788 objects degraded (17.954%); 35591/217788 objects misplaced (16.342%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:59:11.709 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3332440298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:11.712 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:11.919 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.920 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.921 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.921 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.921 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:59:11.921 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:59:12.171 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:12.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:12.275 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:12.324 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:12.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:12.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:12.492 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:12.531 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:12.761 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:59:13.547 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:13.550 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:59:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:59:13] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-09-23T22:59:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:13.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:13.653 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:13.688 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:13.751 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:13.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:13.840 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:13 smithi052 ceph-mon[187743]: pgmap v1094: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 38647/215682 objects degraded (17.919%); 35403/215682 objects misplaced (16.414%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:13.840 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:13 smithi052 ceph-mon[180979]: pgmap v1094: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 38647/215682 objects degraded (17.919%); 35403/215682 objects misplaced (16.414%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:13.856 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:13.891 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:13.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:13 smithi151 ceph-mon[159210]: pgmap v1094: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 38647/215682 objects degraded (17.919%); 35403/215682 objects misplaced (16.414%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:14.390 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:14.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:14.464 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:14.495 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:14.550 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:14.618 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:14.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:14.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:14.723 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:14.832 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:59:15.348 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:15.411 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:15.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:15.522 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:15.541 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:15.543 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:59:15.567 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:15.621 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:15.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:15.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:15.658 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[180979]: pgmap v1095: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 38647/215682 objects degraded (17.919%); 35403/215682 objects misplaced (16.414%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:15.659 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1461137941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:15.659 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:15.659 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 38647/215682 objects degraded (17.919%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:15.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:15 smithi151 ceph-mon[159210]: pgmap v1095: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 38647/215682 objects degraded (17.919%); 35403/215682 objects misplaced (16.414%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:15.660 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1461137941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:15.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:15.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 38647/215682 objects degraded (17.919%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:15.666 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:59:15.725 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:15.760 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:15.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[187743]: pgmap v1095: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 38647/215682 objects degraded (17.919%); 35403/215682 objects misplaced (16.414%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:15.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1461137941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:15.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:15.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 38647/215682 objects degraded (17.919%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:16.560 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1095,"stamp":"2023-09-23T22:59:14.546682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4710602680,"num_objects":71894,"num_object_clones":0,"num_object_copies":215682,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":38647,"num_objects_misplaced":35403,"num_objects_unfound":0,"num_objects_dirty":71894,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":121988,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7456,"num_bytes_recovered":485112764,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95222,"ondisk_log_size":95222,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14018256,"kb_used_data":11158048,"kb_used_omap":137,"kb_used_meta":2860022,"kb_avail":548452656,"statfs":{"total":575970213888,"available":561615519744,"internally_reserved":0,"allocated":11425841152,"data_stored":11418571792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":2928662712},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-141164544,"num_objects":-2154,"num_object_clones":0,"num_object_copies":-6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1044,"num_objects_misplaced":-663,"num_objects_unfound":0,"num_objects_dirty":-2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2154,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":766,"num_bytes_recovered":50200576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005606"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":386,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.247486+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:59:01.247486+0000","last_peered":"2023-09-23T22:59:01.247486+0000","last_clean":"2023-09-23T22:59:01.247486+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:59:01.247486+0000","last_undegraded":"2023-09-23T22:59:01.247486+0000","last_fullsized":"2023-09-23T22:59:01.247486+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":42,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.371640+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:59:01.371640+0000","last_peered":"2023-09-23T22:59:01.371640+0000","last_clean":"2023-09-23T22:59:01.371640+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:59:01.371640+0000","last_undegraded":"2023-09-23T22:59:01.371640+0000","last_fullsized":"2023-09-23T22:59:01.371640+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":41,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239042+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:59:01.239042+0000","last_peered":"2023-09-23T22:59:01.239042+0000","last_clean":"2023-09-23T22:59:01.239042+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:59:01.239042+0000","last_undegraded":"2023-09-23T22:59:01.239042+0000","last_fullsized":"2023-09-23T22:59:01.239042+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":369,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239724+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:59:01.239724+0000","last_peered":"2023-09-23T22:59:01.239724+0000","last_clean":"2023-09-23T22:59:01.239724+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:59:01.239724+0000","last_undegraded":"2023-09-23T22:59:01.239724+0000","last_fullsized":"2023-09-23T22:59:01.239724+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":57,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.238840+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:59:01.238840+0000","last_peered":"2023-09-23T22:59:01.238840+0000","last_clean":"2023-09-23T22:59:01.238840+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:59:01.238840+0000","last_undegraded":"2023-09-23T22:59:01.238840+0000","last_fullsized":"2023-09-23T22:59:01.238840+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":43,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.374976+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T22:59:01.374976+0000","last_peered":"2023-09-23T22:59:01.374976+0000","last_clean":"2023-09-23T22:59:01.374976+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T22:59:01.374976+0000","last_undegraded":"2023-09-23T22:59:01.374976+0000","last_fullsized":"2023-09-23T22:59:01.374976+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":309,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.242794+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:59:01.242794+0000","last_peered":"2023-09-23T22:59:01.242794+0000","last_clean":"2023-09-23T22:59:01.242794+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:59:01.242794+0000","last_undegraded":"2023-09-23T22:59:01.242794+0000","last_fullsized":"2023-09-23T22:59:01.242794+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":51,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.238767+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:59:01.238767+0000","last_peered":"2023-09-23T22:59:01.238767+0000","last_clean":"2023-09-23T22:59:01.238767+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:59:01.238767+0000","last_undegraded":"2023-09-23T22:59:01.238767+0000","last_fullsized":"2023-09-23T22:59:01.238767+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":270,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.245194+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:59:01.245194+0000","last_peered":"2023-09-23T22:59:01.245194+0000","last_clean":"2023-09-23T22:59:01.245194+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:59:01.245194+0000","last_undegraded":"2023-09-23T22:59:01.245194+0000","last_fullsized":"2023-09-23T22:59:01.245194+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.238907+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:59:01.238907+0000","last_peered":"2023-09-23T22:59:01.238907+0000","last_clean":"2023-09-23T22:59:01.238907+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:59:01.238907+0000","last_undegraded":"2023-09-23T22:59:01.238907+0000","last_fullsized":"2023-09-23T22:59:01.238907+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":62,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.371598+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:59:01.371598+0000","last_peered":"2023-09-23T22:59:01.371598+0000","last_clean":"2023-09-23T22:59:01.371598+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:59:01.371598+0000","last_undegraded":"2023-09-23T22:59:01.371598+0000","last_fullsized":"2023-09-23T22:59:01.371598+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":118,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272065+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:59:01.272065+0000","last_peered":"2023-09-23T22:59:01.272065+0000","last_clean":"2023-09-23T22:59:01.272065+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:59:01.272065+0000","last_undegraded":"2023-09-23T22:59:01.272065+0000","last_fullsized":"2023-09-23T22:59:01.272065+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":445,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.254631+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:59:01.254631+0000","last_peered":"2023-09-23T22:59:01.254631+0000","last_clean":"2023-09-23T22:59:01.254631+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:59:01.254631+0000","last_undegraded":"2023-09-23T22:59:01.254631+0000","last_fullsized":"2023-09-23T22:59:01.254631+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.249713+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:59:01.249713+0000","last_peered":"2023-09-23T22:59:01.249713+0000","last_clean":"2023-09-23T22:59:01.249713+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:59:01.249713+0000","last_undegraded":"2023-09-23T22:59:01.249713+0000","last_fullsized":"2023-09-23T22:59:01.249713+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":274,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.249799+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:59:01.249799+0000","last_peered":"2023-09-23T22:59:01.249799+0000","last_clean":"2023-09-23T22:59:01.249799+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:59:01.249799+0000","last_undegraded":"2023-09-23T22:59:01.249799+0000","last_fullsized":"2023-09-23T22:59:01.249799+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":53,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.374133+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:59:01.374133+0000","last_peered":"2023-09-23T22:59:01.374133+0000","last_clean":"2023-09-23T22:59:01.374133+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:59:01.374133+0000","last_undegraded":"2023-09-23T22:59:01.374133+0000","last_fullsized":"2023-09-23T22:59:01.374133+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":362,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.238779+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:59:01.238779+0000","last_peered":"2023-09-23T22:59:01.238779+0000","last_clean":"2023-09-23T22:59:01.238779+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:59:01.238779+0000","last_undegraded":"2023-09-23T22:59:01.238779+0000","last_fullsized":"2023-09-23T22:59:01.238779+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":274,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.242039+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:59:01.242039+0000","last_peered":"2023-09-23T22:59:01.242039+0000","last_clean":"2023-09-23T22:59:01.242039+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:59:01.242039+0000","last_undegraded":"2023-09-23T22:59:01.242039+0000","last_fullsized":"2023-09-23T22:59:01.242039+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":259,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.252224+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:59:01.252224+0000","last_peered":"2023-09-23T22:59:01.252224+0000","last_clean":"2023-09-23T22:59:01.252224+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:59:01.252224+0000","last_undegraded":"2023-09-23T22:59:01.252224+0000","last_fullsized":"2023-09-23T22:59:01.252224+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":386,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239811+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T22:59:01.239811+0000","last_peered":"2023-09-23T22:59:01.239811+0000","last_clean":"2023-09-23T22:59:01.239811+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T22:59:01.239811+0000","last_undegraded":"2023-09-23T22:59:01.239811+0000","last_fullsized":"2023-09-23T22:59:01.239811+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":91,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239035+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:59:01.239035+0000","last_peered":"2023-09-23T22:59:01.239035+0000","last_clean":"2023-09-23T22:59:01.239035+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:59:01.239035+0000","last_undegraded":"2023-09-23T22:59:01.239035+0000","last_fullsized":"2023-09-23T22:59:01.239035+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.371892+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:59:01.371892+0000","last_peered":"2023-09-23T22:59:01.371892+0000","last_clean":"2023-09-23T22:59:01.371892+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:59:01.371892+0000","last_undegraded":"2023-09-23T22:59:01.371892+0000","last_fullsized":"2023-09-23T22:59:01.371892+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"374'7576","reported_seq":7653,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T22:59:09.718825+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T22:59:09.718825+0000","last_peered":"2023-09-23T22:59:09.718825+0000","last_clean":"2023-09-23T22:59:09.718825+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T22:59:09.718825+0000","last_undegraded":"2023-09-23T22:59:09.718825+0000","last_fullsized":"2023-09-23T22:59:09.718825+0000","mapping_epoch":353,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295567360,"num_objects":4510,"num_object_clones":0,"num_object_copies":13530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7576,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":258,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.252712+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:59:01.252712+0000","last_peered":"2023-09-23T22:59:01.252712+0000","last_clean":"2023-09-23T22:59:01.252712+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:59:01.252712+0000","last_undegraded":"2023-09-23T22:59:01.252712+0000","last_fullsized":"2023-09-23T22:59:01.252712+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":244,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.241633+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:59:01.241633+0000","last_peered":"2023-09-23T22:59:01.241633+0000","last_clean":"2023-09-23T22:59:01.241633+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:59:01.241633+0000","last_undegraded":"2023-09-23T22:59:01.241633+0000","last_fullsized":"2023-09-23T22:59:01.241633+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"374'7492","reported_seq":9899,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:10.828839+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T22:59:10.828839+0000","last_peered":"2023-09-23T22:59:10.828839+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T22:59:10.828839+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7092,"log_dups_size":0,"ondisk_log_size":7092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292159488,"num_objects":4458,"num_object_clones":0,"num_object_copies":13374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4458,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":4458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7492,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":891}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.249086+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:59:01.249086+0000","last_peered":"2023-09-23T22:59:01.249086+0000","last_clean":"2023-09-23T22:59:01.249086+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:59:01.249086+0000","last_undegraded":"2023-09-23T22:59:01.249086+0000","last_fullsized":"2023-09-23T22:59:01.249086+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":381,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.252156+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:59:01.252156+0000","last_peered":"2023-09-23T22:59:01.252156+0000","last_clean":"2023-09-23T22:59:01.252156+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:59:01.252156+0000","last_undegraded":"2023-09-23T22:59:01.252156+0000","last_fullsized":"2023-09-23T22:59:01.252156+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"374'7588","reported_seq":7659,"reported_epoch":374,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:09.194688+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T22:59:09.194688+0000","last_peered":"2023-09-23T22:59:09.194688+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T22:59:09.194688+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T22:59:09.194688+0000","mapping_epoch":354,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":4484,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7588,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":4484}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":125,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.242122+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:59:01.242122+0000","last_peered":"2023-09-23T22:59:01.242122+0000","last_clean":"2023-09-23T22:59:01.242122+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:59:01.242122+0000","last_undegraded":"2023-09-23T22:59:01.242122+0000","last_fullsized":"2023-09-23T22:59:01.242122+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":243,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.242406+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:59:01.242406+0000","last_peered":"2023-09-23T22:59:01.242406+0000","last_clean":"2023-09-23T22:59:01.242406+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:59:01.242406+0000","last_undegraded":"2023-09-23T22:59:01.242406+0000","last_fullsized":"2023-09-23T22:59:01.242406+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"374'7615","reported_seq":7872,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:09.317155+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T22:59:09.317155+0000","last_peered":"2023-09-23T22:59:09.317155+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T22:59:09.317155+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3848,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7615,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":949}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":42,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.374127+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:59:01.374127+0000","last_peered":"2023-09-23T22:59:01.374127+0000","last_clean":"2023-09-23T22:59:01.374127+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:59:01.374127+0000","last_undegraded":"2023-09-23T22:59:01.374127+0000","last_fullsized":"2023-09-23T22:59:01.374127+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":272,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239386+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:59:01.239386+0000","last_peered":"2023-09-23T22:59:01.239386+0000","last_clean":"2023-09-23T22:59:01.239386+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:59:01.239386+0000","last_undegraded":"2023-09-23T22:59:01.239386+0000","last_fullsized":"2023-09-23T22:59:01.239386+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"374'7609","reported_seq":7688,"reported_epoch":374,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:11.996141+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T22:59:11.996141+0000","last_peered":"2023-09-23T22:59:11.996141+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T22:59:11.996141+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T22:59:11.996141+0000","mapping_epoch":353,"log_start":"322'1900","ondisk_log_start":"322'1900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293928960,"num_objects":4485,"num_object_clones":0,"num_object_copies":13455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7609,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4485}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.373820+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:59:01.373820+0000","last_peered":"2023-09-23T22:59:01.373820+0000","last_clean":"2023-09-23T22:59:01.373820+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:59:01.373820+0000","last_undegraded":"2023-09-23T22:59:01.373820+0000","last_fullsized":"2023-09-23T22:59:01.373820+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":367,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.254536+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:59:01.254536+0000","last_peered":"2023-09-23T22:59:01.254536+0000","last_clean":"2023-09-23T22:59:01.254536+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:59:01.254536+0000","last_undegraded":"2023-09-23T22:59:01.254536+0000","last_fullsized":"2023-09-23T22:59:01.254536+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"374'7673","reported_seq":7760,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:12.077794+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T22:59:12.077794+0000","last_peered":"2023-09-23T22:59:12.077794+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T22:59:12.077794+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296812544,"num_objects":4529,"num_object_clones":0,"num_object_copies":13587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4529,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":4529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7673,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":910}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":157,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.247643+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:59:01.247643+0000","last_peered":"2023-09-23T22:59:01.247643+0000","last_clean":"2023-09-23T22:59:01.247643+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:59:01.247643+0000","last_undegraded":"2023-09-23T22:59:01.247643+0000","last_fullsized":"2023-09-23T22:59:01.247643+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":384,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.246868+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:59:01.246868+0000","last_peered":"2023-09-23T22:59:01.246868+0000","last_clean":"2023-09-23T22:59:01.246868+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:59:01.246868+0000","last_undegraded":"2023-09-23T22:59:01.246868+0000","last_fullsized":"2023-09-23T22:59:01.246868+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"374'7600","reported_seq":10361,"reported_epoch":374,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:12.054235+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T22:59:12.054235+0000","last_peered":"2023-09-23T22:59:12.054235+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T22:59:12.054235+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T22:59:12.054235+0000","mapping_epoch":354,"log_start":"322'1900","ondisk_log_start":"322'1900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5700,"log_dups_size":0,"ondisk_log_size":5700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292290560,"num_objects":4460,"num_object_clones":0,"num_object_copies":13380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3267,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7600,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1333,"num_bytes_recovered":87359488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":3026}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":397,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.251978+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:59:01.251978+0000","last_peered":"2023-09-23T22:59:01.251978+0000","last_clean":"2023-09-23T22:59:01.251978+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:59:01.251978+0000","last_undegraded":"2023-09-23T22:59:01.251978+0000","last_fullsized":"2023-09-23T22:59:01.251978+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":375,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.249160+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:59:01.249160+0000","last_peered":"2023-09-23T22:59:01.249160+0000","last_clean":"2023-09-23T22:59:01.249160+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:59:01.249160+0000","last_undegraded":"2023-09-23T22:59:01.249160+0000","last_fullsized":"2023-09-23T22:59:01.249160+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"374'7504","reported_seq":7583,"reported_epoch":374,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:12.074724+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T22:59:12.074724+0000","last_peered":"2023-09-23T22:59:12.074724+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T22:59:12.074724+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T22:59:12.074724+0000","mapping_epoch":353,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7504,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4464}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":42,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.238961+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:59:01.238961+0000","last_peered":"2023-09-23T22:59:01.238961+0000","last_clean":"2023-09-23T22:59:01.238961+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:59:01.238961+0000","last_undegraded":"2023-09-23T22:59:01.238961+0000","last_fullsized":"2023-09-23T22:59:01.238961+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":276,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272237+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:59:01.272237+0000","last_peered":"2023-09-23T22:59:01.272237+0000","last_clean":"2023-09-23T22:59:01.272237+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:59:01.272237+0000","last_undegraded":"2023-09-23T22:59:01.272237+0000","last_fullsized":"2023-09-23T22:59:01.272237+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"374'7616","reported_seq":11095,"reported_epoch":374,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:59:12.094825+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T22:59:12.094825+0000","last_peered":"2023-09-23T22:59:12.094825+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T22:59:12.094825+0000","last_undegraded":"2023-09-23T22:59:12.094825+0000","last_fullsized":"2023-09-23T22:59:12.094825+0000","mapping_epoch":354,"log_start":"322'1900","ondisk_log_start":"322'1900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292290560,"num_objects":4460,"num_object_clones":0,"num_object_copies":13380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1982,"num_objects_unfound":0,"num_objects_dirty":4460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7616,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1683,"num_bytes_recovered":109772800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2478},{"shards":"0,4,6","objects":1982}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":280,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272207+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:59:01.272207+0000","last_peered":"2023-09-23T22:59:01.272207+0000","last_clean":"2023-09-23T22:59:01.272207+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:59:01.272207+0000","last_undegraded":"2023-09-23T22:59:01.272207+0000","last_fullsized":"2023-09-23T22:59:01.272207+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.241966+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:59:01.241966+0000","last_peered":"2023-09-23T22:59:01.241966+0000","last_clean":"2023-09-23T22:59:01.241966+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:59:01.241966+0000","last_undegraded":"2023-09-23T22:59:01.241966+0000","last_fullsized":"2023-09-23T22:59:01.241966+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":97,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.242272+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:59:01.242272+0000","last_peered":"2023-09-23T22:59:01.242272+0000","last_clean":"2023-09-23T22:59:01.242272+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:59:01.242272+0000","last_undegraded":"2023-09-23T22:59:01.242272+0000","last_fullsized":"2023-09-23T22:59:01.242272+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"374'7629","reported_seq":7719,"reported_epoch":374,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:59:09.328146+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T22:59:09.328146+0000","last_peered":"2023-09-23T22:59:09.328146+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T22:59:09.328146+0000","last_undegraded":"2023-09-23T22:59:09.328146+0000","last_fullsized":"2023-09-23T22:59:09.328146+0000","mapping_epoch":359,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294584320,"num_objects":4495,"num_object_clones":0,"num_object_copies":13485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4495,"num_objects_unfound":0,"num_objects_dirty":4495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7629,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4495}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":43,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239367+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T22:59:01.239367+0000","last_peered":"2023-09-23T22:59:01.239367+0000","last_clean":"2023-09-23T22:59:01.239367+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T22:59:01.239367+0000","last_undegraded":"2023-09-23T22:59:01.239367+0000","last_fullsized":"2023-09-23T22:59:01.239367+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":394,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.249870+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:59:01.249870+0000","last_peered":"2023-09-23T22:59:01.249870+0000","last_clean":"2023-09-23T22:59:01.249870+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:59:01.249870+0000","last_undegraded":"2023-09-23T22:59:01.249870+0000","last_fullsized":"2023-09-23T22:59:01.249870+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":265,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272253+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:59:01.272253+0000","last_peered":"2023-09-23T22:59:01.272253+0000","last_clean":"2023-09-23T22:59:01.272253+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:59:01.272253+0000","last_undegraded":"2023-09-23T22:59:01.272253+0000","last_fullsized":"2023-09-23T22:59:01.272253+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"374'7603","reported_seq":9851,"reported_epoch":374,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:11.119640+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T22:59:11.119640+0000","last_peered":"2023-09-23T22:59:11.119640+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T22:59:11.119640+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7103,"log_dups_size":0,"ondisk_log_size":7103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292356096,"num_objects":4461,"num_object_clones":0,"num_object_copies":13383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2490,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7603,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1081,"num_bytes_recovered":70844416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1971}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":340,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.271933+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:59:01.271933+0000","last_peered":"2023-09-23T22:59:01.271933+0000","last_clean":"2023-09-23T22:59:01.271933+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:59:01.271933+0000","last_undegraded":"2023-09-23T22:59:01.271933+0000","last_fullsized":"2023-09-23T22:59:01.271933+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":217,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.271898+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:59:01.271898+0000","last_peered":"2023-09-23T22:59:01.271898+0000","last_clean":"2023-09-23T22:59:01.271898+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:59:01.271898+0000","last_undegraded":"2023-09-23T22:59:01.271898+0000","last_fullsized":"2023-09-23T22:59:01.271898+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":284,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.242860+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:59:01.242860+0000","last_peered":"2023-09-23T22:59:01.242860+0000","last_clean":"2023-09-23T22:59:01.242860+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:59:01.242860+0000","last_undegraded":"2023-09-23T22:59:01.242860+0000","last_fullsized":"2023-09-23T22:59:01.242860+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"374'7692","reported_seq":9568,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:11.776729+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T22:59:11.776729+0000","last_peered":"2023-09-23T22:59:11.776729+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T22:59:11.776729+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7692,"log_dups_size":0,"ondisk_log_size":7692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297009152,"num_objects":4532,"num_object_clones":0,"num_object_copies":13596,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":4532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7692,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":3099},{"shards":"0,6","objects":-449}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":357,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.241916+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:59:01.241916+0000","last_peered":"2023-09-23T22:59:01.241916+0000","last_clean":"2023-09-23T22:59:01.241916+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:59:01.241916+0000","last_undegraded":"2023-09-23T22:59:01.241916+0000","last_fullsized":"2023-09-23T22:59:01.241916+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":469,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:01.374577+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T22:59:01.374577+0000","last_peered":"2023-09-23T22:59:01.374577+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T22:59:01.374577+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":42,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239668+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:59:01.239668+0000","last_peered":"2023-09-23T22:59:01.239668+0000","last_clean":"2023-09-23T22:59:01.239668+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:59:01.239668+0000","last_undegraded":"2023-09-23T22:59:01.239668+0000","last_fullsized":"2023-09-23T22:59:01.239668+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":284,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.247093+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:59:01.247093+0000","last_peered":"2023-09-23T22:59:01.247093+0000","last_clean":"2023-09-23T22:59:01.247093+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:59:01.247093+0000","last_undegraded":"2023-09-23T22:59:01.247093+0000","last_fullsized":"2023-09-23T22:59:01.247093+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"374'7599","reported_seq":7681,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:09.577223+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T22:59:09.577223+0000","last_peered":"2023-09-23T22:59:09.577223+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T22:59:09.577223+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293928960,"num_objects":4485,"num_object_clones":0,"num_object_copies":13455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7599,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":978}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":363,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239128+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T22:59:01.239128+0000","last_peered":"2023-09-23T22:59:01.239128+0000","last_clean":"2023-09-23T22:59:01.239128+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T22:59:01.239128+0000","last_undegraded":"2023-09-23T22:59:01.239128+0000","last_fullsized":"2023-09-23T22:59:01.239128+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":334,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239102+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T22:59:01.239102+0000","last_peered":"2023-09-23T22:59:01.239102+0000","last_clean":"2023-09-23T22:59:01.239102+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T22:59:01.239102+0000","last_undegraded":"2023-09-23T22:59:01.239102+0000","last_fullsized":"2023-09-23T22:59:01.239102+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":294,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.375586+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:59:01.375586+0000","last_peered":"2023-09-23T22:59:01.375586+0000","last_clean":"2023-09-23T22:59:01.375586+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:59:01.375586+0000","last_undegraded":"2023-09-23T22:59:01.375586+0000","last_fullsized":"2023-09-23T22:59:01.375586+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"374'7689","reported_seq":7787,"reported_epoch":374,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:11.924749+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T22:59:11.924749+0000","last_peered":"2023-09-23T22:59:11.924749+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T22:59:11.924749+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T22:59:11.924749+0000","mapping_epoch":354,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295829528,"num_objects":4515,"num_object_clones":0,"num_object_copies":13545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":4515,"num_objects_unfound":0,"num_objects_dirty":4515,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7689,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":4515}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":309,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239318+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T22:59:01.239318+0000","last_peered":"2023-09-23T22:59:01.239318+0000","last_clean":"2023-09-23T22:59:01.239318+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T22:59:01.239318+0000","last_undegraded":"2023-09-23T22:59:01.239318+0000","last_fullsized":"2023-09-23T22:59:01.239318+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":217,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272299+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:59:01.272299+0000","last_peered":"2023-09-23T22:59:01.272299+0000","last_clean":"2023-09-23T22:59:01.272299+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:59:01.272299+0000","last_undegraded":"2023-09-23T22:59:01.272299+0000","last_fullsized":"2023-09-23T22:59:01.272299+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":187,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.241992+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:59:01.241992+0000","last_peered":"2023-09-23T22:59:01.241992+0000","last_clean":"2023-09-23T22:59:01.241992+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:59:01.241992+0000","last_undegraded":"2023-09-23T22:59:01.241992+0000","last_fullsized":"2023-09-23T22:59:01.241992+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"374'7642","reported_seq":9694,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:12.039659+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T22:59:12.039659+0000","last_peered":"2023-09-23T22:59:12.039659+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T22:59:12.039659+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"322'1900","ondisk_log_start":"322'1900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5742,"log_dups_size":0,"ondisk_log_size":5742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295174144,"num_objects":4504,"num_object_clones":0,"num_object_copies":13512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8372,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7642,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":980,"num_bytes_recovered":64225280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":923},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":66,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.372102+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:59:01.372102+0000","last_peered":"2023-09-23T22:59:01.372102+0000","last_clean":"2023-09-23T22:59:01.372102+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:59:01.372102+0000","last_undegraded":"2023-09-23T22:59:01.372102+0000","last_fullsized":"2023-09-23T22:59:01.372102+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":82,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.255022+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:59:01.255022+0000","last_peered":"2023-09-23T22:59:01.255022+0000","last_clean":"2023-09-23T22:59:01.255022+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:59:01.255022+0000","last_undegraded":"2023-09-23T22:59:01.255022+0000","last_fullsized":"2023-09-23T22:59:01.255022+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.238862+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:59:01.238862+0000","last_peered":"2023-09-23T22:59:01.238862+0000","last_clean":"2023-09-23T22:59:01.238862+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:59:01.238862+0000","last_undegraded":"2023-09-23T22:59:01.238862+0000","last_fullsized":"2023-09-23T22:59:01.238862+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"374'7601","reported_seq":8109,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:11.848398+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T22:59:11.848398+0000","last_peered":"2023-09-23T22:59:11.848398+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T22:59:11.848398+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"322'1900","ondisk_log_start":"322'1900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292880384,"num_objects":4469,"num_object_clones":0,"num_object_copies":13407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4469,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":4469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7601,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":918}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":329,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272364+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:59:01.272364+0000","last_peered":"2023-09-23T22:59:01.272364+0000","last_clean":"2023-09-23T22:59:01.272364+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:59:01.272364+0000","last_undegraded":"2023-09-23T22:59:01.272364+0000","last_fullsized":"2023-09-23T22:59:01.272364+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":276,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.375498+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:59:01.375498+0000","last_peered":"2023-09-23T22:59:01.375498+0000","last_clean":"2023-09-23T22:59:01.375498+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:59:01.375498+0000","last_undegraded":"2023-09-23T22:59:01.375498+0000","last_fullsized":"2023-09-23T22:59:01.375498+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":348,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.252042+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:59:01.252042+0000","last_peered":"2023-09-23T22:59:01.252042+0000","last_clean":"2023-09-23T22:59:01.252042+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:59:01.252042+0000","last_undegraded":"2023-09-23T22:59:01.252042+0000","last_fullsized":"2023-09-23T22:59:01.252042+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":311,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239625+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:59:01.239625+0000","last_peered":"2023-09-23T22:59:01.239625+0000","last_clean":"2023-09-23T22:59:01.239625+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:59:01.239625+0000","last_undegraded":"2023-09-23T22:59:01.239625+0000","last_fullsized":"2023-09-23T22:59:01.239625+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":121,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.272066+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:59:01.272066+0000","last_peered":"2023-09-23T22:59:01.272066+0000","last_clean":"2023-09-23T22:59:01.272066+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:59:01.272066+0000","last_undegraded":"2023-09-23T22:59:01.272066+0000","last_fullsized":"2023-09-23T22:59:01.272066+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":183,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.254455+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:59:01.254455+0000","last_peered":"2023-09-23T22:59:01.254455+0000","last_clean":"2023-09-23T22:59:01.254455+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:59:01.254455+0000","last_undegraded":"2023-09-23T22:59:01.254455+0000","last_fullsized":"2023-09-23T22:59:01.254455+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":397,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.249935+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:59:01.249935+0000","last_peered":"2023-09-23T22:59:01.249935+0000","last_clean":"2023-09-23T22:59:01.249935+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:59:01.249935+0000","last_undegraded":"2023-09-23T22:59:01.249935+0000","last_fullsized":"2023-09-23T22:59:01.249935+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":197,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.246955+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:59:01.246955+0000","last_peered":"2023-09-23T22:59:01.246955+0000","last_clean":"2023-09-23T22:59:01.246955+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:59:01.246955+0000","last_undegraded":"2023-09-23T22:59:01.246955+0000","last_fullsized":"2023-09-23T22:59:01.246955+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":284,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.247561+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:59:01.247561+0000","last_peered":"2023-09-23T22:59:01.247561+0000","last_clean":"2023-09-23T22:59:01.247561+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:59:01.247561+0000","last_undegraded":"2023-09-23T22:59:01.247561+0000","last_fullsized":"2023-09-23T22:59:01.247561+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":66,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239904+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:59:01.239904+0000","last_peered":"2023-09-23T22:59:01.239904+0000","last_clean":"2023-09-23T22:59:01.239904+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:59:01.239904+0000","last_undegraded":"2023-09-23T22:59:01.239904+0000","last_fullsized":"2023-09-23T22:59:01.239904+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":69,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.239186+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:59:01.239186+0000","last_peered":"2023-09-23T22:59:01.239186+0000","last_clean":"2023-09-23T22:59:01.239186+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:59:01.239186+0000","last_undegraded":"2023-09-23T22:59:01.239186+0000","last_fullsized":"2023-09-23T22:59:01.239186+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":315,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.375280+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T22:59:01.375280+0000","last_peered":"2023-09-23T22:59:01.375280+0000","last_clean":"2023-09-23T22:59:01.375280+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T22:59:01.375280+0000","last_undegraded":"2023-09-23T22:59:01.375280+0000","last_fullsized":"2023-09-23T22:59:01.375280+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":359,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T22:59:01.251820+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:59:01.251820+0000","last_peered":"2023-09-23T22:59:01.251820+0000","last_clean":"2023-09-23T22:59:01.251820+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:59:01.251820+0000","last_undegraded":"2023-09-23T22:59:01.251820+0000","last_fullsized":"2023-09-23T22:59:01.251820+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4710006808,"num_objects":71870,"num_object_clones":0,"num_object_copies":215610,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":38645,"num_objects_misplaced":35403,"num_objects_unfound":0,"num_objects_dirty":71870,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121728,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7421,"num_bytes_recovered":485097472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13772271616,"data_stored":13772259400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95028,"ondisk_log_size":95028,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430594,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626819,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253132,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2369356,"kb_used_data":1689156,"kb_used_omap":9,"kb_used_meta":680182,"kb_avail":91375796,"statfs":{"total":95995035648,"available":93568815104,"internally_reserved":0,"allocated":1729695744,"data_stored":1728304415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":696507369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.5960000000000001,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.49099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:55:30 2023","interfaces":[{"interface":"back","average":{"1min":9.8800000000000008,"5min":4.593,"15min":3.8439999999999999},"min":{"1min":0.246,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":111.452,"5min":111.452,"15min":111.452},"last":0.53900000000000003},{"interface":"front","average":{"1min":10.167,"5min":4.4580000000000002,"15min":3.9420000000000002},"min":{"1min":0.36199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":108.22799999999999,"5min":108.22799999999999,"15min":108.22799999999999},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":1.0629999999999999,"15min":2.3849999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63100000000000001,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":1.147,"15min":2.8340000000000001},"min":{"1min":0.27400000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.624,"5min":8.0099999999999998,"15min":89.861999999999995},"last":0.68200000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5730000000000004,"5min":3.1190000000000002,"15min":2.766},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":151.036,"5min":151.036,"15min":151.036},"last":0.76700000000000002},{"interface":"front","average":{"1min":7.5510000000000002,"5min":3.2469999999999999,"15min":2.7709999999999999},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":151.06899999999999,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.71799999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":3.4449999999999998,"15min":2.5329999999999999},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":0.95099999999999996,"5min":116.098,"15min":116.098},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":3.1200000000000001,"15min":2.7890000000000001},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":2.698,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.80700000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.8600000000000001,"15min":2.431},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.85499999999999998,"5min":20.742000000000001,"15min":79.311000000000007},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.9119999999999999,"15min":2.125},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":4.4219999999999997,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.3580000000000001,"15min":2.028},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":0.87,"5min":8.1400000000000006,"15min":29.539000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.6990000000000001,"15min":2.2120000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.753}]}]},{"osd":0,"up_from":353,"seq":1516123455502,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3113028,"kb_used_data":2987964,"kb_used_omap":37,"kb_used_meta":125018,"kb_avail":90632124,"statfs":{"total":95995035648,"available":92807294976,"internally_reserved":0,"allocated":3059675136,"data_stored":3058175004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":128018594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.6309999999999998,"5min":2.6309999999999998,"15min":2.6309999999999998},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.0539999999999998,"5min":3.0539999999999998,"15min":3.0539999999999998},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.373},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0179999999999998,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.2770000000000001,"5min":3.2770000000000001,"15min":3.2770000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.982,"5min":1.982,"15min":1.982},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.998,"5min":0.998,"15min":0.998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.0920000000000001,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.379},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":4.3120000000000003,"5min":4.3120000000000003,"15min":4.3120000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.3579999999999997,"5min":4.3579999999999997,"15min":4.3579999999999997},"last":0.498}]}]},{"osd":2,"up_from":359,"seq":1541893259275,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072272,"kb_used_data":981000,"kb_used_omap":83,"kb_used_meta":91180,"kb_avail":92672880,"statfs":{"total":95995035648,"available":94897029120,"internally_reserved":0,"allocated":1004544000,"data_stored":1003093743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":93368773},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26600000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4790000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73099999999999998}]}]},{"osd":3,"up_from":94,"seq":403726926079,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532072,"kb_used_data":49568,"kb_used_omap":7,"kb_used_meta":482488,"kb_avail":93213080,"statfs":{"total":95995035648,"available":95450193920,"internally_reserved":0,"allocated":50757632,"data_stored":49279640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":494068328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":0.21299999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.972,"5min":1.972,"15min":1.972},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.097,"15min":2.786},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.60899999999999999,"5min":16.596,"15min":123.866},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.35499999999999998,"5min":1.179,"15min":2.4449999999999998},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":0.47499999999999998,"5min":16.649999999999999,"15min":122.265},"last":0.25700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:55:36 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":2.431,"15min":3.1230000000000002},"min":{"1min":0.251,"5min":0.222,"15min":0.222},"max":{"1min":6.8920000000000003,"5min":38.292999999999999,"15min":125.446},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":2.544,"15min":3.7759999999999998},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.217},"max":{"1min":5.9269999999999996,"5min":38.225999999999999,"15min":182.483},"last":0.23400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.3129999999999999,"15min":1.8640000000000001},"min":{"1min":0.30399999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":1.0700000000000001,"5min":7.8929999999999998,"15min":18.757000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.3129999999999999,"15min":1.629},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.27500000000000002},"max":{"1min":0.72199999999999998,"5min":7.8780000000000001,"15min":15.037000000000001},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.393,"15min":1.52},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.59399999999999997,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":1.546,"15min":1.954},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75600000000000001,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7709999999999999,"15min":1.788},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":14.097,"15min":14.097},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.677,"15min":2.0390000000000001},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5,"5min":12.887,"15min":12.887},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.3480000000000001,"15min":2.2109999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79700000000000004,"5min":6.9950000000000001,"15min":91.757999999999996},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.238,"15min":2.2240000000000002},"min":{"1min":0.434,"5min":0.41699999999999998,"15min":0.33200000000000002},"max":{"1min":1.1479999999999999,"5min":7.1219999999999999,"15min":91.709999999999994},"last":0.43099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019276,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3935988,"kb_used_data":3346284,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":89809164,"statfs":{"total":95995035648,"available":91964583936,"internally_reserved":0,"allocated":3426594816,"data_stored":3425856097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.64800000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":3.7690000000000001,"15min":3.786},"min":{"1min":0.40000000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":1.966,"5min":98.846999999999994,"15min":132.304},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.71699999999999997,"5min":3.9740000000000002,"15min":3.6259999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":3.2269999999999999,"5min":98.938999999999993,"15min":132.453},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:55:40 2023","interfaces":[{"interface":"back","average":{"1min":11.44,"5min":4.0789999999999997,"15min":3.7349999999999999},"min":{"1min":0.33800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":137.922,"5min":137.922,"15min":137.922},"last":5.2439999999999998},{"interface":"front","average":{"1min":11.497,"5min":4.1219999999999999,"15min":3.734},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":140.994,"5min":140.994,"15min":140.994},"last":0.72599999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.536,"15min":4.5099999999999998},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":3.399,"5min":8.5510000000000002,"15min":164.453},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.478,"15min":4.5720000000000001},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.99,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.69999999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:58:27 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":3.0800000000000001,"15min":1.641},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":12.961,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.5019999999999998,"15min":1.7869999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.248},"max":{"1min":5.4619999999999997,"5min":151.905,"15min":151.905},"last":0.82099999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5820000000000001,"15min":1.1559999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":2.113,"5min":16.943000000000001,"15min":16.943000000000001},"last":2.3639999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.423,"15min":1.101},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0350000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.752}]},{"osd":7,"last update":"Sat Sep 23 22:58:12 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.4299999999999999,"15min":1.0820000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.84,"5min":12.84,"15min":12.84},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.397,"5min":1.278,"15min":0.98399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8700000000000001,"5min":8.5190000000000001,"15min":8.6739999999999995},"last":0.45200000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823050,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2995540,"kb_used_data":2104076,"kb_used_omap":0,"kb_used_meta":891456,"kb_avail":90749612,"statfs":{"total":95995035648,"available":92927602688,"internally_reserved":0,"allocated":2154573824,"data_stored":2153862893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912850944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":0.315}]},{"osd":1,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":2.9249999999999998,"15min":2.3330000000000002},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":3.2429999999999999,"5min":110.36499999999999,"15min":110.36499999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":3.4340000000000002,"15min":2.8410000000000002},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":10.353,"5min":110.89,"15min":110.89},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:55:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":4.992,"15min":5.04},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5309999999999997,"5min":189.126,"15min":189.126},"last":0.374},{"interface":"front","average":{"1min":1.4710000000000001,"5min":4.6379999999999999,"15min":4.8920000000000003},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":6.2530000000000001,"5min":188.74799999999999,"15min":188.74799999999999},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.6160000000000001,"15min":5.117},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":3.149,"5min":11.298999999999999,"15min":257.05000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.5469999999999999,"15min":4.9409999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.4940000000000002,"5min":12.185,"15min":257.35000000000002},"last":0.5}]},{"osd":4,"last update":"Sat Sep 23 22:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":2.7559999999999998,"15min":1.4650000000000001},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.603,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":2.6909999999999998,"15min":1.4419999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":3.093,"5min":114.908,"15min":114.908},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.3300000000000001,"15min":1.2649999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.498,"5min":11.121,"15min":48.360999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.272,"15min":1.2709999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.7759999999999998,"5min":12.137,"15min":48.292999999999999},"last":1.0840000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.24,"15min":1.012},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.4390000000000001,"5min":12.23,"15min":12.23},"last":1.052},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.2889999999999999,"15min":0.98899999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.3760000000000003,"5min":11.112,"15min":11.112},"last":6.093}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3053391872,"data_stored":3053387800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1722089472,"data_stored":1722089472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1000407040,"data_stored":1000407040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":46399488,"data_stored":46399488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3422162944,"data_stored":3422158872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2150367232,"data_stored":2150367232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2377453568,"data_stored":2377449496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:59:16.566 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:59:16.598 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:16.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:16.708 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:16.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:16.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: osdmap e375: 8 total, 8 up, 6 in 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[180979]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:59:16.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: osdmap e375: 8 total, 8 up, 6 in 2023-09-23T22:59:16.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:16 smithi052 ceph-mon[187743]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.871 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: osdmap e375: 8 total, 8 up, 6 in 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:16 smithi151 ceph-mon[159210]: failed to encode map e375 with expected crc 2023-09-23T22:59:16.952 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:16.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:16.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:17.061 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:17.090 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:17.163 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:17.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:17.266 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:17.304 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:17.358 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:17.367 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:17.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:17.472 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:17.516 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:17.591 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:17.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:17.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:17.602 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:59:17.602 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:59:17.602 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:59:17.602 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:59:17.602 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:59:17.608 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:17.702 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:17.739 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:17.818 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:17.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:17.928 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:17 smithi151 ceph-mon[159210]: pgmap v1097: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 38405/214275 objects degraded (17.923%); 35261/214275 objects misplaced (16.456%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:59:17.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:17 smithi052 ceph-mon[187743]: pgmap v1097: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 38405/214275 objects degraded (17.923%); 35261/214275 objects misplaced (16.456%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:59:17.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:17 smithi052 ceph-mon[180979]: pgmap v1097: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 38405/214275 objects degraded (17.923%); 35261/214275 objects misplaced (16.456%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:59:17.962 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:18.063 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:18.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:18.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:18.170 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:18.210 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:18.281 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:18.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:18.390 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:18.527 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:18.618 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:18.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:18.726 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:18.771 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:18.874 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:18.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:18.990 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:19.039 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:19.122 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:59:19.123 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:19.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:19.230 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:19.286 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:19.383 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:59:19.391 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:19.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:19.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:19.500 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:19.568 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:19.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:19 smithi151 ceph-mon[159210]: pgmap v1098: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 38093/212283 objects degraded (17.944%); 35077/212283 objects misplaced (16.524%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:19.674 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:19.905 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:19.926 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:19.954 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:19.980 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:19 smithi052 ceph-mon[187743]: pgmap v1098: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 38093/212283 objects degraded (17.944%); 35077/212283 objects misplaced (16.524%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:19 smithi052 ceph-mon[180979]: pgmap v1098: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 38093/212283 objects degraded (17.944%); 35077/212283 objects misplaced (16.524%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:20.004 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:20.025 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:20.046 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:20.067 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.139 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:59:20.742 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:20.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:20.784 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4115352429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:20.784 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 38093/212283 objects degraded (17.944%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:20.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4115352429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:20.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 38093/212283 objects degraded (17.944%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:20.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:20.848 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:20.879 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:20.904 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4115352429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 38093/212283 objects degraded (17.944%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:59:20.964 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:20.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:21.070 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:21.106 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:21.178 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:21.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:21.289 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:21.320 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:21.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:21.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:21.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:21.488 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:21.518 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:21.574 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:21.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:21.625 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:21.625 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 38023/211080 objects degraded (18.014%), 14 pgs degraded, 1 pg undersized","count":15},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1494,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":375,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":70360,"data_bytes":4610070456,"bytes_used":14151483392,"bytes_avail":561818730496,"bytes_total":575970213888,"degraded_objects":38023,"degraded_total":211080,"degraded_ratio":0.18013549365169601,"misplaced_objects":34899,"misplaced_total":211080,"misplaced_ratio":0.16533541785105174,"recovering_objects_per_sec":76,"recovering_bytes_per_sec":5024843,"recovering_keys_per_sec":0,"num_objects_recovered":767,"num_bytes_recovered":50266112,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":223},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":29,"modified":"2023-09-23T22:59:06.541737+0000","services":{"osd":{"daemons":{"summary":"","2":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (3m)\n [======================......] (remaining: 42s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:59:21.680 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:21.721 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:21 smithi151 ceph-mon[159210]: pgmap v1099: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 38023/211080 objects degraded (18.014%); 34899/211080 objects misplaced (16.534%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2743619083' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:59:21.950 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:21 smithi052 ceph-mon[180979]: pgmap v1099: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 38023/211080 objects degraded (18.014%); 34899/211080 objects misplaced (16.534%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:21.950 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2743619083' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:59:21.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:21 smithi052 ceph-mon[187743]: pgmap v1099: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 38023/211080 objects degraded (18.014%); 34899/211080 objects misplaced (16.534%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:21.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2743619083' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:59:22.761 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:59:23.121 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:23.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:23.152 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:59:23.152 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:59:23.229 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:23.266 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:23.341 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:23.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:23.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:23.449 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:23.484 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:23.601 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:23.686 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:59:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:59:23] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T22:59:23.686 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:23 smithi052 ceph-mon[180979]: pgmap v1100: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 232 op/s; 37550/208692 objects degraded (17.993%); 34706/208692 objects misplaced (16.630%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:23.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:23 smithi151 ceph-mon[159210]: pgmap v1100: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 232 op/s; 37550/208692 objects degraded (17.993%); 34706/208692 objects misplaced (16.630%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:23.714 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:59:23.714 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:59:23.947 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:23.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:23 smithi052 ceph-mon[187743]: pgmap v1100: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 232 op/s; 37550/208692 objects degraded (17.993%); 34706/208692 objects misplaced (16.630%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:23.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:24.051 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:24.086 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:24.699 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:59:24.762 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:59:24.820 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3825895572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:24.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3825895572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:24.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3825895572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:25.512 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:25.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:25.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:25.616 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:25.656 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: pgmap v1101: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 232 op/s; 37550/208692 objects degraded (17.993%); 34706/208692 objects misplaced (16.630%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:25.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: failed to encode map e376 with expected crc 2023-09-23T22:59:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: failed to encode map e376 with expected crc 2023-09-23T22:59:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3825895572' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: osdmap e376: 8 total, 8 up, 6 in 2023-09-23T22:59:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 37550/208692 objects degraded (17.993%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: pgmap v1101: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 232 op/s; 37550/208692 objects degraded (17.993%); 34706/208692 objects misplaced (16.630%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: failed to encode map e376 with expected crc 2023-09-23T22:59:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: failed to encode map e376 with expected crc 2023-09-23T22:59:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3825895572' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: osdmap e376: 8 total, 8 up, 6 in 2023-09-23T22:59:25.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 37550/208692 objects degraded (17.993%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: pgmap v1101: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 232 op/s; 37550/208692 objects degraded (17.993%); 34706/208692 objects misplaced (16.630%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:25.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:25.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: failed to encode map e376 with expected crc 2023-09-23T22:59:25.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: failed to encode map e376 with expected crc 2023-09-23T22:59:25.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3825895572' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:25.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: osdmap e376: 8 total, 8 up, 6 in 2023-09-23T22:59:25.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 37550/208692 objects degraded (17.993%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:26.064 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:26.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:26.172 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:26.217 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:26.285 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:26.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:26.394 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:26.395 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:26.441 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:26.508 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:26.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:26.618 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:26.621 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:26.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:26.671 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:26.677 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:26.696 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/960946594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:26.696 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/960946594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:26.702 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:26.732 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:26.750 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:26.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:26.762 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:26.786 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:26.810 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.812 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:59:26.856 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:26.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/960946594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:26.892 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:26.945 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:26.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:27.054 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:27.088 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:27.166 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:27.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:27.274 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:27.309 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:27.632 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:27.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:27.738 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:27.777 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:27.843 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:27.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:27.949 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:27.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:27 smithi151 ceph-mon[159210]: pgmap v1103: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 37325/207249 objects degraded (18.010%); 34567/207249 objects misplaced (16.679%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:27.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:27 smithi052 ceph-mon[187743]: pgmap v1103: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 37325/207249 objects degraded (18.010%); 34567/207249 objects misplaced (16.679%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:27.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:27 smithi052 ceph-mon[180979]: pgmap v1103: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 37325/207249 objects degraded (18.010%); 34567/207249 objects misplaced (16.679%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:27.984 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:59:27.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:28.221 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:28.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:28.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:28.325 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:28.352 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:28.400 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:28.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:28.508 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:28.540 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:28.698 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:28.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:28.802 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:28.840 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:28.926 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:28.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:29.034 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:29.069 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:29.175 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:29.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:29.282 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:29.284 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:29.300 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:59:29.300 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:59:29.300 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T22:59:29.300 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:59:29.319 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:29.373 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:29.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:29.481 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:29.518 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:29.582 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:29.584 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:59:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:29 smithi151 ceph-mon[159210]: pgmap v1104: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 37064/205392 objects degraded (18.045%); 34381/205392 objects misplaced (16.739%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:29.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:29 smithi052 ceph-mon[180979]: pgmap v1104: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 37064/205392 objects degraded (18.045%); 34381/205392 objects misplaced (16.739%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:29.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:29 smithi052 ceph-mon[187743]: pgmap v1104: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 37064/205392 objects degraded (18.045%); 34381/205392 objects misplaced (16.739%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:30.204 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:30.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:30.235 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:59:30.310 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:30.368 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:30.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:30.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:30.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:30.544 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:30.589 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:30.640 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:30.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:30.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:30.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:30 smithi052 ceph-mon[187743]: from='client.36288 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:30.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 37064/205392 objects degraded (18.045%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:30 smithi052 ceph-mon[180979]: from='client.36288 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 37064/205392 objects degraded (18.045%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:30.748 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:30.795 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:30.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:59:30.860 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:30.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:30 smithi151 ceph-mon[159210]: from='client.36288 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:30.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 37064/205392 objects degraded (18.045%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:30.970 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:31.009 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:31.480 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:31.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:31.584 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:31.612 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:31.617 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1104,"stamp":"2023-09-23T22:59:28.551211+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4485814200,"num_objects":68464,"num_object_clones":0,"num_object_copies":205392,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":37064,"num_objects_misplaced":34381,"num_objects_unfound":0,"num_objects_dirty":68464,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":125418,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8574,"num_bytes_recovered":558382012,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96052,"ondisk_log_size":96052,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13519388,"kb_used_data":10638700,"kb_used_omap":137,"kb_used_meta":2880502,"kb_avail":548951524,"statfs":{"total":575970213888,"available":562126360576,"internally_reserved":0,"allocated":10894028800,"data_stored":10886708298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":2949634232},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-150536192,"num_objects":-2297,"num_object_clones":0,"num_object_copies":-6891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1029,"num_objects_misplaced":-696,"num_objects_unfound":0,"num_objects_dirty":-2297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":725,"num_bytes_recovered":47513600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003145"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":388,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665917+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:59:15.665917+0000","last_peered":"2023-09-23T22:59:15.665917+0000","last_clean":"2023-09-23T22:59:15.665917+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:59:15.665917+0000","last_undegraded":"2023-09-23T22:59:15.665917+0000","last_fullsized":"2023-09-23T22:59:15.665917+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":44,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.654290+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:59:15.654290+0000","last_peered":"2023-09-23T22:59:15.654290+0000","last_clean":"2023-09-23T22:59:15.654290+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:59:15.654290+0000","last_undegraded":"2023-09-23T22:59:15.654290+0000","last_fullsized":"2023-09-23T22:59:15.654290+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":43,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.653039+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:59:15.653039+0000","last_peered":"2023-09-23T22:59:15.653039+0000","last_clean":"2023-09-23T22:59:15.653039+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:59:15.653039+0000","last_undegraded":"2023-09-23T22:59:15.653039+0000","last_fullsized":"2023-09-23T22:59:15.653039+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":371,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.653830+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:59:15.653830+0000","last_peered":"2023-09-23T22:59:15.653830+0000","last_clean":"2023-09-23T22:59:15.653830+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:59:15.653830+0000","last_undegraded":"2023-09-23T22:59:15.653830+0000","last_fullsized":"2023-09-23T22:59:15.653830+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":59,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.652926+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:59:15.652926+0000","last_peered":"2023-09-23T22:59:15.652926+0000","last_clean":"2023-09-23T22:59:15.652926+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:59:15.652926+0000","last_undegraded":"2023-09-23T22:59:15.652926+0000","last_fullsized":"2023-09-23T22:59:15.652926+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.662873+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T22:59:15.662873+0000","last_peered":"2023-09-23T22:59:15.662873+0000","last_clean":"2023-09-23T22:59:15.662873+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T22:59:15.662873+0000","last_undegraded":"2023-09-23T22:59:15.662873+0000","last_fullsized":"2023-09-23T22:59:15.662873+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":311,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.664196+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:59:15.664196+0000","last_peered":"2023-09-23T22:59:15.664196+0000","last_clean":"2023-09-23T22:59:15.664196+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:59:15.664196+0000","last_undegraded":"2023-09-23T22:59:15.664196+0000","last_fullsized":"2023-09-23T22:59:15.664196+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":53,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.652856+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:59:15.652856+0000","last_peered":"2023-09-23T22:59:15.652856+0000","last_clean":"2023-09-23T22:59:15.652856+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:59:15.652856+0000","last_undegraded":"2023-09-23T22:59:15.652856+0000","last_fullsized":"2023-09-23T22:59:15.652856+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":272,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.658911+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:59:15.658911+0000","last_peered":"2023-09-23T22:59:15.658911+0000","last_clean":"2023-09-23T22:59:15.658911+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:59:15.658911+0000","last_undegraded":"2023-09-23T22:59:15.658911+0000","last_fullsized":"2023-09-23T22:59:15.658911+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":58,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.652927+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:59:15.652927+0000","last_peered":"2023-09-23T22:59:15.652927+0000","last_clean":"2023-09-23T22:59:15.652927+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:59:15.652927+0000","last_undegraded":"2023-09-23T22:59:15.652927+0000","last_fullsized":"2023-09-23T22:59:15.652927+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.654215+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:59:15.654215+0000","last_peered":"2023-09-23T22:59:15.654215+0000","last_clean":"2023-09-23T22:59:15.654215+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:59:15.654215+0000","last_undegraded":"2023-09-23T22:59:15.654215+0000","last_fullsized":"2023-09-23T22:59:15.654215+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673133+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:59:15.673133+0000","last_peered":"2023-09-23T22:59:15.673133+0000","last_clean":"2023-09-23T22:59:15.673133+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:59:15.673133+0000","last_undegraded":"2023-09-23T22:59:15.673133+0000","last_fullsized":"2023-09-23T22:59:15.673133+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":447,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.672734+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:59:15.672734+0000","last_peered":"2023-09-23T22:59:15.672734+0000","last_clean":"2023-09-23T22:59:15.672734+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:59:15.672734+0000","last_undegraded":"2023-09-23T22:59:15.672734+0000","last_fullsized":"2023-09-23T22:59:15.672734+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":350,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665433+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:59:15.665433+0000","last_peered":"2023-09-23T22:59:15.665433+0000","last_clean":"2023-09-23T22:59:15.665433+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:59:15.665433+0000","last_undegraded":"2023-09-23T22:59:15.665433+0000","last_fullsized":"2023-09-23T22:59:15.665433+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":276,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665522+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:59:15.665522+0000","last_peered":"2023-09-23T22:59:15.665522+0000","last_clean":"2023-09-23T22:59:15.665522+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:59:15.665522+0000","last_undegraded":"2023-09-23T22:59:15.665522+0000","last_fullsized":"2023-09-23T22:59:15.665522+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":55,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.654990+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:59:15.654990+0000","last_peered":"2023-09-23T22:59:15.654990+0000","last_clean":"2023-09-23T22:59:15.654990+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:59:15.654990+0000","last_undegraded":"2023-09-23T22:59:15.654990+0000","last_fullsized":"2023-09-23T22:59:15.654990+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":364,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.653028+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:59:15.653028+0000","last_peered":"2023-09-23T22:59:15.653028+0000","last_clean":"2023-09-23T22:59:15.653028+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:59:15.653028+0000","last_undegraded":"2023-09-23T22:59:15.653028+0000","last_fullsized":"2023-09-23T22:59:15.653028+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.658210+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:59:15.658210+0000","last_peered":"2023-09-23T22:59:15.658210+0000","last_clean":"2023-09-23T22:59:15.658210+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:59:15.658210+0000","last_undegraded":"2023-09-23T22:59:15.658210+0000","last_fullsized":"2023-09-23T22:59:15.658210+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":261,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665227+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:59:15.665227+0000","last_peered":"2023-09-23T22:59:15.665227+0000","last_clean":"2023-09-23T22:59:15.665227+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:59:15.665227+0000","last_undegraded":"2023-09-23T22:59:15.665227+0000","last_fullsized":"2023-09-23T22:59:15.665227+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":388,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.653959+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T22:59:15.653959+0000","last_peered":"2023-09-23T22:59:15.653959+0000","last_clean":"2023-09-23T22:59:15.653959+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T22:59:15.653959+0000","last_undegraded":"2023-09-23T22:59:15.653959+0000","last_fullsized":"2023-09-23T22:59:15.653959+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":93,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.661511+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:59:15.661511+0000","last_peered":"2023-09-23T22:59:15.661511+0000","last_clean":"2023-09-23T22:59:15.661511+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:59:15.661511+0000","last_undegraded":"2023-09-23T22:59:15.661511+0000","last_fullsized":"2023-09-23T22:59:15.661511+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":68,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.655806+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:59:15.655806+0000","last_peered":"2023-09-23T22:59:15.655806+0000","last_clean":"2023-09-23T22:59:15.655806+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:59:15.655806+0000","last_undegraded":"2023-09-23T22:59:15.655806+0000","last_fullsized":"2023-09-23T22:59:15.655806+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"375'7786","reported_seq":7865,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.548062+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T22:59:24.548062+0000","last_peered":"2023-09-23T22:59:24.548062+0000","last_clean":"2023-09-23T22:59:24.548062+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T22:59:24.548062+0000","last_undegraded":"2023-09-23T22:59:24.548062+0000","last_fullsized":"2023-09-23T22:59:24.548062+0000","mapping_epoch":353,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804800,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7786,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":260,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.675781+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:59:15.675781+0000","last_peered":"2023-09-23T22:59:15.675781+0000","last_clean":"2023-09-23T22:59:15.675781+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:59:15.675781+0000","last_undegraded":"2023-09-23T22:59:15.675781+0000","last_fullsized":"2023-09-23T22:59:15.675781+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.657239+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:59:15.657239+0000","last_peered":"2023-09-23T22:59:15.657239+0000","last_clean":"2023-09-23T22:59:15.657239+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:59:15.657239+0000","last_undegraded":"2023-09-23T22:59:15.657239+0000","last_fullsized":"2023-09-23T22:59:15.657239+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"376'7690","reported_seq":10099,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:26.107619+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T22:59:26.107619+0000","last_peered":"2023-09-23T22:59:26.107619+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T22:59:26.107619+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7290,"log_dups_size":0,"ondisk_log_size":7290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279183360,"num_objects":4260,"num_object_clones":0,"num_object_copies":12780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4260,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":4260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7690,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":693}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":270,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.670240+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:59:15.670240+0000","last_peered":"2023-09-23T22:59:15.670240+0000","last_clean":"2023-09-23T22:59:15.670240+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:59:15.670240+0000","last_undegraded":"2023-09-23T22:59:15.670240+0000","last_fullsized":"2023-09-23T22:59:15.670240+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":383,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665211+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:59:15.665211+0000","last_peered":"2023-09-23T22:59:15.665211+0000","last_clean":"2023-09-23T22:59:15.665211+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:59:15.665211+0000","last_undegraded":"2023-09-23T22:59:15.665211+0000","last_fullsized":"2023-09-23T22:59:15.665211+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"376'7822","reported_seq":7895,"reported_epoch":376,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:24.739352+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T22:59:24.739352+0000","last_peered":"2023-09-23T22:59:24.739352+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T22:59:24.739352+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T22:59:24.739352+0000","mapping_epoch":354,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5522,"log_dups_size":0,"ondisk_log_size":5522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278528000,"num_objects":4250,"num_object_clones":0,"num_object_copies":12750,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":4250,"num_objects_unfound":0,"num_objects_dirty":4250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7822,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":4250}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":127,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.658290+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:59:15.658290+0000","last_peered":"2023-09-23T22:59:15.658290+0000","last_clean":"2023-09-23T22:59:15.658290+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:59:15.658290+0000","last_undegraded":"2023-09-23T22:59:15.658290+0000","last_fullsized":"2023-09-23T22:59:15.658290+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":245,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.658561+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:59:15.658561+0000","last_peered":"2023-09-23T22:59:15.658561+0000","last_clean":"2023-09-23T22:59:15.658561+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:59:15.658561+0000","last_undegraded":"2023-09-23T22:59:15.658561+0000","last_fullsized":"2023-09-23T22:59:15.658561+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"376'7836","reported_seq":8095,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:24.693750+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T22:59:24.693750+0000","last_peered":"2023-09-23T22:59:24.693750+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T22:59:24.693750+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5536,"log_dups_size":0,"ondisk_log_size":5536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284295168,"num_objects":4338,"num_object_clones":0,"num_object_copies":13014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3664,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":4338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7836,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":728}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":44,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.654921+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:59:15.654921+0000","last_peered":"2023-09-23T22:59:15.654921+0000","last_clean":"2023-09-23T22:59:15.654921+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:59:15.654921+0000","last_undegraded":"2023-09-23T22:59:15.654921+0000","last_fullsized":"2023-09-23T22:59:15.654921+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":274,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.662164+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:59:15.662164+0000","last_peered":"2023-09-23T22:59:15.662164+0000","last_clean":"2023-09-23T22:59:15.662164+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:59:15.662164+0000","last_undegraded":"2023-09-23T22:59:15.662164+0000","last_fullsized":"2023-09-23T22:59:15.662164+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"376'7840","reported_seq":7921,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:27.090699+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T22:59:27.090699+0000","last_peered":"2023-09-23T22:59:27.090699+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T22:59:27.090699+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T22:59:27.090699+0000","mapping_epoch":353,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278790144,"num_objects":4254,"num_object_clones":0,"num_object_copies":12762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7840,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4254}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":63,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.654882+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:59:15.654882+0000","last_peered":"2023-09-23T22:59:15.654882+0000","last_clean":"2023-09-23T22:59:15.654882+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:59:15.654882+0000","last_undegraded":"2023-09-23T22:59:15.654882+0000","last_fullsized":"2023-09-23T22:59:15.654882+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":369,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.672654+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:59:15.672654+0000","last_peered":"2023-09-23T22:59:15.672654+0000","last_clean":"2023-09-23T22:59:15.672654+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:59:15.672654+0000","last_undegraded":"2023-09-23T22:59:15.672654+0000","last_fullsized":"2023-09-23T22:59:15.672654+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"376'7894","reported_seq":7983,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:27.082939+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T22:59:27.082939+0000","last_peered":"2023-09-23T22:59:27.082939+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T22:59:27.082939+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282329088,"num_objects":4308,"num_object_clones":0,"num_object_copies":12924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4308,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":4308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7894,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":689}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.666060+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:59:15.666060+0000","last_peered":"2023-09-23T22:59:15.666060+0000","last_clean":"2023-09-23T22:59:15.666060+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:59:15.666060+0000","last_undegraded":"2023-09-23T22:59:15.666060+0000","last_fullsized":"2023-09-23T22:59:15.666060+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":386,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.669287+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:59:15.669287+0000","last_peered":"2023-09-23T22:59:15.669287+0000","last_clean":"2023-09-23T22:59:15.669287+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:59:15.669287+0000","last_undegraded":"2023-09-23T22:59:15.669287+0000","last_fullsized":"2023-09-23T22:59:15.669287+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"376'7802","reported_seq":10969,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:26.952355+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T22:59:26.952355+0000","last_peered":"2023-09-23T22:59:26.952355+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T22:59:26.952355+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T22:59:26.952355+0000","mapping_epoch":354,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279052288,"num_objects":4258,"num_object_clones":0,"num_object_copies":12774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7802,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1535,"num_bytes_recovered":100597760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":2824}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":399,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665067+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:59:15.665067+0000","last_peered":"2023-09-23T22:59:15.665067+0000","last_clean":"2023-09-23T22:59:15.665067+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:59:15.665067+0000","last_undegraded":"2023-09-23T22:59:15.665067+0000","last_fullsized":"2023-09-23T22:59:15.665067+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":377,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.670313+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:59:15.670313+0000","last_peered":"2023-09-23T22:59:15.670313+0000","last_clean":"2023-09-23T22:59:15.670313+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:59:15.670313+0000","last_undegraded":"2023-09-23T22:59:15.670313+0000","last_fullsized":"2023-09-23T22:59:15.670313+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"376'7732","reported_seq":7813,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:26.987350+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T22:59:26.987350+0000","last_peered":"2023-09-23T22:59:26.987350+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T22:59:26.987350+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T22:59:26.987350+0000","mapping_epoch":353,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5732,"log_dups_size":0,"ondisk_log_size":5732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277610496,"num_objects":4236,"num_object_clones":0,"num_object_copies":12708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7732,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4236}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":44,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.653041+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:59:15.653041+0000","last_peered":"2023-09-23T22:59:15.653041+0000","last_clean":"2023-09-23T22:59:15.653041+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:59:15.653041+0000","last_undegraded":"2023-09-23T22:59:15.653041+0000","last_fullsized":"2023-09-23T22:59:15.653041+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":278,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673570+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:59:15.673570+0000","last_peered":"2023-09-23T22:59:15.673570+0000","last_clean":"2023-09-23T22:59:15.673570+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:59:15.673570+0000","last_undegraded":"2023-09-23T22:59:15.673570+0000","last_fullsized":"2023-09-23T22:59:15.673570+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"376'7824","reported_seq":12003,"reported_epoch":376,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:59:27.087416+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T22:59:27.087416+0000","last_peered":"2023-09-23T22:59:27.087416+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T22:59:27.087416+0000","last_undegraded":"2023-09-23T22:59:27.087416+0000","last_fullsized":"2023-09-23T22:59:27.087416+0000","mapping_epoch":354,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278659072,"num_objects":4252,"num_object_clones":0,"num_object_copies":12756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1633,"num_objects_unfound":0,"num_objects_dirty":4252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7824,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2032,"num_bytes_recovered":132644864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2619},{"shards":"0,4,6","objects":1633}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":282,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673334+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:59:15.673334+0000","last_peered":"2023-09-23T22:59:15.673334+0000","last_clean":"2023-09-23T22:59:15.673334+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:59:15.673334+0000","last_undegraded":"2023-09-23T22:59:15.673334+0000","last_fullsized":"2023-09-23T22:59:15.673334+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.658142+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:59:15.658142+0000","last_peered":"2023-09-23T22:59:15.658142+0000","last_clean":"2023-09-23T22:59:15.658142+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:59:15.658142+0000","last_undegraded":"2023-09-23T22:59:15.658142+0000","last_fullsized":"2023-09-23T22:59:15.658142+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":99,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.658435+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:59:15.658435+0000","last_peered":"2023-09-23T22:59:15.658435+0000","last_clean":"2023-09-23T22:59:15.658435+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:59:15.658435+0000","last_undegraded":"2023-09-23T22:59:15.658435+0000","last_fullsized":"2023-09-23T22:59:15.658435+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"375'7854","reported_seq":7946,"reported_epoch":375,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:59:24.597242+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T22:59:24.597242+0000","last_peered":"2023-09-23T22:59:24.597242+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T22:59:24.597242+0000","last_undegraded":"2023-09-23T22:59:24.597242+0000","last_fullsized":"2023-09-23T22:59:24.597242+0000","mapping_epoch":359,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5554,"log_dups_size":0,"ondisk_log_size":5554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279838720,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4270,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7854,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4270}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":45,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.662158+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T22:59:15.662158+0000","last_peered":"2023-09-23T22:59:15.662158+0000","last_clean":"2023-09-23T22:59:15.662158+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T22:59:15.662158+0000","last_undegraded":"2023-09-23T22:59:15.662158+0000","last_fullsized":"2023-09-23T22:59:15.662158+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":396,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665609+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:59:15.665609+0000","last_peered":"2023-09-23T22:59:15.665609+0000","last_clean":"2023-09-23T22:59:15.665609+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:59:15.665609+0000","last_undegraded":"2023-09-23T22:59:15.665609+0000","last_fullsized":"2023-09-23T22:59:15.665609+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":267,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673368+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:59:15.673368+0000","last_peered":"2023-09-23T22:59:15.673368+0000","last_clean":"2023-09-23T22:59:15.673368+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:59:15.673368+0000","last_undegraded":"2023-09-23T22:59:15.673368+0000","last_fullsized":"2023-09-23T22:59:15.673368+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"376'7798","reported_seq":10746,"reported_epoch":376,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:26.121632+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T22:59:26.121632+0000","last_peered":"2023-09-23T22:59:26.121632+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T22:59:26.121632+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7298,"log_dups_size":0,"ondisk_log_size":7298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279576576,"num_objects":4266,"num_object_clones":0,"num_object_copies":12798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2141,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7798,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1430,"num_bytes_recovered":93716480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2125}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":342,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673171+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:59:15.673171+0000","last_peered":"2023-09-23T22:59:15.673171+0000","last_clean":"2023-09-23T22:59:15.673171+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:59:15.673171+0000","last_undegraded":"2023-09-23T22:59:15.673171+0000","last_fullsized":"2023-09-23T22:59:15.673171+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":219,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673212+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:59:15.673212+0000","last_peered":"2023-09-23T22:59:15.673212+0000","last_clean":"2023-09-23T22:59:15.673212+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:59:15.673212+0000","last_undegraded":"2023-09-23T22:59:15.673212+0000","last_fullsized":"2023-09-23T22:59:15.673212+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.664270+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:59:15.664270+0000","last_peered":"2023-09-23T22:59:15.664270+0000","last_clean":"2023-09-23T22:59:15.664270+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:59:15.664270+0000","last_undegraded":"2023-09-23T22:59:15.664270+0000","last_fullsized":"2023-09-23T22:59:15.664270+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"376'7889","reported_seq":9768,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:26.913504+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T22:59:26.913504+0000","last_peered":"2023-09-23T22:59:26.913504+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T22:59:26.913504+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7889,"log_dups_size":0,"ondisk_log_size":7889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284098560,"num_objects":4335,"num_object_clones":0,"num_object_copies":13005,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7889,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2902},{"shards":"0,6","objects":-646}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":359,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.657715+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:59:15.657715+0000","last_peered":"2023-09-23T22:59:15.657715+0000","last_clean":"2023-09-23T22:59:15.657715+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:59:15.657715+0000","last_undegraded":"2023-09-23T22:59:15.657715+0000","last_fullsized":"2023-09-23T22:59:15.657715+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":471,"reported_epoch":375,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:15.661954+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T22:59:15.661954+0000","last_peered":"2023-09-23T22:59:15.661954+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T22:59:15.661954+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.655618+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:59:15.655618+0000","last_peered":"2023-09-23T22:59:15.655618+0000","last_clean":"2023-09-23T22:59:15.655618+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:59:15.655618+0000","last_undegraded":"2023-09-23T22:59:15.655618+0000","last_fullsized":"2023-09-23T22:59:15.655618+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.669505+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:59:15.669505+0000","last_peered":"2023-09-23T22:59:15.669505+0000","last_clean":"2023-09-23T22:59:15.669505+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:59:15.669505+0000","last_undegraded":"2023-09-23T22:59:15.669505+0000","last_fullsized":"2023-09-23T22:59:15.669505+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"375'7817","reported_seq":7901,"reported_epoch":375,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:24.617565+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T22:59:24.617565+0000","last_peered":"2023-09-23T22:59:24.617565+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T22:59:24.617565+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5517,"log_dups_size":0,"ondisk_log_size":5517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279642112,"num_objects":4267,"num_object_clones":0,"num_object_copies":12801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7817,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":760}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":365,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.661573+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T22:59:15.661573+0000","last_peered":"2023-09-23T22:59:15.661573+0000","last_clean":"2023-09-23T22:59:15.661573+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T22:59:15.661573+0000","last_undegraded":"2023-09-23T22:59:15.661573+0000","last_fullsized":"2023-09-23T22:59:15.661573+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":336,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.661578+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T22:59:15.661578+0000","last_peered":"2023-09-23T22:59:15.661578+0000","last_clean":"2023-09-23T22:59:15.661578+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T22:59:15.661578+0000","last_undegraded":"2023-09-23T22:59:15.661578+0000","last_fullsized":"2023-09-23T22:59:15.661578+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":296,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.662619+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:59:15.662619+0000","last_peered":"2023-09-23T22:59:15.662619+0000","last_clean":"2023-09-23T22:59:15.662619+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:59:15.662619+0000","last_undegraded":"2023-09-23T22:59:15.662619+0000","last_fullsized":"2023-09-23T22:59:15.662619+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"376'7903","reported_seq":8003,"reported_epoch":376,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:26.831990+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T22:59:26.831990+0000","last_peered":"2023-09-23T22:59:26.831990+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T22:59:26.831990+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T22:59:26.831990+0000","mapping_epoch":354,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804824,"num_objects":4301,"num_object_clones":0,"num_object_copies":12903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":4301,"num_objects_unfound":0,"num_objects_dirty":4301,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7903,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":4301}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":311,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.663322+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T22:59:15.663322+0000","last_peered":"2023-09-23T22:59:15.663322+0000","last_clean":"2023-09-23T22:59:15.663322+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T22:59:15.663322+0000","last_undegraded":"2023-09-23T22:59:15.663322+0000","last_fullsized":"2023-09-23T22:59:15.663322+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":219,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673427+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:59:15.673427+0000","last_peered":"2023-09-23T22:59:15.673427+0000","last_clean":"2023-09-23T22:59:15.673427+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:59:15.673427+0000","last_undegraded":"2023-09-23T22:59:15.673427+0000","last_fullsized":"2023-09-23T22:59:15.673427+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":189,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.657777+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:59:15.657777+0000","last_peered":"2023-09-23T22:59:15.657777+0000","last_clean":"2023-09-23T22:59:15.657777+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:59:15.657777+0000","last_undegraded":"2023-09-23T22:59:15.657777+0000","last_fullsized":"2023-09-23T22:59:15.657777+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"376'7859","reported_seq":10349,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:27.103529+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T22:59:27.103529+0000","last_peered":"2023-09-23T22:59:27.103529+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T22:59:27.103529+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280952832,"num_objects":4287,"num_object_clones":0,"num_object_copies":12861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8154,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7859,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1198,"num_bytes_recovered":78512128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":705},{"shards":"4,5","objects":1},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":68,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.655969+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:59:15.655969+0000","last_peered":"2023-09-23T22:59:15.655969+0000","last_clean":"2023-09-23T22:59:15.655969+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:59:15.655969+0000","last_undegraded":"2023-09-23T22:59:15.655969+0000","last_fullsized":"2023-09-23T22:59:15.655969+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.674863+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:59:15.674863+0000","last_peered":"2023-09-23T22:59:15.674863+0000","last_clean":"2023-09-23T22:59:15.674863+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:59:15.674863+0000","last_undegraded":"2023-09-23T22:59:15.674863+0000","last_fullsized":"2023-09-23T22:59:15.674863+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708817+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:59:24.708817+0000","last_peered":"2023-09-23T22:59:24.708817+0000","last_clean":"2023-09-23T22:59:24.708817+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:59:24.708817+0000","last_undegraded":"2023-09-23T22:59:24.708817+0000","last_fullsized":"2023-09-23T22:59:24.708817+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"376'7812","reported_seq":8322,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:26.893674+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T22:59:26.893674+0000","last_peered":"2023-09-23T22:59:26.893674+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T22:59:26.893674+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279052288,"num_objects":4258,"num_object_clones":0,"num_object_copies":12774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4258,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":4258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7812,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":707}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":331,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673812+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:59:15.673812+0000","last_peered":"2023-09-23T22:59:15.673812+0000","last_clean":"2023-09-23T22:59:15.673812+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:59:15.673812+0000","last_undegraded":"2023-09-23T22:59:15.673812+0000","last_fullsized":"2023-09-23T22:59:15.673812+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":278,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.662550+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:59:15.662550+0000","last_peered":"2023-09-23T22:59:15.662550+0000","last_clean":"2023-09-23T22:59:15.662550+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:59:15.662550+0000","last_undegraded":"2023-09-23T22:59:15.662550+0000","last_fullsized":"2023-09-23T22:59:15.662550+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":350,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665125+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:59:15.665125+0000","last_peered":"2023-09-23T22:59:15.665125+0000","last_clean":"2023-09-23T22:59:15.665125+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:59:15.665125+0000","last_undegraded":"2023-09-23T22:59:15.665125+0000","last_fullsized":"2023-09-23T22:59:15.665125+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":313,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.662422+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:59:15.662422+0000","last_peered":"2023-09-23T22:59:15.662422+0000","last_clean":"2023-09-23T22:59:15.662422+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:59:15.662422+0000","last_undegraded":"2023-09-23T22:59:15.662422+0000","last_fullsized":"2023-09-23T22:59:15.662422+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":123,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.673084+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:59:15.673084+0000","last_peered":"2023-09-23T22:59:15.673084+0000","last_clean":"2023-09-23T22:59:15.673084+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:59:15.673084+0000","last_undegraded":"2023-09-23T22:59:15.673084+0000","last_fullsized":"2023-09-23T22:59:15.673084+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":185,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.672555+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:59:15.672555+0000","last_peered":"2023-09-23T22:59:15.672555+0000","last_clean":"2023-09-23T22:59:15.672555+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:59:15.672555+0000","last_undegraded":"2023-09-23T22:59:15.672555+0000","last_fullsized":"2023-09-23T22:59:15.672555+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":399,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665686+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:59:15.665686+0000","last_peered":"2023-09-23T22:59:15.665686+0000","last_clean":"2023-09-23T22:59:15.665686+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:59:15.665686+0000","last_undegraded":"2023-09-23T22:59:15.665686+0000","last_fullsized":"2023-09-23T22:59:15.665686+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":199,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.669372+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:59:15.669372+0000","last_peered":"2023-09-23T22:59:15.669372+0000","last_clean":"2023-09-23T22:59:15.669372+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:59:15.669372+0000","last_undegraded":"2023-09-23T22:59:15.669372+0000","last_fullsized":"2023-09-23T22:59:15.669372+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665988+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:59:15.665988+0000","last_peered":"2023-09-23T22:59:15.665988+0000","last_clean":"2023-09-23T22:59:15.665988+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:59:15.665988+0000","last_undegraded":"2023-09-23T22:59:15.665988+0000","last_fullsized":"2023-09-23T22:59:15.665988+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":68,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.654110+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:59:15.654110+0000","last_peered":"2023-09-23T22:59:15.654110+0000","last_clean":"2023-09-23T22:59:15.654110+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:59:15.654110+0000","last_undegraded":"2023-09-23T22:59:15.654110+0000","last_fullsized":"2023-09-23T22:59:15.654110+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":71,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.653189+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:59:15.653189+0000","last_peered":"2023-09-23T22:59:15.653189+0000","last_clean":"2023-09-23T22:59:15.653189+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:59:15.653189+0000","last_undegraded":"2023-09-23T22:59:15.653189+0000","last_fullsized":"2023-09-23T22:59:15.653189+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":317,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.663214+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T22:59:15.663214+0000","last_peered":"2023-09-23T22:59:15.663214+0000","last_clean":"2023-09-23T22:59:15.663214+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T22:59:15.663214+0000","last_undegraded":"2023-09-23T22:59:15.663214+0000","last_fullsized":"2023-09-23T22:59:15.663214+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665080+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:59:15.665080+0000","last_peered":"2023-09-23T22:59:15.665080+0000","last_clean":"2023-09-23T22:59:15.665080+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:59:15.665080+0000","last_undegraded":"2023-09-23T22:59:15.665080+0000","last_fullsized":"2023-09-23T22:59:15.665080+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4485218328,"num_objects":68440,"num_object_clones":0,"num_object_copies":205320,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":37062,"num_objects_misplaced":34381,"num_objects_unfound":0,"num_objects_dirty":68440,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":125158,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8539,"num_bytes_recovered":558366720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13169733632,"data_stored":13169721416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95858,"ondisk_log_size":95858,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430597,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626822,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253135,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2313572,"kb_used_data":1625180,"kb_used_omap":9,"kb_used_meta":688374,"kb_avail":91431580,"statfs":{"total":95995035648,"available":93625937920,"internally_reserved":0,"allocated":1664184320,"data_stored":1662785051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":704895977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.5960000000000001,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:59:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.048,"5min":4.048,"15min":4.048},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":1.0629999999999999,"15min":2.3849999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63100000000000001,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.40200000000000002,"5min":1.147,"15min":2.8340000000000001},"min":{"1min":0.27400000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.624,"5min":8.0099999999999998,"15min":89.861999999999995},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5730000000000004,"5min":3.1190000000000002,"15min":2.766},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":151.036,"5min":151.036,"15min":151.036},"last":0.49299999999999999},{"interface":"front","average":{"1min":7.5510000000000002,"5min":3.2469999999999999,"15min":2.7709999999999999},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":151.06899999999999,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":3.4449999999999998,"15min":2.5329999999999999},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":0.95099999999999996,"5min":116.098,"15min":116.098},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":3.1200000000000001,"15min":2.7890000000000001},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":2.698,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.61099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.8600000000000001,"15min":2.431},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.85499999999999998,"5min":20.742000000000001,"15min":79.311000000000007},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.9119999999999999,"15min":2.125},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":4.4219999999999997,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.3580000000000001,"15min":2.028},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":0.87,"5min":8.1400000000000006,"15min":29.539000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.6990000000000001,"15min":2.2120000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.60099999999999998}]}]},{"osd":0,"up_from":353,"seq":1516123455505,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2973396,"kb_used_data":2852428,"kb_used_omap":37,"kb_used_meta":120922,"kb_avail":90771756,"statfs":{"total":95995035648,"available":92950278144,"internally_reserved":0,"allocated":2920886272,"data_stored":2919378434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":123824290},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.6309999999999998,"5min":2.6309999999999998,"15min":2.6309999999999998},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.0539999999999998,"5min":3.0539999999999998,"15min":3.0539999999999998},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2280000000000002,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.51900000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":1.7889999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0179999999999998,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.2770000000000001,"5min":3.2770000000000001,"15min":3.2770000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":1.5800000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.982,"5min":1.982,"15min":1.982},"last":3.4220000000000002},{"interface":"front","average":{"1min":0.998,"5min":0.998,"15min":0.998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":2.1320000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.0920000000000001,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":1.5329999999999999},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":4.3120000000000003,"5min":4.3120000000000003,"15min":4.3120000000000003},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.3579999999999997,"5min":4.3579999999999997,"15min":4.3579999999999997},"last":0.49399999999999999}]}]},{"osd":2,"up_from":359,"seq":1541893259278,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1008360,"kb_used_data":908896,"kb_used_omap":83,"kb_used_meta":99372,"kb_avail":92736792,"statfs":{"total":95995035648,"available":94962475008,"internally_reserved":0,"allocated":930709504,"data_stored":929251307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":101757381},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.442},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.308}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":4.1079999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":4.1529999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":4.1920000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.81299999999999994}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":4.2370000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.754}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":1.5780000000000001},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":0.59899999999999998}]}]},{"osd":3,"up_from":94,"seq":403726926082,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534020,"kb_used_data":47420,"kb_used_omap":7,"kb_used_meta":486584,"kb_avail":93211132,"statfs":{"total":95995035648,"available":95448199168,"internally_reserved":0,"allocated":48558080,"data_stored":47068412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":498262632},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":2.746},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.972,"5min":1.972,"15min":1.972},"last":0.26100000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.097,"15min":2.786},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.60899999999999999,"5min":16.596,"15min":123.866},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.35499999999999998,"5min":1.179,"15min":2.4449999999999998},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":0.47499999999999998,"5min":16.649999999999999,"15min":122.265},"last":0.373}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.1469999999999998,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"last":2.274}]},{"osd":4,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.3129999999999999,"15min":1.8640000000000001},"min":{"1min":0.30399999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":1.0700000000000001,"5min":7.8929999999999998,"15min":18.757000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.3129999999999999,"15min":1.629},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.27500000000000002},"max":{"1min":0.72199999999999998,"5min":7.8780000000000001,"15min":15.037000000000001},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.393,"15min":1.52},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.59399999999999997,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.46800000000000003,"5min":1.546,"15min":1.954},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75600000000000001,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7709999999999999,"15min":1.788},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":14.097,"15min":14.097},"last":2.6970000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.677,"15min":2.0390000000000001},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5,"5min":12.887,"15min":12.887},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.3480000000000001,"15min":2.2109999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79700000000000004,"5min":6.9950000000000001,"15min":91.757999999999996},"last":0.623},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.238,"15min":2.2240000000000002},"min":{"1min":0.434,"5min":0.41699999999999998,"15min":0.33200000000000002},"max":{"1min":1.1479999999999999,"5min":7.1219999999999999,"15min":91.709999999999994},"last":0.79800000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019279,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3779532,"kb_used_data":3193924,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":89965620,"statfs":{"total":95995035648,"available":92124794880,"internally_reserved":0,"allocated":3270578176,"data_stored":3269831517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":3.7690000000000001,"15min":3.786},"min":{"1min":0.40000000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":1.966,"5min":98.846999999999994,"15min":132.304},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.71699999999999997,"5min":3.9740000000000002,"15min":3.6259999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":3.2269999999999999,"5min":98.938999999999993,"15min":132.453},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.536,"15min":4.5099999999999998},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":3.399,"5min":8.5510000000000002,"15min":164.453},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.478,"15min":4.5720000000000001},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.99,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:58:27 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":3.0800000000000001,"15min":1.641},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":12.961,"5min":148.61600000000001,"15min":148.61600000000001},"last":2.3359999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.5019999999999998,"15min":1.7869999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.248},"max":{"1min":5.4619999999999997,"5min":151.905,"15min":151.905},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5820000000000001,"15min":1.1559999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":2.113,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.423,"15min":1.101},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0350000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.3080000000000001,"15min":1.0960000000000001},"min":{"1min":0.46400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0150000000000001,"5min":12.84,"15min":12.84},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.371,"5min":1.3180000000000001,"15min":1.02},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.587,"5min":10.587,"15min":10.587},"last":0.51500000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823053,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2910508,"kb_used_data":2010852,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":90834644,"statfs":{"total":95995035648,"available":93014675456,"internally_reserved":0,"allocated":2059112448,"data_stored":2058393577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":0.439},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":0.35999999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":2.9249999999999998,"15min":2.3330000000000002},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":3.2429999999999999,"5min":110.36499999999999,"15min":110.36499999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.93600000000000005,"5min":3.4340000000000002,"15min":2.8410000000000002},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":10.353,"5min":110.89,"15min":110.89},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.6160000000000001,"15min":5.117},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":3.149,"5min":11.298999999999999,"15min":257.05000000000001},"last":0.375},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.5469999999999999,"15min":4.9409999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.4940000000000002,"5min":12.185,"15min":257.35000000000002},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":2.7559999999999998,"15min":1.4650000000000001},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.603,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":2.6909999999999998,"15min":1.4419999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":3.093,"5min":114.908,"15min":114.908},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.3300000000000001,"15min":1.2649999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.498,"5min":11.121,"15min":48.360999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.272,"15min":1.2709999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.7759999999999998,"5min":12.137,"15min":48.292999999999999},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.159,"15min":1.0389999999999999},"min":{"1min":0.44600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.3029999999999999,"5min":12.23,"15min":12.23},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.651,"5min":1.381,"15min":1.0389999999999999},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":10.638999999999999,"5min":11.112,"15min":11.112},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2912096256,"data_stored":2912092184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1659305984,"data_stored":1659305984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":926416896,"data_stored":926416896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":44171264,"data_stored":44171264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3266383872,"data_stored":3266379800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2053308416,"data_stored":2053308416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2308050944,"data_stored":2308046872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:59:31.663 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:31.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:31.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:31.737 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:31 smithi052 ceph-mon[180979]: pgmap v1105: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 37003/204432 objects degraded (18.100%); 34254/204432 objects misplaced (16.756%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:31.737 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:31 smithi052 ceph-mon[187743]: pgmap v1105: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 37003/204432 objects degraded (18.100%); 34254/204432 objects misplaced (16.756%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:31.769 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:31.796 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:31.826 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:31.839 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:31.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:31.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:31.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:31 smithi151 ceph-mon[159210]: pgmap v1105: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 37003/204432 objects degraded (18.100%); 34254/204432 objects misplaced (16.756%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:31.955 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:31.996 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:32.127 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:32.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:32.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:32.236 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:32.289 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:32.385 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:32.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:32.496 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:32.530 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:32.721 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T22:59:32.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2367621068' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:59:32.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2367621068' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:59:33.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2367621068' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:59:33.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:59:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:59:33] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T22:59:33.520 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:33.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:33.624 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:33.657 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:33.846 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[180979]: pgmap v1106: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 217 op/s; 36520/202164 objects degraded (18.065%); 34079/202164 objects misplaced (16.857%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:33.846 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:33.846 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[180979]: failed to encode map e377 with expected crc 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[180979]: failed to encode map e377 with expected crc 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2367621068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[180979]: osdmap e377: 8 total, 8 up, 6 in 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[187743]: pgmap v1106: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 217 op/s; 36520/202164 objects degraded (18.065%); 34079/202164 objects misplaced (16.857%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[187743]: failed to encode map e377 with expected crc 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[187743]: failed to encode map e377 with expected crc 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2367621068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:59:33.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:33 smithi052 ceph-mon[187743]: osdmap e377: 8 total, 8 up, 6 in 2023-09-23T22:59:34.076 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:34.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:34.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:34.179 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:33 smithi151 ceph-mon[159210]: pgmap v1106: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 217 op/s; 36520/202164 objects degraded (18.065%); 34079/202164 objects misplaced (16.857%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:33 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:33 smithi151 ceph-mon[159210]: failed to encode map e377 with expected crc 2023-09-23T22:59:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:33 smithi151 ceph-mon[159210]: failed to encode map e377 with expected crc 2023-09-23T22:59:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2367621068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:59:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:33 smithi151 ceph-mon[159210]: osdmap e377: 8 total, 8 up, 6 in 2023-09-23T22:59:34.208 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:34.592 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:34.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:34.624 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:59:34.674 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:59:34.674 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:59:34.674 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:59:34.675 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:59:34.675 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:59:34.696 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:34.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:34.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:34.740 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:35.468 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:35.492 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:35.518 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:35.538 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:35.564 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:35.586 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:35.611 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:35.612 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:35.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:35.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:59:35.636 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:35.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.643 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:59:35.655 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:59:35.716 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:35.750 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:35.788 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:35.855 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:35.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:35 smithi052 ceph-mon[187743]: pgmap v1108: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 36520/202164 objects degraded (18.065%); 34079/202164 objects misplaced (16.857%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:35.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 36520/202164 objects degraded (18.065%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:35.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1823240123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:35.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:35 smithi052 ceph-mon[180979]: pgmap v1108: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 36520/202164 objects degraded (18.065%); 34079/202164 objects misplaced (16.857%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:35.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 36520/202164 objects degraded (18.065%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:35.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1823240123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:35.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:35.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:35.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:35 smithi151 ceph-mon[159210]: pgmap v1108: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 36520/202164 objects degraded (18.065%); 34079/202164 objects misplaced (16.857%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:59:35.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 36520/202164 objects degraded (18.065%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:35.987 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1823240123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:35.990 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:36.041 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:36.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:36.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:36.152 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:36.194 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:36.272 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:36.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:36.377 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:36.411 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:36.478 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:36.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:36.585 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:36.622 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:36.670 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:36.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:36.781 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:36.811 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:36.877 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:36.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:36.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:36.986 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:37.021 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:37.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:37.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:37.220 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:37.255 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:37.266 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:37.320 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:37.326 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:37.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:37.435 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:37.477 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:37.556 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:37.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:37.668 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:37.704 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:37.793 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:37.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:37.839 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:37 smithi052 ceph-mon[187743]: pgmap v1109: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 36263/200649 objects degraded (18.073%); 33947/200649 objects misplaced (16.919%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:37.839 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:37 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:59:37.840 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:37 smithi052 ceph-mon[180979]: pgmap v1109: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 36263/200649 objects degraded (18.073%); 33947/200649 objects misplaced (16.919%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:37.840 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:37 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:59:37.901 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:37 smithi151 ceph-mon[159210]: pgmap v1109: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 36263/200649 objects degraded (18.073%); 33947/200649 objects misplaced (16.919%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:59:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:37 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:59:37.960 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:38.074 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:38.076 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:59:38.094 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:38.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:38.198 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:38.258 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:38.383 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:38.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:38.501 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:38.534 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:38.590 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:38.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:38.695 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:38.751 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:38.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:38.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:38.953 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:38.997 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:59:39.003 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:39.121 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:39.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:39.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:39.228 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:39.259 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:39.435 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1109,"stamp":"2023-09-23T22:59:36.554119+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4382201784,"num_objects":66883,"num_object_clones":0,"num_object_copies":200649,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":36263,"num_objects_misplaced":33947,"num_objects_unfound":0,"num_objects_dirty":66883,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":126999,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9089,"num_bytes_recovered":592133052,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96033,"ondisk_log_size":96033,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13240732,"kb_used_data":10372332,"kb_used_omap":137,"kb_used_meta":2868214,"kb_avail":549230180,"statfs":{"total":575970213888,"available":562411704320,"internally_reserved":0,"allocated":10621267968,"data_stored":10613923880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":2937051320},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-144179200,"num_objects":-2200,"num_object_clones":0,"num_object_copies":-6600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1062,"num_objects_misplaced":-620,"num_objects_unfound":0,"num_objects_dirty":-2200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2200,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":758,"num_bytes_recovered":49676288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003703"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":388,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665917+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:59:15.665917+0000","last_peered":"2023-09-23T22:59:15.665917+0000","last_clean":"2023-09-23T22:59:15.665917+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:59:15.665917+0000","last_undegraded":"2023-09-23T22:59:15.665917+0000","last_fullsized":"2023-09-23T22:59:15.665917+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":45,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709895+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:59:24.709895+0000","last_peered":"2023-09-23T22:59:24.709895+0000","last_clean":"2023-09-23T22:59:24.709895+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:59:24.709895+0000","last_undegraded":"2023-09-23T22:59:24.709895+0000","last_fullsized":"2023-09-23T22:59:24.709895+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":44,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709310+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:59:24.709310+0000","last_peered":"2023-09-23T22:59:24.709310+0000","last_clean":"2023-09-23T22:59:24.709310+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:59:24.709310+0000","last_undegraded":"2023-09-23T22:59:24.709310+0000","last_fullsized":"2023-09-23T22:59:24.709310+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":373,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.728539+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:59:32.728539+0000","last_peered":"2023-09-23T22:59:32.728539+0000","last_clean":"2023-09-23T22:59:32.728539+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:59:32.728539+0000","last_undegraded":"2023-09-23T22:59:32.728539+0000","last_fullsized":"2023-09-23T22:59:32.728539+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":60,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708739+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:59:24.708739+0000","last_peered":"2023-09-23T22:59:24.708739+0000","last_clean":"2023-09-23T22:59:24.708739+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:59:24.708739+0000","last_undegraded":"2023-09-23T22:59:24.708739+0000","last_fullsized":"2023-09-23T22:59:24.708739+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":46,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692439+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T22:59:24.692439+0000","last_peered":"2023-09-23T22:59:24.692439+0000","last_clean":"2023-09-23T22:59:24.692439+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T22:59:24.692439+0000","last_undegraded":"2023-09-23T22:59:24.692439+0000","last_fullsized":"2023-09-23T22:59:24.692439+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":312,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695960+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:59:24.695960+0000","last_peered":"2023-09-23T22:59:24.695960+0000","last_clean":"2023-09-23T22:59:24.695960+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:59:24.695960+0000","last_undegraded":"2023-09-23T22:59:24.695960+0000","last_fullsized":"2023-09-23T22:59:24.695960+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":54,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708604+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:59:24.708604+0000","last_peered":"2023-09-23T22:59:24.708604+0000","last_clean":"2023-09-23T22:59:24.708604+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:59:24.708604+0000","last_undegraded":"2023-09-23T22:59:24.708604+0000","last_fullsized":"2023-09-23T22:59:24.708604+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":273,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695695+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:59:24.695695+0000","last_peered":"2023-09-23T22:59:24.695695+0000","last_clean":"2023-09-23T22:59:24.695695+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:59:24.695695+0000","last_undegraded":"2023-09-23T22:59:24.695695+0000","last_fullsized":"2023-09-23T22:59:24.695695+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":59,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709180+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:59:24.709180+0000","last_peered":"2023-09-23T22:59:24.709180+0000","last_clean":"2023-09-23T22:59:24.709180+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:59:24.709180+0000","last_undegraded":"2023-09-23T22:59:24.709180+0000","last_fullsized":"2023-09-23T22:59:24.709180+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":65,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709821+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:59:24.709821+0000","last_peered":"2023-09-23T22:59:24.709821+0000","last_clean":"2023-09-23T22:59:24.709821+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:59:24.709821+0000","last_undegraded":"2023-09-23T22:59:24.709821+0000","last_fullsized":"2023-09-23T22:59:24.709821+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708491+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:59:24.708491+0000","last_peered":"2023-09-23T22:59:24.708491+0000","last_clean":"2023-09-23T22:59:24.708491+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:59:24.708491+0000","last_undegraded":"2023-09-23T22:59:24.708491+0000","last_fullsized":"2023-09-23T22:59:24.708491+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":447,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.672734+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:59:15.672734+0000","last_peered":"2023-09-23T22:59:15.672734+0000","last_clean":"2023-09-23T22:59:15.672734+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:59:15.672734+0000","last_undegraded":"2023-09-23T22:59:15.672734+0000","last_fullsized":"2023-09-23T22:59:15.672734+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":350,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665433+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:59:15.665433+0000","last_peered":"2023-09-23T22:59:15.665433+0000","last_clean":"2023-09-23T22:59:15.665433+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:59:15.665433+0000","last_undegraded":"2023-09-23T22:59:15.665433+0000","last_fullsized":"2023-09-23T22:59:15.665433+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":276,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665522+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:59:15.665522+0000","last_peered":"2023-09-23T22:59:15.665522+0000","last_clean":"2023-09-23T22:59:15.665522+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:59:15.665522+0000","last_undegraded":"2023-09-23T22:59:15.665522+0000","last_fullsized":"2023-09-23T22:59:15.665522+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":56,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.710958+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:59:24.710958+0000","last_peered":"2023-09-23T22:59:24.710958+0000","last_clean":"2023-09-23T22:59:24.710958+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:59:24.710958+0000","last_undegraded":"2023-09-23T22:59:24.710958+0000","last_fullsized":"2023-09-23T22:59:24.710958+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":365,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709537+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:59:24.709537+0000","last_peered":"2023-09-23T22:59:24.709537+0000","last_clean":"2023-09-23T22:59:24.709537+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:59:24.709537+0000","last_undegraded":"2023-09-23T22:59:24.709537+0000","last_fullsized":"2023-09-23T22:59:24.709537+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":277,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695507+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:59:24.695507+0000","last_peered":"2023-09-23T22:59:24.695507+0000","last_clean":"2023-09-23T22:59:24.695507+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:59:24.695507+0000","last_undegraded":"2023-09-23T22:59:24.695507+0000","last_fullsized":"2023-09-23T22:59:24.695507+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":261,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665227+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:59:15.665227+0000","last_peered":"2023-09-23T22:59:15.665227+0000","last_clean":"2023-09-23T22:59:15.665227+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:59:15.665227+0000","last_undegraded":"2023-09-23T22:59:15.665227+0000","last_fullsized":"2023-09-23T22:59:15.665227+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":389,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709198+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T22:59:24.709198+0000","last_peered":"2023-09-23T22:59:24.709198+0000","last_clean":"2023-09-23T22:59:24.709198+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T22:59:24.709198+0000","last_undegraded":"2023-09-23T22:59:24.709198+0000","last_fullsized":"2023-09-23T22:59:24.709198+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":94,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.691818+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:59:24.691818+0000","last_peered":"2023-09-23T22:59:24.691818+0000","last_clean":"2023-09-23T22:59:24.691818+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:59:24.691818+0000","last_undegraded":"2023-09-23T22:59:24.691818+0000","last_fullsized":"2023-09-23T22:59:24.691818+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709951+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:59:24.709951+0000","last_peered":"2023-09-23T22:59:24.709951+0000","last_clean":"2023-09-23T22:59:24.709951+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:59:24.709951+0000","last_undegraded":"2023-09-23T22:59:24.709951+0000","last_fullsized":"2023-09-23T22:59:24.709951+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"377'7922","reported_seq":8002,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:34.478535+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T22:59:34.478535+0000","last_peered":"2023-09-23T22:59:34.478535+0000","last_clean":"2023-09-23T22:59:34.478535+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T22:59:34.478535+0000","last_undegraded":"2023-09-23T22:59:34.478535+0000","last_fullsized":"2023-09-23T22:59:34.478535+0000","mapping_epoch":353,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5522,"log_dups_size":0,"ondisk_log_size":5522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272891904,"num_objects":4164,"num_object_clones":0,"num_object_copies":12492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7922,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":260,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.675781+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:59:15.675781+0000","last_peered":"2023-09-23T22:59:15.675781+0000","last_clean":"2023-09-23T22:59:15.675781+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:59:15.675781+0000","last_undegraded":"2023-09-23T22:59:15.675781+0000","last_fullsized":"2023-09-23T22:59:15.675781+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.694985+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:59:24.694985+0000","last_peered":"2023-09-23T22:59:24.694985+0000","last_clean":"2023-09-23T22:59:24.694985+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:59:24.694985+0000","last_undegraded":"2023-09-23T22:59:24.694985+0000","last_fullsized":"2023-09-23T22:59:24.694985+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"377'7836","reported_seq":10246,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:36.004437+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T22:59:36.004437+0000","last_peered":"2023-09-23T22:59:36.004437+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T22:59:36.004437+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7436,"log_dups_size":0,"ondisk_log_size":7436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269615104,"num_objects":4114,"num_object_clones":0,"num_object_copies":12342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4114,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":4114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7836,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":547}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":270,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.670240+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:59:15.670240+0000","last_peered":"2023-09-23T22:59:15.670240+0000","last_clean":"2023-09-23T22:59:15.670240+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:59:15.670240+0000","last_undegraded":"2023-09-23T22:59:15.670240+0000","last_fullsized":"2023-09-23T22:59:15.670240+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":383,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665211+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:59:15.665211+0000","last_peered":"2023-09-23T22:59:15.665211+0000","last_clean":"2023-09-23T22:59:15.665211+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:59:15.665211+0000","last_undegraded":"2023-09-23T22:59:15.665211+0000","last_fullsized":"2023-09-23T22:59:15.665211+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"377'7959","reported_seq":8033,"reported_epoch":377,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:34.704560+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T22:59:34.704560+0000","last_peered":"2023-09-23T22:59:34.704560+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T22:59:34.704560+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T22:59:34.704560+0000","mapping_epoch":354,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269549568,"num_objects":4113,"num_object_clones":0,"num_object_copies":12339,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":4113,"num_objects_unfound":0,"num_objects_dirty":4113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7959,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":4113}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":128,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695599+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:59:24.695599+0000","last_peered":"2023-09-23T22:59:24.695599+0000","last_clean":"2023-09-23T22:59:24.695599+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:59:24.695599+0000","last_undegraded":"2023-09-23T22:59:24.695599+0000","last_fullsized":"2023-09-23T22:59:24.695599+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":246,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695142+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:59:24.695142+0000","last_peered":"2023-09-23T22:59:24.695142+0000","last_clean":"2023-09-23T22:59:24.695142+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:59:24.695142+0000","last_undegraded":"2023-09-23T22:59:24.695142+0000","last_fullsized":"2023-09-23T22:59:24.695142+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"377'7980","reported_seq":8240,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:34.676474+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T22:59:34.676474+0000","last_peered":"2023-09-23T22:59:34.676474+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T22:59:34.676474+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274857984,"num_objects":4194,"num_object_clones":0,"num_object_copies":12582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3544,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":4194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7980,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":584}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":45,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.710881+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:59:24.710881+0000","last_peered":"2023-09-23T22:59:24.710881+0000","last_clean":"2023-09-23T22:59:24.710881+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:59:24.710881+0000","last_undegraded":"2023-09-23T22:59:24.710881+0000","last_fullsized":"2023-09-23T22:59:24.710881+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":275,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692195+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:59:24.692195+0000","last_peered":"2023-09-23T22:59:24.692195+0000","last_clean":"2023-09-23T22:59:24.692195+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:59:24.692195+0000","last_undegraded":"2023-09-23T22:59:24.692195+0000","last_fullsized":"2023-09-23T22:59:24.692195+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"376'7904","reported_seq":7985,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:31.697328+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T22:59:31.697328+0000","last_peered":"2023-09-23T22:59:31.697328+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T22:59:31.697328+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T22:59:31.697328+0000","mapping_epoch":353,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274595840,"num_objects":4190,"num_object_clones":0,"num_object_copies":12570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7904,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4190}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":64,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.710493+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:59:24.710493+0000","last_peered":"2023-09-23T22:59:24.710493+0000","last_clean":"2023-09-23T22:59:24.710493+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:59:24.710493+0000","last_undegraded":"2023-09-23T22:59:24.710493+0000","last_fullsized":"2023-09-23T22:59:24.710493+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":369,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.672654+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:59:15.672654+0000","last_peered":"2023-09-23T22:59:15.672654+0000","last_clean":"2023-09-23T22:59:15.672654+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:59:15.672654+0000","last_undegraded":"2023-09-23T22:59:15.672654+0000","last_fullsized":"2023-09-23T22:59:15.672654+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"376'7957","reported_seq":8046,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:31.441200+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T22:59:31.441200+0000","last_peered":"2023-09-23T22:59:31.441200+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T22:59:31.441200+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278200320,"num_objects":4245,"num_object_clones":0,"num_object_copies":12735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4245,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":4245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7957,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":626}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.666060+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:59:15.666060+0000","last_peered":"2023-09-23T22:59:15.666060+0000","last_clean":"2023-09-23T22:59:15.666060+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:59:15.666060+0000","last_undegraded":"2023-09-23T22:59:15.666060+0000","last_fullsized":"2023-09-23T22:59:15.666060+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":386,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.669287+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:59:15.669287+0000","last_peered":"2023-09-23T22:59:15.669287+0000","last_clean":"2023-09-23T22:59:15.669287+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:59:15.669287+0000","last_undegraded":"2023-09-23T22:59:15.669287+0000","last_fullsized":"2023-09-23T22:59:15.669287+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"376'7877","reported_seq":11193,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:32.112267+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T22:59:32.112267+0000","last_peered":"2023-09-23T22:59:32.112267+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T22:59:32.112267+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T22:59:32.112267+0000","mapping_epoch":354,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274137088,"num_objects":4183,"num_object_clones":0,"num_object_copies":12549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2990,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7877,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":2749}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":399,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665067+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:59:15.665067+0000","last_peered":"2023-09-23T22:59:15.665067+0000","last_clean":"2023-09-23T22:59:15.665067+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:59:15.665067+0000","last_undegraded":"2023-09-23T22:59:15.665067+0000","last_fullsized":"2023-09-23T22:59:15.665067+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":377,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.670313+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:59:15.670313+0000","last_peered":"2023-09-23T22:59:15.670313+0000","last_clean":"2023-09-23T22:59:15.670313+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:59:15.670313+0000","last_undegraded":"2023-09-23T22:59:15.670313+0000","last_fullsized":"2023-09-23T22:59:15.670313+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"376'7796","reported_seq":7877,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:31.588511+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T22:59:31.588511+0000","last_peered":"2023-09-23T22:59:31.588511+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T22:59:31.588511+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T22:59:31.588511+0000","mapping_epoch":353,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273416192,"num_objects":4172,"num_object_clones":0,"num_object_copies":12516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7796,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":4172}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":46,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.728021+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:59:32.728021+0000","last_peered":"2023-09-23T22:59:32.728021+0000","last_clean":"2023-09-23T22:59:32.728021+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:59:32.728021+0000","last_undegraded":"2023-09-23T22:59:32.728021+0000","last_fullsized":"2023-09-23T22:59:32.728021+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":279,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709597+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:59:24.709597+0000","last_peered":"2023-09-23T22:59:24.709597+0000","last_clean":"2023-09-23T22:59:24.709597+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:59:24.709597+0000","last_undegraded":"2023-09-23T22:59:24.709597+0000","last_fullsized":"2023-09-23T22:59:24.709597+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"376'7889","reported_seq":12302,"reported_epoch":376,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:59:32.117152+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T22:59:32.117152+0000","last_peered":"2023-09-23T22:59:32.117152+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T22:59:32.117152+0000","last_undegraded":"2023-09-23T22:59:32.117152+0000","last_fullsized":"2023-09-23T22:59:32.117152+0000","mapping_epoch":354,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274399232,"num_objects":4187,"num_object_clones":0,"num_object_copies":12561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1516,"num_objects_unfound":0,"num_objects_dirty":4187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7889,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2149,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2671},{"shards":"0,4,6","objects":1516}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":283,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708845+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:59:24.708845+0000","last_peered":"2023-09-23T22:59:24.708845+0000","last_clean":"2023-09-23T22:59:24.708845+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:59:24.708845+0000","last_undegraded":"2023-09-23T22:59:24.708845+0000","last_fullsized":"2023-09-23T22:59:24.708845+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":287,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695424+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:59:24.695424+0000","last_peered":"2023-09-23T22:59:24.695424+0000","last_clean":"2023-09-23T22:59:24.695424+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:59:24.695424+0000","last_undegraded":"2023-09-23T22:59:24.695424+0000","last_fullsized":"2023-09-23T22:59:24.695424+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":100,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695055+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:59:24.695055+0000","last_peered":"2023-09-23T22:59:24.695055+0000","last_clean":"2023-09-23T22:59:24.695055+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:59:24.695055+0000","last_undegraded":"2023-09-23T22:59:24.695055+0000","last_fullsized":"2023-09-23T22:59:24.695055+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"377'7976","reported_seq":8069,"reported_epoch":377,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:59:34.669677+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T22:59:34.669677+0000","last_peered":"2023-09-23T22:59:34.669677+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T22:59:34.669677+0000","last_undegraded":"2023-09-23T22:59:34.669677+0000","last_fullsized":"2023-09-23T22:59:34.669677+0000","mapping_epoch":359,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271843328,"num_objects":4148,"num_object_clones":0,"num_object_copies":12444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4148,"num_objects_unfound":0,"num_objects_dirty":4148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7976,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":4148}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":46,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692047+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T22:59:24.692047+0000","last_peered":"2023-09-23T22:59:24.692047+0000","last_clean":"2023-09-23T22:59:24.692047+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T22:59:24.692047+0000","last_undegraded":"2023-09-23T22:59:24.692047+0000","last_fullsized":"2023-09-23T22:59:24.692047+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":396,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665609+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:59:15.665609+0000","last_peered":"2023-09-23T22:59:15.665609+0000","last_clean":"2023-09-23T22:59:15.665609+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:59:15.665609+0000","last_undegraded":"2023-09-23T22:59:15.665609+0000","last_fullsized":"2023-09-23T22:59:15.665609+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":268,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708917+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:59:24.708917+0000","last_peered":"2023-09-23T22:59:24.708917+0000","last_clean":"2023-09-23T22:59:24.708917+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:59:24.708917+0000","last_undegraded":"2023-09-23T22:59:24.708917+0000","last_fullsized":"2023-09-23T22:59:24.708917+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"377'7932","reported_seq":11347,"reported_epoch":377,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:36.133904+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T22:59:36.133904+0000","last_peered":"2023-09-23T22:59:36.133904+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T22:59:36.133904+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1908,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7932,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1663,"num_bytes_recovered":108986368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2224}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":343,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708960+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:59:24.708960+0000","last_peered":"2023-09-23T22:59:24.708960+0000","last_clean":"2023-09-23T22:59:24.708960+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:59:24.708960+0000","last_undegraded":"2023-09-23T22:59:24.708960+0000","last_fullsized":"2023-09-23T22:59:24.708960+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":220,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708497+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:59:24.708497+0000","last_peered":"2023-09-23T22:59:24.708497+0000","last_clean":"2023-09-23T22:59:24.708497+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:59:24.708497+0000","last_undegraded":"2023-09-23T22:59:24.708497+0000","last_fullsized":"2023-09-23T22:59:24.708497+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":287,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.700361+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:59:24.700361+0000","last_peered":"2023-09-23T22:59:24.700361+0000","last_clean":"2023-09-23T22:59:24.700361+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:59:24.700361+0000","last_undegraded":"2023-09-23T22:59:24.700361+0000","last_fullsized":"2023-09-23T22:59:24.700361+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"376'7956","reported_seq":9835,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:31.939704+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T22:59:31.939704+0000","last_peered":"2023-09-23T22:59:31.939704+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T22:59:31.939704+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7956,"log_dups_size":0,"ondisk_log_size":7956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279707648,"num_objects":4268,"num_object_clones":0,"num_object_copies":12804,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":4268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7956,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2835},{"shards":"0,6","objects":-713}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":360,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695225+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:59:24.695225+0000","last_peered":"2023-09-23T22:59:24.695225+0000","last_clean":"2023-09-23T22:59:24.695225+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:59:24.695225+0000","last_undegraded":"2023-09-23T22:59:24.695225+0000","last_fullsized":"2023-09-23T22:59:24.695225+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":472,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:24.694505+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T22:59:24.694505+0000","last_peered":"2023-09-23T22:59:24.694505+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T22:59:24.694505+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":46,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:31.229778+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:59:31.229778+0000","last_peered":"2023-09-23T22:59:31.229778+0000","last_clean":"2023-09-23T22:59:31.229778+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:59:31.229778+0000","last_undegraded":"2023-09-23T22:59:31.229778+0000","last_fullsized":"2023-09-23T22:59:31.229778+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.669505+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:59:15.669505+0000","last_peered":"2023-09-23T22:59:15.669505+0000","last_clean":"2023-09-23T22:59:15.669505+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:59:15.669505+0000","last_undegraded":"2023-09-23T22:59:15.669505+0000","last_fullsized":"2023-09-23T22:59:15.669505+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"377'7958","reported_seq":8043,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:34.733993+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T22:59:34.733993+0000","last_peered":"2023-09-23T22:59:34.733993+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T22:59:34.733993+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"323'2500","ondisk_log_start":"323'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5458,"log_dups_size":0,"ondisk_log_size":5458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270401536,"num_objects":4126,"num_object_clones":0,"num_object_copies":12378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7958,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":619}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":366,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.691815+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T22:59:24.691815+0000","last_peered":"2023-09-23T22:59:24.691815+0000","last_clean":"2023-09-23T22:59:24.691815+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T22:59:24.691815+0000","last_undegraded":"2023-09-23T22:59:24.691815+0000","last_fullsized":"2023-09-23T22:59:24.691815+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":337,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.691921+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T22:59:24.691921+0000","last_peered":"2023-09-23T22:59:24.691921+0000","last_clean":"2023-09-23T22:59:24.691921+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T22:59:24.691921+0000","last_undegraded":"2023-09-23T22:59:24.691921+0000","last_fullsized":"2023-09-23T22:59:24.691921+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":297,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692785+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:59:24.692785+0000","last_peered":"2023-09-23T22:59:24.692785+0000","last_clean":"2023-09-23T22:59:24.692785+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:59:24.692785+0000","last_undegraded":"2023-09-23T22:59:24.692785+0000","last_fullsized":"2023-09-23T22:59:24.692785+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"376'7961","reported_seq":8061,"reported_epoch":376,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:31.440371+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T22:59:31.440371+0000","last_peered":"2023-09-23T22:59:31.440371+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T22:59:31.440371+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T22:59:31.440371+0000","mapping_epoch":354,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278003736,"num_objects":4243,"num_object_clones":0,"num_object_copies":12729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":4243,"num_objects_unfound":0,"num_objects_dirty":4243,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7961,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":4243}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":312,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692317+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T22:59:24.692317+0000","last_peered":"2023-09-23T22:59:24.692317+0000","last_clean":"2023-09-23T22:59:24.692317+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T22:59:24.692317+0000","last_undegraded":"2023-09-23T22:59:24.692317+0000","last_fullsized":"2023-09-23T22:59:24.692317+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":220,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708997+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:59:24.708997+0000","last_peered":"2023-09-23T22:59:24.708997+0000","last_clean":"2023-09-23T22:59:24.708997+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:59:24.708997+0000","last_undegraded":"2023-09-23T22:59:24.708997+0000","last_fullsized":"2023-09-23T22:59:24.708997+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":190,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.695311+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:59:24.695311+0000","last_peered":"2023-09-23T22:59:24.695311+0000","last_clean":"2023-09-23T22:59:24.695311+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:59:24.695311+0000","last_undegraded":"2023-09-23T22:59:24.695311+0000","last_fullsized":"2023-09-23T22:59:24.695311+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"376'7950","reported_seq":10620,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:32.001555+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T22:59:32.001555+0000","last_peered":"2023-09-23T22:59:32.001555+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T22:59:32.001555+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274989056,"num_objects":4196,"num_object_clones":0,"num_object_copies":12588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8064,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7950,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1288,"num_bytes_recovered":84410368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":615},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":70,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.728541+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:59:32.728541+0000","last_peered":"2023-09-23T22:59:32.728541+0000","last_clean":"2023-09-23T22:59:32.728541+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:59:32.728541+0000","last_undegraded":"2023-09-23T22:59:32.728541+0000","last_fullsized":"2023-09-23T22:59:32.728541+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":84,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.674863+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:59:15.674863+0000","last_peered":"2023-09-23T22:59:15.674863+0000","last_clean":"2023-09-23T22:59:15.674863+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:59:15.674863+0000","last_undegraded":"2023-09-23T22:59:15.674863+0000","last_fullsized":"2023-09-23T22:59:15.674863+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":61,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:26.229147+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:59:26.229147+0000","last_peered":"2023-09-23T22:59:26.229147+0000","last_clean":"2023-09-23T22:59:26.229147+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:59:26.229147+0000","last_undegraded":"2023-09-23T22:59:26.229147+0000","last_fullsized":"2023-09-23T22:59:26.229147+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"376'7886","reported_seq":8396,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:32.075355+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T22:59:32.075355+0000","last_peered":"2023-09-23T22:59:32.075355+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T22:59:32.075355+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274202624,"num_objects":4184,"num_object_clones":0,"num_object_copies":12552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4184,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":4184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7886,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":633}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709867+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:59:24.709867+0000","last_peered":"2023-09-23T22:59:24.709867+0000","last_clean":"2023-09-23T22:59:24.709867+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:59:24.709867+0000","last_undegraded":"2023-09-23T22:59:24.709867+0000","last_fullsized":"2023-09-23T22:59:24.709867+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":279,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692705+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:59:24.692705+0000","last_peered":"2023-09-23T22:59:24.692705+0000","last_clean":"2023-09-23T22:59:24.692705+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:59:24.692705+0000","last_undegraded":"2023-09-23T22:59:24.692705+0000","last_fullsized":"2023-09-23T22:59:24.692705+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":350,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665125+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:59:15.665125+0000","last_peered":"2023-09-23T22:59:15.665125+0000","last_clean":"2023-09-23T22:59:15.665125+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:59:15.665125+0000","last_undegraded":"2023-09-23T22:59:15.665125+0000","last_fullsized":"2023-09-23T22:59:15.665125+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":314,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692454+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:59:24.692454+0000","last_peered":"2023-09-23T22:59:24.692454+0000","last_clean":"2023-09-23T22:59:24.692454+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:59:24.692454+0000","last_undegraded":"2023-09-23T22:59:24.692454+0000","last_fullsized":"2023-09-23T22:59:24.692454+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.708654+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:59:24.708654+0000","last_peered":"2023-09-23T22:59:24.708654+0000","last_clean":"2023-09-23T22:59:24.708654+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:59:24.708654+0000","last_undegraded":"2023-09-23T22:59:24.708654+0000","last_fullsized":"2023-09-23T22:59:24.708654+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":185,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.672555+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:59:15.672555+0000","last_peered":"2023-09-23T22:59:15.672555+0000","last_clean":"2023-09-23T22:59:15.672555+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:59:15.672555+0000","last_undegraded":"2023-09-23T22:59:15.672555+0000","last_fullsized":"2023-09-23T22:59:15.672555+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":399,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665686+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:59:15.665686+0000","last_peered":"2023-09-23T22:59:15.665686+0000","last_clean":"2023-09-23T22:59:15.665686+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:59:15.665686+0000","last_undegraded":"2023-09-23T22:59:15.665686+0000","last_fullsized":"2023-09-23T22:59:15.665686+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":199,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.669372+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:59:15.669372+0000","last_peered":"2023-09-23T22:59:15.669372+0000","last_clean":"2023-09-23T22:59:15.669372+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:59:15.669372+0000","last_undegraded":"2023-09-23T22:59:15.669372+0000","last_fullsized":"2023-09-23T22:59:15.669372+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665988+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:59:15.665988+0000","last_peered":"2023-09-23T22:59:15.665988+0000","last_clean":"2023-09-23T22:59:15.665988+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:59:15.665988+0000","last_undegraded":"2023-09-23T22:59:15.665988+0000","last_fullsized":"2023-09-23T22:59:15.665988+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":70,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:31.229887+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:59:31.229887+0000","last_peered":"2023-09-23T22:59:31.229887+0000","last_clean":"2023-09-23T22:59:31.229887+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:59:31.229887+0000","last_undegraded":"2023-09-23T22:59:31.229887+0000","last_fullsized":"2023-09-23T22:59:31.229887+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":72,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.709440+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:59:24.709440+0000","last_peered":"2023-09-23T22:59:24.709440+0000","last_clean":"2023-09-23T22:59:24.709440+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:59:24.709440+0000","last_undegraded":"2023-09-23T22:59:24.709440+0000","last_fullsized":"2023-09-23T22:59:24.709440+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":318,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:59:24.692872+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T22:59:24.692872+0000","last_peered":"2023-09-23T22:59:24.692872+0000","last_clean":"2023-09-23T22:59:24.692872+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T22:59:24.692872+0000","last_undegraded":"2023-09-23T22:59:24.692872+0000","last_fullsized":"2023-09-23T22:59:24.692872+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T22:59:15.665080+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:59:15.665080+0000","last_peered":"2023-09-23T22:59:15.665080+0000","last_clean":"2023-09-23T22:59:15.665080+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:59:15.665080+0000","last_undegraded":"2023-09-23T22:59:15.665080+0000","last_fullsized":"2023-09-23T22:59:15.665080+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4381605912,"num_objects":66859,"num_object_clones":0,"num_object_copies":200577,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":36261,"num_objects_misplaced":33947,"num_objects_unfound":0,"num_objects_dirty":66859,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126739,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9054,"num_bytes_recovered":592117760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12874166272,"data_stored":12874154056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95839,"ondisk_log_size":95839,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430599,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626823,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253136,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295588,"kb_used_data":1607196,"kb_used_omap":9,"kb_used_meta":688374,"kb_avail":91449564,"statfs":{"total":95995035648,"available":93644353536,"internally_reserved":0,"allocated":1645768704,"data_stored":1644369435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":704895977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.214},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.5960000000000001,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.048,"5min":4.048,"15min":4.048},"last":0.38400000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":1.0629999999999999,"15min":2.3849999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63100000000000001,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.307},{"interface":"front","average":{"1min":0.40200000000000002,"5min":1.147,"15min":2.8340000000000001},"min":{"1min":0.27400000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.624,"5min":8.0099999999999998,"15min":89.861999999999995},"last":0.34699999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":7.5730000000000004,"5min":3.1190000000000002,"15min":2.766},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":151.036,"5min":151.036,"15min":151.036},"last":2.605},{"interface":"front","average":{"1min":7.5510000000000002,"5min":3.2469999999999999,"15min":2.7709999999999999},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":151.06899999999999,"5min":151.06899999999999,"15min":151.06899999999999},"last":2.5859999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":3.4449999999999998,"15min":2.5329999999999999},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":0.95099999999999996,"5min":116.098,"15min":116.098},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":3.1200000000000001,"15min":2.7890000000000001},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":2.698,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.8600000000000001,"15min":2.431},"min":{"1min":0.33600000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.85499999999999998,"5min":20.742000000000001,"15min":79.311000000000007},"last":3.2789999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.9119999999999999,"15min":2.125},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":4.4219999999999997,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.37}]},{"osd":7,"last update":"Sat Sep 23 22:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.3580000000000001,"15min":2.028},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":0.87,"5min":8.1400000000000006,"15min":29.539000000000001},"last":1.2849999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.6990000000000001,"15min":2.2120000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.57699999999999996}]}]},{"osd":0,"up_from":353,"seq":1516123455507,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2890160,"kb_used_data":2765096,"kb_used_omap":37,"kb_used_meta":125018,"kb_avail":90854992,"statfs":{"total":95995035648,"available":93035511808,"internally_reserved":0,"allocated":2831458304,"data_stored":2829938790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":128018594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.6309999999999998,"5min":2.6309999999999998,"15min":2.6309999999999998},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.0539999999999998,"5min":3.0539999999999998,"15min":3.0539999999999998},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.876},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2280000000000002,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0179999999999998,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.2770000000000001,"5min":3.2770000000000001,"15min":3.2770000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.982,"5min":1.982,"15min":1.982},"last":2.0030000000000001},{"interface":"front","average":{"1min":0.998,"5min":0.998,"15min":0.998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.0920000000000001,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":2.371},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":2.3860000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":4.3120000000000003,"5min":4.3120000000000003,"15min":4.3120000000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.3579999999999997,"5min":4.3579999999999997,"15min":4.3579999999999997},"last":1.004}]}]},{"osd":2,"up_from":359,"seq":1541893259280,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944244,"kb_used_data":857068,"kb_used_omap":83,"kb_used_meta":87084,"kb_avail":92800908,"statfs":{"total":95995035648,"available":95028129792,"internally_reserved":0,"allocated":877637632,"data_stored":876175465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":89174469},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":2.706},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.96699999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":0.435},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":1.627},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":0.55700000000000005}]}]},{"osd":3,"up_from":94,"seq":403726926084,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":520080,"kb_used_data":45768,"kb_used_omap":7,"kb_used_meta":474296,"kb_avail":93225072,"statfs":{"total":95995035648,"available":95462473728,"internally_reserved":0,"allocated":46866432,"data_stored":45372794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":485679720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.972,"5min":1.972,"15min":1.972},"last":0.23400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.097,"15min":2.786},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.60899999999999999,"5min":16.596,"15min":123.866},"last":0.313},{"interface":"front","average":{"1min":0.35499999999999998,"5min":1.179,"15min":2.4449999999999998},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":0.47499999999999998,"5min":16.649999999999999,"15min":122.265},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.1469999999999998,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"last":0.24399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.3129999999999999,"15min":1.8640000000000001},"min":{"1min":0.30399999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":1.0700000000000001,"5min":7.8929999999999998,"15min":18.757000000000001},"last":0.375},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.3129999999999999,"15min":1.629},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.27500000000000002},"max":{"1min":0.72199999999999998,"5min":7.8780000000000001,"15min":15.037000000000001},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.393,"15min":1.52},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.59399999999999997,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.46800000000000003,"5min":1.546,"15min":1.954},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75600000000000001,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7709999999999999,"15min":1.788},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":14.097,"15min":14.097},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.677,"15min":2.0390000000000001},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5,"5min":12.887,"15min":12.887},"last":0.38700000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.3480000000000001,"15min":2.2109999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79700000000000004,"5min":6.9950000000000001,"15min":91.757999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.238,"15min":2.2240000000000002},"min":{"1min":0.434,"5min":0.41699999999999998,"15min":0.33200000000000002},"max":{"1min":1.1479999999999999,"5min":7.1219999999999999,"15min":91.709999999999994},"last":0.51300000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019280,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3737420,"kb_used_data":3147716,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90007732,"statfs":{"total":95995035648,"available":92167917568,"internally_reserved":0,"allocated":3223261184,"data_stored":3222514525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":3.0649999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":3.7690000000000001,"15min":3.786},"min":{"1min":0.40000000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":1.966,"5min":98.846999999999994,"15min":132.304},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.71699999999999997,"5min":3.9740000000000002,"15min":3.6259999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":3.2269999999999999,"5min":98.938999999999993,"15min":132.453},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.536,"15min":4.5099999999999998},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":3.399,"5min":8.5510000000000002,"15min":164.453},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.478,"15min":4.5720000000000001},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.99,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:59:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.9630000000000001,"15min":1.6319999999999999},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3359999999999999,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.3599999999999999,"15min":1.7669999999999999},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.248},"max":{"1min":1.6040000000000001,"5min":151.905,"15min":151.905},"last":0.318}]},{"osd":6,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5820000000000001,"15min":1.1559999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":2.113,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.423,"15min":1.101},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0350000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.372}]},{"osd":7,"last update":"Sat Sep 23 22:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.3080000000000001,"15min":1.0960000000000001},"min":{"1min":0.46400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0150000000000001,"5min":12.84,"15min":12.84},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.371,"5min":1.3180000000000001,"15min":1.02},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.587,"5min":10.587,"15min":10.587},"last":0.48599999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823055,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2853240,"kb_used_data":1949488,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":90891912,"statfs":{"total":95995035648,"available":93073317888,"internally_reserved":0,"allocated":1996275712,"data_stored":1995552871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":2.1709999999999998},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":2.5419999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":2.9249999999999998,"15min":2.3330000000000002},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.24199999999999999},"max":{"1min":3.2429999999999999,"5min":110.36499999999999,"15min":110.36499999999999},"last":2.5800000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":3.4340000000000002,"15min":2.8410000000000002},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":10.353,"5min":110.89,"15min":110.89},"last":6.7690000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":2.6259999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.6160000000000001,"15min":5.117},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":3.149,"5min":11.298999999999999,"15min":257.05000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.5469999999999999,"15min":4.9409999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.4940000000000002,"5min":12.185,"15min":257.35000000000002},"last":9.532}]},{"osd":4,"last update":"Sat Sep 23 22:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.802,"15min":1.4570000000000001},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6850000000000001,"5min":115.06699999999999,"15min":115.06699999999999},"last":6.8440000000000003},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.661,"15min":1.4510000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.1880000000000002,"5min":114.908,"15min":114.908},"last":1.579}]},{"osd":6,"last update":"Sat Sep 23 22:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.3300000000000001,"15min":1.2649999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.498,"5min":11.121,"15min":48.360999999999997},"last":6.8150000000000004},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.272,"15min":1.2709999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.7759999999999998,"5min":12.137,"15min":48.292999999999999},"last":2.6040000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.159,"15min":1.0389999999999999},"min":{"1min":0.44600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.3029999999999999,"5min":12.23,"15min":12.23},"last":9.5980000000000008},{"interface":"front","average":{"1min":1.651,"5min":1.381,"15min":1.0389999999999999},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":10.638999999999999,"5min":11.112,"15min":11.112},"last":6.8710000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2821591040,"data_stored":2821586968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1640628224,"data_stored":1640628224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":873201664,"data_stored":873201664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":42401792,"data_stored":42401792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3218083840,"data_stored":3218079768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1991704576,"data_stored":1991704576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2286555136,"data_stored":2286551064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:59:39.441 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:59:39.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:39 smithi052 ceph-mon[180979]: from='client.46373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:39.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:39 smithi052 ceph-mon[180979]: pgmap v1110: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 35948/198720 objects degraded (18.090%); 33762/198720 objects misplaced (16.990%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:59:39.828 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:39 smithi052 ceph-mon[187743]: from='client.46373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:39.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:39 smithi052 ceph-mon[187743]: pgmap v1110: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 35948/198720 objects degraded (18.090%); 33762/198720 objects misplaced (16.990%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:59:39.864 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:39.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:39.972 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:40.021 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:39 smithi151 ceph-mon[159210]: from='client.46373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:40.021 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:39 smithi151 ceph-mon[159210]: pgmap v1110: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 35948/198720 objects degraded (18.090%); 33762/198720 objects misplaced (16.990%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:59:40.024 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:40.092 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:40.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:40.141 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:40.197 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:40.237 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:40.861 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:40.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:40.867 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 35948/198720 objects degraded (18.090%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:40.868 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 35948/198720 objects degraded (18.090%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:40.902 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:59:40.902 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:59:40.969 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:41.010 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 35948/198720 objects degraded (18.090%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:41.014 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:41.082 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:41.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:41.191 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:41.224 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:41.298 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:41.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:41.410 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:41.458 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:41.543 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:41.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:41.656 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:41.688 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:41.748 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T22:59:41.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[187743]: pgmap v1111: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 35890/197838 objects degraded (18.141%); 33647/197838 objects misplaced (17.007%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/426085258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[180979]: pgmap v1111: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 35890/197838 objects degraded (18.141%); 33647/197838 objects misplaced (17.007%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/426085258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:41.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:41 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:42.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:59:42.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:42.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:41 smithi151 ceph-mon[159210]: pgmap v1111: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 35890/197838 objects degraded (18.141%); 33647/197838 objects misplaced (17.007%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:59:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/426085258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:41 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:42.277 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:42.338 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:42.429 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:42.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:42.539 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:42.572 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:42.620 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:42.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:42.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:42.724 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:42.763 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:42.826 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:42.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:42.933 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:42.938 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:42 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:59:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:42 smithi151 ceph-mon[159210]: osdmap e378: 8 total, 8 up, 6 in 2023-09-23T22:59:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:42 smithi151 ceph-mon[159210]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:42 smithi151 ceph-mon[159210]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:42 smithi151 ceph-mon[159210]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.959 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:59:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[187743]: osdmap e378: 8 total, 8 up, 6 in 2023-09-23T22:59:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[187743]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[187743]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[187743]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[180979]: osdmap e378: 8 total, 8 up, 6 in 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[180979]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[180979]: failed to encode map e378 with expected crc 2023-09-23T22:59:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:42 smithi052 ceph-mon[180979]: failed to encode map e378 with expected crc 2023-09-23T22:59:43.004 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:43.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:43.047 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:43.066 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:43.088 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:43.110 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:43.111 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:43.114 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:43.134 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:43.143 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:43.169 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:43.202 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:43.216 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:43.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:43.232 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:43.232 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.232 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.232 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.232 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.233 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.233 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.233 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.233 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:59:43.321 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:43.376 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:43.450 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:43.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:43.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:59:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:59:43] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-09-23T22:59:43.561 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:43.596 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:43.948 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:43 smithi052 ceph-mon[180979]: pgmap v1113: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 211 op/s; 35482/195924 objects degraded (18.110%); 33474/195924 objects misplaced (17.085%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:59:43.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/289982263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:43.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:43 smithi052 ceph-mon[187743]: pgmap v1113: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 211 op/s; 35482/195924 objects degraded (18.110%); 33474/195924 objects misplaced (17.085%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:59:43.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/289982263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:43 smithi151 ceph-mon[159210]: pgmap v1113: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 211 op/s; 35482/195924 objects degraded (18.110%); 33474/195924 objects misplaced (17.085%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:59:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/289982263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:44.406 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:44.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:44.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:44.510 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:44.589 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:44.697 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:59:44.700 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:44.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:44.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:44.847 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:44.912 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:44.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:45.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:45.059 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:45.326 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:45.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:45.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:45.429 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:45.457 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:45.515 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:45.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:45.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:45.653 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:45.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:45 smithi151 ceph-mon[159210]: pgmap v1114: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 35482/195924 objects degraded (18.110%); 33474/195924 objects misplaced (17.085%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:45.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 35482/195924 objects degraded (18.110%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:45.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:45 smithi052 ceph-mon[187743]: pgmap v1114: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 35482/195924 objects degraded (18.110%); 33474/195924 objects misplaced (17.085%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:45.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 35482/195924 objects degraded (18.110%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:45.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:45 smithi052 ceph-mon[180979]: pgmap v1114: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 35482/195924 objects degraded (18.110%); 33474/195924 objects misplaced (17.085%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:45.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 35482/195924 objects degraded (18.110%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:45.966 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:45.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:45.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:46.073 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:46.112 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:46.228 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:46.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:46.335 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:46.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:46.435 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:46.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:46.482 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:46.544 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:129408 10.5 e378/359 10:a4d57977:::benchmark_data_smithi052_461816_object32607:head v 378'8138, mlcod=323'2477)", 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:46.576429+0000", 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.00080273300000000003, 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00082554299999999998, 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T22:59:46.579 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576429+0000", 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576429+0000", 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T22:59:46.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576427+0000", 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576445+0000", 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.821e-05 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576478+0000", 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2778999999999999e-05 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576492+0000", 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3631e-05 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T22:59:46.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576588+0000", 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6310000000000005e-05 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:46.576615+0000", 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7169e-05 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T22:59:46.583 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T22:59:46.584 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 1 2023-09-23T22:59:46.584 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:59:46.584 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:59:46.597 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:46.602 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:46.602 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35482/195924 objects degraded (18.110%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1519,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":378,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":65308,"data_bytes":4278982584,"bytes_used":13352697856,"bytes_avail":562617516032,"bytes_total":575970213888,"degraded_objects":35482,"degraded_total":195924,"degraded_ratio":0.18110083501765992,"misplaced_objects":33474,"misplaced_total":195924,"misplaced_ratio":0.17085196300606356,"recovering_objects_per_sec":72,"recovering_bytes_per_sec":4743118,"recovering_keys_per_sec":0,"num_objects_recovered":724,"num_bytes_recovered":47448064,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":207},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (3m)\n [======================......] (remaining: 47s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:59:46.853 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2216307827' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:59:46.854 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:46.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2216307827' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:59:46.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:46.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:46.959 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:46.993 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2216307827' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:59:47.927 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:47.929 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:47 smithi052 ceph-mon[180979]: pgmap v1115: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 35238/194610 objects degraded (18.107%); 33345/194610 objects misplaced (17.134%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:47.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:47 smithi052 ceph-mon[187743]: pgmap v1115: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 35238/194610 objects degraded (18.107%); 33345/194610 objects misplaced (17.134%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:47.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:48.031 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:48.059 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:47 smithi151 ceph-mon[159210]: pgmap v1115: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 35238/194610 objects degraded (18.107%); 33345/194610 objects misplaced (17.134%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:48.062 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:48.130 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:48.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:48.246 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:48.295 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:48.325 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:59:48.362 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:48.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:48.469 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:48.510 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:48.575 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:59:48.575 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:59:48.590 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:48.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:48.702 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:48.729 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:48.786 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:48.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:48.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:48.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:48.937 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:49.097 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:49.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:49.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:49.186 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:49.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:49.238 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:49.321 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:49.402 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:49.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:49.507 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:49.554 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:49.608 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:49.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:49.713 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:49.744 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:49.806 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: pgmap v1116: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 34997/192891 objects degraded (18.143%); 33154/192891 objects misplaced (17.188%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:59:49.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3635692159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: pgmap v1116: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 34997/192891 objects degraded (18.143%); 33154/192891 objects misplaced (17.188%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3635692159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:59:49.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:50.108 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:50.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: pgmap v1116: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 34997/192891 objects degraded (18.143%); 33154/192891 objects misplaced (17.188%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:59:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:59:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3635692159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:59:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:59:50.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:59:50.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T22:59:50.215 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:50.275 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:50.329 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:50.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:50.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:50.479 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:50.523 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T22:59:50.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:50.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:50.670 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:50.704 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 34997/192891 objects degraded (18.143%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:50 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3635692159' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:59:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:50 smithi151 ceph-mon[159210]: osdmap e379: 8 total, 8 up, 6 in 2023-09-23T22:59:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:50 smithi151 ceph-mon[159210]: failed to encode map e379 with expected crc 2023-09-23T22:59:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 34997/192891 objects degraded (18.143%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3635692159' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[180979]: osdmap e379: 8 total, 8 up, 6 in 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[180979]: failed to encode map e379 with expected crc 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 34997/192891 objects degraded (18.143%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3635692159' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:59:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[187743]: osdmap e379: 8 total, 8 up, 6 in 2023-09-23T22:59:50.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:50 smithi052 ceph-mon[187743]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.262 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:51.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:51.366 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:51.404 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:51.463 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:51.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:51.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:51.569 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:51.598 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:51.672 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:51.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:51.781 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:51.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:51 smithi151 ceph-mon[159210]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:51 smithi151 ceph-mon[159210]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:51 smithi151 ceph-mon[159210]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:51 smithi151 ceph-mon[159210]: pgmap v1118: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 34934/191901 objects degraded (18.204%); 33035/191901 objects misplaced (17.215%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:59:51.972 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[187743]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[187743]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[187743]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[187743]: pgmap v1118: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 34934/191901 objects degraded (18.204%); 33035/191901 objects misplaced (17.215%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[180979]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[180979]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[180979]: failed to encode map e379 with expected crc 2023-09-23T22:59:51.975 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:51 smithi052 ceph-mon[180979]: pgmap v1118: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 34934/191901 objects degraded (18.204%); 33035/191901 objects misplaced (17.215%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:59:51.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:51.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:52.077 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:52.113 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:52.161 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:52.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:52.269 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:52.309 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:52.555 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:52.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:52.584 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:52.609 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:52.629 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:52.649 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:52.668 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:52.688 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.690 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:59:52.726 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T22:59:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T22:59:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T22:59:52.727 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T22:59:52.727 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T22:59:52.733 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:52.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:52.838 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:59:52.877 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:52.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3954837683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3954837683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3954837683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:59:53.478 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:59:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:53.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:59:53.543 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:59:53.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:53.614 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 22:59:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:22:59:53] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.33.4" 2023-09-23T22:59:53.618 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:53.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:53 smithi052 ceph-mon[180979]: pgmap v1119: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 34464/189627 objects degraded (18.175%); 32845/189627 objects misplaced (17.321%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:53.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:53 smithi052 ceph-mon[187743]: pgmap v1119: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 34464/189627 objects degraded (18.175%); 32845/189627 objects misplaced (17.321%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:53.876 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:53.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:53.981 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:54.020 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:54.058 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:54.074 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:54.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:54.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:54.088 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:53 smithi151 ceph-mon[159210]: pgmap v1119: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 34464/189627 objects degraded (18.175%); 32845/189627 objects misplaced (17.321%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:54.129 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:59:54.186 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:54.232 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:54.307 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:54.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:54.412 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:54.442 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:54.498 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:54.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:54.605 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:54.671 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:54.722 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T22:59:54.724 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T22:59:55.752 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:55.788 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:55.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:55.813 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:59:55.827 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1120,"stamp":"2023-09-23T22:59:54.560478+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4141422520,"num_objects":63209,"num_object_clones":0,"num_object_copies":189627,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":34464,"num_objects_misplaced":32845,"num_objects_unfound":0,"num_objects_dirty":63209,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":130673,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10423,"num_bytes_recovered":679558076,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97107,"ondisk_log_size":97107,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12762644,"kb_used_data":9855716,"kb_used_omap":137,"kb_used_meta":2906742,"kb_avail":549708268,"statfs":{"total":575970213888,"available":562901266432,"internally_reserved":0,"allocated":10092253184,"data_stored":10084831334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":2976503992},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-137560064,"num_objects":-2099,"num_object_clones":0,"num_object_copies":-6297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1018,"num_objects_misplaced":-629,"num_objects_unfound":0,"num_objects_dirty":-2099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2099,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":726,"num_bytes_recovered":47579136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003367"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":391,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.766713+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:59:41.766713+0000","last_peered":"2023-09-23T22:59:41.766713+0000","last_clean":"2023-09-23T22:59:41.766713+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:59:41.766713+0000","last_undegraded":"2023-09-23T22:59:41.766713+0000","last_fullsized":"2023-09-23T22:59:41.766713+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":47,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.741400+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:59:41.741400+0000","last_peered":"2023-09-23T22:59:41.741400+0000","last_clean":"2023-09-23T22:59:41.741400+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:59:41.741400+0000","last_undegraded":"2023-09-23T22:59:41.741400+0000","last_fullsized":"2023-09-23T22:59:41.741400+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740138+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:59:41.740138+0000","last_peered":"2023-09-23T22:59:41.740138+0000","last_clean":"2023-09-23T22:59:41.740138+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:59:41.740138+0000","last_undegraded":"2023-09-23T22:59:41.740138+0000","last_fullsized":"2023-09-23T22:59:41.740138+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":375,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740647+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:59:41.740647+0000","last_peered":"2023-09-23T22:59:41.740647+0000","last_clean":"2023-09-23T22:59:41.740647+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:59:41.740647+0000","last_undegraded":"2023-09-23T22:59:41.740647+0000","last_fullsized":"2023-09-23T22:59:41.740647+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":63,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.525949+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:59:50.525949+0000","last_peered":"2023-09-23T22:59:50.525949+0000","last_clean":"2023-09-23T22:59:50.525949+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:59:50.525949+0000","last_undegraded":"2023-09-23T22:59:50.525949+0000","last_fullsized":"2023-09-23T22:59:50.525949+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":47,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.723348+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T22:59:32.723348+0000","last_peered":"2023-09-23T22:59:32.723348+0000","last_clean":"2023-09-23T22:59:32.723348+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T22:59:32.723348+0000","last_undegraded":"2023-09-23T22:59:32.723348+0000","last_fullsized":"2023-09-23T22:59:32.723348+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":314,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753620+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:59:41.753620+0000","last_peered":"2023-09-23T22:59:41.753620+0000","last_clean":"2023-09-23T22:59:41.753620+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:59:41.753620+0000","last_undegraded":"2023-09-23T22:59:41.753620+0000","last_fullsized":"2023-09-23T22:59:41.753620+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":57,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.525821+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:59:50.525821+0000","last_peered":"2023-09-23T22:59:50.525821+0000","last_clean":"2023-09-23T22:59:50.525821+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:59:50.525821+0000","last_undegraded":"2023-09-23T22:59:50.525821+0000","last_fullsized":"2023-09-23T22:59:50.525821+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":275,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753270+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:59:41.753270+0000","last_peered":"2023-09-23T22:59:41.753270+0000","last_clean":"2023-09-23T22:59:41.753270+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:59:41.753270+0000","last_undegraded":"2023-09-23T22:59:41.753270+0000","last_fullsized":"2023-09-23T22:59:41.753270+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.526726+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:59:50.526726+0000","last_peered":"2023-09-23T22:59:50.526726+0000","last_clean":"2023-09-23T22:59:50.526726+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:59:50.526726+0000","last_undegraded":"2023-09-23T22:59:50.526726+0000","last_fullsized":"2023-09-23T22:59:50.526726+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":67,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.741335+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:59:41.741335+0000","last_peered":"2023-09-23T22:59:41.741335+0000","last_clean":"2023-09-23T22:59:41.741335+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:59:41.741335+0000","last_undegraded":"2023-09-23T22:59:41.741335+0000","last_fullsized":"2023-09-23T22:59:41.741335+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750689+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:59:41.750689+0000","last_peered":"2023-09-23T22:59:41.750689+0000","last_clean":"2023-09-23T22:59:41.750689+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:59:41.750689+0000","last_undegraded":"2023-09-23T22:59:41.750689+0000","last_fullsized":"2023-09-23T22:59:41.750689+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":450,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767786+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:59:41.767786+0000","last_peered":"2023-09-23T22:59:41.767786+0000","last_clean":"2023-09-23T22:59:41.767786+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:59:41.767786+0000","last_undegraded":"2023-09-23T22:59:41.767786+0000","last_fullsized":"2023-09-23T22:59:41.767786+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":353,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.768924+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:59:41.768924+0000","last_peered":"2023-09-23T22:59:41.768924+0000","last_clean":"2023-09-23T22:59:41.768924+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:59:41.768924+0000","last_undegraded":"2023-09-23T22:59:41.768924+0000","last_fullsized":"2023-09-23T22:59:41.768924+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":279,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769007+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:59:41.769007+0000","last_peered":"2023-09-23T22:59:41.769007+0000","last_clean":"2023-09-23T22:59:41.769007+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:59:41.769007+0000","last_undegraded":"2023-09-23T22:59:41.769007+0000","last_fullsized":"2023-09-23T22:59:41.769007+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:46.231649+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:59:46.231649+0000","last_peered":"2023-09-23T22:59:46.231649+0000","last_clean":"2023-09-23T22:59:46.231649+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:59:46.231649+0000","last_undegraded":"2023-09-23T22:59:46.231649+0000","last_fullsized":"2023-09-23T22:59:46.231649+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":367,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740348+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:59:41.740348+0000","last_peered":"2023-09-23T22:59:41.740348+0000","last_clean":"2023-09-23T22:59:41.740348+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:59:41.740348+0000","last_undegraded":"2023-09-23T22:59:41.740348+0000","last_fullsized":"2023-09-23T22:59:41.740348+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":279,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753028+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:59:41.753028+0000","last_peered":"2023-09-23T22:59:41.753028+0000","last_clean":"2023-09-23T22:59:41.753028+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:59:41.753028+0000","last_undegraded":"2023-09-23T22:59:41.753028+0000","last_fullsized":"2023-09-23T22:59:41.753028+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":264,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770150+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:59:41.770150+0000","last_peered":"2023-09-23T22:59:41.770150+0000","last_clean":"2023-09-23T22:59:41.770150+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:59:41.770150+0000","last_undegraded":"2023-09-23T22:59:41.770150+0000","last_fullsized":"2023-09-23T22:59:41.770150+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":391,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740760+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T22:59:41.740760+0000","last_peered":"2023-09-23T22:59:41.740760+0000","last_clean":"2023-09-23T22:59:41.740760+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T22:59:41.740760+0000","last_undegraded":"2023-09-23T22:59:41.740760+0000","last_fullsized":"2023-09-23T22:59:41.740760+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":95,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.721966+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:59:32.721966+0000","last_peered":"2023-09-23T22:59:32.721966+0000","last_clean":"2023-09-23T22:59:32.721966+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:59:32.721966+0000","last_undegraded":"2023-09-23T22:59:32.721966+0000","last_fullsized":"2023-09-23T22:59:32.721966+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.741590+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:59:41.741590+0000","last_peered":"2023-09-23T22:59:41.741590+0000","last_clean":"2023-09-23T22:59:41.741590+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:59:41.741590+0000","last_undegraded":"2023-09-23T22:59:41.741590+0000","last_fullsized":"2023-09-23T22:59:41.741590+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"378'8097","reported_seq":8178,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:49.507286+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T22:59:49.507286+0000","last_peered":"2023-09-23T22:59:49.507286+0000","last_clean":"2023-09-23T22:59:49.507286+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T22:59:49.507286+0000","last_undegraded":"2023-09-23T22:59:49.507286+0000","last_fullsized":"2023-09-23T22:59:49.507286+0000","mapping_epoch":353,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261423104,"num_objects":3989,"num_object_clones":0,"num_object_copies":11967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8097,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":263,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770839+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:59:41.770839+0000","last_peered":"2023-09-23T22:59:41.770839+0000","last_clean":"2023-09-23T22:59:41.770839+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:59:41.770839+0000","last_undegraded":"2023-09-23T22:59:41.770839+0000","last_fullsized":"2023-09-23T22:59:41.770839+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.752552+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:59:41.752552+0000","last_peered":"2023-09-23T22:59:41.752552+0000","last_clean":"2023-09-23T22:59:41.752552+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:59:41.752552+0000","last_undegraded":"2023-09-23T22:59:41.752552+0000","last_fullsized":"2023-09-23T22:59:41.752552+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"379'8051","reported_seq":10463,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:51.072953+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T22:59:51.072953+0000","last_peered":"2023-09-23T22:59:51.072953+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T22:59:51.072953+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7651,"log_dups_size":0,"ondisk_log_size":7651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524864,"num_objects":3899,"num_object_clones":0,"num_object_copies":11697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3899,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8051,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":332}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":273,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767985+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:59:41.767985+0000","last_peered":"2023-09-23T22:59:41.767985+0000","last_clean":"2023-09-23T22:59:41.767985+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:59:41.767985+0000","last_undegraded":"2023-09-23T22:59:41.767985+0000","last_fullsized":"2023-09-23T22:59:41.767985+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":386,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770083+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:59:41.770083+0000","last_peered":"2023-09-23T22:59:41.770083+0000","last_clean":"2023-09-23T22:59:41.770083+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:59:41.770083+0000","last_undegraded":"2023-09-23T22:59:41.770083+0000","last_fullsized":"2023-09-23T22:59:41.770083+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"378'8147","reported_seq":8222,"reported_epoch":378,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:49.133205+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T22:59:49.133205+0000","last_peered":"2023-09-23T22:59:49.133205+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T22:59:49.133205+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T22:59:49.133205+0000","mapping_epoch":354,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5547,"log_dups_size":0,"ondisk_log_size":5547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257228800,"num_objects":3925,"num_object_clones":0,"num_object_copies":11775,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3925,"num_objects_unfound":0,"num_objects_dirty":3925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8147,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3925}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":130,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753054+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:59:41.753054+0000","last_peered":"2023-09-23T22:59:41.753054+0000","last_clean":"2023-09-23T22:59:41.753054+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:59:41.753054+0000","last_undegraded":"2023-09-23T22:59:41.753054+0000","last_fullsized":"2023-09-23T22:59:41.753054+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753050+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:59:41.753050+0000","last_peered":"2023-09-23T22:59:41.753050+0000","last_clean":"2023-09-23T22:59:41.753050+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:59:41.753050+0000","last_undegraded":"2023-09-23T22:59:41.753050+0000","last_fullsized":"2023-09-23T22:59:41.753050+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"378'8180","reported_seq":8441,"reported_epoch":378,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:49.635476+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T22:59:49.635476+0000","last_peered":"2023-09-23T22:59:49.635476+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T22:59:49.635476+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261750784,"num_objects":3994,"num_object_clones":0,"num_object_copies":11982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3369,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8180,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":384}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":48,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.742480+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:59:41.742480+0000","last_peered":"2023-09-23T22:59:41.742480+0000","last_clean":"2023-09-23T22:59:41.742480+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:59:41.742480+0000","last_undegraded":"2023-09-23T22:59:41.742480+0000","last_fullsized":"2023-09-23T22:59:41.742480+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":276,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.722827+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:59:32.722827+0000","last_peered":"2023-09-23T22:59:32.722827+0000","last_clean":"2023-09-23T22:59:32.722827+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:59:32.722827+0000","last_undegraded":"2023-09-23T22:59:32.722827+0000","last_fullsized":"2023-09-23T22:59:32.722827+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"379'8153","reported_seq":8237,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:51.772224+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T22:59:51.772224+0000","last_peered":"2023-09-23T22:59:51.772224+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T22:59:51.772224+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T22:59:51.772224+0000","mapping_epoch":353,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5753,"log_dups_size":0,"ondisk_log_size":5753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258277376,"num_objects":3941,"num_object_clones":0,"num_object_copies":11823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8153,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3941}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":67,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740410+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:59:41.740410+0000","last_peered":"2023-09-23T22:59:41.740410+0000","last_clean":"2023-09-23T22:59:41.740410+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:59:41.740410+0000","last_undegraded":"2023-09-23T22:59:41.740410+0000","last_fullsized":"2023-09-23T22:59:41.740410+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":372,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767704+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:59:41.767704+0000","last_peered":"2023-09-23T22:59:41.767704+0000","last_clean":"2023-09-23T22:59:41.767704+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:59:41.767704+0000","last_undegraded":"2023-09-23T22:59:41.767704+0000","last_fullsized":"2023-09-23T22:59:41.767704+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"379'8224","reported_seq":8316,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:51.869715+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T22:59:51.869715+0000","last_peered":"2023-09-23T22:59:51.869715+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T22:59:51.869715+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260702208,"num_objects":3978,"num_object_clones":0,"num_object_copies":11934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3978,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8224,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":359}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":162,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.766798+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:59:41.766798+0000","last_peered":"2023-09-23T22:59:41.766798+0000","last_clean":"2023-09-23T22:59:41.766798+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:59:41.766798+0000","last_undegraded":"2023-09-23T22:59:41.766798+0000","last_fullsized":"2023-09-23T22:59:41.766798+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":389,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767233+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:59:41.767233+0000","last_peered":"2023-09-23T22:59:41.767233+0000","last_clean":"2023-09-23T22:59:41.767233+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:59:41.767233+0000","last_undegraded":"2023-09-23T22:59:41.767233+0000","last_fullsized":"2023-09-23T22:59:41.767233+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"379'8138","reported_seq":11982,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:52.123052+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T22:59:52.123052+0000","last_peered":"2023-09-23T22:59:52.123052+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T22:59:52.123052+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T22:59:52.123052+0000","mapping_epoch":354,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257032192,"num_objects":3922,"num_object_clones":0,"num_object_copies":11766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2728,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8138,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1872,"num_bytes_recovered":122683392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1435},{"shards":"1,4,6","objects":2487}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":402,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769907+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:59:41.769907+0000","last_peered":"2023-09-23T22:59:41.769907+0000","last_clean":"2023-09-23T22:59:41.769907+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:59:41.769907+0000","last_undegraded":"2023-09-23T22:59:41.769907+0000","last_fullsized":"2023-09-23T22:59:41.769907+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":380,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.768316+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:59:41.768316+0000","last_peered":"2023-09-23T22:59:41.768316+0000","last_clean":"2023-09-23T22:59:41.768316+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:59:41.768316+0000","last_undegraded":"2023-09-23T22:59:41.768316+0000","last_fullsized":"2023-09-23T22:59:41.768316+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"379'8040","reported_seq":8124,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:51.757650+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T22:59:51.757650+0000","last_peered":"2023-09-23T22:59:51.757650+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T22:59:51.757650+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T22:59:51.757650+0000","mapping_epoch":353,"log_start":"323'2300","ondisk_log_start":"323'2300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257425408,"num_objects":3928,"num_object_clones":0,"num_object_copies":11784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8040,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3928}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":48,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740113+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:59:41.740113+0000","last_peered":"2023-09-23T22:59:41.740113+0000","last_clean":"2023-09-23T22:59:41.740113+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:59:41.740113+0000","last_undegraded":"2023-09-23T22:59:41.740113+0000","last_fullsized":"2023-09-23T22:59:41.740113+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":281,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750881+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:59:41.750881+0000","last_peered":"2023-09-23T22:59:41.750881+0000","last_clean":"2023-09-23T22:59:41.750881+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:59:41.750881+0000","last_undegraded":"2023-09-23T22:59:41.750881+0000","last_fullsized":"2023-09-23T22:59:41.750881+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"379'8141","reported_seq":13487,"reported_epoch":379,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:59:52.106180+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T22:59:52.106180+0000","last_peered":"2023-09-23T22:59:52.106180+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T22:59:52.106180+0000","last_undegraded":"2023-09-23T22:59:52.106180+0000","last_fullsized":"2023-09-23T22:59:52.106180+0000","mapping_epoch":354,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5741,"log_dups_size":0,"ondisk_log_size":5741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1051,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8141,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2614,"num_bytes_recovered":170786816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2884},{"shards":"0,4,6","objects":1051}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":285,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751675+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:59:41.751675+0000","last_peered":"2023-09-23T22:59:41.751675+0000","last_clean":"2023-09-23T22:59:41.751675+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:59:41.751675+0000","last_undegraded":"2023-09-23T22:59:41.751675+0000","last_fullsized":"2023-09-23T22:59:41.751675+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.752953+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:59:41.752953+0000","last_peered":"2023-09-23T22:59:41.752953+0000","last_clean":"2023-09-23T22:59:41.752953+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:59:41.752953+0000","last_undegraded":"2023-09-23T22:59:41.752953+0000","last_fullsized":"2023-09-23T22:59:41.752953+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.752900+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:59:41.752900+0000","last_peered":"2023-09-23T22:59:41.752900+0000","last_clean":"2023-09-23T22:59:41.752900+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:59:41.752900+0000","last_undegraded":"2023-09-23T22:59:41.752900+0000","last_fullsized":"2023-09-23T22:59:41.752900+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"378'8151","reported_seq":8245,"reported_epoch":378,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:59:49.314436+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T22:59:49.314436+0000","last_peered":"2023-09-23T22:59:49.314436+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T22:59:49.314436+0000","last_undegraded":"2023-09-23T22:59:49.314436+0000","last_fullsized":"2023-09-23T22:59:49.314436+0000","mapping_epoch":359,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5551,"log_dups_size":0,"ondisk_log_size":5551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260374528,"num_objects":3973,"num_object_clones":0,"num_object_copies":11919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3973,"num_objects_unfound":0,"num_objects_dirty":3973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8151,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3973}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":47,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.724379+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T22:59:32.724379+0000","last_peered":"2023-09-23T22:59:32.724379+0000","last_clean":"2023-09-23T22:59:32.724379+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T22:59:32.724379+0000","last_undegraded":"2023-09-23T22:59:32.724379+0000","last_fullsized":"2023-09-23T22:59:32.724379+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":399,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769077+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:59:41.769077+0000","last_peered":"2023-09-23T22:59:41.769077+0000","last_clean":"2023-09-23T22:59:41.769077+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:59:41.769077+0000","last_undegraded":"2023-09-23T22:59:41.769077+0000","last_fullsized":"2023-09-23T22:59:41.769077+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":270,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751755+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:59:41.751755+0000","last_peered":"2023-09-23T22:59:41.751755+0000","last_clean":"2023-09-23T22:59:41.751755+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:59:41.751755+0000","last_undegraded":"2023-09-23T22:59:41.751755+0000","last_fullsized":"2023-09-23T22:59:41.751755+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"379'8107","reported_seq":12222,"reported_epoch":379,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:51.137775+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T22:59:51.137775+0000","last_peered":"2023-09-23T22:59:51.137775+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T22:59:51.137775+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7607,"log_dups_size":0,"ondisk_log_size":7607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259325952,"num_objects":3957,"num_object_clones":0,"num_object_copies":11871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8107,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2012,"num_bytes_recovered":131858432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2398}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":345,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750579+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:59:41.750579+0000","last_peered":"2023-09-23T22:59:41.750579+0000","last_clean":"2023-09-23T22:59:41.750579+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:59:41.750579+0000","last_undegraded":"2023-09-23T22:59:41.750579+0000","last_fullsized":"2023-09-23T22:59:41.750579+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":222,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750520+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:59:41.750520+0000","last_peered":"2023-09-23T22:59:41.750520+0000","last_clean":"2023-09-23T22:59:41.750520+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:59:41.750520+0000","last_undegraded":"2023-09-23T22:59:41.750520+0000","last_fullsized":"2023-09-23T22:59:41.750520+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753663+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:59:41.753663+0000","last_peered":"2023-09-23T22:59:41.753663+0000","last_clean":"2023-09-23T22:59:41.753663+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:59:41.753663+0000","last_undegraded":"2023-09-23T22:59:41.753663+0000","last_fullsized":"2023-09-23T22:59:41.753663+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"379'8226","reported_seq":10108,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:52.181217+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T22:59:52.181217+0000","last_peered":"2023-09-23T22:59:52.181217+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T22:59:52.181217+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8226,"log_dups_size":0,"ondisk_log_size":8226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012928,"num_objects":3998,"num_object_clones":0,"num_object_copies":11994,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8226,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2565},{"shards":"0,6","objects":-983}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":362,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753281+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:59:41.753281+0000","last_peered":"2023-09-23T22:59:41.753281+0000","last_clean":"2023-09-23T22:59:41.753281+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:59:41.753281+0000","last_undegraded":"2023-09-23T22:59:41.753281+0000","last_fullsized":"2023-09-23T22:59:41.753281+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":473,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:32.725435+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T22:59:32.725435+0000","last_peered":"2023-09-23T22:59:32.725435+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T22:59:32.725435+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":48,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740517+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:59:41.740517+0000","last_peered":"2023-09-23T22:59:41.740517+0000","last_clean":"2023-09-23T22:59:41.740517+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:59:41.740517+0000","last_undegraded":"2023-09-23T22:59:41.740517+0000","last_fullsized":"2023-09-23T22:59:41.740517+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767454+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:59:41.767454+0000","last_peered":"2023-09-23T22:59:41.767454+0000","last_clean":"2023-09-23T22:59:41.767454+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:59:41.767454+0000","last_undegraded":"2023-09-23T22:59:41.767454+0000","last_fullsized":"2023-09-23T22:59:41.767454+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"378'8156","reported_seq":8242,"reported_epoch":378,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:49.564128+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T22:59:49.564128+0000","last_peered":"2023-09-23T22:59:49.564128+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T22:59:49.564128+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257425408,"num_objects":3928,"num_object_clones":0,"num_object_copies":11784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8156,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":421}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":367,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.722104+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T22:59:32.722104+0000","last_peered":"2023-09-23T22:59:32.722104+0000","last_clean":"2023-09-23T22:59:32.722104+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T22:59:32.722104+0000","last_undegraded":"2023-09-23T22:59:32.722104+0000","last_fullsized":"2023-09-23T22:59:32.722104+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":338,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.722345+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T22:59:32.722345+0000","last_peered":"2023-09-23T22:59:32.722345+0000","last_clean":"2023-09-23T22:59:32.722345+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T22:59:32.722345+0000","last_undegraded":"2023-09-23T22:59:32.722345+0000","last_fullsized":"2023-09-23T22:59:32.722345+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":298,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.724109+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:59:32.724109+0000","last_peered":"2023-09-23T22:59:32.724109+0000","last_clean":"2023-09-23T22:59:32.724109+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:59:32.724109+0000","last_undegraded":"2023-09-23T22:59:32.724109+0000","last_fullsized":"2023-09-23T22:59:32.724109+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"379'8235","reported_seq":8338,"reported_epoch":379,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:52.037743+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T22:59:52.037743+0000","last_peered":"2023-09-23T22:59:52.037743+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T22:59:52.037743+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T22:59:52.037743+0000","mapping_epoch":354,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260046872,"num_objects":3969,"num_object_clones":0,"num_object_copies":11907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3969,"num_objects_unfound":0,"num_objects_dirty":3969,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8235,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3969}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":313,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.723296+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T22:59:32.723296+0000","last_peered":"2023-09-23T22:59:32.723296+0000","last_clean":"2023-09-23T22:59:32.723296+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T22:59:32.723296+0000","last_undegraded":"2023-09-23T22:59:32.723296+0000","last_fullsized":"2023-09-23T22:59:32.723296+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":222,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751833+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:59:41.751833+0000","last_peered":"2023-09-23T22:59:41.751833+0000","last_clean":"2023-09-23T22:59:41.751833+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:59:41.751833+0000","last_undegraded":"2023-09-23T22:59:41.751833+0000","last_fullsized":"2023-09-23T22:59:41.751833+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":192,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753357+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:59:41.753357+0000","last_peered":"2023-09-23T22:59:41.753357+0000","last_clean":"2023-09-23T22:59:41.753357+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:59:41.753357+0000","last_undegraded":"2023-09-23T22:59:41.753357+0000","last_fullsized":"2023-09-23T22:59:41.753357+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"379'8208","reported_seq":11397,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:52.062169+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T22:59:52.062169+0000","last_peered":"2023-09-23T22:59:52.062169+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T22:59:52.062169+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258080768,"num_objects":3938,"num_object_clones":0,"num_object_copies":11814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8208,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1546,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":357},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":72,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.741773+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:59:41.741773+0000","last_peered":"2023-09-23T22:59:41.741773+0000","last_clean":"2023-09-23T22:59:41.741773+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:59:41.741773+0000","last_undegraded":"2023-09-23T22:59:41.741773+0000","last_fullsized":"2023-09-23T22:59:41.741773+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":87,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770741+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:59:41.770741+0000","last_peered":"2023-09-23T22:59:41.770741+0000","last_clean":"2023-09-23T22:59:41.770741+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:59:41.770741+0000","last_undegraded":"2023-09-23T22:59:41.770741+0000","last_fullsized":"2023-09-23T22:59:41.770741+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.739967+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:59:41.739967+0000","last_peered":"2023-09-23T22:59:41.739967+0000","last_clean":"2023-09-23T22:59:41.739967+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:59:41.739967+0000","last_undegraded":"2023-09-23T22:59:41.739967+0000","last_fullsized":"2023-09-23T22:59:41.739967+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"379'8159","reported_seq":8672,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:52.048865+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T22:59:52.048865+0000","last_peered":"2023-09-23T22:59:52.048865+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T22:59:52.048865+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256311296,"num_objects":3911,"num_object_clones":0,"num_object_copies":11733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3911,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8159,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":360}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":334,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751018+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:59:41.751018+0000","last_peered":"2023-09-23T22:59:41.751018+0000","last_clean":"2023-09-23T22:59:41.751018+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:59:41.751018+0000","last_undegraded":"2023-09-23T22:59:41.751018+0000","last_fullsized":"2023-09-23T22:59:41.751018+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":280,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.724010+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:59:32.724010+0000","last_peered":"2023-09-23T22:59:32.724010+0000","last_clean":"2023-09-23T22:59:32.724010+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:59:32.724010+0000","last_undegraded":"2023-09-23T22:59:32.724010+0000","last_fullsized":"2023-09-23T22:59:32.724010+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":353,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769972+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:59:41.769972+0000","last_peered":"2023-09-23T22:59:41.769972+0000","last_clean":"2023-09-23T22:59:41.769972+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:59:41.769972+0000","last_undegraded":"2023-09-23T22:59:41.769972+0000","last_fullsized":"2023-09-23T22:59:41.769972+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":315,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.723070+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:59:32.723070+0000","last_peered":"2023-09-23T22:59:32.723070+0000","last_clean":"2023-09-23T22:59:32.723070+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:59:32.723070+0000","last_undegraded":"2023-09-23T22:59:32.723070+0000","last_fullsized":"2023-09-23T22:59:32.723070+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":126,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751455+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:59:41.751455+0000","last_peered":"2023-09-23T22:59:41.751455+0000","last_clean":"2023-09-23T22:59:41.751455+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:59:41.751455+0000","last_undegraded":"2023-09-23T22:59:41.751455+0000","last_fullsized":"2023-09-23T22:59:41.751455+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":188,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767635+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:59:41.767635+0000","last_peered":"2023-09-23T22:59:41.767635+0000","last_clean":"2023-09-23T22:59:41.767635+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:59:41.767635+0000","last_undegraded":"2023-09-23T22:59:41.767635+0000","last_fullsized":"2023-09-23T22:59:41.767635+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":402,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769137+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:59:41.769137+0000","last_peered":"2023-09-23T22:59:41.769137+0000","last_clean":"2023-09-23T22:59:41.769137+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:59:41.769137+0000","last_undegraded":"2023-09-23T22:59:41.769137+0000","last_fullsized":"2023-09-23T22:59:41.769137+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":202,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767316+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:59:41.767316+0000","last_peered":"2023-09-23T22:59:41.767316+0000","last_clean":"2023-09-23T22:59:41.767316+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:59:41.767316+0000","last_undegraded":"2023-09-23T22:59:41.767316+0000","last_fullsized":"2023-09-23T22:59:41.767316+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.766715+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:59:41.766715+0000","last_peered":"2023-09-23T22:59:41.766715+0000","last_clean":"2023-09-23T22:59:41.766715+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:59:41.766715+0000","last_undegraded":"2023-09-23T22:59:41.766715+0000","last_fullsized":"2023-09-23T22:59:41.766715+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":72,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740885+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:59:41.740885+0000","last_peered":"2023-09-23T22:59:41.740885+0000","last_clean":"2023-09-23T22:59:41.740885+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:59:41.740885+0000","last_undegraded":"2023-09-23T22:59:41.740885+0000","last_fullsized":"2023-09-23T22:59:41.740885+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":75,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.526961+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:59:50.526961+0000","last_peered":"2023-09-23T22:59:50.526961+0000","last_clean":"2023-09-23T22:59:50.526961+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:59:50.526961+0000","last_undegraded":"2023-09-23T22:59:50.526961+0000","last_fullsized":"2023-09-23T22:59:50.526961+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":319,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:59:32.723761+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T22:59:32.723761+0000","last_peered":"2023-09-23T22:59:32.723761+0000","last_clean":"2023-09-23T22:59:32.723761+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T22:59:32.723761+0000","last_undegraded":"2023-09-23T22:59:32.723761+0000","last_fullsized":"2023-09-23T22:59:32.723761+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":364,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.768796+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:59:41.768796+0000","last_peered":"2023-09-23T22:59:41.768796+0000","last_clean":"2023-09-23T22:59:41.768796+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:59:41.768796+0000","last_undegraded":"2023-09-23T22:59:41.768796+0000","last_fullsized":"2023-09-23T22:59:41.768796+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4140826648,"num_objects":63185,"num_object_clones":0,"num_object_copies":189555,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":34462,"num_objects_misplaced":32845,"num_objects_unfound":0,"num_objects_dirty":63185,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130413,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10388,"num_bytes_recovered":679542784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12253540352,"data_stored":12253528136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96913,"ondisk_log_size":96913,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430602,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626827,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253140,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2223376,"kb_used_data":1543176,"kb_used_omap":9,"kb_used_meta":680182,"kb_avail":91521776,"statfs":{"total":95995035648,"available":93718298624,"internally_reserved":0,"allocated":1580212224,"data_stored":1578793573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":696507369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.5960000000000001,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 22:59:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.048,"5min":4.048,"15min":4.048},"last":0.73699999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.74299999999999999,"15min":2.153},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.827,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.754},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.83099999999999996,"15min":2.5960000000000001},"min":{"1min":0.29099999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.109,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.69699999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.7850000000000001,"15min":2.6309999999999998},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5470000000000002,"5min":151.036,"15min":151.036},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.972,"15min":2.6419999999999999},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.488,"5min":151.06899999999999,"15min":151.06899999999999},"last":1.7969999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":3.1960000000000002,"15min":2.391},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.6909999999999998,"5min":116.098,"15min":116.098},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.23,"5min":2.992,"15min":2.6890000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":7.431,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.649,"15min":2.2400000000000002},"min":{"1min":0.41599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5939999999999999,"5min":20.742000000000001,"15min":79.311000000000007},"last":1.851},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.532,"15min":1.9319999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":1.857,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.1140000000000001,"15min":1.8899999999999999},"min":{"1min":0.46700000000000003,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":2.8039999999999998,"5min":5.8410000000000002,"15min":29.539000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.395,"15min":2.0459999999999998},"min":{"1min":0.45100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7849999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":1.9019999999999999}]}]},{"osd":0,"up_from":353,"seq":1516123455510,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2781760,"kb_used_data":2644408,"kb_used_omap":37,"kb_used_meta":137306,"kb_avail":90963392,"statfs":{"total":95995035648,"available":93146513408,"internally_reserved":0,"allocated":2707873792,"data_stored":2706346572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":140601506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.6309999999999998,"5min":2.6309999999999998,"15min":2.6309999999999998},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.0539999999999998,"5min":3.0539999999999998,"15min":3.0539999999999998},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2280000000000002,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.22700000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.316},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0179999999999998,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.2770000000000001,"5min":3.2770000000000001,"15min":3.2770000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.982,"5min":1.982,"15min":1.982},"last":0.377},{"interface":"front","average":{"1min":0.998,"5min":0.998,"15min":0.998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.0920000000000001,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":4.3120000000000003,"5min":4.3120000000000003,"15min":4.3120000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.3579999999999997,"5min":4.3579999999999997,"15min":4.3579999999999997},"last":0.63100000000000001}]}]},{"osd":2,"up_from":359,"seq":1541893259283,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907412,"kb_used_data":793996,"kb_used_omap":83,"kb_used_meta":113324,"kb_avail":92837740,"statfs":{"total":95995035648,"available":95065845760,"internally_reserved":0,"allocated":813051904,"data_stored":811574325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":116044229},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":1.2}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.38},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":5.6269999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":9.2140000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":1.365},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":5.6029999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":5.5679999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":3.5819999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":1.4610000000000001},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":0.503}]}]},{"osd":3,"up_from":94,"seq":403726926087,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522672,"kb_used_data":44264,"kb_used_omap":7,"kb_used_meta":478392,"kb_avail":93222480,"statfs":{"total":95995035648,"available":95459819520,"internally_reserved":0,"allocated":45326336,"data_stored":43817286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":489874024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.972,"5min":1.972,"15min":1.972},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.78300000000000003,"15min":2.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":3.355,"5min":16.596,"15min":123.866},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95599999999999996,"15min":2.2429999999999999},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":1.746,"5min":16.649999999999999,"15min":122.265},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.1469999999999998,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":4.6769999999999996,"5min":1.798,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":87.046999999999997,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":4.6580000000000004,"5min":1.972,"15min":1.7430000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":87.066000000000003,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.76100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2410000000000001,"15min":1.3839999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5910000000000002,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.032,"5min":1.2150000000000001,"15min":1.861},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3390000000000004,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.5649999999999999,"15min":1.7090000000000001},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2520000000000007,"5min":11.116,"15min":14.097},"last":0.83399999999999996},{"interface":"front","average":{"1min":1.613,"5min":1.397,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.2810000000000006,"5min":10.859,"15min":12.887},"last":0.434}]},{"osd":7,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97899999999999998,"15min":2.1179999999999999},"min":{"1min":0.443,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1040000000000001,"5min":6.0270000000000001,"15min":91.757999999999996},"last":0.498},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.92300000000000004,"15min":2.097},"min":{"1min":0.37,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":3.4820000000000002,"5min":6.468,"15min":91.709999999999994},"last":0.55100000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019284,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3568240,"kb_used_data":2966248,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":90176912,"statfs":{"total":95995035648,"available":92341157888,"internally_reserved":0,"allocated":3037437952,"data_stored":3036679383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":2.3100000000000001},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.50700000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":3.7690000000000001,"15min":3.786},"min":{"1min":0.40000000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":1.966,"5min":98.846999999999994,"15min":132.304},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.71699999999999997,"5min":3.9740000000000002,"15min":3.6259999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":3.2269999999999999,"5min":98.938999999999993,"15min":132.453},"last":2.3540000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":2.4020000000000001},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.536,"15min":4.5099999999999998},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":3.399,"5min":8.5510000000000002,"15min":164.453},"last":2.3799999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.478,"15min":4.5720000000000001},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.99,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:59:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.9630000000000001,"15min":1.6319999999999999},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3359999999999999,"5min":148.61600000000001,"15min":148.61600000000001},"last":2.1680000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.3599999999999999,"15min":1.7669999999999999},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.248},"max":{"1min":1.6040000000000001,"5min":151.905,"15min":151.905},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5820000000000001,"15min":1.1559999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":2.113,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.423,"15min":1.101},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0350000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":1.0469999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.3080000000000001,"15min":1.0960000000000001},"min":{"1min":0.46400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0150000000000001,"5min":12.84,"15min":12.84},"last":11.045999999999999},{"interface":"front","average":{"1min":1.371,"5min":1.3180000000000001,"15min":1.02},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.587,"5min":10.587,"15min":10.587},"last":2.4239999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823058,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2759184,"kb_used_data":1863624,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":90985968,"statfs":{"total":95995035648,"available":93169631232,"internally_reserved":0,"allocated":1908350976,"data_stored":1907620195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":2.6150000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2150000000000001,"15min":2.2850000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24199999999999999},"max":{"1min":3.21,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.748,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":6.7690000000000001,"5min":10.353,"15min":110.89},"last":2.6669999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"last":2.7570000000000001},{"interface":"front","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":2.7669999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.304,"15min":4.9619999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":3.1400000000000001,"5min":8.6489999999999991,"15min":257.05000000000001},"last":9.2710000000000008},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.4850000000000001,"15min":4.9610000000000003},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.965,"5min":12.185,"15min":257.35000000000002},"last":9.3399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.802,"15min":1.4570000000000001},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6850000000000001,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.661,"15min":1.4510000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.1880000000000002,"5min":114.908,"15min":114.908},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.345,"15min":1.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":6.8150000000000004,"5min":11.121,"15min":12.615},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.2130000000000001,"15min":1.123},"min":{"1min":0.433,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.2719999999999998,"5min":12.137,"15min":12.74},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.159,"15min":1.0389999999999999},"min":{"1min":0.44600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.3029999999999999,"5min":12.23,"15min":12.23},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.651,"5min":1.381,"15min":1.0389999999999999},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":10.638999999999999,"5min":11.112,"15min":11.112},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2699890688,"data_stored":2699886616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1572470784,"data_stored":1572470784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":807206912,"data_stored":807206912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":40763392,"data_stored":40763392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3033141248,"data_stored":3033137176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1902182400,"data_stored":1902182400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2197884928,"data_stored":2197880856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:59:55.893 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:55.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:56.137 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:56.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:55 smithi052 ceph-mon[187743]: pgmap v1120: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 34464/189627 objects degraded (18.175%); 32845/189627 objects misplaced (17.321%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:56.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:55 smithi052 ceph-mon[187743]: from='client.46406 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:56.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 34464/189627 objects degraded (18.175%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:56.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:56.140 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:55 smithi052 ceph-mon[180979]: pgmap v1120: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 34464/189627 objects degraded (18.175%); 32845/189627 objects misplaced (17.321%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:56.140 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:55 smithi052 ceph-mon[180979]: from='client.46406 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:56.140 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 34464/189627 objects degraded (18.175%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:55 smithi151 ceph-mon[159210]: pgmap v1120: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 34464/189627 objects degraded (18.175%); 32845/189627 objects misplaced (17.321%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:55 smithi151 ceph-mon[159210]: from='client.46406 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:59:56.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 34464/189627 objects degraded (18.175%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:59:56.227 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T22:59:56.228 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:59:56.241 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:56.268 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:56.335 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:56.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:56.452 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:56.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:56.557 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:56.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:56.663 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:56.697 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:56.766 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:56.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:56.877 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:56.907 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:56.974 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:56.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:57.090 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:59:57.143 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:57.226 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:59:57.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:57.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:59:57.335 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:57.395 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:57.422 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:59:57.449 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:59:57.468 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:57.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:57.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:59:57.637 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:57.712 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:59:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:57.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:59:57.818 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:57.864 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:57.922 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:57.925 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:57 smithi151 ceph-mon[159210]: pgmap v1121: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 34242/188448 objects degraded (18.171%); 32738/188448 objects misplaced (17.372%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:57.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:57.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:57.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:57 smithi052 ceph-mon[180979]: pgmap v1121: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 34242/188448 objects degraded (18.171%); 32738/188448 objects misplaced (17.372%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:57.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:57 smithi052 ceph-mon[187743]: pgmap v1121: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 34242/188448 objects degraded (18.171%); 32738/188448 objects misplaced (17.372%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:59:58.027 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:59:58.062 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:58.125 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:59:58.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:58.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:59:58.230 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:59:58.262 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:58.314 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:59:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:58.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:59:58.361 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:59:58.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:59:58.473 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:58.588 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:59:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:58.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:59:58.700 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:59:58.742 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:58.820 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T22:59:58.834 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:59:58.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3209124231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:58.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:58 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:58.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3209124231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:58.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:58 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:58.990 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3209124231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:58.991 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:58 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:59:59.025 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:59:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:59.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:59:59.131 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:59:59.165 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:59:59.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T22:59:59.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T22:59:59.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T22:59:59.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T22:59:59.465 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T22:59:59.490 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T22:59:59.511 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T22:59:59.529 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.531 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:59:59.876 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:59:59.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:59:59.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:59:59.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:00.010 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:00.078 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: pgmap v1122: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 33986/186855 objects degraded (18.188%); 32564/186855 objects misplaced (17.427%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:00.078 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: osdmap e380: 8 total, 8 up, 6 in 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 22:59:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/326870314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:00.079 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: pgmap v1122: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 33986/186855 objects degraded (18.188%); 32564/186855 objects misplaced (17.427%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: osdmap e380: 8 total, 8 up, 6 in 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/326870314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: pgmap v1122: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 33986/186855 objects degraded (18.188%); 32564/186855 objects misplaced (17.427%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:00.080 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:00.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:00.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: osdmap e380: 8 total, 8 up, 6 in 2023-09-23T23:00:00.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: failed to encode map e380 with expected crc 2023-09-23T23:00:00.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 22:59:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/326870314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:00.404 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:00.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:00.453 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:00.509 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:00.529 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:132242 10.f 10:fe165625:::benchmark_data_smithi052_461816_object35441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e380)", 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:00:00.525334+0000", 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0035958629999999999, 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036295640000000001, 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:00.530 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 132242 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525334+0000", 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:00.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525334+0000", 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525333+0000", 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4200000000000004e-06 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525339+0000", 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2272e-05 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525351+0000", 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.024e-06 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525358+0000", 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5823999999999999e-05 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:00.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525414+0000", 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114654 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525529+0000", 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014269099999999999 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 2,3,6", 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.525672+0000", 2023-09-23T23:00:00.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031601860000000002 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.528832+0000", 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011480999999999999 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:00:00.528947+0000", 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036127149999999999 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:00.535 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:00.536 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:00.536 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T23:00:00.536 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 1 2023-09-23T23:00:00.536 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:00:00.547 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:00.600 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:00.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:00.704 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:00.745 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:00.812 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:00.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:00.816 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: [WRN] PG_DEGRADED: Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 1.0 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,1] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.0 is active+recovery_wait+degraded, acting [4,1,0] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.1 is active+recovery_wait+degraded, acting [4,0,1] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.2 is active+recovery_wait+degraded, acting [4,1,0] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.3 is stuck undersized for 95s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,5] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.4 is active+recovery_wait+degraded+remapped, acting [0,4,1] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.5 is stuck undersized for 95s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.7 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.8 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-23T23:00:00.817 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.9 is stuck undersized for 94s, current state active+recovering+undersized+degraded+remapped, last acting [5,4] 2023-09-23T23:00:00.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.b is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-09-23T23:00:00.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.d is active+recovery_wait+degraded+remapped, acting [4,0,6] 2023-09-23T23:00:00.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.e is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,5] 2023-09-23T23:00:00.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: pg 10.f is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2023-09-23T23:00:00.818 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:00.917 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:00.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized 2023-09-23T23:00:00.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T23:00:00.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: [WRN] PG_DEGRADED: Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized 2023-09-23T23:00:00.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 1.0 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,1] 2023-09-23T23:00:00.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.0 is active+recovery_wait+degraded, acting [4,1,0] 2023-09-23T23:00:00.949 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.1 is active+recovery_wait+degraded, acting [4,0,1] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.2 is active+recovery_wait+degraded, acting [4,1,0] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.3 is stuck undersized for 95s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,5] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.4 is active+recovery_wait+degraded+remapped, acting [0,4,1] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.5 is stuck undersized for 95s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.7 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.8 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.9 is stuck undersized for 94s, current state active+recovering+undersized+degraded+remapped, last acting [5,4] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.b is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.d is active+recovery_wait+degraded+remapped, acting [4,0,6] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.e is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,5] 2023-09-23T23:00:00.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: pg 10.f is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: [WRN] PG_DEGRADED: Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 1.0 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,1] 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.0 is active+recovery_wait+degraded, acting [4,1,0] 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.1 is active+recovery_wait+degraded, acting [4,0,1] 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.2 is active+recovery_wait+degraded, acting [4,1,0] 2023-09-23T23:00:00.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.3 is stuck undersized for 95s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,5] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.4 is active+recovery_wait+degraded+remapped, acting [0,4,1] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.5 is stuck undersized for 95s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.7 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.8 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.9 is stuck undersized for 94s, current state active+recovering+undersized+degraded+remapped, last acting [5,4] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.b is stuck undersized for 110s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.d is active+recovery_wait+degraded+remapped, acting [4,0,6] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.e is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,5] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: pg 10.f is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2023-09-23T23:00:00.952 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 33986/186855 objects degraded (18.188%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:00.953 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:01.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:01.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:01.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:01.294 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:01.363 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:01.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:01.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:01.469 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:01.496 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:00:01.502 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:01.616 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:01.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:01.717 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:01.755 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:01.811 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:01.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:01.815 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:01 smithi151 ceph-mon[159210]: pgmap v1124: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 33919/185856 objects degraded (18.250%); 32436/185856 objects misplaced (17.452%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:01.877 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:01 smithi052 ceph-mon[187743]: pgmap v1124: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 33919/185856 objects degraded (18.250%); 32436/185856 objects misplaced (17.452%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:01.877 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:01 smithi052 ceph-mon[180979]: pgmap v1124: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 33919/185856 objects degraded (18.250%); 32436/185856 objects misplaced (17.452%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:01.917 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:01.953 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:01.957 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:02.015 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:02.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:02.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:02.098 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:00:02.123 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:02.168 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:02.293 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:02.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:02.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:02.336 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:02.347 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:00:02.403 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:02.411 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:00:02.460 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:02.542 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:02.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:02.650 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:02.678 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:02.741 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:02.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:02.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:02.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:02.905 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:03.001 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:03.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:03.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:03.105 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:03.126 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1124,"stamp":"2023-09-23T23:00:00.562573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4059043768,"num_objects":61952,"num_object_clones":0,"num_object_copies":185856,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":33919,"num_objects_misplaced":32436,"num_objects_unfound":0,"num_objects_dirty":61952,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":131930,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10792,"num_bytes_recovered":703740860,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96864,"ondisk_log_size":96864,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12580852,"kb_used_data":9680388,"kb_used_omap":137,"kb_used_meta":2900214,"kb_avail":549890060,"statfs":{"total":575970213888,"available":563087421440,"internally_reserved":0,"allocated":9912717312,"data_stored":9905280050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":2969819320},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-132055040,"num_objects":-2015,"num_object_clones":0,"num_object_copies":-6045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1015,"num_objects_misplaced":-599,"num_objects_unfound":0,"num_objects_dirty":-2015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2015,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":734,"num_bytes_recovered":48103424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003323"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":391,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.766713+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:59:41.766713+0000","last_peered":"2023-09-23T22:59:41.766713+0000","last_clean":"2023-09-23T22:59:41.766713+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:59:41.766713+0000","last_undegraded":"2023-09-23T22:59:41.766713+0000","last_fullsized":"2023-09-23T22:59:41.766713+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":48,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.527706+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:59:50.527706+0000","last_peered":"2023-09-23T22:59:50.527706+0000","last_clean":"2023-09-23T22:59:50.527706+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:59:50.527706+0000","last_undegraded":"2023-09-23T22:59:50.527706+0000","last_fullsized":"2023-09-23T22:59:50.527706+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.526841+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:59:50.526841+0000","last_peered":"2023-09-23T22:59:50.526841+0000","last_clean":"2023-09-23T22:59:50.526841+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:59:50.526841+0000","last_undegraded":"2023-09-23T22:59:50.526841+0000","last_fullsized":"2023-09-23T22:59:50.526841+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":375,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740647+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:59:41.740647+0000","last_peered":"2023-09-23T22:59:41.740647+0000","last_clean":"2023-09-23T22:59:41.740647+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:59:41.740647+0000","last_undegraded":"2023-09-23T22:59:41.740647+0000","last_fullsized":"2023-09-23T22:59:41.740647+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":63,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.525949+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:59:50.525949+0000","last_peered":"2023-09-23T22:59:50.525949+0000","last_clean":"2023-09-23T22:59:50.525949+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:59:50.525949+0000","last_undegraded":"2023-09-23T22:59:50.525949+0000","last_fullsized":"2023-09-23T22:59:50.525949+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":49,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515346+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T22:59:50.515346+0000","last_peered":"2023-09-23T22:59:50.515346+0000","last_clean":"2023-09-23T22:59:50.515346+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T22:59:50.515346+0000","last_undegraded":"2023-09-23T22:59:50.515346+0000","last_fullsized":"2023-09-23T22:59:50.515346+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":314,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753620+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:59:41.753620+0000","last_peered":"2023-09-23T22:59:41.753620+0000","last_clean":"2023-09-23T22:59:41.753620+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:59:41.753620+0000","last_undegraded":"2023-09-23T22:59:41.753620+0000","last_fullsized":"2023-09-23T22:59:41.753620+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":57,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.525821+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:59:50.525821+0000","last_peered":"2023-09-23T22:59:50.525821+0000","last_clean":"2023-09-23T22:59:50.525821+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:59:50.525821+0000","last_undegraded":"2023-09-23T22:59:50.525821+0000","last_fullsized":"2023-09-23T22:59:50.525821+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":275,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753270+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:59:41.753270+0000","last_peered":"2023-09-23T22:59:41.753270+0000","last_clean":"2023-09-23T22:59:41.753270+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:59:41.753270+0000","last_undegraded":"2023-09-23T22:59:41.753270+0000","last_fullsized":"2023-09-23T22:59:41.753270+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.526726+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:59:50.526726+0000","last_peered":"2023-09-23T22:59:50.526726+0000","last_clean":"2023-09-23T22:59:50.526726+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:59:50.526726+0000","last_undegraded":"2023-09-23T22:59:50.526726+0000","last_fullsized":"2023-09-23T22:59:50.526726+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":68,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.527622+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:59:50.527622+0000","last_peered":"2023-09-23T22:59:50.527622+0000","last_clean":"2023-09-23T22:59:50.527622+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:59:50.527622+0000","last_undegraded":"2023-09-23T22:59:50.527622+0000","last_fullsized":"2023-09-23T22:59:50.527622+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750689+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:59:41.750689+0000","last_peered":"2023-09-23T22:59:41.750689+0000","last_clean":"2023-09-23T22:59:41.750689+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:59:41.750689+0000","last_undegraded":"2023-09-23T22:59:41.750689+0000","last_fullsized":"2023-09-23T22:59:41.750689+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":450,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767786+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:59:41.767786+0000","last_peered":"2023-09-23T22:59:41.767786+0000","last_clean":"2023-09-23T22:59:41.767786+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:59:41.767786+0000","last_undegraded":"2023-09-23T22:59:41.767786+0000","last_fullsized":"2023-09-23T22:59:41.767786+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":353,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.768924+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:59:41.768924+0000","last_peered":"2023-09-23T22:59:41.768924+0000","last_clean":"2023-09-23T22:59:41.768924+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:59:41.768924+0000","last_undegraded":"2023-09-23T22:59:41.768924+0000","last_fullsized":"2023-09-23T22:59:41.768924+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":279,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769007+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:59:41.769007+0000","last_peered":"2023-09-23T22:59:41.769007+0000","last_clean":"2023-09-23T22:59:41.769007+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:59:41.769007+0000","last_undegraded":"2023-09-23T22:59:41.769007+0000","last_fullsized":"2023-09-23T22:59:41.769007+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:46.231649+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:59:46.231649+0000","last_peered":"2023-09-23T22:59:46.231649+0000","last_clean":"2023-09-23T22:59:46.231649+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:59:46.231649+0000","last_undegraded":"2023-09-23T22:59:46.231649+0000","last_fullsized":"2023-09-23T22:59:46.231649+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":368,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.525946+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:59:50.525946+0000","last_peered":"2023-09-23T22:59:50.525946+0000","last_clean":"2023-09-23T22:59:50.525946+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:59:50.525946+0000","last_undegraded":"2023-09-23T22:59:50.525946+0000","last_fullsized":"2023-09-23T22:59:50.525946+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":279,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753028+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:59:41.753028+0000","last_peered":"2023-09-23T22:59:41.753028+0000","last_clean":"2023-09-23T22:59:41.753028+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:59:41.753028+0000","last_undegraded":"2023-09-23T22:59:41.753028+0000","last_fullsized":"2023-09-23T22:59:41.753028+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":264,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770150+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:59:41.770150+0000","last_peered":"2023-09-23T22:59:41.770150+0000","last_clean":"2023-09-23T22:59:41.770150+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:59:41.770150+0000","last_undegraded":"2023-09-23T22:59:41.770150+0000","last_fullsized":"2023-09-23T22:59:41.770150+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":391,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740760+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T22:59:41.740760+0000","last_peered":"2023-09-23T22:59:41.740760+0000","last_clean":"2023-09-23T22:59:41.740760+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T22:59:41.740760+0000","last_undegraded":"2023-09-23T22:59:41.740760+0000","last_fullsized":"2023-09-23T22:59:41.740760+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":97,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.514981+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:59:50.514981+0000","last_peered":"2023-09-23T22:59:50.514981+0000","last_clean":"2023-09-23T22:59:50.514981+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:59:50.514981+0000","last_undegraded":"2023-09-23T22:59:50.514981+0000","last_fullsized":"2023-09-23T22:59:50.514981+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.527640+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:59:50.527640+0000","last_peered":"2023-09-23T22:59:50.527640+0000","last_clean":"2023-09-23T22:59:50.527640+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:59:50.527640+0000","last_undegraded":"2023-09-23T22:59:50.527640+0000","last_fullsized":"2023-09-23T22:59:50.527640+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"380'8206","reported_seq":8289,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:59.617543+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T22:59:59.617543+0000","last_peered":"2023-09-23T22:59:59.617543+0000","last_clean":"2023-09-23T22:59:59.617543+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T22:59:59.617543+0000","last_undegraded":"2023-09-23T22:59:59.617543+0000","last_fullsized":"2023-09-23T22:59:59.617543+0000","mapping_epoch":353,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254279680,"num_objects":3880,"num_object_clones":0,"num_object_copies":11640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8206,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":263,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770839+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:59:41.770839+0000","last_peered":"2023-09-23T22:59:41.770839+0000","last_clean":"2023-09-23T22:59:41.770839+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:59:41.770839+0000","last_undegraded":"2023-09-23T22:59:41.770839+0000","last_fullsized":"2023-09-23T22:59:41.770839+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.752552+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:59:41.752552+0000","last_peered":"2023-09-23T22:59:41.752552+0000","last_clean":"2023-09-23T22:59:41.752552+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:59:41.752552+0000","last_undegraded":"2023-09-23T22:59:41.752552+0000","last_fullsized":"2023-09-23T22:59:41.752552+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"379'8105","reported_seq":10517,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:55.869873+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T22:59:55.869873+0000","last_peered":"2023-09-23T22:59:55.869873+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T22:59:55.869873+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251985920,"num_objects":3845,"num_object_clones":0,"num_object_copies":11535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3845,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8105,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":278}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":273,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767985+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:59:41.767985+0000","last_peered":"2023-09-23T22:59:41.767985+0000","last_clean":"2023-09-23T22:59:41.767985+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:59:41.767985+0000","last_undegraded":"2023-09-23T22:59:41.767985+0000","last_fullsized":"2023-09-23T22:59:41.767985+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":386,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770083+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:59:41.770083+0000","last_peered":"2023-09-23T22:59:41.770083+0000","last_clean":"2023-09-23T22:59:41.770083+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:59:41.770083+0000","last_undegraded":"2023-09-23T22:59:41.770083+0000","last_fullsized":"2023-09-23T22:59:41.770083+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"380'8260","reported_seq":8337,"reported_epoch":380,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:59.755564+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T22:59:59.755564+0000","last_peered":"2023-09-23T22:59:59.755564+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T22:59:59.755564+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T22:59:59.755564+0000","mapping_epoch":354,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5460,"log_dups_size":0,"ondisk_log_size":5460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249823232,"num_objects":3812,"num_object_clones":0,"num_object_copies":11436,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3812,"num_objects_unfound":0,"num_objects_dirty":3812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8260,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3812}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":130,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753054+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:59:41.753054+0000","last_peered":"2023-09-23T22:59:41.753054+0000","last_clean":"2023-09-23T22:59:41.753054+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:59:41.753054+0000","last_undegraded":"2023-09-23T22:59:41.753054+0000","last_fullsized":"2023-09-23T22:59:41.753054+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753050+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:59:41.753050+0000","last_peered":"2023-09-23T22:59:41.753050+0000","last_clean":"2023-09-23T22:59:41.753050+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:59:41.753050+0000","last_undegraded":"2023-09-23T22:59:41.753050+0000","last_fullsized":"2023-09-23T22:59:41.753050+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"380'8322","reported_seq":8585,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:59.732600+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T22:59:59.732600+0000","last_peered":"2023-09-23T22:59:59.732600+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T22:59:59.732600+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5522,"log_dups_size":0,"ondisk_log_size":5522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252444672,"num_objects":3852,"num_object_clones":0,"num_object_copies":11556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3251,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8322,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":242}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":48,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.742480+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:59:41.742480+0000","last_peered":"2023-09-23T22:59:41.742480+0000","last_clean":"2023-09-23T22:59:41.742480+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:59:41.742480+0000","last_undegraded":"2023-09-23T22:59:41.742480+0000","last_fullsized":"2023-09-23T22:59:41.742480+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":278,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515644+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:59:50.515644+0000","last_peered":"2023-09-23T22:59:50.515644+0000","last_clean":"2023-09-23T22:59:50.515644+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:59:50.515644+0000","last_undegraded":"2023-09-23T22:59:50.515644+0000","last_fullsized":"2023-09-23T22:59:50.515644+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"379'8203","reported_seq":8287,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:57.022047+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T22:59:57.022047+0000","last_peered":"2023-09-23T22:59:57.022047+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T22:59:57.022047+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T22:59:57.022047+0000","mapping_epoch":353,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255000576,"num_objects":3891,"num_object_clones":0,"num_object_copies":11673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8203,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3891}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":67,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740410+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:59:41.740410+0000","last_peered":"2023-09-23T22:59:41.740410+0000","last_clean":"2023-09-23T22:59:41.740410+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:59:41.740410+0000","last_undegraded":"2023-09-23T22:59:41.740410+0000","last_fullsized":"2023-09-23T22:59:41.740410+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":372,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767704+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:59:41.767704+0000","last_peered":"2023-09-23T22:59:41.767704+0000","last_clean":"2023-09-23T22:59:41.767704+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:59:41.767704+0000","last_undegraded":"2023-09-23T22:59:41.767704+0000","last_fullsized":"2023-09-23T22:59:41.767704+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"379'8276","reported_seq":8368,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:57.051002+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T22:59:57.051002+0000","last_peered":"2023-09-23T22:59:57.051002+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T22:59:57.051002+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257294336,"num_objects":3926,"num_object_clones":0,"num_object_copies":11778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3926,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8276,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":307}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":162,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.766798+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:59:41.766798+0000","last_peered":"2023-09-23T22:59:41.766798+0000","last_clean":"2023-09-23T22:59:41.766798+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:59:41.766798+0000","last_undegraded":"2023-09-23T22:59:41.766798+0000","last_fullsized":"2023-09-23T22:59:41.766798+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":389,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767233+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:59:41.767233+0000","last_peered":"2023-09-23T22:59:41.767233+0000","last_clean":"2023-09-23T22:59:41.767233+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:59:41.767233+0000","last_undegraded":"2023-09-23T22:59:41.767233+0000","last_fullsized":"2023-09-23T22:59:41.767233+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"379'8201","reported_seq":12169,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:56.833410+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T22:59:56.833410+0000","last_peered":"2023-09-23T22:59:56.833410+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T22:59:56.833410+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T22:59:56.833410+0000","mapping_epoch":354,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252903424,"num_objects":3859,"num_object_clones":0,"num_object_copies":11577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8201,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1934,"num_bytes_recovered":126746624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":2425}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":402,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769907+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:59:41.769907+0000","last_peered":"2023-09-23T22:59:41.769907+0000","last_clean":"2023-09-23T22:59:41.769907+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:59:41.769907+0000","last_undegraded":"2023-09-23T22:59:41.769907+0000","last_fullsized":"2023-09-23T22:59:41.769907+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":380,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.768316+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:59:41.768316+0000","last_peered":"2023-09-23T22:59:41.768316+0000","last_clean":"2023-09-23T22:59:41.768316+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:59:41.768316+0000","last_undegraded":"2023-09-23T22:59:41.768316+0000","last_fullsized":"2023-09-23T22:59:41.768316+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"379'8074","reported_seq":8158,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:59:56.901742+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T22:59:56.901742+0000","last_peered":"2023-09-23T22:59:56.901742+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T22:59:56.901742+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T22:59:56.901742+0000","mapping_epoch":353,"log_start":"325'2400","ondisk_log_start":"325'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255197184,"num_objects":3894,"num_object_clones":0,"num_object_copies":11682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8074,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3894}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":48,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740113+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:59:41.740113+0000","last_peered":"2023-09-23T22:59:41.740113+0000","last_clean":"2023-09-23T22:59:41.740113+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:59:41.740113+0000","last_undegraded":"2023-09-23T22:59:41.740113+0000","last_fullsized":"2023-09-23T22:59:41.740113+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":281,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750881+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:59:41.750881+0000","last_peered":"2023-09-23T22:59:41.750881+0000","last_clean":"2023-09-23T22:59:41.750881+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:59:41.750881+0000","last_undegraded":"2023-09-23T22:59:41.750881+0000","last_fullsized":"2023-09-23T22:59:41.750881+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"379'8197","reported_seq":13775,"reported_epoch":379,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:59:57.084795+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T22:59:57.084795+0000","last_peered":"2023-09-23T22:59:57.084795+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T22:59:57.084795+0000","last_undegraded":"2023-09-23T22:59:57.084795+0000","last_fullsized":"2023-09-23T22:59:57.084795+0000","mapping_epoch":354,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254214144,"num_objects":3879,"num_object_clones":0,"num_object_copies":11637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":935,"num_objects_unfound":0,"num_objects_dirty":3879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8197,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2730,"num_bytes_recovered":178388992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2944},{"shards":"0,4,6","objects":935}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":285,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751675+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:59:41.751675+0000","last_peered":"2023-09-23T22:59:41.751675+0000","last_clean":"2023-09-23T22:59:41.751675+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:59:41.751675+0000","last_undegraded":"2023-09-23T22:59:41.751675+0000","last_fullsized":"2023-09-23T22:59:41.751675+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.752953+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:59:41.752953+0000","last_peered":"2023-09-23T22:59:41.752953+0000","last_clean":"2023-09-23T22:59:41.752953+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:59:41.752953+0000","last_undegraded":"2023-09-23T22:59:41.752953+0000","last_fullsized":"2023-09-23T22:59:41.752953+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.752900+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:59:41.752900+0000","last_peered":"2023-09-23T22:59:41.752900+0000","last_clean":"2023-09-23T22:59:41.752900+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:59:41.752900+0000","last_undegraded":"2023-09-23T22:59:41.752900+0000","last_fullsized":"2023-09-23T22:59:41.752900+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"380'8273","reported_seq":8369,"reported_epoch":380,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:59:59.726608+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T22:59:59.726608+0000","last_peered":"2023-09-23T22:59:59.726608+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T22:59:59.726608+0000","last_undegraded":"2023-09-23T22:59:59.726608+0000","last_fullsized":"2023-09-23T22:59:59.726608+0000","mapping_epoch":359,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252379136,"num_objects":3851,"num_object_clones":0,"num_object_copies":11553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3851,"num_objects_unfound":0,"num_objects_dirty":3851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8273,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3851}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":49,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515868+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T22:59:50.515868+0000","last_peered":"2023-09-23T22:59:50.515868+0000","last_clean":"2023-09-23T22:59:50.515868+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T22:59:50.515868+0000","last_undegraded":"2023-09-23T22:59:50.515868+0000","last_fullsized":"2023-09-23T22:59:50.515868+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":399,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769077+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:59:41.769077+0000","last_peered":"2023-09-23T22:59:41.769077+0000","last_clean":"2023-09-23T22:59:41.769077+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:59:41.769077+0000","last_undegraded":"2023-09-23T22:59:41.769077+0000","last_fullsized":"2023-09-23T22:59:41.769077+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":270,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751755+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:59:41.751755+0000","last_peered":"2023-09-23T22:59:41.751755+0000","last_clean":"2023-09-23T22:59:41.751755+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:59:41.751755+0000","last_undegraded":"2023-09-23T22:59:41.751755+0000","last_fullsized":"2023-09-23T22:59:41.751755+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"379'8156","reported_seq":12505,"reported_epoch":379,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:56.159532+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T22:59:56.159532+0000","last_peered":"2023-09-23T22:59:56.159532+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T22:59:56.159532+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7656,"log_dups_size":0,"ondisk_log_size":7656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256114688,"num_objects":3908,"num_object_clones":0,"num_object_copies":11724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1442,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8156,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2129,"num_bytes_recovered":139526144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2466}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":345,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750579+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:59:41.750579+0000","last_peered":"2023-09-23T22:59:41.750579+0000","last_clean":"2023-09-23T22:59:41.750579+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:59:41.750579+0000","last_undegraded":"2023-09-23T22:59:41.750579+0000","last_fullsized":"2023-09-23T22:59:41.750579+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":222,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.750520+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:59:41.750520+0000","last_peered":"2023-09-23T22:59:41.750520+0000","last_clean":"2023-09-23T22:59:41.750520+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:59:41.750520+0000","last_undegraded":"2023-09-23T22:59:41.750520+0000","last_fullsized":"2023-09-23T22:59:41.750520+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753663+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:59:41.753663+0000","last_peered":"2023-09-23T22:59:41.753663+0000","last_clean":"2023-09-23T22:59:41.753663+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:59:41.753663+0000","last_undegraded":"2023-09-23T22:59:41.753663+0000","last_fullsized":"2023-09-23T22:59:41.753663+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"379'8302","reported_seq":10184,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:57.184752+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T22:59:57.184752+0000","last_peered":"2023-09-23T22:59:57.184752+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T22:59:57.184752+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8302,"log_dups_size":0,"ondisk_log_size":8302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257032192,"num_objects":3922,"num_object_clones":0,"num_object_copies":11766,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8302,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2489},{"shards":"0,6","objects":-1059}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":362,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753281+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:59:41.753281+0000","last_peered":"2023-09-23T22:59:41.753281+0000","last_clean":"2023-09-23T22:59:41.753281+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:59:41.753281+0000","last_undegraded":"2023-09-23T22:59:41.753281+0000","last_fullsized":"2023-09-23T22:59:41.753281+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":475,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:50.515466+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T22:59:50.515466+0000","last_peered":"2023-09-23T22:59:50.515466+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T22:59:50.515466+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":48,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740517+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:59:41.740517+0000","last_peered":"2023-09-23T22:59:41.740517+0000","last_clean":"2023-09-23T22:59:41.740517+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:59:41.740517+0000","last_undegraded":"2023-09-23T22:59:41.740517+0000","last_fullsized":"2023-09-23T22:59:41.740517+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767454+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:59:41.767454+0000","last_peered":"2023-09-23T22:59:41.767454+0000","last_clean":"2023-09-23T22:59:41.767454+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:59:41.767454+0000","last_undegraded":"2023-09-23T22:59:41.767454+0000","last_fullsized":"2023-09-23T22:59:41.767454+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"380'8293","reported_seq":8381,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:59.752623+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T22:59:59.752623+0000","last_peered":"2023-09-23T22:59:59.752623+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T22:59:59.752623+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248446976,"num_objects":3791,"num_object_clones":0,"num_object_copies":11373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8293,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":284}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":369,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515100+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T22:59:50.515100+0000","last_peered":"2023-09-23T22:59:50.515100+0000","last_clean":"2023-09-23T22:59:50.515100+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T22:59:50.515100+0000","last_undegraded":"2023-09-23T22:59:50.515100+0000","last_fullsized":"2023-09-23T22:59:50.515100+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":340,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515158+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T22:59:50.515158+0000","last_peered":"2023-09-23T22:59:50.515158+0000","last_clean":"2023-09-23T22:59:50.515158+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T22:59:50.515158+0000","last_undegraded":"2023-09-23T22:59:50.515158+0000","last_fullsized":"2023-09-23T22:59:50.515158+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":300,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515858+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:59:50.515858+0000","last_peered":"2023-09-23T22:59:50.515858+0000","last_clean":"2023-09-23T22:59:50.515858+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:59:50.515858+0000","last_undegraded":"2023-09-23T22:59:50.515858+0000","last_fullsized":"2023-09-23T22:59:50.515858+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"379'8293","reported_seq":8396,"reported_epoch":379,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:59:56.763224+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T22:59:56.763224+0000","last_peered":"2023-09-23T22:59:56.763224+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T22:59:56.763224+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T22:59:56.763224+0000","mapping_epoch":354,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256245784,"num_objects":3911,"num_object_clones":0,"num_object_copies":11733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3911,"num_objects_unfound":0,"num_objects_dirty":3911,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8293,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3911}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":315,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.516045+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T22:59:50.516045+0000","last_peered":"2023-09-23T22:59:50.516045+0000","last_clean":"2023-09-23T22:59:50.516045+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T22:59:50.516045+0000","last_undegraded":"2023-09-23T22:59:50.516045+0000","last_fullsized":"2023-09-23T22:59:50.516045+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":222,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751833+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:59:41.751833+0000","last_peered":"2023-09-23T22:59:41.751833+0000","last_clean":"2023-09-23T22:59:41.751833+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:59:41.751833+0000","last_undegraded":"2023-09-23T22:59:41.751833+0000","last_fullsized":"2023-09-23T22:59:41.751833+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":192,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.753357+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:59:41.753357+0000","last_peered":"2023-09-23T22:59:41.753357+0000","last_clean":"2023-09-23T22:59:41.753357+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:59:41.753357+0000","last_undegraded":"2023-09-23T22:59:41.753357+0000","last_fullsized":"2023-09-23T22:59:41.753357+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"379'8282","reported_seq":11619,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:57.094002+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T22:59:57.094002+0000","last_peered":"2023-09-23T22:59:57.094002+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T22:59:57.094002+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253231104,"num_objects":3864,"num_object_clones":0,"num_object_copies":11592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7732,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8282,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1620,"num_bytes_recovered":106168320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":283},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":72,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.741773+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:59:41.741773+0000","last_peered":"2023-09-23T22:59:41.741773+0000","last_clean":"2023-09-23T22:59:41.741773+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:59:41.741773+0000","last_undegraded":"2023-09-23T22:59:41.741773+0000","last_fullsized":"2023-09-23T22:59:41.741773+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":87,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.770741+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:59:41.770741+0000","last_peered":"2023-09-23T22:59:41.770741+0000","last_clean":"2023-09-23T22:59:41.770741+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:59:41.770741+0000","last_undegraded":"2023-09-23T22:59:41.770741+0000","last_fullsized":"2023-09-23T22:59:41.770741+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.739967+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:59:41.739967+0000","last_peered":"2023-09-23T22:59:41.739967+0000","last_clean":"2023-09-23T22:59:41.739967+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:59:41.739967+0000","last_undegraded":"2023-09-23T22:59:41.739967+0000","last_fullsized":"2023-09-23T22:59:41.739967+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"379'8227","reported_seq":8740,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:59:57.062154+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T22:59:57.062154+0000","last_peered":"2023-09-23T22:59:57.062154+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T22:59:57.062154+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251854848,"num_objects":3843,"num_object_clones":0,"num_object_copies":11529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3843,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8227,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":292}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":334,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751018+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:59:41.751018+0000","last_peered":"2023-09-23T22:59:41.751018+0000","last_clean":"2023-09-23T22:59:41.751018+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:59:41.751018+0000","last_undegraded":"2023-09-23T22:59:41.751018+0000","last_fullsized":"2023-09-23T22:59:41.751018+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":282,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515773+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:59:50.515773+0000","last_peered":"2023-09-23T22:59:50.515773+0000","last_clean":"2023-09-23T22:59:50.515773+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:59:50.515773+0000","last_undegraded":"2023-09-23T22:59:50.515773+0000","last_fullsized":"2023-09-23T22:59:50.515773+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":353,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769972+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:59:41.769972+0000","last_peered":"2023-09-23T22:59:41.769972+0000","last_clean":"2023-09-23T22:59:41.769972+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:59:41.769972+0000","last_undegraded":"2023-09-23T22:59:41.769972+0000","last_fullsized":"2023-09-23T22:59:41.769972+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":317,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515773+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:59:50.515773+0000","last_peered":"2023-09-23T22:59:50.515773+0000","last_clean":"2023-09-23T22:59:50.515773+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:59:50.515773+0000","last_undegraded":"2023-09-23T22:59:50.515773+0000","last_fullsized":"2023-09-23T22:59:50.515773+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":126,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.751455+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:59:41.751455+0000","last_peered":"2023-09-23T22:59:41.751455+0000","last_clean":"2023-09-23T22:59:41.751455+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:59:41.751455+0000","last_undegraded":"2023-09-23T22:59:41.751455+0000","last_fullsized":"2023-09-23T22:59:41.751455+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":188,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767635+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:59:41.767635+0000","last_peered":"2023-09-23T22:59:41.767635+0000","last_clean":"2023-09-23T22:59:41.767635+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:59:41.767635+0000","last_undegraded":"2023-09-23T22:59:41.767635+0000","last_fullsized":"2023-09-23T22:59:41.767635+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":402,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.769137+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:59:41.769137+0000","last_peered":"2023-09-23T22:59:41.769137+0000","last_clean":"2023-09-23T22:59:41.769137+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:59:41.769137+0000","last_undegraded":"2023-09-23T22:59:41.769137+0000","last_fullsized":"2023-09-23T22:59:41.769137+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":202,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.767316+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:59:41.767316+0000","last_peered":"2023-09-23T22:59:41.767316+0000","last_clean":"2023-09-23T22:59:41.767316+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:59:41.767316+0000","last_undegraded":"2023-09-23T22:59:41.767316+0000","last_fullsized":"2023-09-23T22:59:41.767316+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.766715+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:59:41.766715+0000","last_peered":"2023-09-23T22:59:41.766715+0000","last_clean":"2023-09-23T22:59:41.766715+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:59:41.766715+0000","last_undegraded":"2023-09-23T22:59:41.766715+0000","last_fullsized":"2023-09-23T22:59:41.766715+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":72,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.740885+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:59:41.740885+0000","last_peered":"2023-09-23T22:59:41.740885+0000","last_clean":"2023-09-23T22:59:41.740885+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:59:41.740885+0000","last_undegraded":"2023-09-23T22:59:41.740885+0000","last_fullsized":"2023-09-23T22:59:41.740885+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":75,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.526961+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:59:50.526961+0000","last_peered":"2023-09-23T22:59:50.526961+0000","last_clean":"2023-09-23T22:59:50.526961+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:59:50.526961+0000","last_undegraded":"2023-09-23T22:59:50.526961+0000","last_fullsized":"2023-09-23T22:59:50.526961+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":321,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-09-23T22:59:50.515718+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T22:59:50.515718+0000","last_peered":"2023-09-23T22:59:50.515718+0000","last_clean":"2023-09-23T22:59:50.515718+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T22:59:50.515718+0000","last_undegraded":"2023-09-23T22:59:50.515718+0000","last_fullsized":"2023-09-23T22:59:50.515718+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":364,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:59:41.768796+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:59:41.768796+0000","last_peered":"2023-09-23T22:59:41.768796+0000","last_clean":"2023-09-23T22:59:41.768796+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:59:41.768796+0000","last_undegraded":"2023-09-23T22:59:41.768796+0000","last_fullsized":"2023-09-23T22:59:41.768796+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4058447896,"num_objects":61928,"num_object_clones":0,"num_object_copies":185784,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":33917,"num_objects_misplaced":32436,"num_objects_unfound":0,"num_objects_dirty":61928,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131670,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10757,"num_bytes_recovered":703725568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12053327872,"data_stored":12053315656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96670,"ondisk_log_size":96670,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430603,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626828,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253141,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2209040,"kb_used_data":1528840,"kb_used_omap":9,"kb_used_meta":680182,"kb_avail":91536112,"statfs":{"total":95995035648,"available":93732978688,"internally_reserved":0,"allocated":1565532160,"data_stored":1564113509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":696507369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.5960000000000001,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.80700000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:59:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.048,"5min":4.048,"15min":4.048},"last":0.91800000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.74299999999999999,"15min":2.153},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.827,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.83099999999999996,"15min":2.5960000000000001},"min":{"1min":0.29099999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.109,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.95499999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.7850000000000001,"15min":2.6309999999999998},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5470000000000002,"5min":151.036,"15min":151.036},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.972,"15min":2.6419999999999999},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.488,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":3.1960000000000002,"15min":2.391},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.6909999999999998,"5min":116.098,"15min":116.098},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.23,"5min":2.992,"15min":2.6890000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":7.431,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.83799999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.649,"15min":2.2400000000000002},"min":{"1min":0.41599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5939999999999999,"5min":20.742000000000001,"15min":79.311000000000007},"last":0.62},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.532,"15min":1.9319999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":1.857,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.501}]},{"osd":7,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.1140000000000001,"15min":1.8899999999999999},"min":{"1min":0.46700000000000003,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":2.8039999999999998,"5min":5.8410000000000002,"15min":29.539000000000001},"last":0.875},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.395,"15min":2.0459999999999998},"min":{"1min":0.45100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7849999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.66700000000000004}]}]},{"osd":0,"up_from":353,"seq":1516123455512,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2710880,"kb_used_data":2567768,"kb_used_omap":37,"kb_used_meta":143066,"kb_avail":91034272,"statfs":{"total":95995035648,"available":93219094528,"internally_reserved":0,"allocated":2629394432,"data_stored":2627851800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":146499746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.6309999999999998,"5min":2.6309999999999998,"15min":2.6309999999999998},"last":0.22},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.0539999999999998,"5min":3.0539999999999998,"15min":3.0539999999999998},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2280000000000002,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.318},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0179999999999998,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.34100000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.2770000000000001,"5min":3.2770000000000001,"15min":3.2770000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":1.3100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.982,"5min":1.982,"15min":1.982},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.998,"5min":0.998,"15min":0.998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.3330000000000002,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.0920000000000001,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.0600000000000001,"5min":2.0600000000000001,"15min":2.0600000000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":4.3120000000000003,"5min":4.3120000000000003,"15min":4.3120000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.3579999999999997,"5min":4.3579999999999997,"15min":4.3579999999999997},"last":0.47499999999999998}]}]},{"osd":2,"up_from":359,"seq":1541893259284,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":887636,"kb_used_data":774220,"kb_used_omap":83,"kb_used_meta":113324,"kb_avail":92857516,"statfs":{"total":95995035648,"available":95086096384,"internally_reserved":0,"allocated":792801280,"data_stored":791323701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":116044229},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":2.8319999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":0.497},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.85999999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":0.67900000000000005}]}]},{"osd":3,"up_from":94,"seq":403726926088,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":526192,"kb_used_data":43688,"kb_used_omap":7,"kb_used_meta":482488,"kb_avail":93218960,"statfs":{"total":95995035648,"available":95456215040,"internally_reserved":0,"allocated":44736512,"data_stored":43227462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":494068328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.972,"5min":1.972,"15min":1.972},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.78300000000000003,"15min":2.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":3.355,"5min":16.596,"15min":123.866},"last":0.25},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95599999999999996,"15min":2.2429999999999999},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":1.746,"5min":16.649999999999999,"15min":122.265},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.1469999999999998,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":4.6769999999999996,"5min":1.798,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":87.046999999999997,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":4.6580000000000004,"5min":1.972,"15min":1.7430000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":87.066000000000003,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2410000000000001,"15min":1.3839999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5910000000000002,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.443},{"interface":"front","average":{"1min":1.032,"5min":1.2150000000000001,"15min":1.861},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3390000000000004,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.5}]},{"osd":6,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.5649999999999999,"15min":1.7090000000000001},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2520000000000007,"5min":11.116,"15min":14.097},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.613,"5min":1.397,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.2810000000000006,"5min":10.859,"15min":12.887},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97899999999999998,"15min":2.1179999999999999},"min":{"1min":0.443,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1040000000000001,"5min":6.0270000000000001,"15min":91.757999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.92300000000000004,"15min":2.097},"min":{"1min":0.37,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":3.4820000000000002,"5min":6.468,"15min":91.709999999999994},"last":0.47699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019285,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3513712,"kb_used_data":2928040,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90231440,"statfs":{"total":95995035648,"available":92396994560,"internally_reserved":0,"allocated":2998312960,"data_stored":2997554391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":3.7690000000000001,"15min":3.786},"min":{"1min":0.40000000000000002,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":1.966,"5min":98.846999999999994,"15min":132.304},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.71699999999999997,"5min":3.9740000000000002,"15min":3.6259999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":3.2269999999999999,"5min":98.938999999999993,"15min":132.453},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.536,"15min":4.5099999999999998},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":3.399,"5min":8.5510000000000002,"15min":164.453},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.478,"15min":4.5720000000000001},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.99,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:59:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.9630000000000001,"15min":1.6319999999999999},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3359999999999999,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.5},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.3599999999999999,"15min":1.7669999999999999},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.248},"max":{"1min":1.6040000000000001,"5min":151.905,"15min":151.905},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5820000000000001,"15min":1.1559999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":2.113,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.423,"15min":1.101},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.0350000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.57899999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.3080000000000001,"15min":1.0960000000000001},"min":{"1min":0.46400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0150000000000001,"5min":12.84,"15min":12.84},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.371,"5min":1.3180000000000001,"15min":1.02},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.587,"5min":10.587,"15min":10.587},"last":0.60999999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823059,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2733392,"kb_used_data":1837832,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":91011760,"statfs":{"total":95995035648,"available":93196042240,"internally_reserved":0,"allocated":1881939968,"data_stored":1881209187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":0.52900000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2150000000000001,"15min":2.2850000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24199999999999999},"max":{"1min":3.21,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.748,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":6.7690000000000001,"5min":10.353,"15min":110.89},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.304,"15min":4.9619999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":3.1400000000000001,"5min":8.6489999999999991,"15min":257.05000000000001},"last":0.5},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.4850000000000001,"15min":4.9610000000000003},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.965,"5min":12.185,"15min":257.35000000000002},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.802,"15min":1.4570000000000001},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6850000000000001,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.661,"15min":1.4510000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.1880000000000002,"5min":114.908,"15min":114.908},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.345,"15min":1.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":6.8150000000000004,"5min":11.121,"15min":12.615},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.2130000000000001,"15min":1.123},"min":{"1min":0.433,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.2719999999999998,"5min":12.137,"15min":12.74},"last":0.432}]},{"osd":7,"last update":"Sat Sep 23 22:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.159,"15min":1.0389999999999999},"min":{"1min":0.44600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.3029999999999999,"5min":12.23,"15min":12.23},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.651,"5min":1.381,"15min":1.0389999999999999},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":10.638999999999999,"5min":11.112,"15min":11.112},"last":0.61399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2619150336,"data_stored":2619146264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1557200896,"data_stored":1557200896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":787939328,"data_stored":787939328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":40239104,"data_stored":40239104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2994016256,"data_stored":2994012184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1877671936,"data_stored":1877671936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2177110016,"data_stored":2177105944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:00:03.132 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:00:03.140 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:03.193 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:03.290 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:03.364 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:03.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:03.443 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:00:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:00:03] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.33.4" 2023-09-23T23:00:03.468 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:03.500 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:03.669 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:03.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:03.774 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:03.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:03.899 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:00:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:00:03.899 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:00:03.900 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:00:03.900 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:00:03.918 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:03.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:03.924 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:03 smithi151 ceph-mon[159210]: from='client.36348 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:03.924 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:03 smithi151 ceph-mon[159210]: pgmap v1125: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 33494/183795 objects degraded (18.224%); 32259/183795 objects misplaced (17.552%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:04.022 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:04.060 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:03 smithi052 ceph-mon[187743]: from='client.36348 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:04.060 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:03 smithi052 ceph-mon[187743]: pgmap v1125: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 33494/183795 objects degraded (18.224%); 32259/183795 objects misplaced (17.552%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:04.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:03 smithi052 ceph-mon[180979]: from='client.36348 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:04.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:03 smithi052 ceph-mon[180979]: pgmap v1125: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 33494/183795 objects degraded (18.224%); 32259/183795 objects misplaced (17.552%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:04.061 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:04.168 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:04.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:04.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:04.273 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:04.313 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:04.377 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:04.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:04.484 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:04.516 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:04.895 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:00:05.087 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:05.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:05.146 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:05.192 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:05.225 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:05.339 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:05.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:05.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:05.378 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:05.443 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:05.492 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:05.601 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:05.611 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:05.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:05.625 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:05.646 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:05.666 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:05.694 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:05.718 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:05.718 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:05.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[180979]: pgmap v1126: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 33494/183795 objects degraded (18.224%); 32259/183795 objects misplaced (17.552%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 33494/183795 objects degraded (18.224%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[187743]: pgmap v1126: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 33494/183795 objects degraded (18.224%); 32259/183795 objects misplaced (17.552%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 33494/183795 objects degraded (18.224%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:05.750 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:05.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:05.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:05.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:05 smithi151 ceph-mon[159210]: pgmap v1126: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 33494/183795 objects degraded (18.224%); 32259/183795 objects misplaced (17.552%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:05.772 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 33494/183795 objects degraded (18.224%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:05.773 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:05.777 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:05.778 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.779 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:00:05.942 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:05.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:06.048 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:06.086 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:06.538 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:00:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2659777553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2346473086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:06 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:06.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2659777553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:06.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2346473086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:06.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:06 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:06.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2659777553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:06.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2346473086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:06.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:06 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:07.203 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:07.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:07.277 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:07.277 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:07.308 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:07.348 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:07.493 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:07.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:07.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:07.599 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:07.631 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: osdmap e381: 8 total, 8 up, 6 in 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:07 smithi151 ceph-mon[159210]: pgmap v1128: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 33258/182346 objects degraded (18.239%); 32120/182346 objects misplaced (17.615%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:00:07.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:07.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:07.874 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:07.894 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:07.894 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:00:07.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: osdmap e381: 8 total, 8 up, 6 in 2023-09-23T23:00:07.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[187743]: pgmap v1128: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 33258/182346 objects degraded (18.239%); 32120/182346 objects misplaced (17.615%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:00:07.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:07.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:00:07.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: osdmap e381: 8 total, 8 up, 6 in 2023-09-23T23:00:07.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: failed to encode map e381 with expected crc 2023-09-23T23:00:07.896 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:07 smithi052 ceph-mon[180979]: pgmap v1128: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 33258/182346 objects degraded (18.239%); 32120/182346 objects misplaced (17.615%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T23:00:07.911 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:08.089 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:08.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:08.197 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:08.255 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:08.257 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:00:08.336 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:08.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:08.446 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:08.504 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:08.507 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:08.507 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 33258/182346 objects degraded (18.239%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1541,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":381,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":60782,"data_bytes":3982366648,"bytes_used":12993290240,"bytes_avail":562976923648,"bytes_total":575970213888,"degraded_objects":33258,"degraded_total":182346,"degraded_ratio":0.18238952321410945,"misplaced_objects":32120,"misplaced_total":182346,"misplaced_ratio":0.17614864049663825,"recovering_objects_per_sec":57,"recovering_bytes_per_sec":3791634,"recovering_keys_per_sec":0,"num_objects_recovered":463,"num_bytes_recovered":30343168,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":187},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (4m)\n [======================......] (remaining: 52s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:00:08.608 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:08.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:08.717 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:08.750 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:08.798 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:08.799 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2874108889' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:08.800 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2874108889' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:08.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:08.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:08.824 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:08.901 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:08.928 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:08.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2874108889' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:08.985 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:08.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:08.993 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:00:09.093 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:09.124 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:09.247 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:09.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:09.298 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:00:09.298 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:00:09.360 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:09.392 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:09.446 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:09.452 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:09.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:09.560 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:09.594 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:09.642 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:09.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:09 smithi151 ceph-mon[159210]: pgmap v1129: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 32986/180552 objects degraded (18.270%); 31939/180552 objects misplaced (17.690%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:09.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:09.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:09 smithi052 ceph-mon[180979]: pgmap v1129: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 32986/180552 objects degraded (18.270%); 31939/180552 objects misplaced (17.690%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:09.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:09 smithi052 ceph-mon[187743]: pgmap v1129: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 215 op/s; 32986/180552 objects degraded (18.270%); 31939/180552 objects misplaced (17.690%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T23:00:09.698 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:00:09.746 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:09.773 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:09.818 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:09.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:09.921 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:09.974 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:10.162 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:10.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:10.265 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:10.304 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:10.551 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:10.557 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 32986/180552 objects degraded (18.270%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:10.557 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 32986/180552 objects degraded (18.270%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:10.577 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:10.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:10.685 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:10.723 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:10.725 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:10.797 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:10.797 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:10.823 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 32986/180552 objects degraded (18.270%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:10.856 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:10.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:10.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:10.962 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:11.022 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:11.090 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:11.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:11.195 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:11.232 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:11.298 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:11.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:11.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:11.401 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:11.433 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:11.479 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:11.594 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:11.594 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:11.594 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:11.594 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:11.620 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:11 smithi052 ceph-mon[180979]: pgmap v1130: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 32932/179688 objects degraded (18.327%); 31827/179688 objects misplaced (17.712%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:00:11.621 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1770632355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:11.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:11 smithi052 ceph-mon[187743]: pgmap v1130: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 32932/179688 objects degraded (18.327%); 31827/179688 objects misplaced (17.712%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:00:11.621 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1770632355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:11.678 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:11.678 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:11.678 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:11.678 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.679 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:00:11.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:11 smithi151 ceph-mon[159210]: pgmap v1130: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 32932/179688 objects degraded (18.327%); 31827/179688 objects misplaced (17.712%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T23:00:11.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1770632355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:11.857 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:00:12.386 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:12.689 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:12.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:12.792 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:12.836 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:13.037 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:13.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:13.142 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:13.188 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:13.261 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:13.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:13.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:13.366 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:13.395 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:13.447 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:13.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:13.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:00:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:00:13] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-09-23T23:00:13.551 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:13.600 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:13.638 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:13.640 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:00:13.902 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:13 smithi151 ceph-mon[159210]: pgmap v1131: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 32463/177354 objects degraded (18.304%); 31652/177354 objects misplaced (17.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:13.938 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:13.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:13.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:13 smithi052 ceph-mon[187743]: pgmap v1131: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 32463/177354 objects degraded (18.304%); 31652/177354 objects misplaced (17.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:13.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:13 smithi052 ceph-mon[180979]: pgmap v1131: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 32463/177354 objects degraded (18.304%); 31652/177354 objects misplaced (17.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:14.044 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:14.081 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:14.206 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:14.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:14.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:14.348 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:14.360 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1131,"stamp":"2023-09-23T23:00:12.566278+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3873314744,"num_objects":59118,"num_object_clones":0,"num_object_copies":177354,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":32463,"num_objects_misplaced":31652,"num_objects_unfound":0,"num_objects_dirty":59118,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":134764,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11874,"num_bytes_recovered":774650812,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97598,"ondisk_log_size":97598,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12509256,"kb_used_data":9270616,"kb_used_omap":137,"kb_used_meta":3238454,"kb_avail":549961656,"statfs":{"total":575970213888,"available":563160735744,"internally_reserved":0,"allocated":9493110784,"data_stored":9485614908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3316177080},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-140705792,"num_objects":-2147,"num_object_clones":0,"num_object_copies":-6441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1031,"num_objects_misplaced":-607,"num_objects_unfound":0,"num_objects_dirty":-2147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2147,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":735,"num_bytes_recovered":48168960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002978"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":393,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818415+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T22:59:58.818415+0000","last_peered":"2023-09-23T22:59:58.818415+0000","last_clean":"2023-09-23T22:59:58.818415+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T22:59:58.818415+0000","last_undegraded":"2023-09-23T22:59:58.818415+0000","last_fullsized":"2023-09-23T22:59:58.818415+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":50,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850777+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T22:59:58.850777+0000","last_peered":"2023-09-23T22:59:58.850777+0000","last_clean":"2023-09-23T22:59:58.850777+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T22:59:58.850777+0000","last_undegraded":"2023-09-23T22:59:58.850777+0000","last_fullsized":"2023-09-23T22:59:58.850777+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":49,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849503+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T22:59:58.849503+0000","last_peered":"2023-09-23T22:59:58.849503+0000","last_clean":"2023-09-23T22:59:58.849503+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T22:59:58.849503+0000","last_undegraded":"2023-09-23T22:59:58.849503+0000","last_fullsized":"2023-09-23T22:59:58.849503+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":377,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850493+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T22:59:58.850493+0000","last_peered":"2023-09-23T22:59:58.850493+0000","last_clean":"2023-09-23T22:59:58.850493+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T22:59:58.850493+0000","last_undegraded":"2023-09-23T22:59:58.850493+0000","last_fullsized":"2023-09-23T22:59:58.850493+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":65,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849462+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T22:59:58.849462+0000","last_peered":"2023-09-23T22:59:58.849462+0000","last_clean":"2023-09-23T22:59:58.849462+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T22:59:58.849462+0000","last_undegraded":"2023-09-23T22:59:58.849462+0000","last_fullsized":"2023-09-23T22:59:58.849462+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":51,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.541156+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:06.541156+0000","last_peered":"2023-09-23T23:00:06.541156+0000","last_clean":"2023-09-23T23:00:06.541156+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:06.541156+0000","last_undegraded":"2023-09-23T23:00:06.541156+0000","last_fullsized":"2023-09-23T23:00:06.541156+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":316,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841812+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T22:59:58.841812+0000","last_peered":"2023-09-23T22:59:58.841812+0000","last_clean":"2023-09-23T22:59:58.841812+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T22:59:58.841812+0000","last_undegraded":"2023-09-23T22:59:58.841812+0000","last_fullsized":"2023-09-23T22:59:58.841812+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":59,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849334+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T22:59:58.849334+0000","last_peered":"2023-09-23T22:59:58.849334+0000","last_clean":"2023-09-23T22:59:58.849334+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T22:59:58.849334+0000","last_undegraded":"2023-09-23T22:59:58.849334+0000","last_fullsized":"2023-09-23T22:59:58.849334+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":277,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842178+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T22:59:58.842178+0000","last_peered":"2023-09-23T22:59:58.842178+0000","last_clean":"2023-09-23T22:59:58.842178+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T22:59:58.842178+0000","last_undegraded":"2023-09-23T22:59:58.842178+0000","last_fullsized":"2023-09-23T22:59:58.842178+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849429+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T22:59:58.849429+0000","last_peered":"2023-09-23T22:59:58.849429+0000","last_clean":"2023-09-23T22:59:58.849429+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T22:59:58.849429+0000","last_undegraded":"2023-09-23T22:59:58.849429+0000","last_fullsized":"2023-09-23T22:59:58.849429+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":70,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850690+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T22:59:58.850690+0000","last_peered":"2023-09-23T22:59:58.850690+0000","last_clean":"2023-09-23T22:59:58.850690+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T22:59:58.850690+0000","last_undegraded":"2023-09-23T22:59:58.850690+0000","last_fullsized":"2023-09-23T22:59:58.850690+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842655+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T22:59:58.842655+0000","last_peered":"2023-09-23T22:59:58.842655+0000","last_clean":"2023-09-23T22:59:58.842655+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T22:59:58.842655+0000","last_undegraded":"2023-09-23T22:59:58.842655+0000","last_fullsized":"2023-09-23T22:59:58.842655+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":452,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818833+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T22:59:58.818833+0000","last_peered":"2023-09-23T22:59:58.818833+0000","last_clean":"2023-09-23T22:59:58.818833+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T22:59:58.818833+0000","last_undegraded":"2023-09-23T22:59:58.818833+0000","last_fullsized":"2023-09-23T22:59:58.818833+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":355,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.825571+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T22:59:58.825571+0000","last_peered":"2023-09-23T22:59:58.825571+0000","last_clean":"2023-09-23T22:59:58.825571+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T22:59:58.825571+0000","last_undegraded":"2023-09-23T22:59:58.825571+0000","last_fullsized":"2023-09-23T22:59:58.825571+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":281,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.825680+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T22:59:58.825680+0000","last_peered":"2023-09-23T22:59:58.825680+0000","last_clean":"2023-09-23T22:59:58.825680+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T22:59:58.825680+0000","last_undegraded":"2023-09-23T22:59:58.825680+0000","last_fullsized":"2023-09-23T22:59:58.825680+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.851016+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T22:59:58.851016+0000","last_peered":"2023-09-23T22:59:58.851016+0000","last_clean":"2023-09-23T22:59:58.851016+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T22:59:58.851016+0000","last_undegraded":"2023-09-23T22:59:58.851016+0000","last_fullsized":"2023-09-23T22:59:58.851016+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":370,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849334+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T22:59:58.849334+0000","last_peered":"2023-09-23T22:59:58.849334+0000","last_clean":"2023-09-23T22:59:58.849334+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T22:59:58.849334+0000","last_undegraded":"2023-09-23T22:59:58.849334+0000","last_fullsized":"2023-09-23T22:59:58.849334+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":281,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841385+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T22:59:58.841385+0000","last_peered":"2023-09-23T22:59:58.841385+0000","last_clean":"2023-09-23T22:59:58.841385+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T22:59:58.841385+0000","last_undegraded":"2023-09-23T22:59:58.841385+0000","last_fullsized":"2023-09-23T22:59:58.841385+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":266,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.817980+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T22:59:58.817980+0000","last_peered":"2023-09-23T22:59:58.817980+0000","last_clean":"2023-09-23T22:59:58.817980+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T22:59:58.817980+0000","last_undegraded":"2023-09-23T22:59:58.817980+0000","last_fullsized":"2023-09-23T22:59:58.817980+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":394,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T23:00:01.233683+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:01.233683+0000","last_peered":"2023-09-23T23:00:01.233683+0000","last_clean":"2023-09-23T23:00:01.233683+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:01.233683+0000","last_undegraded":"2023-09-23T23:00:01.233683+0000","last_fullsized":"2023-09-23T23:00:01.233683+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":98,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.815222+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T22:59:58.815222+0000","last_peered":"2023-09-23T22:59:58.815222+0000","last_clean":"2023-09-23T22:59:58.815222+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T22:59:58.815222+0000","last_undegraded":"2023-09-23T22:59:58.815222+0000","last_fullsized":"2023-09-23T22:59:58.815222+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":74,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850711+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T22:59:58.850711+0000","last_peered":"2023-09-23T22:59:58.850711+0000","last_clean":"2023-09-23T22:59:58.850711+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T22:59:58.850711+0000","last_undegraded":"2023-09-23T22:59:58.850711+0000","last_fullsized":"2023-09-23T22:59:58.850711+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"381'8329","reported_seq":8413,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:09.406814+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:00:09.406814+0000","last_peered":"2023-09-23T23:00:09.406814+0000","last_clean":"2023-09-23T23:00:09.406814+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:00:09.406814+0000","last_undegraded":"2023-09-23T23:00:09.406814+0000","last_fullsized":"2023-09-23T23:00:09.406814+0000","mapping_epoch":353,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246218752,"num_objects":3757,"num_object_clones":0,"num_object_copies":11271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8329,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":265,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.819526+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T22:59:58.819526+0000","last_peered":"2023-09-23T22:59:58.819526+0000","last_clean":"2023-09-23T22:59:58.819526+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T22:59:58.819526+0000","last_undegraded":"2023-09-23T22:59:58.819526+0000","last_fullsized":"2023-09-23T22:59:58.819526+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.835878+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T22:59:58.835878+0000","last_peered":"2023-09-23T22:59:58.835878+0000","last_clean":"2023-09-23T22:59:58.835878+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T22:59:58.835878+0000","last_undegraded":"2023-09-23T22:59:58.835878+0000","last_fullsized":"2023-09-23T22:59:58.835878+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"381'8286","reported_seq":10700,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:10.688391+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:00:10.688391+0000","last_peered":"2023-09-23T23:00:10.688391+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:00:10.688391+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7886,"log_dups_size":0,"ondisk_log_size":7886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240123904,"num_objects":3664,"num_object_clones":0,"num_object_copies":10992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3664,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8286,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":97}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":275,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.823904+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T22:59:58.823904+0000","last_peered":"2023-09-23T22:59:58.823904+0000","last_clean":"2023-09-23T22:59:58.823904+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T22:59:58.823904+0000","last_undegraded":"2023-09-23T22:59:58.823904+0000","last_fullsized":"2023-09-23T22:59:58.823904+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":388,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.817912+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T22:59:58.817912+0000","last_peered":"2023-09-23T22:59:58.817912+0000","last_clean":"2023-09-23T22:59:58.817912+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T22:59:58.817912+0000","last_undegraded":"2023-09-23T22:59:58.817912+0000","last_fullsized":"2023-09-23T22:59:58.817912+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"381'8390","reported_seq":8468,"reported_epoch":381,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:09.341865+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:00:09.341865+0000","last_peered":"2023-09-23T23:00:09.341865+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:00:09.341865+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:00:09.341865+0000","mapping_epoch":354,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241303552,"num_objects":3682,"num_object_clones":0,"num_object_copies":11046,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3682,"num_objects_unfound":0,"num_objects_dirty":3682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8390,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3682}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":132,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841471+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T22:59:58.841471+0000","last_peered":"2023-09-23T22:59:58.841471+0000","last_clean":"2023-09-23T22:59:58.841471+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T22:59:58.841471+0000","last_undegraded":"2023-09-23T22:59:58.841471+0000","last_fullsized":"2023-09-23T22:59:58.841471+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841199+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T22:59:58.841199+0000","last_peered":"2023-09-23T22:59:58.841199+0000","last_clean":"2023-09-23T22:59:58.841199+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T22:59:58.841199+0000","last_undegraded":"2023-09-23T22:59:58.841199+0000","last_fullsized":"2023-09-23T22:59:58.841199+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"381'8443","reported_seq":8708,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:09.585113+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:00:09.585113+0000","last_peered":"2023-09-23T23:00:09.585113+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:00:09.585113+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5543,"log_dups_size":0,"ondisk_log_size":5543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244514816,"num_objects":3731,"num_object_clones":0,"num_object_copies":11193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3146,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8443,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":570},{"shards":"0,5","objects":121}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":50,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850999+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T22:59:58.850999+0000","last_peered":"2023-09-23T22:59:58.850999+0000","last_clean":"2023-09-23T22:59:58.850999+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T22:59:58.850999+0000","last_undegraded":"2023-09-23T22:59:58.850999+0000","last_fullsized":"2023-09-23T22:59:58.850999+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":279,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.815451+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T22:59:58.815451+0000","last_peered":"2023-09-23T22:59:58.815451+0000","last_clean":"2023-09-23T22:59:58.815451+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T22:59:58.815451+0000","last_undegraded":"2023-09-23T22:59:58.815451+0000","last_fullsized":"2023-09-23T22:59:58.815451+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"381'8401","reported_seq":8487,"reported_epoch":381,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:11.855674+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:00:11.855674+0000","last_peered":"2023-09-23T23:00:11.855674+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:00:11.855674+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:00:11.855674+0000","mapping_epoch":353,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242024448,"num_objects":3693,"num_object_clones":0,"num_object_copies":11079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8401,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3693}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850218+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T22:59:58.850218+0000","last_peered":"2023-09-23T22:59:58.850218+0000","last_clean":"2023-09-23T22:59:58.850218+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T22:59:58.850218+0000","last_undegraded":"2023-09-23T22:59:58.850218+0000","last_fullsized":"2023-09-23T22:59:58.850218+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":374,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818758+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T22:59:58.818758+0000","last_peered":"2023-09-23T22:59:58.818758+0000","last_clean":"2023-09-23T22:59:58.818758+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T22:59:58.818758+0000","last_undegraded":"2023-09-23T22:59:58.818758+0000","last_fullsized":"2023-09-23T22:59:58.818758+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"381'8496","reported_seq":8590,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:11.992218+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:00:11.992218+0000","last_peered":"2023-09-23T23:00:11.992218+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:00:11.992218+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242876416,"num_objects":3706,"num_object_clones":0,"num_object_copies":11118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3706,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8496,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":87}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":164,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818556+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T22:59:58.818556+0000","last_peered":"2023-09-23T22:59:58.818556+0000","last_clean":"2023-09-23T22:59:58.818556+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T22:59:58.818556+0000","last_undegraded":"2023-09-23T22:59:58.818556+0000","last_fullsized":"2023-09-23T22:59:58.818556+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":391,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818299+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T22:59:58.818299+0000","last_peered":"2023-09-23T22:59:58.818299+0000","last_clean":"2023-09-23T22:59:58.818299+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T22:59:58.818299+0000","last_undegraded":"2023-09-23T22:59:58.818299+0000","last_fullsized":"2023-09-23T22:59:58.818299+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"381'8417","reported_seq":12819,"reported_epoch":381,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:11.966355+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:00:11.966355+0000","last_peered":"2023-09-23T23:00:11.966355+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:00:11.966355+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:00:11.966355+0000","mapping_epoch":354,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238747648,"num_objects":3643,"num_object_clones":0,"num_object_copies":10929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2450,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8417,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2150,"num_bytes_recovered":140902400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":2209}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":404,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.817728+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T22:59:58.817728+0000","last_peered":"2023-09-23T22:59:58.817728+0000","last_clean":"2023-09-23T22:59:58.817728+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T22:59:58.817728+0000","last_undegraded":"2023-09-23T22:59:58.817728+0000","last_fullsized":"2023-09-23T22:59:58.817728+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":382,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.823995+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T22:59:58.823995+0000","last_peered":"2023-09-23T22:59:58.823995+0000","last_clean":"2023-09-23T22:59:58.823995+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T22:59:58.823995+0000","last_undegraded":"2023-09-23T22:59:58.823995+0000","last_fullsized":"2023-09-23T22:59:58.823995+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"381'8269","reported_seq":8355,"reported_epoch":381,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:11.841883+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:00:11.841883+0000","last_peered":"2023-09-23T23:00:11.841883+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:00:11.841883+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:00:11.841883+0000","mapping_epoch":353,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242417664,"num_objects":3699,"num_object_clones":0,"num_object_copies":11097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8269,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3699}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":50,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849629+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T22:59:58.849629+0000","last_peered":"2023-09-23T22:59:58.849629+0000","last_clean":"2023-09-23T22:59:58.849629+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T22:59:58.849629+0000","last_undegraded":"2023-09-23T22:59:58.849629+0000","last_fullsized":"2023-09-23T22:59:58.849629+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":283,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842521+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T22:59:58.842521+0000","last_peered":"2023-09-23T22:59:58.842521+0000","last_clean":"2023-09-23T22:59:58.842521+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T22:59:58.842521+0000","last_undegraded":"2023-09-23T22:59:58.842521+0000","last_fullsized":"2023-09-23T22:59:58.842521+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"381'8368","reported_seq":14612,"reported_epoch":381,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:00:12.114287+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:00:12.114287+0000","last_peered":"2023-09-23T23:00:12.114287+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:00:12.114287+0000","last_undegraded":"2023-09-23T23:00:12.114287+0000","last_fullsized":"2023-09-23T23:00:12.114287+0000","mapping_epoch":354,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5668,"log_dups_size":0,"ondisk_log_size":5668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243007488,"num_objects":3708,"num_object_clones":0,"num_object_copies":11124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":603,"num_objects_unfound":0,"num_objects_dirty":3708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8368,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3062,"num_bytes_recovered":200146944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":3105},{"shards":"0,4,6","objects":603}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":287,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842298+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T22:59:58.842298+0000","last_peered":"2023-09-23T22:59:58.842298+0000","last_clean":"2023-09-23T22:59:58.842298+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T22:59:58.842298+0000","last_undegraded":"2023-09-23T22:59:58.842298+0000","last_fullsized":"2023-09-23T22:59:58.842298+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":291,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841311+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T22:59:58.841311+0000","last_peered":"2023-09-23T22:59:58.841311+0000","last_clean":"2023-09-23T22:59:58.841311+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T22:59:58.841311+0000","last_undegraded":"2023-09-23T22:59:58.841311+0000","last_fullsized":"2023-09-23T22:59:58.841311+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":104,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.835943+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T22:59:58.835943+0000","last_peered":"2023-09-23T22:59:58.835943+0000","last_clean":"2023-09-23T22:59:58.835943+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T22:59:58.835943+0000","last_undegraded":"2023-09-23T22:59:58.835943+0000","last_fullsized":"2023-09-23T22:59:58.835943+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"381'8394","reported_seq":8491,"reported_epoch":381,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:00:09.158160+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:00:09.158160+0000","last_peered":"2023-09-23T23:00:09.158160+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:00:09.158160+0000","last_undegraded":"2023-09-23T23:00:09.158160+0000","last_fullsized":"2023-09-23T23:00:09.158160+0000","mapping_epoch":359,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5494,"log_dups_size":0,"ondisk_log_size":5494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3730,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8394,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3730}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":51,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.535722+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:06.535722+0000","last_peered":"2023-09-23T23:00:06.535722+0000","last_clean":"2023-09-23T23:00:06.535722+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:06.535722+0000","last_undegraded":"2023-09-23T23:00:06.535722+0000","last_fullsized":"2023-09-23T23:00:06.535722+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":401,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.825772+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T22:59:58.825772+0000","last_peered":"2023-09-23T22:59:58.825772+0000","last_clean":"2023-09-23T22:59:58.825772+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T22:59:58.825772+0000","last_undegraded":"2023-09-23T22:59:58.825772+0000","last_fullsized":"2023-09-23T22:59:58.825772+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":272,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842344+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T22:59:58.842344+0000","last_peered":"2023-09-23T22:59:58.842344+0000","last_clean":"2023-09-23T22:59:58.842344+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T22:59:58.842344+0000","last_undegraded":"2023-09-23T22:59:58.842344+0000","last_fullsized":"2023-09-23T22:59:58.842344+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"381'8387","reported_seq":13436,"reported_epoch":381,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:11.156978+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T23:00:11.156978+0000","last_peered":"2023-09-23T23:00:11.156978+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:00:11.156978+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7887,"log_dups_size":0,"ondisk_log_size":7887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240975872,"num_objects":3677,"num_object_clones":0,"num_object_copies":11031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8387,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2478,"num_bytes_recovered":162398208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2584}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":347,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842294+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T22:59:58.842294+0000","last_peered":"2023-09-23T22:59:58.842294+0000","last_clean":"2023-09-23T22:59:58.842294+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T22:59:58.842294+0000","last_undegraded":"2023-09-23T22:59:58.842294+0000","last_fullsized":"2023-09-23T22:59:58.842294+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":224,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841977+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T22:59:58.841977+0000","last_peered":"2023-09-23T22:59:58.841977+0000","last_clean":"2023-09-23T22:59:58.841977+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T22:59:58.841977+0000","last_undegraded":"2023-09-23T22:59:58.841977+0000","last_fullsized":"2023-09-23T22:59:58.841977+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":291,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.841878+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T22:59:58.841878+0000","last_peered":"2023-09-23T22:59:58.841878+0000","last_clean":"2023-09-23T22:59:58.841878+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T22:59:58.841878+0000","last_undegraded":"2023-09-23T22:59:58.841878+0000","last_fullsized":"2023-09-23T22:59:58.841878+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"381'8509","reported_seq":10393,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:12.238064+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:00:12.238064+0000","last_peered":"2023-09-23T23:00:12.238064+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:00:12.238064+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8509,"log_dups_size":0,"ondisk_log_size":8509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2282},{"shards":"0,6","objects":-1266}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":364,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.840911+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T22:59:58.840911+0000","last_peered":"2023-09-23T22:59:58.840911+0000","last_clean":"2023-09-23T22:59:58.840911+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T22:59:58.840911+0000","last_undegraded":"2023-09-23T22:59:58.840911+0000","last_fullsized":"2023-09-23T22:59:58.840911+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":477,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:04.765501+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:04.765501+0000","last_peered":"2023-09-23T23:00:04.765501+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:04.765501+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":50,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849888+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T22:59:58.849888+0000","last_peered":"2023-09-23T22:59:58.849888+0000","last_clean":"2023-09-23T22:59:58.849888+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T22:59:58.849888+0000","last_undegraded":"2023-09-23T22:59:58.849888+0000","last_fullsized":"2023-09-23T22:59:58.849888+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":291,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818510+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T22:59:58.818510+0000","last_peered":"2023-09-23T22:59:58.818510+0000","last_clean":"2023-09-23T22:59:58.818510+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T22:59:58.818510+0000","last_undegraded":"2023-09-23T22:59:58.818510+0000","last_fullsized":"2023-09-23T22:59:58.818510+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"381'8428","reported_seq":8517,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:09.574542+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:00:09.574542+0000","last_peered":"2023-09-23T23:00:09.574542+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:00:09.574542+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5528,"log_dups_size":0,"ondisk_log_size":5528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239599616,"num_objects":3656,"num_object_clones":0,"num_object_copies":10968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8428,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":149}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":371,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T23:00:04.765479+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:04.765479+0000","last_peered":"2023-09-23T23:00:04.765479+0000","last_clean":"2023-09-23T23:00:04.765479+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:04.765479+0000","last_undegraded":"2023-09-23T23:00:04.765479+0000","last_fullsized":"2023-09-23T23:00:04.765479+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":342,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.536667+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:06.536667+0000","last_peered":"2023-09-23T23:00:06.536667+0000","last_clean":"2023-09-23T23:00:06.536667+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:06.536667+0000","last_undegraded":"2023-09-23T23:00:06.536667+0000","last_fullsized":"2023-09-23T23:00:06.536667+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.816145+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T22:59:58.816145+0000","last_peered":"2023-09-23T22:59:58.816145+0000","last_clean":"2023-09-23T22:59:58.816145+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T22:59:58.816145+0000","last_undegraded":"2023-09-23T22:59:58.816145+0000","last_fullsized":"2023-09-23T22:59:58.816145+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"381'8494","reported_seq":8599,"reported_epoch":381,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:11.978023+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:00:11.978023+0000","last_peered":"2023-09-23T23:00:11.978023+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:00:11.978023+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:00:11.978023+0000","mapping_epoch":354,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243073048,"num_objects":3710,"num_object_clones":0,"num_object_copies":11130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3710,"num_objects_unfound":0,"num_objects_dirty":3710,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8494,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3710}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":317,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542125+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:06.542125+0000","last_peered":"2023-09-23T23:00:06.542125+0000","last_clean":"2023-09-23T23:00:06.542125+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:06.542125+0000","last_undegraded":"2023-09-23T23:00:06.542125+0000","last_fullsized":"2023-09-23T23:00:06.542125+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":224,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842403+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T22:59:58.842403+0000","last_peered":"2023-09-23T22:59:58.842403+0000","last_clean":"2023-09-23T22:59:58.842403+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T22:59:58.842403+0000","last_undegraded":"2023-09-23T22:59:58.842403+0000","last_fullsized":"2023-09-23T22:59:58.842403+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":194,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.840982+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T22:59:58.840982+0000","last_peered":"2023-09-23T22:59:58.840982+0000","last_clean":"2023-09-23T22:59:58.840982+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T22:59:58.840982+0000","last_undegraded":"2023-09-23T22:59:58.840982+0000","last_fullsized":"2023-09-23T22:59:58.840982+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"381'8466","reported_seq":12174,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:12.133867+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:00:12.133867+0000","last_peered":"2023-09-23T23:00:12.133867+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:00:12.133867+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241172480,"num_objects":3680,"num_object_clones":0,"num_object_copies":11040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7547,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8466,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1805,"num_bytes_recovered":118292480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":98},{"shards":"4,5","objects":1},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850927+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T22:59:58.850927+0000","last_peered":"2023-09-23T22:59:58.850927+0000","last_clean":"2023-09-23T22:59:58.850927+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T22:59:58.850927+0000","last_undegraded":"2023-09-23T22:59:58.850927+0000","last_fullsized":"2023-09-23T22:59:58.850927+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.826206+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T22:59:58.826206+0000","last_peered":"2023-09-23T22:59:58.826206+0000","last_clean":"2023-09-23T22:59:58.826206+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T22:59:58.826206+0000","last_undegraded":"2023-09-23T22:59:58.826206+0000","last_fullsized":"2023-09-23T22:59:58.826206+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849534+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T22:59:58.849534+0000","last_peered":"2023-09-23T22:59:58.849534+0000","last_clean":"2023-09-23T22:59:58.849534+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T22:59:58.849534+0000","last_undegraded":"2023-09-23T22:59:58.849534+0000","last_fullsized":"2023-09-23T22:59:58.849534+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"381'8427","reported_seq":8942,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:12.012928+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:00:12.012928+0000","last_peered":"2023-09-23T23:00:12.012928+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:00:12.012928+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238747648,"num_objects":3643,"num_object_clones":0,"num_object_copies":10929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3643,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8427,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":92}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":336,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842712+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T22:59:58.842712+0000","last_peered":"2023-09-23T22:59:58.842712+0000","last_clean":"2023-09-23T22:59:58.842712+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T22:59:58.842712+0000","last_undegraded":"2023-09-23T22:59:58.842712+0000","last_fullsized":"2023-09-23T22:59:58.842712+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":283,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.816050+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T22:59:58.816050+0000","last_peered":"2023-09-23T22:59:58.816050+0000","last_clean":"2023-09-23T22:59:58.816050+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T22:59:58.816050+0000","last_undegraded":"2023-09-23T22:59:58.816050+0000","last_fullsized":"2023-09-23T22:59:58.816050+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":355,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.817797+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T22:59:58.817797+0000","last_peered":"2023-09-23T22:59:58.817797+0000","last_clean":"2023-09-23T22:59:58.817797+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T22:59:58.817797+0000","last_undegraded":"2023-09-23T22:59:58.817797+0000","last_fullsized":"2023-09-23T22:59:58.817797+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":318,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.816119+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T22:59:58.816119+0000","last_peered":"2023-09-23T22:59:58.816119+0000","last_clean":"2023-09-23T22:59:58.816119+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T22:59:58.816119+0000","last_undegraded":"2023-09-23T22:59:58.816119+0000","last_fullsized":"2023-09-23T22:59:58.816119+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":128,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.842099+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T22:59:58.842099+0000","last_peered":"2023-09-23T22:59:58.842099+0000","last_clean":"2023-09-23T22:59:58.842099+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T22:59:58.842099+0000","last_undegraded":"2023-09-23T22:59:58.842099+0000","last_fullsized":"2023-09-23T22:59:58.842099+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":190,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818692+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T22:59:58.818692+0000","last_peered":"2023-09-23T22:59:58.818692+0000","last_clean":"2023-09-23T22:59:58.818692+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T22:59:58.818692+0000","last_undegraded":"2023-09-23T22:59:58.818692+0000","last_fullsized":"2023-09-23T22:59:58.818692+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":404,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.825837+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T22:59:58.825837+0000","last_peered":"2023-09-23T22:59:58.825837+0000","last_clean":"2023-09-23T22:59:58.825837+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T22:59:58.825837+0000","last_undegraded":"2023-09-23T22:59:58.825837+0000","last_fullsized":"2023-09-23T22:59:58.825837+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":204,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818378+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T22:59:58.818378+0000","last_peered":"2023-09-23T22:59:58.818378+0000","last_clean":"2023-09-23T22:59:58.818378+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T22:59:58.818378+0000","last_undegraded":"2023-09-23T22:59:58.818378+0000","last_fullsized":"2023-09-23T22:59:58.818378+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":291,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.818487+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T22:59:58.818487+0000","last_peered":"2023-09-23T22:59:58.818487+0000","last_clean":"2023-09-23T22:59:58.818487+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T22:59:58.818487+0000","last_undegraded":"2023-09-23T22:59:58.818487+0000","last_fullsized":"2023-09-23T22:59:58.818487+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":74,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.850775+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T22:59:58.850775+0000","last_peered":"2023-09-23T22:59:58.850775+0000","last_clean":"2023-09-23T22:59:58.850775+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T22:59:58.850775+0000","last_undegraded":"2023-09-23T22:59:58.850775+0000","last_fullsized":"2023-09-23T22:59:58.850775+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.849573+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T22:59:58.849573+0000","last_peered":"2023-09-23T22:59:58.849573+0000","last_clean":"2023-09-23T22:59:58.849573+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T22:59:58.849573+0000","last_undegraded":"2023-09-23T22:59:58.849573+0000","last_fullsized":"2023-09-23T22:59:58.849573+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":323,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.541540+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:06.541540+0000","last_peered":"2023-09-23T23:00:06.541540+0000","last_clean":"2023-09-23T23:00:06.541540+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:06.541540+0000","last_undegraded":"2023-09-23T23:00:06.541540+0000","last_fullsized":"2023-09-23T23:00:06.541540+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":366,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-09-23T22:59:58.817496+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T22:59:58.817496+0000","last_peered":"2023-09-23T22:59:58.817496+0000","last_clean":"2023-09-23T22:59:58.817496+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T22:59:58.817496+0000","last_undegraded":"2023-09-23T22:59:58.817496+0000","last_fullsized":"2023-09-23T22:59:58.817496+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3872718872,"num_objects":59094,"num_object_clones":0,"num_object_copies":177282,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":32461,"num_objects_misplaced":31652,"num_objects_unfound":0,"num_objects_dirty":59094,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134504,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11839,"num_bytes_recovered":774635520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11563249664,"data_stored":11563237448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97404,"ondisk_log_size":97404,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430606,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626831,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253143,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2178668,"kb_used_data":1494372,"kb_used_omap":9,"kb_used_meta":684278,"kb_avail":91566484,"statfs":{"total":95995035648,"available":93764079616,"internally_reserved":0,"allocated":1530236928,"data_stored":1528806601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":700701673},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.5960000000000001,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:59:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.379},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.048,"5min":4.048,"15min":4.048},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.74299999999999999,"15min":2.153},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.827,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.83099999999999996,"15min":2.5960000000000001},"min":{"1min":0.29099999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.109,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.35999999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.7850000000000001,"15min":2.6309999999999998},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5470000000000002,"5min":151.036,"15min":151.036},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.972,"15min":2.6419999999999999},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.488,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":3.1960000000000002,"15min":2.391},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.6909999999999998,"5min":116.098,"15min":116.098},"last":3.1720000000000002},{"interface":"front","average":{"1min":1.23,"5min":2.992,"15min":2.6890000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":7.431,"5min":116.08499999999999,"15min":116.08499999999999},"last":2.9769999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.649,"15min":2.2400000000000002},"min":{"1min":0.41599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5939999999999999,"5min":20.742000000000001,"15min":79.311000000000007},"last":1.4770000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.532,"15min":1.9319999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":1.857,"5min":20.614000000000001,"15min":79.328999999999994},"last":1.3600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.1140000000000001,"15min":1.8899999999999999},"min":{"1min":0.46700000000000003,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":2.8039999999999998,"5min":5.8410000000000002,"15min":29.539000000000001},"last":1.581},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.395,"15min":2.0459999999999998},"min":{"1min":0.45100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7849999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":1.399}]}]},{"osd":0,"up_from":353,"seq":1516123455514,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2635888,"kb_used_data":2484584,"kb_used_omap":37,"kb_used_meta":151258,"kb_avail":91109264,"statfs":{"total":95995035648,"available":93295886336,"internally_reserved":0,"allocated":2544214016,"data_stored":2542663678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":154888354},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2280000000000002,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.437}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":3.6309999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":0.52100000000000002}]}]},{"osd":2,"up_from":359,"seq":1541893259287,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":811184,"kb_used_data":705960,"kb_used_omap":83,"kb_used_meta":105132,"kb_avail":92933968,"statfs":{"total":95995035648,"available":95164383232,"internally_reserved":0,"allocated":722903040,"data_stored":721413785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":107655621},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.438},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":1.256}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":0.75800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":0.495},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":1.8580000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":3.8690000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":0.52000000000000002}]}]},{"osd":3,"up_from":94,"seq":403726926091,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528396,"kb_used_data":41796,"kb_used_omap":7,"kb_used_meta":486584,"kb_avail":93216756,"statfs":{"total":95995035648,"available":95453958144,"internally_reserved":0,"allocated":42799104,"data_stored":41278378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":498262632},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.25},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.78300000000000003,"15min":2.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":3.355,"5min":16.596,"15min":123.866},"last":1.8919999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95599999999999996,"15min":2.2429999999999999},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":1.746,"5min":16.649999999999999,"15min":122.265},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.1469999999999998,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"last":1.9570000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":4.6769999999999996,"5min":1.798,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":87.046999999999997,"5min":87.046999999999997,"15min":87.046999999999997},"last":1.9910000000000001},{"interface":"front","average":{"1min":4.6580000000000004,"5min":1.972,"15min":1.7430000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":87.066000000000003,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2410000000000001,"15min":1.3839999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5910000000000002,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.2150000000000001,"15min":1.861},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3390000000000004,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.5649999999999999,"15min":1.7090000000000001},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2520000000000007,"5min":11.116,"15min":14.097},"last":2.4780000000000002},{"interface":"front","average":{"1min":1.613,"5min":1.397,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.2810000000000006,"5min":10.859,"15min":12.887},"last":2.4630000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97899999999999998,"15min":2.1179999999999999},"min":{"1min":0.443,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1040000000000001,"5min":6.0270000000000001,"15min":91.757999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.92300000000000004,"15min":2.097},"min":{"1min":0.37,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":3.4820000000000002,"5min":6.468,"15min":91.709999999999994},"last":2.0259999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019288,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3701448,"kb_used_data":2793984,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":90043704,"statfs":{"total":95995035648,"available":92204752896,"internally_reserved":0,"allocated":2861039616,"data_stored":2860273107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":0.70599999999999996,"15min":0.55000000000000004},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.65300000000000002,"15min":0.61499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":3.5110000000000001,"15min":3.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":2.77,"5min":98.846999999999994,"15min":132.304},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.302,"5min":3.6749999999999998,"15min":3.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":10.361000000000001,"5min":98.938999999999993,"15min":132.453},"last":0.501}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.378}]},{"osd":3,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.4450000000000001,"15min":4.3140000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.32400000000000001},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.2110000000000001,"15min":4.3049999999999997},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":3.2490000000000001,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:59:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.9630000000000001,"15min":1.6319999999999999},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3359999999999999,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.3599999999999999,"15min":1.7669999999999999},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.248},"max":{"1min":1.6040000000000001,"5min":151.905,"15min":151.905},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.585,"15min":1.1839999999999999},"min":{"1min":0.373,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":4.1269999999999998,"5min":16.943000000000001,"15min":16.943000000000001},"last":1.764},{"interface":"front","average":{"1min":1.155,"5min":1.371,"15min":1.105},"min":{"1min":0.372,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2130000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":1.1100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.3080000000000001,"15min":1.0960000000000001},"min":{"1min":0.46400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0150000000000001,"5min":12.84,"15min":12.84},"last":1.008},{"interface":"front","average":{"1min":1.371,"5min":1.3180000000000001,"15min":1.02},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.587,"5min":10.587,"15min":10.587},"last":0.745}]}]},{"osd":5,"up_from":36,"seq":154618823062,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2653672,"kb_used_data":1749920,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":91091480,"statfs":{"total":95995035648,"available":93277675520,"internally_reserved":0,"allocated":1791918080,"data_stored":1791179359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.73599999999999999,"15min":0.64900000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":14.42,"5min":14.42,"15min":14.42},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.274,"15min":1.1459999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.496,"5min":14.496,"15min":14.496},"last":0.81799999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2150000000000001,"15min":2.2850000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24199999999999999},"max":{"1min":3.21,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.748,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":6.7690000000000001,"5min":10.353,"15min":110.89},"last":2.5609999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.88300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.304,"15min":4.9619999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":3.1400000000000001,"5min":8.6489999999999991,"15min":257.05000000000001},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.4850000000000001,"15min":4.9610000000000003},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.965,"5min":12.185,"15min":257.35000000000002},"last":0.91500000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.802,"15min":1.4570000000000001},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6850000000000001,"5min":115.06699999999999,"15min":115.06699999999999},"last":2.6299999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.661,"15min":1.4510000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.1880000000000002,"5min":114.908,"15min":114.908},"last":0.89900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.345,"15min":1.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":6.8150000000000004,"5min":11.121,"15min":12.615},"last":7.4630000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.2130000000000001,"15min":1.123},"min":{"1min":0.433,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.2719999999999998,"5min":12.137,"15min":12.74},"last":5.4909999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.159,"15min":1.0389999999999999},"min":{"1min":0.44600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.3029999999999999,"5min":12.23,"15min":12.23},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.651,"5min":1.381,"15min":1.0389999999999999},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":10.638999999999999,"5min":11.112,"15min":11.112},"last":2.5990000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2536640512,"data_stored":2536636440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1522597888,"data_stored":1522597888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":717160448,"data_stored":717160448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":38338560,"data_stored":38338560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2849771520,"data_stored":2849767448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1785397248,"data_stored":1785397248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2113343488,"data_stored":2113339416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:00:14.401 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:14.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:14.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:14.500 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:00:14.500 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:00:14.500 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:00:14.500 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:00:14.501 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:00:14.507 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:14.541 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:14.606 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:14.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:14.646 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:00:14.652 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:00:14.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:14 smithi052 ceph-mon[180979]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:14.671 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4090377575' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:14.672 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:14 smithi052 ceph-mon[187743]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:14.672 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4090377575' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:14.713 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:14.763 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:14.882 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:14.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:14 smithi151 ceph-mon[159210]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:14.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4090377575' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:14.987 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:15.021 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:15.288 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:15.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:15.393 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:15.457 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:15.458 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:15.560 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:00:15.566 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:15.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:15.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: pgmap v1132: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 32463/177354 objects degraded (18.304%); 31652/177354 objects misplaced (17.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:15.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:00:15.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: failed to encode map e382 with expected crc 2023-09-23T23:00:15.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: failed to encode map e382 with expected crc 2023-09-23T23:00:15.636 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4090377575' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: osdmap e382: 8 total, 8 up, 6 in 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 32463/177354 objects degraded (18.304%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: pgmap v1132: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 32463/177354 objects degraded (18.304%); 31652/177354 objects misplaced (17.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: failed to encode map e382 with expected crc 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: failed to encode map e382 with expected crc 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4090377575' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:15.637 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: osdmap e382: 8 total, 8 up, 6 in 2023-09-23T23:00:15.638 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 32463/177354 objects degraded (18.304%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:15.673 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:15.710 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:15.764 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:15.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:15.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:15.868 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:15.914 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:15.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: pgmap v1132: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 32463/177354 objects degraded (18.304%); 31652/177354 objects misplaced (17.847%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:15.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:00:15.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: failed to encode map e382 with expected crc 2023-09-23T23:00:15.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: failed to encode map e382 with expected crc 2023-09-23T23:00:15.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4090377575' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:15.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: osdmap e382: 8 total, 8 up, 6 in 2023-09-23T23:00:15.916 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 32463/177354 objects degraded (18.304%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:16.017 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:16.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:16.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:16.101 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:16.132 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:16.179 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:16.310 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:16.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:16.419 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:16.456 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:16.590 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:16.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:16.702 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:16.738 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:16.896 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:16.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:17.000 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:17.028 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:17.079 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:17.080 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:17.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:17.099 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:17.118 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:17.137 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:17.184 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:17.214 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:17.246 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:17.250 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:17.271 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:17.292 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.294 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:00:17.357 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:00:17.368 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:00:17.635 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:17.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:17.649 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:17 smithi052 ceph-mon[187743]: pgmap v1134: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 32224/175935 objects degraded (18.316%); 31514/175935 objects misplaced (17.912%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:17.649 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4132306041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:17.649 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:17 smithi052 ceph-mon[180979]: pgmap v1134: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 32224/175935 objects degraded (18.316%); 31514/175935 objects misplaced (17.912%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:17.649 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4132306041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:17.749 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:17.792 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:17.874 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:17 smithi151 ceph-mon[159210]: pgmap v1134: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 32224/175935 objects degraded (18.316%); 31514/175935 objects misplaced (17.912%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:17.875 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4132306041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:18.090 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:18.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:18.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:18.195 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:18.212 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:18.212 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:18.255 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:18.284 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:18.324 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:18.360 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:18.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:18.466 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:18.498 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:18.578 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:18.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:18.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:18.723 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:19.174 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:00:19.339 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:19.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:19.443 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:19.480 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:19.559 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:19.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:19.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:19.679 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:19.713 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:19.790 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:19.792 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:19 smithi151 ceph-mon[159210]: pgmap v1135: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 31950/174102 objects degraded (18.351%); 31341/174102 objects misplaced (18.002%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:00:19.793 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:19.793 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:19.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:19.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:19.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:19 smithi052 ceph-mon[180979]: pgmap v1135: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 31950/174102 objects degraded (18.351%); 31341/174102 objects misplaced (18.002%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:00:19.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:19.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:19.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:19 smithi052 ceph-mon[187743]: pgmap v1135: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 31950/174102 objects degraded (18.351%); 31341/174102 objects misplaced (18.002%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:00:19.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:19.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:19.928 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:19.984 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:19.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:20.091 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:20.122 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:20.171 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:20.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:20.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:20.198 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:00:20.275 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:20.313 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:20.389 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:20.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:20.605 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:20.649 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:20.721 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:20.724 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:20.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:20.833 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:20.898 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31950/174102 objects degraded (18.351%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:20.898 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:20.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31950/174102 objects degraded (18.351%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:20.941 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:20.948 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:00:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31950/174102 objects degraded (18.351%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:21.174 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:21.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:21.279 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:21.314 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:21.372 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:21.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:21.478 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:21.511 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:21.625 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:21.626 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:21.648 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:21 smithi151 ceph-mon[159210]: pgmap v1136: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 31886/172956 objects degraded (18.436%); 31192/172956 objects misplaced (18.035%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:00:21.649 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/755612002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:00:21.660 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:00:21.794 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:21 smithi052 ceph-mon[187743]: pgmap v1136: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 31886/172956 objects degraded (18.436%); 31192/172956 objects misplaced (18.035%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:00:21.794 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/755612002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:00:21.794 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:21 smithi052 ceph-mon[180979]: pgmap v1136: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 31886/172956 objects degraded (18.436%); 31192/172956 objects misplaced (18.035%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T23:00:21.794 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/755612002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:00:21.814 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1136,"stamp":"2023-09-23T23:00:20.569121+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3777238968,"num_objects":57652,"num_object_clones":0,"num_object_copies":172956,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":31886,"num_objects_misplaced":31192,"num_objects_unfound":0,"num_objects_dirty":57652,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":136230,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12246,"num_bytes_recovered":799030204,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97964,"ondisk_log_size":97964,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12276304,"kb_used_data":9029344,"kb_used_omap":137,"kb_used_meta":3246646,"kb_avail":550194608,"statfs":{"total":575970213888,"available":563399278592,"internally_reserved":0,"allocated":9246048256,"data_stored":9238528560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3324565688},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-147062784,"num_objects":-2244,"num_object_clones":0,"num_object_copies":-6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1046,"num_objects_misplaced":-635,"num_objects_unfound":0,"num_objects_dirty":-2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2244,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":754,"num_bytes_recovered":49414144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003404"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":394,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542416+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:00:06.542416+0000","last_peered":"2023-09-23T23:00:06.542416+0000","last_clean":"2023-09-23T23:00:06.542416+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:00:06.542416+0000","last_undegraded":"2023-09-23T23:00:06.542416+0000","last_fullsized":"2023-09-23T23:00:06.542416+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":51,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.553204+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:00:06.553204+0000","last_peered":"2023-09-23T23:00:06.553204+0000","last_clean":"2023-09-23T23:00:06.553204+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:00:06.553204+0000","last_undegraded":"2023-09-23T23:00:06.553204+0000","last_fullsized":"2023-09-23T23:00:06.553204+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":50,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.550804+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:00:06.550804+0000","last_peered":"2023-09-23T23:00:06.550804+0000","last_clean":"2023-09-23T23:00:06.550804+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:00:06.550804+0000","last_undegraded":"2023-09-23T23:00:06.550804+0000","last_fullsized":"2023-09-23T23:00:06.550804+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":378,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551059+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:00:06.551059+0000","last_peered":"2023-09-23T23:00:06.551059+0000","last_clean":"2023-09-23T23:00:06.551059+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:00:06.551059+0000","last_undegraded":"2023-09-23T23:00:06.551059+0000","last_fullsized":"2023-09-23T23:00:06.551059+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":66,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551622+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:00:06.551622+0000","last_peered":"2023-09-23T23:00:06.551622+0000","last_clean":"2023-09-23T23:00:06.551622+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:00:06.551622+0000","last_undegraded":"2023-09-23T23:00:06.551622+0000","last_fullsized":"2023-09-23T23:00:06.551622+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":52,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:09.768157+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:09.768157+0000","last_peered":"2023-09-23T23:00:09.768157+0000","last_clean":"2023-09-23T23:00:09.768157+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:09.768157+0000","last_undegraded":"2023-09-23T23:00:09.768157+0000","last_fullsized":"2023-09-23T23:00:09.768157+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":317,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.533684+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:00:06.533684+0000","last_peered":"2023-09-23T23:00:06.533684+0000","last_clean":"2023-09-23T23:00:06.533684+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:00:06.533684+0000","last_undegraded":"2023-09-23T23:00:06.533684+0000","last_fullsized":"2023-09-23T23:00:06.533684+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551504+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:00:06.551504+0000","last_peered":"2023-09-23T23:00:06.551504+0000","last_clean":"2023-09-23T23:00:06.551504+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:00:06.551504+0000","last_undegraded":"2023-09-23T23:00:06.551504+0000","last_fullsized":"2023-09-23T23:00:06.551504+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":278,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.532740+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:00:06.532740+0000","last_peered":"2023-09-23T23:00:06.532740+0000","last_clean":"2023-09-23T23:00:06.532740+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:00:06.532740+0000","last_undegraded":"2023-09-23T23:00:06.532740+0000","last_fullsized":"2023-09-23T23:00:06.532740+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.550674+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:00:06.550674+0000","last_peered":"2023-09-23T23:00:06.550674+0000","last_clean":"2023-09-23T23:00:06.550674+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:00:06.550674+0000","last_undegraded":"2023-09-23T23:00:06.550674+0000","last_fullsized":"2023-09-23T23:00:06.550674+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":71,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.553132+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:00:06.553132+0000","last_peered":"2023-09-23T23:00:06.553132+0000","last_clean":"2023-09-23T23:00:06.553132+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:00:06.553132+0000","last_undegraded":"2023-09-23T23:00:06.553132+0000","last_fullsized":"2023-09-23T23:00:06.553132+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":126,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551951+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:00:06.551951+0000","last_peered":"2023-09-23T23:00:06.551951+0000","last_clean":"2023-09-23T23:00:06.551951+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:00:06.551951+0000","last_undegraded":"2023-09-23T23:00:06.551951+0000","last_fullsized":"2023-09-23T23:00:06.551951+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":453,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543164+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:00:06.543164+0000","last_peered":"2023-09-23T23:00:06.543164+0000","last_clean":"2023-09-23T23:00:06.543164+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:00:06.543164+0000","last_undegraded":"2023-09-23T23:00:06.543164+0000","last_fullsized":"2023-09-23T23:00:06.543164+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":356,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543697+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:00:06.543697+0000","last_peered":"2023-09-23T23:00:06.543697+0000","last_clean":"2023-09-23T23:00:06.543697+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:00:06.543697+0000","last_undegraded":"2023-09-23T23:00:06.543697+0000","last_fullsized":"2023-09-23T23:00:06.543697+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":282,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543779+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:00:06.543779+0000","last_peered":"2023-09-23T23:00:06.543779+0000","last_clean":"2023-09-23T23:00:06.543779+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:00:06.543779+0000","last_undegraded":"2023-09-23T23:00:06.543779+0000","last_fullsized":"2023-09-23T23:00:06.543779+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552266+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:00:06.552266+0000","last_peered":"2023-09-23T23:00:06.552266+0000","last_clean":"2023-09-23T23:00:06.552266+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:00:06.552266+0000","last_undegraded":"2023-09-23T23:00:06.552266+0000","last_fullsized":"2023-09-23T23:00:06.552266+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":371,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551909+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:00:06.551909+0000","last_peered":"2023-09-23T23:00:06.551909+0000","last_clean":"2023-09-23T23:00:06.551909+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:00:06.551909+0000","last_undegraded":"2023-09-23T23:00:06.551909+0000","last_fullsized":"2023-09-23T23:00:06.551909+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":282,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.533279+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:00:06.533279+0000","last_peered":"2023-09-23T23:00:06.533279+0000","last_clean":"2023-09-23T23:00:06.533279+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:00:06.533279+0000","last_undegraded":"2023-09-23T23:00:06.533279+0000","last_fullsized":"2023-09-23T23:00:06.533279+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":267,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542725+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:00:06.542725+0000","last_peered":"2023-09-23T23:00:06.542725+0000","last_clean":"2023-09-23T23:00:06.542725+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:00:06.542725+0000","last_undegraded":"2023-09-23T23:00:06.542725+0000","last_fullsized":"2023-09-23T23:00:06.542725+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":395,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551176+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:06.551176+0000","last_peered":"2023-09-23T23:00:06.551176+0000","last_clean":"2023-09-23T23:00:06.551176+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:06.551176+0000","last_undegraded":"2023-09-23T23:00:06.551176+0000","last_fullsized":"2023-09-23T23:00:06.551176+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":99,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.535429+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:00:06.535429+0000","last_peered":"2023-09-23T23:00:06.535429+0000","last_clean":"2023-09-23T23:00:06.535429+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:00:06.535429+0000","last_undegraded":"2023-09-23T23:00:06.535429+0000","last_fullsized":"2023-09-23T23:00:06.535429+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552348+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:00:06.552348+0000","last_peered":"2023-09-23T23:00:06.552348+0000","last_clean":"2023-09-23T23:00:06.552348+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:00:06.552348+0000","last_undegraded":"2023-09-23T23:00:06.552348+0000","last_fullsized":"2023-09-23T23:00:06.552348+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"382'8472","reported_seq":8558,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:19.414924+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:00:19.414924+0000","last_peered":"2023-09-23T23:00:19.414924+0000","last_clean":"2023-09-23T23:00:19.414924+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:00:19.414924+0000","last_undegraded":"2023-09-23T23:00:19.414924+0000","last_fullsized":"2023-09-23T23:00:19.414924+0000","mapping_epoch":353,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236847104,"num_objects":3614,"num_object_clones":0,"num_object_copies":10842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8472,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":266,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.550451+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:00:06.550451+0000","last_peered":"2023-09-23T23:00:06.550451+0000","last_clean":"2023-09-23T23:00:06.550451+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:00:06.550451+0000","last_undegraded":"2023-09-23T23:00:06.550451+0000","last_fullsized":"2023-09-23T23:00:06.550451+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.532362+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:00:06.532362+0000","last_peered":"2023-09-23T23:00:06.532362+0000","last_clean":"2023-09-23T23:00:06.532362+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:00:06.532362+0000","last_undegraded":"2023-09-23T23:00:06.532362+0000","last_fullsized":"2023-09-23T23:00:06.532362+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"382'8351","reported_seq":10766,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:16.089093+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:00:16.089093+0000","last_peered":"2023-09-23T23:00:16.089093+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:00:16.089093+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235864064,"num_objects":3599,"num_object_clones":0,"num_object_copies":10797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3599,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8351,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":32}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":276,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.548440+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:00:06.548440+0000","last_peered":"2023-09-23T23:00:06.548440+0000","last_clean":"2023-09-23T23:00:06.548440+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:00:06.548440+0000","last_undegraded":"2023-09-23T23:00:06.548440+0000","last_fullsized":"2023-09-23T23:00:06.548440+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":389,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542665+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:00:06.542665+0000","last_peered":"2023-09-23T23:00:06.542665+0000","last_clean":"2023-09-23T23:00:06.542665+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:00:06.542665+0000","last_undegraded":"2023-09-23T23:00:06.542665+0000","last_fullsized":"2023-09-23T23:00:06.542665+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"382'8542","reported_seq":8622,"reported_epoch":382,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:19.609231+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:00:19.609231+0000","last_peered":"2023-09-23T23:00:19.609231+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:00:19.609231+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:00:19.609231+0000","mapping_epoch":354,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5542,"log_dups_size":0,"ondisk_log_size":5542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231342080,"num_objects":3530,"num_object_clones":0,"num_object_copies":10590,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3530,"num_objects_unfound":0,"num_objects_dirty":3530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8542,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3530}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":133,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.533359+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:00:06.533359+0000","last_peered":"2023-09-23T23:00:06.533359+0000","last_clean":"2023-09-23T23:00:06.533359+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:00:06.533359+0000","last_undegraded":"2023-09-23T23:00:06.533359+0000","last_fullsized":"2023-09-23T23:00:06.533359+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":251,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.532957+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:00:06.532957+0000","last_peered":"2023-09-23T23:00:06.532957+0000","last_clean":"2023-09-23T23:00:06.532957+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:00:06.532957+0000","last_undegraded":"2023-09-23T23:00:06.532957+0000","last_fullsized":"2023-09-23T23:00:06.532957+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"382'8584","reported_seq":8850,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:19.685911+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:00:19.685911+0000","last_peered":"2023-09-23T23:00:19.685911+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:00:19.685911+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235274240,"num_objects":3590,"num_object_clones":0,"num_object_copies":10770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3024,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8584,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":565},{"shards":"0,5","objects":-20}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":51,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552187+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:00:06.552187+0000","last_peered":"2023-09-23T23:00:06.552187+0000","last_clean":"2023-09-23T23:00:06.552187+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:00:06.552187+0000","last_undegraded":"2023-09-23T23:00:06.552187+0000","last_fullsized":"2023-09-23T23:00:06.552187+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":281,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640751+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:00:14.640751+0000","last_peered":"2023-09-23T23:00:14.640751+0000","last_clean":"2023-09-23T23:00:14.640751+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:00:14.640751+0000","last_undegraded":"2023-09-23T23:00:14.640751+0000","last_fullsized":"2023-09-23T23:00:14.640751+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"382'8477","reported_seq":8564,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:16.612665+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:00:16.612665+0000","last_peered":"2023-09-23T23:00:16.612665+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:00:16.612665+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:00:16.612665+0000","mapping_epoch":353,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237043712,"num_objects":3617,"num_object_clones":0,"num_object_copies":10851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8477,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3617}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":70,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.554072+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:00:06.554072+0000","last_peered":"2023-09-23T23:00:06.554072+0000","last_clean":"2023-09-23T23:00:06.554072+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:00:06.554072+0000","last_undegraded":"2023-09-23T23:00:06.554072+0000","last_fullsized":"2023-09-23T23:00:06.554072+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":375,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543076+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:00:06.543076+0000","last_peered":"2023-09-23T23:00:06.543076+0000","last_clean":"2023-09-23T23:00:06.543076+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:00:06.543076+0000","last_undegraded":"2023-09-23T23:00:06.543076+0000","last_fullsized":"2023-09-23T23:00:06.543076+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"382'8560","reported_seq":8655,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:16.909080+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:00:16.909080+0000","last_peered":"2023-09-23T23:00:16.909080+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:00:16.909080+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238682112,"num_objects":3642,"num_object_clones":0,"num_object_copies":10926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3642,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8560,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":23}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":165,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542567+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:00:06.542567+0000","last_peered":"2023-09-23T23:00:06.542567+0000","last_clean":"2023-09-23T23:00:06.542567+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:00:06.542567+0000","last_undegraded":"2023-09-23T23:00:06.542567+0000","last_fullsized":"2023-09-23T23:00:06.542567+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":392,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.544981+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:00:06.544981+0000","last_peered":"2023-09-23T23:00:06.544981+0000","last_clean":"2023-09-23T23:00:06.544981+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:00:06.544981+0000","last_undegraded":"2023-09-23T23:00:06.544981+0000","last_fullsized":"2023-09-23T23:00:06.544981+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"382'8484","reported_seq":13021,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:17.178693+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:00:17.178693+0000","last_peered":"2023-09-23T23:00:17.178693+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:00:17.178693+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:00:17.178693+0000","mapping_epoch":354,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234356736,"num_objects":3576,"num_object_clones":0,"num_object_copies":10728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2383,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8484,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2217,"num_bytes_recovered":145293312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":2142}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":405,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542456+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:00:06.542456+0000","last_peered":"2023-09-23T23:00:06.542456+0000","last_clean":"2023-09-23T23:00:06.542456+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:00:06.542456+0000","last_undegraded":"2023-09-23T23:00:06.542456+0000","last_fullsized":"2023-09-23T23:00:06.542456+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":383,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543200+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:00:06.543200+0000","last_peered":"2023-09-23T23:00:06.543200+0000","last_clean":"2023-09-23T23:00:06.543200+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:00:06.543200+0000","last_undegraded":"2023-09-23T23:00:06.543200+0000","last_fullsized":"2023-09-23T23:00:06.543200+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"382'8336","reported_seq":8423,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:16.959994+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:00:16.959994+0000","last_peered":"2023-09-23T23:00:16.959994+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:00:16.959994+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:00:16.959994+0000","mapping_epoch":353,"log_start":"325'2600","ondisk_log_start":"325'2600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238026752,"num_objects":3632,"num_object_clones":0,"num_object_copies":10896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8336,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3632}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":51,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551814+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:00:06.551814+0000","last_peered":"2023-09-23T23:00:06.551814+0000","last_clean":"2023-09-23T23:00:06.551814+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:00:06.551814+0000","last_undegraded":"2023-09-23T23:00:06.551814+0000","last_fullsized":"2023-09-23T23:00:06.551814+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":284,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552807+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:00:06.552807+0000","last_peered":"2023-09-23T23:00:06.552807+0000","last_clean":"2023-09-23T23:00:06.552807+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:00:06.552807+0000","last_undegraded":"2023-09-23T23:00:06.552807+0000","last_fullsized":"2023-09-23T23:00:06.552807+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"382'8428","reported_seq":14895,"reported_epoch":382,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:00:17.183947+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:00:17.183947+0000","last_peered":"2023-09-23T23:00:17.183947+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:00:17.183947+0000","last_undegraded":"2023-09-23T23:00:17.183947+0000","last_fullsized":"2023-09-23T23:00:17.183947+0000","mapping_epoch":354,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5728,"log_dups_size":0,"ondisk_log_size":5728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075328,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":492,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8428,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3173,"num_bytes_recovered":207421440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":3156},{"shards":"0,4,6","objects":492}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":288,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552374+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:00:06.552374+0000","last_peered":"2023-09-23T23:00:06.552374+0000","last_clean":"2023-09-23T23:00:06.552374+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:00:06.552374+0000","last_undegraded":"2023-09-23T23:00:06.552374+0000","last_fullsized":"2023-09-23T23:00:06.552374+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":292,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.533211+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:00:06.533211+0000","last_peered":"2023-09-23T23:00:06.533211+0000","last_clean":"2023-09-23T23:00:06.533211+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:00:06.533211+0000","last_undegraded":"2023-09-23T23:00:06.533211+0000","last_fullsized":"2023-09-23T23:00:06.533211+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.532365+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:00:06.532365+0000","last_peered":"2023-09-23T23:00:06.532365+0000","last_clean":"2023-09-23T23:00:06.532365+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:00:06.532365+0000","last_undegraded":"2023-09-23T23:00:06.532365+0000","last_fullsized":"2023-09-23T23:00:06.532365+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"382'8529","reported_seq":8628,"reported_epoch":382,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:00:19.564065+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:00:19.564065+0000","last_peered":"2023-09-23T23:00:19.564065+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:00:19.564065+0000","last_undegraded":"2023-09-23T23:00:19.564065+0000","last_fullsized":"2023-09-23T23:00:19.564065+0000","mapping_epoch":359,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235601920,"num_objects":3595,"num_object_clones":0,"num_object_copies":10785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3595,"num_objects_unfound":0,"num_objects_dirty":3595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8529,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3595}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":52,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:09.766218+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:09.766218+0000","last_peered":"2023-09-23T23:00:09.766218+0000","last_clean":"2023-09-23T23:00:09.766218+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:09.766218+0000","last_undegraded":"2023-09-23T23:00:09.766218+0000","last_fullsized":"2023-09-23T23:00:09.766218+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":402,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543864+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:00:06.543864+0000","last_peered":"2023-09-23T23:00:06.543864+0000","last_clean":"2023-09-23T23:00:06.543864+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:00:06.543864+0000","last_undegraded":"2023-09-23T23:00:06.543864+0000","last_fullsized":"2023-09-23T23:00:06.543864+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":273,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552448+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:00:06.552448+0000","last_peered":"2023-09-23T23:00:06.552448+0000","last_clean":"2023-09-23T23:00:06.552448+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:00:06.552448+0000","last_undegraded":"2023-09-23T23:00:06.552448+0000","last_fullsized":"2023-09-23T23:00:06.552448+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"382'8444","reported_seq":13726,"reported_epoch":382,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:16.150602+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T23:00:16.150602+0000","last_peered":"2023-09-23T23:00:16.150602+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:00:16.150602+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":7944,"log_dups_size":0,"ondisk_log_size":7944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237240320,"num_objects":3620,"num_object_clones":0,"num_object_copies":10860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8444,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2594,"num_bytes_recovered":170000384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2643}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":348,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.554353+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:00:06.554353+0000","last_peered":"2023-09-23T23:00:06.554353+0000","last_clean":"2023-09-23T23:00:06.554353+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:00:06.554353+0000","last_undegraded":"2023-09-23T23:00:06.554353+0000","last_fullsized":"2023-09-23T23:00:06.554353+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":225,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551752+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:00:06.551752+0000","last_peered":"2023-09-23T23:00:06.551752+0000","last_clean":"2023-09-23T23:00:06.551752+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:00:06.551752+0000","last_undegraded":"2023-09-23T23:00:06.551752+0000","last_fullsized":"2023-09-23T23:00:06.551752+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":292,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.533747+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:00:06.533747+0000","last_peered":"2023-09-23T23:00:06.533747+0000","last_clean":"2023-09-23T23:00:06.533747+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:00:06.533747+0000","last_undegraded":"2023-09-23T23:00:06.533747+0000","last_fullsized":"2023-09-23T23:00:06.533747+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"382'8581","reported_seq":10464,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:17.366895+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:00:17.366895+0000","last_peered":"2023-09-23T23:00:17.366895+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:00:17.366895+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8581,"log_dups_size":0,"ondisk_log_size":8581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238747648,"num_objects":3643,"num_object_clones":0,"num_object_copies":10929,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8581,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2210},{"shards":"0,6","objects":-1338}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":365,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.532850+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:00:06.532850+0000","last_peered":"2023-09-23T23:00:06.532850+0000","last_clean":"2023-09-23T23:00:06.532850+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:00:06.532850+0000","last_undegraded":"2023-09-23T23:00:06.532850+0000","last_fullsized":"2023-09-23T23:00:06.532850+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":478,"reported_epoch":381,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:06.536473+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:06.536473+0000","last_peered":"2023-09-23T23:00:06.536473+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:06.536473+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":51,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552108+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:00:06.552108+0000","last_peered":"2023-09-23T23:00:06.552108+0000","last_clean":"2023-09-23T23:00:06.552108+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:00:06.552108+0000","last_undegraded":"2023-09-23T23:00:06.552108+0000","last_fullsized":"2023-09-23T23:00:06.552108+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":292,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.545225+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:00:06.545225+0000","last_peered":"2023-09-23T23:00:06.545225+0000","last_clean":"2023-09-23T23:00:06.545225+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:00:06.545225+0000","last_undegraded":"2023-09-23T23:00:06.545225+0000","last_fullsized":"2023-09-23T23:00:06.545225+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"382'8590","reported_seq":8681,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:19.388140+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:00:19.388140+0000","last_peered":"2023-09-23T23:00:19.388140+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:00:19.388140+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228982784,"num_objects":3494,"num_object_clones":0,"num_object_copies":10482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8590,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-13}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":372,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.535441+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:06.535441+0000","last_peered":"2023-09-23T23:00:06.535441+0000","last_clean":"2023-09-23T23:00:06.535441+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:06.535441+0000","last_undegraded":"2023-09-23T23:00:06.535441+0000","last_fullsized":"2023-09-23T23:00:06.535441+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":343,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:09.766192+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:09.766192+0000","last_peered":"2023-09-23T23:00:09.766192+0000","last_clean":"2023-09-23T23:00:09.766192+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:09.766192+0000","last_undegraded":"2023-09-23T23:00:09.766192+0000","last_fullsized":"2023-09-23T23:00:09.766192+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":303,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.641717+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:00:14.641717+0000","last_peered":"2023-09-23T23:00:14.641717+0000","last_clean":"2023-09-23T23:00:14.641717+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:00:14.641717+0000","last_undegraded":"2023-09-23T23:00:14.641717+0000","last_fullsized":"2023-09-23T23:00:14.641717+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"382'8556","reported_seq":8662,"reported_epoch":382,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:16.764424+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:00:16.764424+0000","last_peered":"2023-09-23T23:00:16.764424+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:00:16.764424+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:00:16.764424+0000","mapping_epoch":354,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239009816,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3648,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8556,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3648}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":318,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:09.766427+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:09.766427+0000","last_peered":"2023-09-23T23:00:09.766427+0000","last_clean":"2023-09-23T23:00:09.766427+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:09.766427+0000","last_undegraded":"2023-09-23T23:00:09.766427+0000","last_fullsized":"2023-09-23T23:00:09.766427+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":225,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552519+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:00:06.552519+0000","last_peered":"2023-09-23T23:00:06.552519+0000","last_clean":"2023-09-23T23:00:06.552519+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:00:06.552519+0000","last_undegraded":"2023-09-23T23:00:06.552519+0000","last_fullsized":"2023-09-23T23:00:06.552519+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":195,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.532858+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:00:06.532858+0000","last_peered":"2023-09-23T23:00:06.532858+0000","last_clean":"2023-09-23T23:00:06.532858+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:00:06.532858+0000","last_undegraded":"2023-09-23T23:00:06.532858+0000","last_fullsized":"2023-09-23T23:00:06.532858+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"382'8544","reported_seq":12410,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:17.180983+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:00:17.180983+0000","last_peered":"2023-09-23T23:00:17.180983+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:00:17.180983+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236060672,"num_objects":3602,"num_object_clones":0,"num_object_copies":10806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7469,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8544,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1883,"num_bytes_recovered":123404288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"2,4,6","objects":20},{"shards":"4,5","objects":1},{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2196}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":75,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552545+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:00:06.552545+0000","last_peered":"2023-09-23T23:00:06.552545+0000","last_clean":"2023-09-23T23:00:06.552545+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:00:06.552545+0000","last_undegraded":"2023-09-23T23:00:06.552545+0000","last_fullsized":"2023-09-23T23:00:06.552545+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543314+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:00:06.543314+0000","last_peered":"2023-09-23T23:00:06.543314+0000","last_clean":"2023-09-23T23:00:06.543314+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:00:06.543314+0000","last_undegraded":"2023-09-23T23:00:06.543314+0000","last_fullsized":"2023-09-23T23:00:06.543314+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551691+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:00:06.551691+0000","last_peered":"2023-09-23T23:00:06.551691+0000","last_clean":"2023-09-23T23:00:06.551691+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:00:06.551691+0000","last_undegraded":"2023-09-23T23:00:06.551691+0000","last_fullsized":"2023-09-23T23:00:06.551691+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"382'8492","reported_seq":9008,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:16.858456+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:00:16.858456+0000","last_peered":"2023-09-23T23:00:16.858456+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:00:16.858456+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234487808,"num_objects":3578,"num_object_clones":0,"num_object_copies":10734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3578,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8492,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":27}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":337,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.553065+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:00:06.553065+0000","last_peered":"2023-09-23T23:00:06.553065+0000","last_clean":"2023-09-23T23:00:06.553065+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:00:06.553065+0000","last_undegraded":"2023-09-23T23:00:06.553065+0000","last_fullsized":"2023-09-23T23:00:06.553065+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.641642+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:00:14.641642+0000","last_peered":"2023-09-23T23:00:14.641642+0000","last_clean":"2023-09-23T23:00:14.641642+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:00:14.641642+0000","last_undegraded":"2023-09-23T23:00:14.641642+0000","last_fullsized":"2023-09-23T23:00:14.641642+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":356,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542536+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:00:06.542536+0000","last_peered":"2023-09-23T23:00:06.542536+0000","last_clean":"2023-09-23T23:00:06.542536+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:00:06.542536+0000","last_undegraded":"2023-09-23T23:00:06.542536+0000","last_fullsized":"2023-09-23T23:00:06.542536+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":319,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.539271+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:00:06.539271+0000","last_peered":"2023-09-23T23:00:06.539271+0000","last_clean":"2023-09-23T23:00:06.539271+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:00:06.539271+0000","last_undegraded":"2023-09-23T23:00:06.539271+0000","last_fullsized":"2023-09-23T23:00:06.539271+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":129,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.552237+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:00:06.552237+0000","last_peered":"2023-09-23T23:00:06.552237+0000","last_clean":"2023-09-23T23:00:06.552237+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:00:06.552237+0000","last_undegraded":"2023-09-23T23:00:06.552237+0000","last_fullsized":"2023-09-23T23:00:06.552237+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":191,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542997+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:00:06.542997+0000","last_peered":"2023-09-23T23:00:06.542997+0000","last_clean":"2023-09-23T23:00:06.542997+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:00:06.542997+0000","last_undegraded":"2023-09-23T23:00:06.542997+0000","last_fullsized":"2023-09-23T23:00:06.542997+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":405,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.543939+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:00:06.543939+0000","last_peered":"2023-09-23T23:00:06.543939+0000","last_clean":"2023-09-23T23:00:06.543939+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:00:06.543939+0000","last_undegraded":"2023-09-23T23:00:06.543939+0000","last_fullsized":"2023-09-23T23:00:06.543939+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":205,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.545075+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:00:06.545075+0000","last_peered":"2023-09-23T23:00:06.545075+0000","last_clean":"2023-09-23T23:00:06.545075+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:00:06.545075+0000","last_undegraded":"2023-09-23T23:00:06.545075+0000","last_fullsized":"2023-09-23T23:00:06.545075+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":292,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542490+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:00:06.542490+0000","last_peered":"2023-09-23T23:00:06.542490+0000","last_clean":"2023-09-23T23:00:06.542490+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:00:06.542490+0000","last_undegraded":"2023-09-23T23:00:06.542490+0000","last_fullsized":"2023-09-23T23:00:06.542490+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":75,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.551303+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:00:06.551303+0000","last_peered":"2023-09-23T23:00:06.551303+0000","last_clean":"2023-09-23T23:00:06.551303+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:00:06.551303+0000","last_undegraded":"2023-09-23T23:00:06.551303+0000","last_fullsized":"2023-09-23T23:00:06.551303+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":78,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.550919+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:00:06.550919+0000","last_peered":"2023-09-23T23:00:06.550919+0000","last_clean":"2023-09-23T23:00:06.550919+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:00:06.550919+0000","last_undegraded":"2023-09-23T23:00:06.550919+0000","last_fullsized":"2023-09-23T23:00:06.550919+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":324,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:09.768191+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:09.768191+0000","last_peered":"2023-09-23T23:00:09.768191+0000","last_clean":"2023-09-23T23:00:09.768191+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:09.768191+0000","last_undegraded":"2023-09-23T23:00:09.768191+0000","last_fullsized":"2023-09-23T23:00:09.768191+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":367,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-09-23T23:00:06.542228+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:00:06.542228+0000","last_peered":"2023-09-23T23:00:06.542228+0000","last_clean":"2023-09-23T23:00:06.542228+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:00:06.542228+0000","last_undegraded":"2023-09-23T23:00:06.542228+0000","last_fullsized":"2023-09-23T23:00:06.542228+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3776643096,"num_objects":57628,"num_object_clones":0,"num_object_copies":172884,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":31884,"num_objects_misplaced":31192,"num_objects_unfound":0,"num_objects_dirty":57628,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135970,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12211,"num_bytes_recovered":799014912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11299336192,"data_stored":11299323976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97770,"ondisk_log_size":97770,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430607,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626832,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253145,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2143864,"kb_used_data":1451248,"kb_used_omap":9,"kb_used_meta":692470,"kb_avail":91601288,"statfs":{"total":95995035648,"available":93799718912,"internally_reserved":0,"allocated":1486077952,"data_stored":1484643655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":709090281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.76900000000000002,"15min":0.82699999999999996},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80700000000000005,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":1.8560000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.048,"5min":4.048,"15min":4.048},"last":1.341}]},{"osd":3,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.74299999999999999,"15min":2.153},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.827,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.83099999999999996,"15min":2.5960000000000001},"min":{"1min":0.29099999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.109,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.7850000000000001,"15min":2.6309999999999998},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5470000000000002,"5min":151.036,"15min":151.036},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.972,"15min":2.6419999999999999},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.488,"5min":151.06899999999999,"15min":151.06899999999999},"last":2.3620000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":3.1960000000000002,"15min":2.391},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.6909999999999998,"5min":116.098,"15min":116.098},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.23,"5min":2.992,"15min":2.6890000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":7.431,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.52000000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.649,"15min":2.2400000000000002},"min":{"1min":0.41599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5939999999999999,"5min":20.742000000000001,"15min":79.311000000000007},"last":2.4319999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.532,"15min":1.9319999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":1.857,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.1140000000000001,"15min":1.8899999999999999},"min":{"1min":0.46700000000000003,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":2.8039999999999998,"5min":5.8410000000000002,"15min":29.539000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.395,"15min":2.0459999999999998},"min":{"1min":0.45100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7849999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":2.4079999999999999}]}]},{"osd":0,"up_from":353,"seq":1516123455516,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2532348,"kb_used_data":2393332,"kb_used_omap":37,"kb_used_meta":138970,"kb_avail":91212804,"statfs":{"total":95995035648,"available":93401911296,"internally_reserved":0,"allocated":2450771968,"data_stored":2449217660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":142305442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.251},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.7000000000000002,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2280000000000002,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.29899999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":0.50900000000000001}]}]},{"osd":2,"up_from":359,"seq":1541893259288,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":790780,"kb_used_data":681460,"kb_used_omap":83,"kb_used_meta":109228,"kb_avail":92954372,"statfs":{"total":95995035648,"available":95185276928,"internally_reserved":0,"allocated":697815040,"data_stored":696321815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":111849925},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.76600000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.93700000000000006}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":0.628},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":0.879}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":3.871},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":2.7080000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":2.5539999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.379}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":2.1339999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":3.0350000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":6.8520000000000003}]}]},{"osd":3,"up_from":94,"seq":403726926092,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":531864,"kb_used_data":41168,"kb_used_omap":7,"kb_used_meta":490680,"kb_avail":93213288,"statfs":{"total":95995035648,"available":95450406912,"internally_reserved":0,"allocated":42156032,"data_stored":40631336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":502456936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.78300000000000003,"15min":2.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":3.355,"5min":16.596,"15min":123.866},"last":0.311},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95599999999999996,"15min":2.2429999999999999},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":1.746,"5min":16.649999999999999,"15min":122.265},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.1469999999999998,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":4.6769999999999996,"5min":1.798,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":87.046999999999997,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.43099999999999999},{"interface":"front","average":{"1min":4.6580000000000004,"5min":1.972,"15min":1.7430000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":87.066000000000003,"5min":87.066000000000003,"15min":87.066000000000003},"last":3.625}]},{"osd":5,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2410000000000001,"15min":1.3839999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5910000000000002,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.2150000000000001,"15min":1.861},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3390000000000004,"5min":6.5510000000000002,"15min":11.951000000000001},"last":3.5449999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.5649999999999999,"15min":1.7090000000000001},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2520000000000007,"5min":11.116,"15min":14.097},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.613,"5min":1.397,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.2810000000000006,"5min":10.859,"15min":12.887},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97899999999999998,"15min":2.1179999999999999},"min":{"1min":0.443,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1040000000000001,"5min":6.0270000000000001,"15min":91.757999999999996},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.92300000000000004,"15min":2.097},"min":{"1min":0.37,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":3.4820000000000002,"5min":6.468,"15min":91.709999999999994},"last":0.68600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019289,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3654612,"kb_used_data":2743052,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":90090540,"statfs":{"total":95995035648,"available":92252712960,"internally_reserved":0,"allocated":2808885248,"data_stored":2808114769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":0.70599999999999996,"15min":0.55000000000000004},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.65300000000000002,"15min":0.61499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":3.5110000000000001,"15min":3.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":2.77,"5min":98.846999999999994,"15min":132.304},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.302,"5min":3.6749999999999998,"15min":3.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":10.361000000000001,"5min":98.938999999999993,"15min":132.453},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.4450000000000001,"15min":4.3140000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.32400000000000001},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.2110000000000001,"15min":4.3049999999999997},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":3.2490000000000001,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.502}]},{"osd":5,"last update":"Sat Sep 23 22:59:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.9630000000000001,"15min":1.6319999999999999},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3359999999999999,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.373},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.3599999999999999,"15min":1.7669999999999999},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.248},"max":{"1min":1.6040000000000001,"5min":151.905,"15min":151.905},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.585,"15min":1.1839999999999999},"min":{"1min":0.373,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":4.1269999999999998,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.438},{"interface":"front","average":{"1min":1.155,"5min":1.371,"15min":1.105},"min":{"1min":0.372,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2130000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":1.3080000000000001,"15min":1.0960000000000001},"min":{"1min":0.46400000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0150000000000001,"5min":12.84,"15min":12.84},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.371,"5min":1.3180000000000001,"15min":1.02},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.587,"5min":10.587,"15min":10.587},"last":0.45700000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823063,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2622836,"kb_used_data":1719084,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":91122316,"statfs":{"total":95995035648,"available":93309251584,"internally_reserved":0,"allocated":1760342016,"data_stored":1759599325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.73599999999999999,"15min":0.64900000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":14.42,"5min":14.42,"15min":14.42},"last":1.9119999999999999},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.274,"15min":1.1459999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.496,"5min":14.496,"15min":14.496},"last":1.8009999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2150000000000001,"15min":2.2850000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24199999999999999},"max":{"1min":3.21,"5min":12.159000000000001,"15min":110.36499999999999},"last":2.0739999999999998},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.748,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":6.7690000000000001,"5min":10.353,"15min":110.89},"last":2.9729999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":3.0569999999999999,"5min":3.0569999999999999,"15min":3.0569999999999999},"last":1.8680000000000001},{"interface":"front","average":{"1min":0.995,"5min":0.995,"15min":0.995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":1.8799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.304,"15min":4.9619999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":3.1400000000000001,"5min":8.6489999999999991,"15min":257.05000000000001},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.4850000000000001,"15min":4.9610000000000003},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.965,"5min":12.185,"15min":257.35000000000002},"last":2.0089999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.802,"15min":1.4570000000000001},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6850000000000001,"5min":115.06699999999999,"15min":115.06699999999999},"last":2.1520000000000001},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.661,"15min":1.4510000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.1880000000000002,"5min":114.908,"15min":114.908},"last":2.2290000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.345,"15min":1.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":6.8150000000000004,"5min":11.121,"15min":12.615},"last":1.895},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.2130000000000001,"15min":1.123},"min":{"1min":0.433,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.2719999999999998,"5min":12.137,"15min":12.74},"last":1.847}]},{"osd":7,"last update":"Sat Sep 23 23:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.1180000000000001,"15min":1.046},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.5980000000000008,"5min":12.23,"15min":12.23},"last":1.921},{"interface":"front","average":{"1min":1.627,"5min":1.4199999999999999,"15min":1.0940000000000001},"min":{"1min":0.45500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.8710000000000004,"5min":11.112,"15min":11.112},"last":2.5049999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2442334208,"data_stored":2442330136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1481113600,"data_stored":1481113600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":692649984,"data_stored":692649984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":37683200,"data_stored":37683200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2800685056,"data_stored":2800680984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1754267648,"data_stored":1754267648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2090602496,"data_stored":2090598424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:00:21.821 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:00:22.420 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:22.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:22.524 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:22.558 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:22.622 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:00:22.624 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:22.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:22.731 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:22.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[180979]: from='client.36399 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:22.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:22.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[180979]: failed to encode map e383 with expected crc 2023-09-23T23:00:22.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[180979]: failed to encode map e383 with expected crc 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/755612002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[180979]: osdmap e383: 8 total, 8 up, 6 in 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[187743]: from='client.36399 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[187743]: failed to encode map e383 with expected crc 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[187743]: failed to encode map e383 with expected crc 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/755612002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:00:22.740 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:22 smithi052 ceph-mon[187743]: osdmap e383: 8 total, 8 up, 6 in 2023-09-23T23:00:22.783 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:22.916 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:22.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:22 smithi151 ceph-mon[159210]: from='client.36399 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:22.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:22 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:22.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:22 smithi151 ceph-mon[159210]: failed to encode map e383 with expected crc 2023-09-23T23:00:22.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:22 smithi151 ceph-mon[159210]: failed to encode map e383 with expected crc 2023-09-23T23:00:22.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/755612002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:00:22.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:22 smithi151 ceph-mon[159210]: osdmap e383: 8 total, 8 up, 6 in 2023-09-23T23:00:22.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:23.042 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:23.053 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:23.095 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:23.164 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:23.215 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:23.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:23.331 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:23.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:23.444 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:23.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:23.521 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:00:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:00:23] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T23:00:23.548 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:23.552 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:23.588 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:23.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:23.614 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:23.640 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:23.664 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:23.685 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:23.706 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:23.727 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:23.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:00:23.754 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:23 smithi151 ceph-mon[159210]: pgmap v1138: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 274 op/s; 31385/170772 objects degraded (18.378%); 31014/170772 objects misplaced (18.161%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:00:23.754 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3092381788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:23.759 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:23.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:23.864 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:23.893 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:23 smithi052 ceph-mon[187743]: pgmap v1138: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 274 op/s; 31385/170772 objects degraded (18.378%); 31014/170772 objects misplaced (18.161%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:00:23.893 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3092381788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:23.893 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:23 smithi052 ceph-mon[180979]: pgmap v1138: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 274 op/s; 31385/170772 objects degraded (18.378%); 31014/170772 objects misplaced (18.161%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:00:23.893 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3092381788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:23.896 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:24.517 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:24.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:24.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:24.660 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:24.761 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:00:24.991 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:24.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:24.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:25.097 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:25.136 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:25.155 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:00:25.155 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:00:25.156 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:00:25.156 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:00:25.158 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:00:25.196 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:25.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:25.320 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:25.364 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:25.393 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:25.433 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:25.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:25.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:25.572 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:25.620 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:25.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:25.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:25.708 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:25.708 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 31385/170772 objects degraded (18.378%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1558,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":383,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":56924,"data_bytes":3729528760,"bytes_used":12425719808,"bytes_avail":563544494080,"bytes_total":575970213888,"degraded_objects":31385,"degraded_total":170772,"degraded_ratio":0.1837830557702668,"misplaced_objects":31014,"misplaced_total":170772,"misplaced_ratio":0.18161056847726795,"recovering_objects_per_sec":74,"recovering_bytes_per_sec":4853796,"recovering_keys_per_sec":0,"num_objects_recovered":736,"num_bytes_recovered":48234496,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":220},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (4m)\n [======================......] (remaining: 55s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:00:25.727 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:25.737 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:25 smithi052 ceph-mon[187743]: pgmap v1139: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 31385/170772 objects degraded (18.378%); 31014/170772 objects misplaced (18.161%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:00:25.738 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31385/170772 objects degraded (18.378%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:25.738 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:25 smithi052 ceph-mon[180979]: pgmap v1139: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 31385/170772 objects degraded (18.378%); 31014/170772 objects misplaced (18.161%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:00:25.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31385/170772 objects degraded (18.378%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:25.766 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:25 smithi151 ceph-mon[159210]: pgmap v1139: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 220 op/s; 31385/170772 objects degraded (18.378%); 31014/170772 objects misplaced (18.161%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T23:00:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31385/170772 objects degraded (18.378%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:26.116 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:00:26.581 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:26.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:26.589 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:00:26.589 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:00:26.674 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:26.686 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:26.720 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:26.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:26.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:26.874 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:26.900 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/46202569' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:26.901 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/46202569' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:26.904 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:26.924 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/46202569' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:27.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:00:27.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:27.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:27.329 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:27.339 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:27.404 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:27.483 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:27.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:27.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:27.632 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:27.670 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:27 smithi052 ceph-mon[180979]: pgmap v1140: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 31134/169359 objects degraded (18.383%); 30883/169359 objects misplaced (18.235%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:27.671 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:27 smithi052 ceph-mon[187743]: pgmap v1140: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 31134/169359 objects degraded (18.383%); 30883/169359 objects misplaced (18.235%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:27.737 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:27.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:27.840 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:27.871 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:27 smithi151 ceph-mon[159210]: pgmap v1140: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 219 op/s; 31134/169359 objects degraded (18.383%); 30883/169359 objects misplaced (18.235%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:27.950 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:28.125 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:28.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:28.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:28.233 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:28.273 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:28.405 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:28.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:28.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:28.511 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:28.572 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:28.695 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:00:28.721 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:28.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:28.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:28.787 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:00:28.787 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:00:28.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:00:28.788 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:00:28.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.788 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115752 10.b 10:dd4f5f0e:::benchmark_data_smithi052_461816_object18951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.788 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:00:28.788 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 106.728538332, 2023-09-23T23:00:28.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.395328154, 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115752 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.789 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052719+0000", 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9670000000000001e-06 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052727+0000", 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4453e-05 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052742+0000", 2023-09-23T23:00:28.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5230000000000001e-06 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052748+0000", 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1202455760000001 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.172994+0000", 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010631813 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.183626+0000", 2023-09-23T23:00:28.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011626968 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.195253+0000", 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.037761408000000003 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.233014+0000", 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097050000000000001 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.233984+0000", 2023-09-23T23:00:28.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029044180000000002 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.236889+0000", 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.182741982 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.419631+0000", 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043122619999999999 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.423943+0000", 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024105932 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.448049+0000", 2023-09-23T23:00:28.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.395328154 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115769 10.b 10:d0413e63:::benchmark_data_smithi052_461816_object18968:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.122591+0000", 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 106.65866785, 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.387893357, 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.799 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115769 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122591+0000", 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122591+0000", 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122590+0000", 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0461e-05 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122600+0000", 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9517999999999998e-05 2023-09-23T23:00:28.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122619+0000", 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3550000000000005e-06 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122628+0000", 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.146729326 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.269357+0000", 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012356867000000001 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.281714+0000", 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013998277 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.295712+0000", 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017133461999999999 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.312846+0000", 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013916410000000001 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.314237+0000", 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028647410000000001 2023-09-23T23:00:28.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.317102+0000", 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16527270599999999 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.482375+0000", 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049430890000000003 2023-09-23T23:00:28.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.487318+0000", 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023166771999999999 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.510485+0000", 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.387893357 2023-09-23T23:00:28.806 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115897 10.b 10:da71a67f:::benchmark_data_smithi052_461816_object19096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 105.822062327, 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511318728, 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.807 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115897 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959194+0000", 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4834000000000001e-05 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959209+0000", 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6504999999999999e-05 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959226+0000", 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.585e-06 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959234+0000", 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.341365068 2023-09-23T23:00:28.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.300599+0000", 2023-09-23T23:00:28.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024846097000000001 2023-09-23T23:00:28.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.325445+0000", 2023-09-23T23:00:28.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035146149000000002 2023-09-23T23:00:28.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.360592+0000", 2023-09-23T23:00:28.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.066987158000000005 2023-09-23T23:00:28.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.427579+0000", 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050410359999999996 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.432620+0000", 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016353240000000001 2023-09-23T23:00:28.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.434255+0000", 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046578970000000003 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.438913+0000", 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054771630000000002 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.444390+0000", 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.026125426 2023-09-23T23:00:28.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.470516+0000", 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511318728 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115903 10.b 10:df88d60c:::benchmark_data_smithi052_461816_object19102:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 105.778928986, 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5939529189999999, 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115903 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.819 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002328+0000", 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5429999999999999e-06 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002338+0000", 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4139e-05 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002352+0000", 2023-09-23T23:00:28.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2870000000000001e-06 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002358+0000", 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4692954739999999 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.471654+0000", 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013024641999999999 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.484678+0000", 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024981533 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.509660+0000", 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.059660008 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.569320+0000", 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073572899999999996 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.570056+0000", 2023-09-23T23:00:28.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021077549999999998 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.572163+0000", 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027725240000000002 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.574936+0000", 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034689930000000001 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.578405+0000", 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017878277000000001 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.596283+0000", 2023-09-23T23:00:28.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5939529189999999 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115918 10.b 10:d203a882:::benchmark_data_smithi052_461816_object19117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 105.660522922, 2023-09-23T23:00:28.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511800968, 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115918 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120735+0000", 2023-09-23T23:00:28.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.841e-06 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120743+0000", 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3618000000000001e-05 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120756+0000", 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6620000000000002e-06 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120762+0000", 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4106586059999999 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.531421+0000", 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012043214 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.543464+0000", 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012684575999999999 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.556148+0000", 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016711621999999999 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.572860+0000", 2023-09-23T23:00:28.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.025699692999999999 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.598560+0000", 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002245637 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.600805+0000", 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003157188 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.603963+0000", 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049459949999999999 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.608909+0000", 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023628764999999999 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.632537+0000", 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511800968 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116102 10.b 10:d60ae293:::benchmark_data_smithi052_461816_object19301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 104.827380451, 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.473893417, 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.835 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116102 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999967 2023-09-23T23:00:28.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953875+0000", 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4865000000000001e-05 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953890+0000", 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1577999999999999e-05 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953912+0000", 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.075e-05 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953923+0000", 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2057103410000001 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.159633+0000", 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0091687120000000007 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.168802+0000", 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011821033999999999 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.180623+0000", 2023-09-23T23:00:28.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018815996000000001 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.199439+0000", 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022259570000000002 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.201665+0000", 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021258484000000001 2023-09-23T23:00:28.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.222923+0000", 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.186530428 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.409453+0000", 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034981790000000001 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.412952+0000", 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014820543 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.427772+0000", 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.473893417 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.843 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116106 10.b 10:dc0e4ffc:::benchmark_data_smithi052_461816_object19305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 104.79653353400001, 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4803338619999999, 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116106 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.844 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984724+0000", 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0258000000000001e-05 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984734+0000", 2023-09-23T23:00:28.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8243000000000001e-05 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984752+0000", 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0596e-05 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984763+0000", 2023-09-23T23:00:28.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.238889275 2023-09-23T23:00:28.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.223652+0000", 2023-09-23T23:00:28.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012527140000000001 2023-09-23T23:00:28.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.236179+0000", 2023-09-23T23:00:28.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015698178 2023-09-23T23:00:28.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.251877+0000", 2023-09-23T23:00:28.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.043569531000000002 2023-09-23T23:00:28.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.295447+0000", 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020602350000000001 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.297507+0000", 2023-09-23T23:00:28.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020648390000000002 2023-09-23T23:00:28.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.299572+0000", 2023-09-23T23:00:28.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.134241317 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.433813+0000", 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068885409999999998 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.440702+0000", 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024357711000000001 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.465060+0000", 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4803338619999999 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116119 10.b 10:dbf3fcdb:::benchmark_data_smithi052_461816_object19318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 104.70347075700001, 2023-09-23T23:00:28.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4178687299999999, 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116119 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077785+0000", 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5787e-05 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077801+0000", 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3853e-05 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077825+0000", 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1316e-05 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077836+0000", 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.185146891 2023-09-23T23:00:28.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.262983+0000", 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095509050000000002 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.272534+0000", 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011856833000000001 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.284391+0000", 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017265624 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.301657+0000", 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002050713 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.303707+0000", 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12684677899999999 2023-09-23T23:00:28.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.430554+0000", 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035632981000000001 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.466187+0000", 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056081689999999997 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.471795+0000", 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023861869000000001 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.495657+0000", 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4178687299999999 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.861 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116267 10.b 10:d6c817ba:::benchmark_data_smithi052_461816_object19466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 103.65688374, 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4895586060000001, 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.862 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.863 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.863 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116267 2023-09-23T23:00:28.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.863 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124373+0000", 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0097e-05 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124383+0000", 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6015000000000002e-05 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124399+0000", 2023-09-23T23:00:28.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4009999999999997e-06 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124408+0000", 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.33608481 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.460493+0000", 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019478128000000001 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.479971+0000", 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023908400999999999 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.503879+0000", 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.084884822999999998 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.588764+0000", 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029924790000000001 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.591756+0000", 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022099440000000001 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.593966+0000", 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00070443600000000002 2023-09-23T23:00:28.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.594671+0000", 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035467929999999999 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.598218+0000", 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015716463 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.613934+0000", 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4895586060000001 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116299 10.b 10:d81ed6a4:::benchmark_data_smithi052_461816_object19498:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.502875+0000", 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 103.278383805, 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.385743757, 2023-09-23T23:00:28.870 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116299 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.502875+0000", 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.502875+0000", 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.502874+0000", 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.652e-06 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.502881+0000", 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3679999999999999e-05 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.502894+0000", 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9819999999999996e-06 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.502900+0000", 2023-09-23T23:00:28.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.192022522 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.694923+0000", 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018165408000000001 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.713088+0000", 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019098754999999999 2023-09-23T23:00:28.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.732187+0000", 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021266337999999999 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.753453+0000", 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022670941999999999 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.776124+0000", 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021795069999999998 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.778304+0000", 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.092004356999999995 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.870308+0000", 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034662709999999999 2023-09-23T23:00:28.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.873774+0000", 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014844843 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.888619+0000", 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.385743757 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.877 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116332 10.b 10:dc130431:::benchmark_data_smithi052_461816_object19531:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 103.177058059, 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001, 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.878 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116332 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:00:28.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604199+0000", 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1632e-05 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604211+0000", 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7592999999999999e-05 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604228+0000", 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0749999999999998e-06 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604236+0000", 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.223173936 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.827410+0000", 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015809969 2023-09-23T23:00:28.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.843220+0000", 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013161054 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.856381+0000", 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035988506000000003 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.892370+0000", 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.027739362 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.920109+0000", 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020550299999999998 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922164+0000", 2023-09-23T23:00:28.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.28725175800000002 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.209416+0000", 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0092034360000000006 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.218619+0000", 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030490359000000002 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.249110+0000", 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116347 10.b 10:db51aae1:::benchmark_data_smithi052_461816_object19546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 103.12672817399999, 2023-09-23T23:00:28.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001, 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116347 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:00:28.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654529+0000", 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5270000000000001e-06 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654537+0000", 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5132e-05 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654552+0000", 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5509999999999999e-06 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654557+0000", 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2683612470000001 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922919+0000", 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0088446870000000004 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.931763+0000", 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011420990000000001 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.943184+0000", 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040857233999999999 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.984042+0000", 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022559659999999999 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.986298+0000", 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002071696 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.988369+0000", 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26508044400000003 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.253450+0000", 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055181509999999998 2023-09-23T23:00:28.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.258968+0000", 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017719586999999998 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.276687+0000", 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.892 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116354 10.b 10:dc4e8e26:::benchmark_data_smithi052_461816_object19553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 103.058548741, 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.590831154, 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.893 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116354 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722708+0000", 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1059e-05 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722720+0000", 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8337000000000001e-05 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722738+0000", 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0482999999999999e-05 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722748+0000", 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2310586699999999 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.953807+0000", 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089079880000000004 2023-09-23T23:00:28.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.962715+0000", 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011595375 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.974310+0000", 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016259483000000002 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.990570+0000", 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002797819 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.993368+0000", 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034075324999999997 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.027443+0000", 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.252294817 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.279738+0000", 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053108809999999999 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.285049+0000", 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.028492905999999998 2023-09-23T23:00:28.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.313542+0000", 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.590831154 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116358 10.b 10:d271a06f:::benchmark_data_smithi052_461816_object19557:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:00:28.900 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 103.023265405, 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.585204461, 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116358 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757992+0000", 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7521e-05 2023-09-23T23:00:28.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758009+0000", 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3963e-05 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758023+0000", 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0438999999999999e-05 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758034+0000", 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2363462219999999 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.994380+0000", 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089635630000000008 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.003343+0000", 2023-09-23T23:00:28.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014499898000000001 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.017843+0000", 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.042956156000000002 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.060800+0000", 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068733800000000001 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.061487+0000", 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024376810000000001 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.063925+0000", 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25300748499999998 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.316932+0000", 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039558340000000001 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.320888+0000", 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022310436999999999 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.343198+0000", 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.585204461 2023-09-23T23:00:28.907 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116372 10.b 10:ddfaedb1:::benchmark_data_smithi052_461816_object19571:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 102.939732951, 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.532253965, 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.908 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116372 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:00:28.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841525+0000", 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5699999999999999e-06 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841534+0000", 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6439999999999998e-05 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841551+0000", 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9019999999999999e-06 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841558+0000", 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1866856050000001 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.028244+0000", 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099314550000000005 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.038176+0000", 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011378353000000001 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.049554+0000", 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015845318000000001 2023-09-23T23:00:28.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.065399+0000", 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062175360000000001 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.071617+0000", 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068547419999999996 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.078471+0000", 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26989495099999999 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.348366+0000", 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045601740000000002 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.352927+0000", 2023-09-23T23:00:28.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020853618000000001 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.373780+0000", 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.532253965 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116379 10.b 10:d308aafa:::benchmark_data_smithi052_461816_object19578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:00:28.915 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 102.822455335, 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4501070380000001, 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116379 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958802+0000", 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0233e-05 2023-09-23T23:00:28.917 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958812+0000", 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5444999999999999e-05 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958828+0000", 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6100000000000006e-06 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.918 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958836+0000", 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.12189563 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.080732+0000", 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021823031 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.102555+0000", 2023-09-23T23:00:28.919 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018619785 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.121175+0000", 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22298790399999999 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.344162+0000", 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035032671000000001 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.920 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.379195+0000", 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068400680000000004 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.386035+0000", 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003642756 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.389678+0000", 2023-09-23T23:00:28.921 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004166154 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.393844+0000", 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015066784 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.408911+0000", 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4501070380000001 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.922 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123112 10.b 10:d1224097:::benchmark_data_smithi052_461816_object26311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 71.540694494999997, 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.389065591, 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.923 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123112 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.924 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240563+0000", 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5110000000000003e-06 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240572+0000", 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5494e-05 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.925 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240597+0000", 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.776e-06 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240604+0000", 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.150625987 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.391230+0000", 2023-09-23T23:00:28.926 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014517555 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.405747+0000", 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011751549 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.417499+0000", 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.041869322 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.927 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.459368+0000", 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001986991 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.461355+0000", 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.046885463000000002 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.928 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.508241+0000", 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.076663249000000003 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.584904+0000", 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011116469 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.596020+0000", 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.033609914999999997 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.929 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.629630+0000", 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.389065591 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123247 10.b 10:df1d14f6:::benchmark_data_smithi052_461816_object26446:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 70.871439068000001, 2023-09-23T23:00:28.930 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4117894469999999, 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123247 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.931 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909818+0000", 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1479999999999995e-06 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.932 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909827+0000", 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4456e-05 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909842+0000", 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0669999999999997e-06 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.933 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909848+0000", 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2526637350000001 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.162512+0000", 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016769646999999999 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.179281+0000", 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015743411999999998 2023-09-23T23:00:28.934 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.195025+0000", 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.080734345999999998 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.275759+0000", 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053276499999999997 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.935 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.281087+0000", 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046729550000000003 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.285760+0000", 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012794950000000001 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.287039+0000", 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011935753 2023-09-23T23:00:28.936 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.298975+0000", 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022634743999999998 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.321610+0000", 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4117894469999999 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.937 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123254 10.b 10:d637fe9a:::benchmark_data_smithi052_461816_object26453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 70.815383389999994, 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.394393993, 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.938 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123254 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:00:28.939 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965874+0000", 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7100000000000002e-06 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965884+0000", 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4491999999999999e-05 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.940 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965898+0000", 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8060000000000001e-06 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965907+0000", 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2457147019999999 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.211622+0000", 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014958720999999999 2023-09-23T23:00:28.941 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.226581+0000", 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023973563 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.250554+0000", 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.072382668999999997 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.942 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.322937+0000", 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045649990000000001 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.327502+0000", 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005907359 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.333409+0000", 2023-09-23T23:00:28.943 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006139238 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.339548+0000", 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055687870000000004 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.345117+0000", 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015152598 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.360270+0000", 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.394393993 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123279 10.b 10:dfdd8540:::benchmark_data_smithi052_461816_object26478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 70.666657448999999, 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4014525230000001, 2023-09-23T23:00:28.945 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123279 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:00:28.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114600+0000", 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0774999999999999e-05 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114610+0000", 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7652999999999999e-05 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114628+0000", 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0809999999999994e-06 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114636+0000", 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2466315459999999 2023-09-23T23:00:28.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.361268+0000", 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017441438 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.378709+0000", 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031131333000000001 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.409840+0000", 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.072885458 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.482726+0000", 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024013120000000001 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.485127+0000", 2023-09-23T23:00:28.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026478410000000002 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.487775+0000", 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012406679999999999 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.489016+0000", 2023-09-23T23:00:28.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.007715375 2023-09-23T23:00:28.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:00:28.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.496731+0000", 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019323248000000001 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.516054+0000", 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4014525230000001 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:00:28.954 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:00:28.955 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:28.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/248444821' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:28.956 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:28 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:28.985 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/248444821' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:28.985 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:28 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:28.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/248444821' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:28.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:28 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:28.995 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:29.302 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:29.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:29.406 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:29.435 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:29.494 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:29.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:29.608 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:29.650 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:29.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: pgmap v1141: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 30794/167436 objects degraded (18.392%); 30702/167436 objects misplaced (18.337%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T23:00:29.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:29.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:29.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: osdmap e384: 8 total, 8 up, 6 in 2023-09-23T23:00:29.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:29 smithi151 ceph-mon[159210]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.857 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:00:29.859 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:29.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:29.969 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:29.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: pgmap v1141: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 30794/167436 objects degraded (18.392%); 30702/167436 objects misplaced (18.337%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T23:00:29.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:29.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: osdmap e384: 8 total, 8 up, 6 in 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[180979]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: pgmap v1141: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 30794/167436 objects degraded (18.392%); 30702/167436 objects misplaced (18.337%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:29.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: osdmap e384: 8 total, 8 up, 6 in 2023-09-23T23:00:29.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: failed to encode map e384 with expected crc 2023-09-23T23:00:29.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:29 smithi052 ceph-mon[187743]: failed to encode map e384 with expected crc 2023-09-23T23:00:30.053 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:30.137 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:30.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:30.244 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:30.280 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:30.339 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:30.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:30.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:30.391 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:30.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:30.483 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:30.554 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:30.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:30.646 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:30.650 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:00:30.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:30.660 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:30.675 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:30.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 30794/167436 objects degraded (18.392%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:30.676 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3777341042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:30.697 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:30.711 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:30.720 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:30.749 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:30.776 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:30.776 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:30.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:30.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:30.802 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:30.828 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.836 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:00:30.882 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:30.919 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 30794/167436 objects degraded (18.392%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3777341042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:30.975 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:30.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 30794/167436 objects degraded (18.392%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:30.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3777341042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:30.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:30.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:31.081 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:31.126 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:31.675 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1143,"stamp":"2023-09-23T23:00:30.572553+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3633649592,"num_objects":55461,"num_object_clones":0,"num_object_copies":166383,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":30737,"num_objects_misplaced":30573,"num_objects_unfound":0,"num_objects_dirty":55461,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":138421,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12975,"num_bytes_recovered":846805948,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98255,"ondisk_log_size":98255,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11995632,"kb_used_data":8741376,"kb_used_omap":137,"kb_used_meta":3254070,"kb_avail":550475280,"statfs":{"total":575970213888,"available":563686686720,"internally_reserved":0,"allocated":8951169024,"data_stored":8943610330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3332167864},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-133758976,"num_objects":-2041,"num_object_clones":0,"num_object_copies":-6123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-962,"num_objects_misplaced":-619,"num_objects_unfound":0,"num_objects_dirty":-2041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2041,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":612,"num_bytes_recovered":40108032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.919754"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":395,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.638197+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:00:14.638197+0000","last_peered":"2023-09-23T23:00:14.638197+0000","last_clean":"2023-09-23T23:00:14.638197+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:00:14.638197+0000","last_undegraded":"2023-09-23T23:00:14.638197+0000","last_fullsized":"2023-09-23T23:00:14.638197+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":52,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.644449+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:00:14.644449+0000","last_peered":"2023-09-23T23:00:14.644449+0000","last_clean":"2023-09-23T23:00:14.644449+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:00:14.644449+0000","last_undegraded":"2023-09-23T23:00:14.644449+0000","last_fullsized":"2023-09-23T23:00:14.644449+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640782+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:00:14.640782+0000","last_peered":"2023-09-23T23:00:14.640782+0000","last_clean":"2023-09-23T23:00:14.640782+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:00:14.640782+0000","last_undegraded":"2023-09-23T23:00:14.640782+0000","last_fullsized":"2023-09-23T23:00:14.640782+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":379,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.641068+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:00:14.641068+0000","last_peered":"2023-09-23T23:00:14.641068+0000","last_clean":"2023-09-23T23:00:14.641068+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:00:14.641068+0000","last_undegraded":"2023-09-23T23:00:14.641068+0000","last_fullsized":"2023-09-23T23:00:14.641068+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":67,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640651+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:00:14.640651+0000","last_peered":"2023-09-23T23:00:14.640651+0000","last_clean":"2023-09-23T23:00:14.640651+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:00:14.640651+0000","last_undegraded":"2023-09-23T23:00:14.640651+0000","last_fullsized":"2023-09-23T23:00:14.640651+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":54,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.656720+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:21.656720+0000","last_peered":"2023-09-23T23:00:21.656720+0000","last_clean":"2023-09-23T23:00:21.656720+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:21.656720+0000","last_undegraded":"2023-09-23T23:00:21.656720+0000","last_fullsized":"2023-09-23T23:00:21.656720+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":318,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.645162+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:00:14.645162+0000","last_peered":"2023-09-23T23:00:14.645162+0000","last_clean":"2023-09-23T23:00:14.645162+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:00:14.645162+0000","last_undegraded":"2023-09-23T23:00:14.645162+0000","last_fullsized":"2023-09-23T23:00:14.645162+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640495+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:00:14.640495+0000","last_peered":"2023-09-23T23:00:14.640495+0000","last_clean":"2023-09-23T23:00:14.640495+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:00:14.640495+0000","last_undegraded":"2023-09-23T23:00:14.640495+0000","last_fullsized":"2023-09-23T23:00:14.640495+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":279,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.646404+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:00:14.646404+0000","last_peered":"2023-09-23T23:00:14.646404+0000","last_clean":"2023-09-23T23:00:14.646404+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:00:14.646404+0000","last_undegraded":"2023-09-23T23:00:14.646404+0000","last_fullsized":"2023-09-23T23:00:14.646404+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":66,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640652+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:00:14.640652+0000","last_peered":"2023-09-23T23:00:14.640652+0000","last_clean":"2023-09-23T23:00:14.640652+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:00:14.640652+0000","last_undegraded":"2023-09-23T23:00:14.640652+0000","last_fullsized":"2023-09-23T23:00:14.640652+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.644427+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:00:14.644427+0000","last_peered":"2023-09-23T23:00:14.644427+0000","last_clean":"2023-09-23T23:00:14.644427+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:00:14.644427+0000","last_undegraded":"2023-09-23T23:00:14.644427+0000","last_fullsized":"2023-09-23T23:00:14.644427+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":127,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.657511+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:00:14.657511+0000","last_peered":"2023-09-23T23:00:14.657511+0000","last_clean":"2023-09-23T23:00:14.657511+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:00:14.657511+0000","last_undegraded":"2023-09-23T23:00:14.657511+0000","last_fullsized":"2023-09-23T23:00:14.657511+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":454,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.646320+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:00:14.646320+0000","last_peered":"2023-09-23T23:00:14.646320+0000","last_clean":"2023-09-23T23:00:14.646320+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:00:14.646320+0000","last_undegraded":"2023-09-23T23:00:14.646320+0000","last_fullsized":"2023-09-23T23:00:14.646320+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":357,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643418+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:00:14.643418+0000","last_peered":"2023-09-23T23:00:14.643418+0000","last_clean":"2023-09-23T23:00:14.643418+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:00:14.643418+0000","last_undegraded":"2023-09-23T23:00:14.643418+0000","last_fullsized":"2023-09-23T23:00:14.643418+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":283,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643497+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:00:14.643497+0000","last_peered":"2023-09-23T23:00:14.643497+0000","last_clean":"2023-09-23T23:00:14.643497+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:00:14.643497+0000","last_undegraded":"2023-09-23T23:00:14.643497+0000","last_fullsized":"2023-09-23T23:00:14.643497+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642188+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:00:14.642188+0000","last_peered":"2023-09-23T23:00:14.642188+0000","last_clean":"2023-09-23T23:00:14.642188+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:00:14.642188+0000","last_undegraded":"2023-09-23T23:00:14.642188+0000","last_fullsized":"2023-09-23T23:00:14.642188+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":372,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640744+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:00:14.640744+0000","last_peered":"2023-09-23T23:00:14.640744+0000","last_clean":"2023-09-23T23:00:14.640744+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:00:14.640744+0000","last_undegraded":"2023-09-23T23:00:14.640744+0000","last_fullsized":"2023-09-23T23:00:14.640744+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.644739+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:00:14.644739+0000","last_peered":"2023-09-23T23:00:14.644739+0000","last_clean":"2023-09-23T23:00:14.644739+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:00:14.644739+0000","last_undegraded":"2023-09-23T23:00:14.644739+0000","last_fullsized":"2023-09-23T23:00:14.644739+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":268,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.645871+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:00:14.645871+0000","last_peered":"2023-09-23T23:00:14.645871+0000","last_clean":"2023-09-23T23:00:14.645871+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:00:14.645871+0000","last_undegraded":"2023-09-23T23:00:14.645871+0000","last_fullsized":"2023-09-23T23:00:14.645871+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":396,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.641209+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:14.641209+0000","last_peered":"2023-09-23T23:00:14.641209+0000","last_clean":"2023-09-23T23:00:14.641209+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:14.641209+0000","last_undegraded":"2023-09-23T23:00:14.641209+0000","last_fullsized":"2023-09-23T23:00:14.641209+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":102,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:24.774485+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:00:24.774485+0000","last_peered":"2023-09-23T23:00:24.774485+0000","last_clean":"2023-09-23T23:00:24.774485+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:00:24.774485+0000","last_undegraded":"2023-09-23T23:00:24.774485+0000","last_fullsized":"2023-09-23T23:00:24.774485+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.641090+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:00:14.641090+0000","last_peered":"2023-09-23T23:00:14.641090+0000","last_clean":"2023-09-23T23:00:14.641090+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:00:14.641090+0000","last_undegraded":"2023-09-23T23:00:14.641090+0000","last_fullsized":"2023-09-23T23:00:14.641090+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"384'8621","reported_seq":8709,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:29.743145+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:00:29.743145+0000","last_peered":"2023-09-23T23:00:29.743145+0000","last_clean":"2023-09-23T23:00:29.743145+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:00:29.743145+0000","last_undegraded":"2023-09-23T23:00:29.743145+0000","last_fullsized":"2023-09-23T23:00:29.743145+0000","mapping_epoch":353,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8621,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":267,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643161+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:00:14.643161+0000","last_peered":"2023-09-23T23:00:14.643161+0000","last_clean":"2023-09-23T23:00:14.643161+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:00:14.643161+0000","last_undegraded":"2023-09-23T23:00:14.643161+0000","last_fullsized":"2023-09-23T23:00:14.643161+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.646034+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:00:14.646034+0000","last_peered":"2023-09-23T23:00:14.646034+0000","last_clean":"2023-09-23T23:00:14.646034+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:00:14.646034+0000","last_undegraded":"2023-09-23T23:00:14.646034+0000","last_fullsized":"2023-09-23T23:00:14.646034+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"383'8492","reported_seq":10908,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:26.040140+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:00:26.040140+0000","last_peered":"2023-09-23T23:00:26.040140+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:00:26.040140+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8092,"log_dups_size":0,"ondisk_log_size":8092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226623488,"num_objects":3458,"num_object_clones":0,"num_object_copies":10374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3458,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8492,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-109}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":277,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643843+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:00:14.643843+0000","last_peered":"2023-09-23T23:00:14.643843+0000","last_clean":"2023-09-23T23:00:14.643843+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:00:14.643843+0000","last_undegraded":"2023-09-23T23:00:14.643843+0000","last_fullsized":"2023-09-23T23:00:14.643843+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":390,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.645799+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:00:14.645799+0000","last_peered":"2023-09-23T23:00:14.645799+0000","last_clean":"2023-09-23T23:00:14.645799+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:00:14.645799+0000","last_undegraded":"2023-09-23T23:00:14.645799+0000","last_fullsized":"2023-09-23T23:00:14.645799+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"384'8667","reported_seq":8748,"reported_epoch":384,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:29.737728+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:00:29.737728+0000","last_peered":"2023-09-23T23:00:29.737728+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:00:29.737728+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:00:29.737728+0000","mapping_epoch":354,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223150080,"num_objects":3405,"num_object_clones":0,"num_object_copies":10215,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3405,"num_objects_unfound":0,"num_objects_dirty":3405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8667,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3405}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.644821+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:00:14.644821+0000","last_peered":"2023-09-23T23:00:14.644821+0000","last_clean":"2023-09-23T23:00:14.644821+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:00:14.644821+0000","last_undegraded":"2023-09-23T23:00:14.644821+0000","last_fullsized":"2023-09-23T23:00:14.644821+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":252,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.647774+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:00:14.647774+0000","last_peered":"2023-09-23T23:00:14.647774+0000","last_clean":"2023-09-23T23:00:14.647774+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:00:14.647774+0000","last_undegraded":"2023-09-23T23:00:14.647774+0000","last_fullsized":"2023-09-23T23:00:14.647774+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"384'8731","reported_seq":8999,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:29.686207+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:00:29.686207+0000","last_peered":"2023-09-23T23:00:29.686207+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:00:29.686207+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225640448,"num_objects":3443,"num_object_clones":0,"num_object_copies":10329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2902,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8731,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":540},{"shards":"0,5","objects":-167}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":52,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642155+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:00:14.642155+0000","last_peered":"2023-09-23T23:00:14.642155+0000","last_clean":"2023-09-23T23:00:14.642155+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:00:14.642155+0000","last_undegraded":"2023-09-23T23:00:14.642155+0000","last_fullsized":"2023-09-23T23:00:14.642155+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":283,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.657372+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:00:21.657372+0000","last_peered":"2023-09-23T23:00:21.657372+0000","last_clean":"2023-09-23T23:00:21.657372+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:00:21.657372+0000","last_undegraded":"2023-09-23T23:00:21.657372+0000","last_fullsized":"2023-09-23T23:00:21.657372+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"383'8615","reported_seq":8703,"reported_epoch":383,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:27.137468+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:00:27.137468+0000","last_peered":"2023-09-23T23:00:27.137468+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:00:27.137468+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:00:27.137468+0000","mapping_epoch":353,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5715,"log_dups_size":0,"ondisk_log_size":5715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227999744,"num_objects":3479,"num_object_clones":0,"num_object_copies":10437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8615,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3479}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642478+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:00:14.642478+0000","last_peered":"2023-09-23T23:00:14.642478+0000","last_clean":"2023-09-23T23:00:14.642478+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:00:14.642478+0000","last_undegraded":"2023-09-23T23:00:14.642478+0000","last_fullsized":"2023-09-23T23:00:14.642478+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":376,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.646241+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:00:14.646241+0000","last_peered":"2023-09-23T23:00:14.646241+0000","last_clean":"2023-09-23T23:00:14.646241+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:00:14.646241+0000","last_undegraded":"2023-09-23T23:00:14.646241+0000","last_fullsized":"2023-09-23T23:00:14.646241+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"383'8686","reported_seq":8782,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:27.055197+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:00:27.055197+0000","last_peered":"2023-09-23T23:00:27.055197+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:00:27.055197+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230424576,"num_objects":3516,"num_object_clones":0,"num_object_copies":10548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3516,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8686,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-103}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":166,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.638347+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:00:14.638347+0000","last_peered":"2023-09-23T23:00:14.638347+0000","last_clean":"2023-09-23T23:00:14.638347+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:00:14.638347+0000","last_undegraded":"2023-09-23T23:00:14.638347+0000","last_fullsized":"2023-09-23T23:00:14.638347+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":393,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642738+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:00:14.642738+0000","last_peered":"2023-09-23T23:00:14.642738+0000","last_clean":"2023-09-23T23:00:14.642738+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:00:14.642738+0000","last_undegraded":"2023-09-23T23:00:14.642738+0000","last_fullsized":"2023-09-23T23:00:14.642738+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"383'8626","reported_seq":13448,"reported_epoch":383,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:27.191739+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:00:27.191739+0000","last_peered":"2023-09-23T23:00:27.191739+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:00:27.191739+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:00:27.191739+0000","mapping_epoch":354,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5726,"log_dups_size":0,"ondisk_log_size":5726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225050624,"num_objects":3434,"num_object_clones":0,"num_object_copies":10302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8626,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2359,"num_bytes_recovered":154599424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":2000}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":406,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.645613+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:00:14.645613+0000","last_peered":"2023-09-23T23:00:14.645613+0000","last_clean":"2023-09-23T23:00:14.645613+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:00:14.645613+0000","last_undegraded":"2023-09-23T23:00:14.645613+0000","last_fullsized":"2023-09-23T23:00:14.645613+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":384,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643917+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:00:14.643917+0000","last_peered":"2023-09-23T23:00:14.643917+0000","last_clean":"2023-09-23T23:00:14.643917+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:00:14.643917+0000","last_undegraded":"2023-09-23T23:00:14.643917+0000","last_fullsized":"2023-09-23T23:00:14.643917+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"383'8457","reported_seq":8545,"reported_epoch":383,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:27.144927+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:00:27.144927+0000","last_peered":"2023-09-23T23:00:27.144927+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:00:27.144927+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:00:27.144927+0000","mapping_epoch":353,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230096896,"num_objects":3511,"num_object_clones":0,"num_object_copies":10533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8457,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3511}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":52,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640849+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:00:14.640849+0000","last_peered":"2023-09-23T23:00:14.640849+0000","last_clean":"2023-09-23T23:00:14.640849+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:00:14.640849+0000","last_undegraded":"2023-09-23T23:00:14.640849+0000","last_fullsized":"2023-09-23T23:00:14.640849+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.657921+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:00:14.657921+0000","last_peered":"2023-09-23T23:00:14.657921+0000","last_clean":"2023-09-23T23:00:14.657921+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:00:14.657921+0000","last_undegraded":"2023-09-23T23:00:14.657921+0000","last_fullsized":"2023-09-23T23:00:14.657921+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"383'8562","reported_seq":15488,"reported_epoch":383,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:00:27.186579+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:00:27.186579+0000","last_peered":"2023-09-23T23:00:27.186579+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:00:27.186579+0000","last_undegraded":"2023-09-23T23:00:27.186579+0000","last_fullsized":"2023-09-23T23:00:27.186579+0000","mapping_epoch":354,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230293504,"num_objects":3514,"num_object_clones":0,"num_object_copies":10542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":263,"num_objects_unfound":0,"num_objects_dirty":3514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8562,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3402,"num_bytes_recovered":222429184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":3251},{"shards":"0,4,6","objects":263}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":289,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.657338+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:00:14.657338+0000","last_peered":"2023-09-23T23:00:14.657338+0000","last_clean":"2023-09-23T23:00:14.657338+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:00:14.657338+0000","last_undegraded":"2023-09-23T23:00:14.657338+0000","last_fullsized":"2023-09-23T23:00:14.657338+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":293,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.644665+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:00:14.644665+0000","last_peered":"2023-09-23T23:00:14.644665+0000","last_clean":"2023-09-23T23:00:14.644665+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:00:14.644665+0000","last_undegraded":"2023-09-23T23:00:14.644665+0000","last_fullsized":"2023-09-23T23:00:14.644665+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":106,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.647652+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:00:14.647652+0000","last_peered":"2023-09-23T23:00:14.647652+0000","last_clean":"2023-09-23T23:00:14.647652+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:00:14.647652+0000","last_undegraded":"2023-09-23T23:00:14.647652+0000","last_fullsized":"2023-09-23T23:00:14.647652+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"384'8664","reported_seq":8765,"reported_epoch":384,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:00:29.733923+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:00:29.733923+0000","last_peered":"2023-09-23T23:00:29.733923+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:00:29.733923+0000","last_undegraded":"2023-09-23T23:00:29.733923+0000","last_fullsized":"2023-09-23T23:00:29.733923+0000","mapping_epoch":359,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5464,"log_dups_size":0,"ondisk_log_size":5464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3460,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8664,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3460}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":54,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.657765+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:21.657765+0000","last_peered":"2023-09-23T23:00:21.657765+0000","last_clean":"2023-09-23T23:00:21.657765+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:21.657765+0000","last_undegraded":"2023-09-23T23:00:21.657765+0000","last_fullsized":"2023-09-23T23:00:21.657765+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":403,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643569+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:00:14.643569+0000","last_peered":"2023-09-23T23:00:14.643569+0000","last_clean":"2023-09-23T23:00:14.643569+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:00:14.643569+0000","last_undegraded":"2023-09-23T23:00:14.643569+0000","last_fullsized":"2023-09-23T23:00:14.643569+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":275,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.742517+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:00:21.742517+0000","last_peered":"2023-09-23T23:00:21.742517+0000","last_clean":"2023-09-23T23:00:21.742517+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:00:21.742517+0000","last_undegraded":"2023-09-23T23:00:21.742517+0000","last_fullsized":"2023-09-23T23:00:21.742517+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"383'8590","reported_seq":14337,"reported_epoch":383,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:26.148883+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T23:00:26.148883+0000","last_peered":"2023-09-23T23:00:26.148883+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:00:26.148883+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8090,"log_dups_size":0,"ondisk_log_size":8090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227672064,"num_objects":3474,"num_object_clones":0,"num_object_copies":10422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8590,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2729}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":350,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.743115+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:00:21.743115+0000","last_peered":"2023-09-23T23:00:21.743115+0000","last_clean":"2023-09-23T23:00:21.743115+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:00:21.743115+0000","last_undegraded":"2023-09-23T23:00:21.743115+0000","last_fullsized":"2023-09-23T23:00:21.743115+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":227,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.260036+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:00:21.260036+0000","last_peered":"2023-09-23T23:00:21.260036+0000","last_clean":"2023-09-23T23:00:21.260036+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:00:21.260036+0000","last_undegraded":"2023-09-23T23:00:21.260036+0000","last_fullsized":"2023-09-23T23:00:21.260036+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":293,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.645229+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:00:14.645229+0000","last_peered":"2023-09-23T23:00:14.645229+0000","last_clean":"2023-09-23T23:00:14.645229+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:00:14.645229+0000","last_undegraded":"2023-09-23T23:00:14.645229+0000","last_fullsized":"2023-09-23T23:00:14.645229+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"383'8728","reported_seq":10614,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:27.274694+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:00:27.274694+0000","last_peered":"2023-09-23T23:00:27.274694+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:00:27.274694+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8728,"log_dups_size":0,"ondisk_log_size":8728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229113856,"num_objects":3496,"num_object_clones":0,"num_object_copies":10488,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8728,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":2063},{"shards":"0,6","objects":-1485}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":366,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.647480+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:00:14.647480+0000","last_peered":"2023-09-23T23:00:14.647480+0000","last_clean":"2023-09-23T23:00:14.647480+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:00:14.647480+0000","last_undegraded":"2023-09-23T23:00:14.647480+0000","last_fullsized":"2023-09-23T23:00:14.647480+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":480,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:21.656785+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:21.656785+0000","last_peered":"2023-09-23T23:00:21.656785+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:21.656785+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":52,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640963+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:00:14.640963+0000","last_peered":"2023-09-23T23:00:14.640963+0000","last_clean":"2023-09-23T23:00:14.640963+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:00:14.640963+0000","last_undegraded":"2023-09-23T23:00:14.640963+0000","last_fullsized":"2023-09-23T23:00:14.640963+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":293,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642959+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:00:14.642959+0000","last_peered":"2023-09-23T23:00:14.642959+0000","last_clean":"2023-09-23T23:00:14.642959+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:00:14.642959+0000","last_undegraded":"2023-09-23T23:00:14.642959+0000","last_fullsized":"2023-09-23T23:00:14.642959+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"384'8719","reported_seq":8812,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:29.757656+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:00:29.757656+0000","last_peered":"2023-09-23T23:00:29.757656+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:00:29.757656+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220528640,"num_objects":3365,"num_object_clones":0,"num_object_copies":10095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8719,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-142}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":374,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.656389+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:21.656389+0000","last_peered":"2023-09-23T23:00:21.656389+0000","last_clean":"2023-09-23T23:00:21.656389+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:21.656389+0000","last_undegraded":"2023-09-23T23:00:21.656389+0000","last_fullsized":"2023-09-23T23:00:21.656389+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":345,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.656448+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:21.656448+0000","last_peered":"2023-09-23T23:00:21.656448+0000","last_clean":"2023-09-23T23:00:21.656448+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:21.656448+0000","last_undegraded":"2023-09-23T23:00:21.656448+0000","last_fullsized":"2023-09-23T23:00:21.656448+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":305,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.657082+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:00:21.657082+0000","last_peered":"2023-09-23T23:00:21.657082+0000","last_clean":"2023-09-23T23:00:21.657082+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:00:21.657082+0000","last_undegraded":"2023-09-23T23:00:21.657082+0000","last_fullsized":"2023-09-23T23:00:21.657082+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"383'8686","reported_seq":8793,"reported_epoch":383,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:27.199507+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:00:27.199507+0000","last_peered":"2023-09-23T23:00:27.199507+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:00:27.199507+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:00:27.199507+0000","mapping_epoch":354,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230490136,"num_objects":3518,"num_object_clones":0,"num_object_copies":10554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3518,"num_objects_unfound":0,"num_objects_dirty":3518,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8686,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3518}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":320,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.658148+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:21.658148+0000","last_peered":"2023-09-23T23:00:21.658148+0000","last_clean":"2023-09-23T23:00:21.658148+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:21.658148+0000","last_undegraded":"2023-09-23T23:00:21.658148+0000","last_fullsized":"2023-09-23T23:00:21.658148+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":227,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.742604+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:00:21.742604+0000","last_peered":"2023-09-23T23:00:21.742604+0000","last_clean":"2023-09-23T23:00:21.742604+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:00:21.742604+0000","last_undegraded":"2023-09-23T23:00:21.742604+0000","last_fullsized":"2023-09-23T23:00:21.742604+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":196,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.647558+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:00:14.647558+0000","last_peered":"2023-09-23T23:00:14.647558+0000","last_clean":"2023-09-23T23:00:14.647558+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:00:14.647558+0000","last_undegraded":"2023-09-23T23:00:14.647558+0000","last_fullsized":"2023-09-23T23:00:14.647558+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"383'8671","reported_seq":12896,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:27.189538+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:00:27.189538+0000","last_peered":"2023-09-23T23:00:27.189538+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:00:27.189538+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227737600,"num_objects":3475,"num_object_clones":0,"num_object_copies":10425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7237,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8671,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2009,"num_bytes_recovered":131661824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":2090}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":76,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.641301+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:00:14.641301+0000","last_peered":"2023-09-23T23:00:14.641301+0000","last_clean":"2023-09-23T23:00:14.641301+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:00:14.641301+0000","last_undegraded":"2023-09-23T23:00:14.641301+0000","last_fullsized":"2023-09-23T23:00:14.641301+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.651643+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:00:14.651643+0000","last_peered":"2023-09-23T23:00:14.651643+0000","last_clean":"2023-09-23T23:00:14.651643+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:00:14.651643+0000","last_undegraded":"2023-09-23T23:00:14.651643+0000","last_fullsized":"2023-09-23T23:00:14.651643+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640724+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:00:14.640724+0000","last_peered":"2023-09-23T23:00:14.640724+0000","last_clean":"2023-09-23T23:00:14.640724+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:00:14.640724+0000","last_undegraded":"2023-09-23T23:00:14.640724+0000","last_fullsized":"2023-09-23T23:00:14.640724+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"383'8646","reported_seq":9163,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:27.055192+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:00:27.055192+0000","last_peered":"2023-09-23T23:00:27.055192+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:00:27.055192+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224395264,"num_objects":3424,"num_object_clones":0,"num_object_copies":10272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3424,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8646,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-127}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":338,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.663533+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:00:14.663533+0000","last_peered":"2023-09-23T23:00:14.663533+0000","last_clean":"2023-09-23T23:00:14.663533+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:00:14.663533+0000","last_undegraded":"2023-09-23T23:00:14.663533+0000","last_fullsized":"2023-09-23T23:00:14.663533+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":287,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.657001+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:00:21.657001+0000","last_peered":"2023-09-23T23:00:21.657001+0000","last_clean":"2023-09-23T23:00:21.657001+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:00:21.657001+0000","last_undegraded":"2023-09-23T23:00:21.657001+0000","last_fullsized":"2023-09-23T23:00:21.657001+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":357,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.645689+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:00:14.645689+0000","last_peered":"2023-09-23T23:00:14.645689+0000","last_clean":"2023-09-23T23:00:14.645689+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:00:14.645689+0000","last_undegraded":"2023-09-23T23:00:14.645689+0000","last_fullsized":"2023-09-23T23:00:14.645689+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":321,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.657510+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:00:21.657510+0000","last_peered":"2023-09-23T23:00:21.657510+0000","last_clean":"2023-09-23T23:00:21.657510+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:00:21.657510+0000","last_undegraded":"2023-09-23T23:00:21.657510+0000","last_fullsized":"2023-09-23T23:00:21.657510+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":130,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.657242+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:00:14.657242+0000","last_peered":"2023-09-23T23:00:14.657242+0000","last_clean":"2023-09-23T23:00:14.657242+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:00:14.657242+0000","last_undegraded":"2023-09-23T23:00:14.657242+0000","last_fullsized":"2023-09-23T23:00:14.657242+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":192,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.646137+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:00:14.646137+0000","last_peered":"2023-09-23T23:00:14.646137+0000","last_clean":"2023-09-23T23:00:14.646137+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:00:14.646137+0000","last_undegraded":"2023-09-23T23:00:14.646137+0000","last_fullsized":"2023-09-23T23:00:14.646137+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":406,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.643642+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:00:14.643642+0000","last_peered":"2023-09-23T23:00:14.643642+0000","last_clean":"2023-09-23T23:00:14.643642+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:00:14.643642+0000","last_undegraded":"2023-09-23T23:00:14.643642+0000","last_fullsized":"2023-09-23T23:00:14.643642+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":206,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642828+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:00:14.642828+0000","last_peered":"2023-09-23T23:00:14.642828+0000","last_clean":"2023-09-23T23:00:14.642828+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:00:14.642828+0000","last_undegraded":"2023-09-23T23:00:14.642828+0000","last_fullsized":"2023-09-23T23:00:14.642828+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.638275+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:00:14.638275+0000","last_peered":"2023-09-23T23:00:14.638275+0000","last_clean":"2023-09-23T23:00:14.638275+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:00:14.638275+0000","last_undegraded":"2023-09-23T23:00:14.638275+0000","last_fullsized":"2023-09-23T23:00:14.638275+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":76,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.642130+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:00:14.642130+0000","last_peered":"2023-09-23T23:00:14.642130+0000","last_clean":"2023-09-23T23:00:14.642130+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:00:14.642130+0000","last_undegraded":"2023-09-23T23:00:14.642130+0000","last_fullsized":"2023-09-23T23:00:14.642130+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":79,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.640909+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:00:14.640909+0000","last_peered":"2023-09-23T23:00:14.640909+0000","last_clean":"2023-09-23T23:00:14.640909+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:00:14.640909+0000","last_undegraded":"2023-09-23T23:00:14.640909+0000","last_fullsized":"2023-09-23T23:00:14.640909+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":326,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-09-23T23:00:21.657203+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:21.657203+0000","last_peered":"2023-09-23T23:00:21.657203+0000","last_clean":"2023-09-23T23:00:21.657203+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:21.657203+0000","last_undegraded":"2023-09-23T23:00:21.657203+0000","last_fullsized":"2023-09-23T23:00:21.657203+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":368,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-09-23T23:00:14.637837+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:00:14.637837+0000","last_peered":"2023-09-23T23:00:14.637837+0000","last_clean":"2023-09-23T23:00:14.637837+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:00:14.637837+0000","last_undegraded":"2023-09-23T23:00:14.637837+0000","last_fullsized":"2023-09-23T23:00:14.637837+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3633053720,"num_objects":55437,"num_object_clones":0,"num_object_copies":166311,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":30735,"num_objects_misplaced":30573,"num_objects_unfound":0,"num_objects_dirty":55437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138161,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12940,"num_bytes_recovered":846790656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10954682368,"data_stored":10954670152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98061,"ondisk_log_size":98061,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430609,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626834,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253147,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2130692,"kb_used_data":1414396,"kb_used_omap":9,"kb_used_meta":716278,"kb_avail":91614460,"statfs":{"total":95995035648,"available":93813207040,"internally_reserved":0,"allocated":1448341504,"data_stored":1446903237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":733469673},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.76900000000000002,"15min":0.82699999999999996},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80700000000000005,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.504,"15min":0.52000000000000002},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.93600000000000005,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.341,"5min":4.048,"15min":4.048},"last":0.26900000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.74299999999999999,"15min":2.153},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.827,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.83099999999999996,"15min":2.5960000000000001},"min":{"1min":0.29099999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.109,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.89600000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.7850000000000001,"15min":2.6309999999999998},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5470000000000002,"5min":151.036,"15min":151.036},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.972,"15min":2.6419999999999999},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.488,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":3.1960000000000002,"15min":2.391},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.6909999999999998,"5min":116.098,"15min":116.098},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.23,"5min":2.992,"15min":2.6890000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":7.431,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.649,"15min":2.2400000000000002},"min":{"1min":0.41599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5939999999999999,"5min":20.742000000000001,"15min":79.311000000000007},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.532,"15min":1.9319999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":1.857,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.1140000000000001,"15min":1.8899999999999999},"min":{"1min":0.46700000000000003,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":2.8039999999999998,"5min":5.8410000000000002,"15min":29.539000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.395,"15min":2.0459999999999998},"min":{"1min":0.45100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7849999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.58199999999999996}]}]},{"osd":0,"up_from":353,"seq":1516123455518,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2456028,"kb_used_data":2308820,"kb_used_omap":37,"kb_used_meta":147162,"kb_avail":91289124,"statfs":{"total":95995035648,"available":93480062976,"internally_reserved":0,"allocated":2364231680,"data_stored":2362661960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":150694050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.214},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.73299999999999998,"15min":0.76400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.9319999999999999,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.0270000000000001,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":2.7589999999999999},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":2.5390000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":4.4249999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":4.415}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":2.4319999999999999},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":2.4470000000000001}]}]},{"osd":2,"up_from":359,"seq":1541893259290,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":786380,"kb_used_data":672964,"kb_used_omap":83,"kb_used_meta":113324,"kb_avail":92958772,"statfs":{"total":95995035648,"available":95189782528,"internally_reserved":0,"allocated":689115136,"data_stored":687614205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":116044229},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.179,"5min":1.179,"15min":1.179},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.5600000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.29299999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.251},{"interface":"front","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.3819999999999997,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.51800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.468,"5min":3.468,"15min":3.468},"last":1.427},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.77500000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.2370000000000001,"5min":4.2370000000000001,"15min":4.2370000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.74199999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:59:25 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":4.1340000000000003,"5min":4.1340000000000003,"15min":4.1340000000000003},"last":1.004},{"interface":"front","average":{"1min":1.179,"5min":1.179,"15min":1.179},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":4.0780000000000003,"5min":4.0780000000000003,"15min":4.0780000000000003},"last":0.54300000000000004}]}]},{"osd":3,"up_from":94,"seq":403726926094,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":517924,"kb_used_data":39516,"kb_used_omap":7,"kb_used_meta":478392,"kb_avail":93227228,"statfs":{"total":95995035648,"available":95464681472,"internally_reserved":0,"allocated":40464384,"data_stored":38935718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":489874024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":3.552}]},{"osd":1,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.78300000000000003,"15min":2.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":3.355,"5min":16.596,"15min":123.866},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95599999999999996,"15min":2.2429999999999999},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":1.746,"5min":16.649999999999999,"15min":122.265},"last":0.38200000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.1339999999999999,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.76600000000000001,"15min":0.75},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":3.8740000000000001,"5min":3.8740000000000001,"15min":3.8740000000000001},"last":3.8740000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":4.6769999999999996,"5min":1.798,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":87.046999999999997,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":4.6580000000000004,"5min":1.972,"15min":1.7430000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":87.066000000000003,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2410000000000001,"15min":1.3839999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5910000000000002,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.032,"5min":1.2150000000000001,"15min":1.861},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3390000000000004,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.92400000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.5649999999999999,"15min":1.7090000000000001},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2520000000000007,"5min":11.116,"15min":14.097},"last":0.504},{"interface":"front","average":{"1min":1.613,"5min":1.397,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.2810000000000006,"5min":10.859,"15min":12.887},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97899999999999998,"15min":2.1179999999999999},"min":{"1min":0.443,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1040000000000001,"5min":6.0270000000000001,"15min":91.757999999999996},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.92300000000000004,"15min":2.097},"min":{"1min":0.37,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":3.4820000000000002,"5min":6.468,"15min":91.709999999999994},"last":0.45200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019291,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3550368,"kb_used_data":2647000,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":90194784,"statfs":{"total":95995035648,"available":92359458816,"internally_reserved":0,"allocated":2710528000,"data_stored":2709753551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":0.70599999999999996,"15min":0.55000000000000004},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.65300000000000002,"15min":0.61499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":0.40500000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":3.5110000000000001,"15min":3.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":2.77,"5min":98.846999999999994,"15min":132.304},"last":0.371},{"interface":"front","average":{"1min":1.302,"5min":3.6749999999999998,"15min":3.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":10.361000000000001,"5min":98.938999999999993,"15min":132.453},"last":0.57899999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.2439999999999998,"5min":5.2439999999999998,"15min":5.2439999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.9379999999999999,"5min":1.9379999999999999,"15min":1.9379999999999999},"last":0.61699999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.4450000000000001,"15min":4.3140000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.32400000000000001},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.2110000000000001,"15min":4.3049999999999997},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":3.2490000000000001,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:59:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.9630000000000001,"15min":1.6319999999999999},"min":{"1min":0.35199999999999998,"5min":0.214,"15min":0.214},"max":{"1min":2.3359999999999999,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.3599999999999999,"15min":1.7669999999999999},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.248},"max":{"1min":1.6040000000000001,"5min":151.905,"15min":151.905},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.585,"15min":1.1839999999999999},"min":{"1min":0.373,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":4.1269999999999998,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.155,"5min":1.371,"15min":1.105},"min":{"1min":0.372,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2130000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.47,"15min":1.127},"min":{"1min":0.40899999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.045999999999999,"5min":12.84,"15min":12.84},"last":1.2729999999999999},{"interface":"front","average":{"1min":1.615,"5min":1.5089999999999999,"15min":1.0740000000000001},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.227,"5min":10.587,"15min":10.587},"last":0.81999999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823065,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2554240,"kb_used_data":1658680,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":91190912,"statfs":{"total":95995035648,"available":93379493888,"internally_reserved":0,"allocated":1698488320,"data_stored":1697741659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.73599999999999999,"15min":0.64900000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":14.42,"5min":14.42,"15min":14.42},"last":0.27300000000000002},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.274,"15min":1.1459999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.496,"5min":14.496,"15min":14.496},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2150000000000001,"15min":2.2850000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24199999999999999},"max":{"1min":3.21,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.748,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":6.7690000000000001,"5min":10.353,"15min":110.89},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.601,"5min":6.601,"15min":6.601},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7669999999999999,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.304,"15min":4.9619999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":3.1400000000000001,"5min":8.6489999999999991,"15min":257.05000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.4850000000000001,"15min":4.9610000000000003},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.965,"5min":12.185,"15min":257.35000000000002},"last":0.60899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.802,"15min":1.4570000000000001},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6850000000000001,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":2.661,"15min":1.4510000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.1880000000000002,"5min":114.908,"15min":114.908},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.345,"15min":1.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":6.8150000000000004,"5min":11.121,"15min":12.615},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.2130000000000001,"15min":1.123},"min":{"1min":0.433,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.2719999999999998,"5min":12.137,"15min":12.74},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 23:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.1180000000000001,"15min":1.046},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.5980000000000008,"5min":12.23,"15min":12.23},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.627,"5min":1.4199999999999999,"15min":1.0940000000000001},"min":{"1min":0.45500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.8710000000000004,"5min":11.112,"15min":11.112},"last":0.68000000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2354515968,"data_stored":2354511896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1442512896,"data_stored":1442512896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":684654592,"data_stored":684654592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":35979264,"data_stored":35979264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2702512128,"data_stored":2702508056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1690697728,"data_stored":1690697728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2043809792,"data_stored":2043805720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:00:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:31 smithi052 ceph-mon[180979]: pgmap v1143: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 30737/166383 objects degraded (18.474%); 30573/166383 objects misplaced (18.375%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:00:31.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:31 smithi052 ceph-mon[180979]: from='client.46499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:31 smithi052 ceph-mon[187743]: pgmap v1143: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 30737/166383 objects degraded (18.474%); 30573/166383 objects misplaced (18.375%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:00:31.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:31 smithi052 ceph-mon[187743]: from='client.46499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:31.859 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:31.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:31.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:31 smithi151 ceph-mon[159210]: pgmap v1143: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 30737/166383 objects degraded (18.474%); 30573/166383 objects misplaced (18.375%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T23:00:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:31 smithi151 ceph-mon[159210]: from='client.46499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:31.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:32.009 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:32.060 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:32.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:32.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:32.078 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:00:32.165 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:32.206 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:32.279 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:32.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:32.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:32.370 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:32.370 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:32.392 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:32.422 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:32.477 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:32.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:32.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:32.523 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:32.581 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:32.611 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:32.759 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:32.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:32.866 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:32.895 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:33.161 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:33.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:33.266 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:33.298 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:33.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:00:33.356 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:33.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:33.460 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:33.485 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:00:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:00:33] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T23:00:33.488 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:33.686 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:33.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:33.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:33.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:33 smithi052 ceph-mon[180979]: pgmap v1144: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 30173/164067 objects degraded (18.391%); 30393/164067 objects misplaced (18.525%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:33.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:33 smithi052 ceph-mon[187743]: pgmap v1144: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 30173/164067 objects degraded (18.391%); 30393/164067 objects misplaced (18.525%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:33.787 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:33.793 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:33.825 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:33.890 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:00:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:33 smithi151 ceph-mon[159210]: pgmap v1144: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 30173/164067 objects degraded (18.391%); 30393/164067 objects misplaced (18.525%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:33.984 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:33.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:34.090 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:34.132 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:34.198 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:34.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:34.302 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:34.341 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:34.350 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:34.420 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:34.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:34.524 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:34.566 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:34.641 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:34.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:34.681 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:00:34.682 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3283353551' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:34.694 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:00:34.745 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:34.856 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:34.940 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3283353551' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:34.940 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:34.940 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:34.940 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:34.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3283353551' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:34.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:34.941 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:34.941 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:00:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:35.155 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:35.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:35.263 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:35.295 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:35.454 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:35.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:35.560 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:35.609 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: pgmap v1145: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 30173/164067 objects degraded (18.391%); 30393/164067 objects misplaced (18.525%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: failed to encode map e385 with expected crc 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: osdmap e385: 8 total, 8 up, 6 in 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: failed to encode map e385 with expected crc 2023-09-23T23:00:35.685 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 30173/164067 objects degraded (18.391%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:35.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: pgmap v1145: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 30173/164067 objects degraded (18.391%); 30393/164067 objects misplaced (18.525%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:35.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:35.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: failed to encode map e385 with expected crc 2023-09-23T23:00:35.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:00:35.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: osdmap e385: 8 total, 8 up, 6 in 2023-09-23T23:00:35.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: failed to encode map e385 with expected crc 2023-09-23T23:00:35.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 30173/164067 objects degraded (18.391%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:35.690 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:35.691 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:35.692 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:35.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:35.798 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:35.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:35.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: pgmap v1145: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 30173/164067 objects degraded (18.391%); 30393/164067 objects misplaced (18.525%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:35.972 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:35.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: failed to encode map e385 with expected crc 2023-09-23T23:00:35.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:00:35.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: osdmap e385: 8 total, 8 up, 6 in 2023-09-23T23:00:35.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: failed to encode map e385 with expected crc 2023-09-23T23:00:35.973 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 30173/164067 objects degraded (18.391%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:36.036 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:36.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:36.141 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:36.160 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:36.172 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:36.339 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:36.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:36.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:36.445 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:36.487 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:36.571 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:00:36.588 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:36.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:36.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:36.604 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:36.626 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:36.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:36.673 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:36.694 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:36.702 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:36.734 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:36.741 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:36.769 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:36.794 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:36.801 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.801 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.802 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.802 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.802 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.802 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.802 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.802 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:00:36.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2895013768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:36.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2895013768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2895013768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:37.105 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:00:37.698 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:37 smithi052 ceph-mon[180979]: pgmap v1147: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 29939/162657 objects degraded (18.406%); 30252/162657 objects misplaced (18.599%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:00:37.871 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:37.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:37.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:37 smithi151 ceph-mon[159210]: pgmap v1147: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 29939/162657 objects degraded (18.406%); 30252/162657 objects misplaced (18.599%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:00:37.977 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:37.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:37 smithi052 ceph-mon[187743]: pgmap v1147: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 29939/162657 objects degraded (18.406%); 30252/162657 objects misplaced (18.599%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T23:00:38.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:00:38.007 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:38.071 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:38.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:38.181 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:38.227 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:38.401 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:38.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:38.504 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:38.517 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:38.555 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:38.604 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:38.612 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:00:38.673 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:38.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:38.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:38.776 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:38.816 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:38.914 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:00:38.914 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:00:38.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:00:38.915 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [] 2023-09-23T23:00:38.915 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:00:38.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:38.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:39.037 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:39.069 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:39.797 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:00:39.861 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:00:39.876 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:39.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:39.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:39.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:39 smithi052 ceph-mon[187743]: pgmap v1148: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 225 op/s; 29580/160749 objects degraded (18.401%); 30073/160749 objects misplaced (18.708%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:39.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:39 smithi052 ceph-mon[187743]: from='client.46523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:39.908 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:39 smithi052 ceph-mon[180979]: pgmap v1148: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 225 op/s; 29580/160749 objects degraded (18.401%); 30073/160749 objects misplaced (18.708%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:39.908 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:39 smithi052 ceph-mon[180979]: from='client.46523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:39.931 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1148,"stamp":"2023-09-23T23:00:38.575114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3510572984,"num_objects":53583,"num_object_clones":0,"num_object_copies":160749,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":29580,"num_objects_misplaced":30073,"num_objects_unfound":0,"num_objects_dirty":53583,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":140299,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13715,"num_bytes_recovered":895302588,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98833,"ondisk_log_size":98833,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11785080,"kb_used_data":8500488,"kb_used_omap":137,"kb_used_meta":3284406,"kb_avail":550685832,"statfs":{"total":575970213888,"available":563902291968,"internally_reserved":0,"allocated":8704499712,"data_stored":8696886374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3363231928},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-146079744,"num_objects":-2229,"num_object_clones":0,"num_object_copies":-6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1214,"num_objects_misplaced":-629,"num_objects_unfound":0,"num_objects_dirty":-2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2229,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":740,"num_bytes_recovered":48496640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.896640"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":397,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.691561+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:00:28.691561+0000","last_peered":"2023-09-23T23:00:28.691561+0000","last_clean":"2023-09-23T23:00:28.691561+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:00:28.691561+0000","last_undegraded":"2023-09-23T23:00:28.691561+0000","last_fullsized":"2023-09-23T23:00:28.691561+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":54,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708784+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:00:28.708784+0000","last_peered":"2023-09-23T23:00:28.708784+0000","last_clean":"2023-09-23T23:00:28.708784+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:00:28.708784+0000","last_undegraded":"2023-09-23T23:00:28.708784+0000","last_fullsized":"2023-09-23T23:00:28.708784+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707541+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:00:28.707541+0000","last_peered":"2023-09-23T23:00:28.707541+0000","last_clean":"2023-09-23T23:00:28.707541+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:00:28.707541+0000","last_undegraded":"2023-09-23T23:00:28.707541+0000","last_fullsized":"2023-09-23T23:00:28.707541+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":381,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708332+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:00:28.708332+0000","last_peered":"2023-09-23T23:00:28.708332+0000","last_clean":"2023-09-23T23:00:28.708332+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:00:28.708332+0000","last_undegraded":"2023-09-23T23:00:28.708332+0000","last_fullsized":"2023-09-23T23:00:28.708332+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":69,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707479+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:00:28.707479+0000","last_peered":"2023-09-23T23:00:28.707479+0000","last_clean":"2023-09-23T23:00:28.707479+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:00:28.707479+0000","last_undegraded":"2023-09-23T23:00:28.707479+0000","last_fullsized":"2023-09-23T23:00:28.707479+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":55,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.683481+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:28.683481+0000","last_peered":"2023-09-23T23:00:28.683481+0000","last_clean":"2023-09-23T23:00:28.683481+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:28.683481+0000","last_undegraded":"2023-09-23T23:00:28.683481+0000","last_fullsized":"2023-09-23T23:00:28.683481+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":320,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.688934+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:00:28.688934+0000","last_peered":"2023-09-23T23:00:28.688934+0000","last_clean":"2023-09-23T23:00:28.688934+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:00:28.688934+0000","last_undegraded":"2023-09-23T23:00:28.688934+0000","last_fullsized":"2023-09-23T23:00:28.688934+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707391+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:00:28.707391+0000","last_peered":"2023-09-23T23:00:28.707391+0000","last_clean":"2023-09-23T23:00:28.707391+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:00:28.707391+0000","last_undegraded":"2023-09-23T23:00:28.707391+0000","last_fullsized":"2023-09-23T23:00:28.707391+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":281,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.687346+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:00:28.687346+0000","last_peered":"2023-09-23T23:00:28.687346+0000","last_clean":"2023-09-23T23:00:28.687346+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:00:28.687346+0000","last_undegraded":"2023-09-23T23:00:28.687346+0000","last_fullsized":"2023-09-23T23:00:28.687346+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707453+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:00:28.707453+0000","last_peered":"2023-09-23T23:00:28.707453+0000","last_clean":"2023-09-23T23:00:28.707453+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:00:28.707453+0000","last_undegraded":"2023-09-23T23:00:28.707453+0000","last_fullsized":"2023-09-23T23:00:28.707453+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708725+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:00:28.708725+0000","last_peered":"2023-09-23T23:00:28.708725+0000","last_clean":"2023-09-23T23:00:28.708725+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:00:28.708725+0000","last_undegraded":"2023-09-23T23:00:28.708725+0000","last_fullsized":"2023-09-23T23:00:28.708725+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":129,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680570+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:00:28.680570+0000","last_peered":"2023-09-23T23:00:28.680570+0000","last_clean":"2023-09-23T23:00:28.680570+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:00:28.680570+0000","last_undegraded":"2023-09-23T23:00:28.680570+0000","last_fullsized":"2023-09-23T23:00:28.680570+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":456,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.692147+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:00:28.692147+0000","last_peered":"2023-09-23T23:00:28.692147+0000","last_clean":"2023-09-23T23:00:28.692147+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:00:28.692147+0000","last_undegraded":"2023-09-23T23:00:28.692147+0000","last_fullsized":"2023-09-23T23:00:28.692147+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":359,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.701009+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:00:28.701009+0000","last_peered":"2023-09-23T23:00:28.701009+0000","last_clean":"2023-09-23T23:00:28.701009+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:00:28.701009+0000","last_undegraded":"2023-09-23T23:00:28.701009+0000","last_fullsized":"2023-09-23T23:00:28.701009+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":285,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.701097+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:00:28.701097+0000","last_peered":"2023-09-23T23:00:28.701097+0000","last_clean":"2023-09-23T23:00:28.701097+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:00:28.701097+0000","last_undegraded":"2023-09-23T23:00:28.701097+0000","last_fullsized":"2023-09-23T23:00:28.701097+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":65,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708492+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:00:28.708492+0000","last_peered":"2023-09-23T23:00:28.708492+0000","last_clean":"2023-09-23T23:00:28.708492+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:00:28.708492+0000","last_undegraded":"2023-09-23T23:00:28.708492+0000","last_fullsized":"2023-09-23T23:00:28.708492+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":374,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707633+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:00:28.707633+0000","last_peered":"2023-09-23T23:00:28.707633+0000","last_clean":"2023-09-23T23:00:28.707633+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:00:28.707633+0000","last_undegraded":"2023-09-23T23:00:28.707633+0000","last_fullsized":"2023-09-23T23:00:28.707633+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":285,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.687061+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:00:28.687061+0000","last_peered":"2023-09-23T23:00:28.687061+0000","last_clean":"2023-09-23T23:00:28.687061+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:00:28.687061+0000","last_undegraded":"2023-09-23T23:00:28.687061+0000","last_fullsized":"2023-09-23T23:00:28.687061+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":270,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696600+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:00:28.696600+0000","last_peered":"2023-09-23T23:00:28.696600+0000","last_clean":"2023-09-23T23:00:28.696600+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:00:28.696600+0000","last_undegraded":"2023-09-23T23:00:28.696600+0000","last_fullsized":"2023-09-23T23:00:28.696600+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":398,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708510+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:28.708510+0000","last_peered":"2023-09-23T23:00:28.708510+0000","last_clean":"2023-09-23T23:00:28.708510+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:28.708510+0000","last_undegraded":"2023-09-23T23:00:28.708510+0000","last_fullsized":"2023-09-23T23:00:28.708510+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":103,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680360+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:00:28.680360+0000","last_peered":"2023-09-23T23:00:28.680360+0000","last_clean":"2023-09-23T23:00:28.680360+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:00:28.680360+0000","last_undegraded":"2023-09-23T23:00:28.680360+0000","last_fullsized":"2023-09-23T23:00:28.680360+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708389+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:00:28.708389+0000","last_peered":"2023-09-23T23:00:28.708389+0000","last_clean":"2023-09-23T23:00:28.708389+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:00:28.708389+0000","last_undegraded":"2023-09-23T23:00:28.708389+0000","last_fullsized":"2023-09-23T23:00:28.708389+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"385'8688","reported_seq":8777,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.688290+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:00:34.688290+0000","last_peered":"2023-09-23T23:00:34.688290+0000","last_clean":"2023-09-23T23:00:34.688290+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:00:34.688290+0000","last_undegraded":"2023-09-23T23:00:34.688290+0000","last_fullsized":"2023-09-23T23:00:34.688290+0000","mapping_epoch":353,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222691328,"num_objects":3398,"num_object_clones":0,"num_object_copies":10194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8688,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":269,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.695833+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:00:28.695833+0000","last_peered":"2023-09-23T23:00:28.695833+0000","last_clean":"2023-09-23T23:00:28.695833+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:00:28.695833+0000","last_undegraded":"2023-09-23T23:00:28.695833+0000","last_fullsized":"2023-09-23T23:00:28.695833+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.686143+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:00:28.686143+0000","last_peered":"2023-09-23T23:00:28.686143+0000","last_clean":"2023-09-23T23:00:28.686143+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:00:28.686143+0000","last_undegraded":"2023-09-23T23:00:28.686143+0000","last_fullsized":"2023-09-23T23:00:28.686143+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"385'8613","reported_seq":11031,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:35.838550+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:00:35.838550+0000","last_peered":"2023-09-23T23:00:35.838550+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:00:35.838550+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8213,"log_dups_size":0,"ondisk_log_size":8213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218693632,"num_objects":3337,"num_object_clones":0,"num_object_copies":10011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3337,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8613,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-230}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":279,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.700319+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:00:28.700319+0000","last_peered":"2023-09-23T23:00:28.700319+0000","last_clean":"2023-09-23T23:00:28.700319+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:00:28.700319+0000","last_undegraded":"2023-09-23T23:00:28.700319+0000","last_fullsized":"2023-09-23T23:00:28.700319+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":392,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696584+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:00:28.696584+0000","last_peered":"2023-09-23T23:00:28.696584+0000","last_clean":"2023-09-23T23:00:28.696584+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:00:28.696584+0000","last_undegraded":"2023-09-23T23:00:28.696584+0000","last_fullsized":"2023-09-23T23:00:28.696584+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"385'8743","reported_seq":8826,"reported_epoch":385,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:34.688793+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:00:34.688793+0000","last_peered":"2023-09-23T23:00:34.688793+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:00:34.688793+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:00:34.688793+0000","mapping_epoch":354,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5543,"log_dups_size":0,"ondisk_log_size":5543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3329,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8743,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3329}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":136,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.687155+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:00:28.687155+0000","last_peered":"2023-09-23T23:00:28.687155+0000","last_clean":"2023-09-23T23:00:28.687155+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:00:28.687155+0000","last_undegraded":"2023-09-23T23:00:28.687155+0000","last_fullsized":"2023-09-23T23:00:28.687155+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":254,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.686520+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:00:28.686520+0000","last_peered":"2023-09-23T23:00:28.686520+0000","last_clean":"2023-09-23T23:00:28.686520+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:00:28.686520+0000","last_undegraded":"2023-09-23T23:00:28.686520+0000","last_fullsized":"2023-09-23T23:00:28.686520+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"385'8804","reported_seq":9073,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:34.743276+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:00:34.743276+0000","last_peered":"2023-09-23T23:00:34.743276+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:00:34.743276+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5504,"log_dups_size":0,"ondisk_log_size":5504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220856320,"num_objects":3370,"num_object_clones":0,"num_object_copies":10110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2834,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8804,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":535},{"shards":"0,5","objects":-240}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":54,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708376+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:00:28.708376+0000","last_peered":"2023-09-23T23:00:28.708376+0000","last_clean":"2023-09-23T23:00:28.708376+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:00:28.708376+0000","last_undegraded":"2023-09-23T23:00:28.708376+0000","last_fullsized":"2023-09-23T23:00:28.708376+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":284,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680814+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:00:28.680814+0000","last_peered":"2023-09-23T23:00:28.680814+0000","last_clean":"2023-09-23T23:00:28.680814+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:00:28.680814+0000","last_undegraded":"2023-09-23T23:00:28.680814+0000","last_fullsized":"2023-09-23T23:00:28.680814+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"385'8737","reported_seq":8827,"reported_epoch":385,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:36.948130+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:00:36.948130+0000","last_peered":"2023-09-23T23:00:36.948130+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:00:36.948130+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:00:36.948130+0000","mapping_epoch":353,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5737,"log_dups_size":0,"ondisk_log_size":5737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220004352,"num_objects":3357,"num_object_clones":0,"num_object_copies":10071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8737,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3357}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":73,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708112+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:00:28.708112+0000","last_peered":"2023-09-23T23:00:28.708112+0000","last_clean":"2023-09-23T23:00:28.708112+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:00:28.708112+0000","last_undegraded":"2023-09-23T23:00:28.708112+0000","last_fullsized":"2023-09-23T23:00:28.708112+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":378,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.692069+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:00:28.692069+0000","last_peered":"2023-09-23T23:00:28.692069+0000","last_clean":"2023-09-23T23:00:28.692069+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:00:28.692069+0000","last_undegraded":"2023-09-23T23:00:28.692069+0000","last_fullsized":"2023-09-23T23:00:28.692069+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"385'8844","reported_seq":8942,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:37.163494+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:00:37.163494+0000","last_peered":"2023-09-23T23:00:37.163494+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:00:37.163494+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220069888,"num_objects":3358,"num_object_clones":0,"num_object_copies":10074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3358,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8844,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-261}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.691709+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:00:28.691709+0000","last_peered":"2023-09-23T23:00:28.691709+0000","last_clean":"2023-09-23T23:00:28.691709+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:00:28.691709+0000","last_undegraded":"2023-09-23T23:00:28.691709+0000","last_fullsized":"2023-09-23T23:00:28.691709+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":395,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696029+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:00:28.696029+0000","last_peered":"2023-09-23T23:00:28.696029+0000","last_clean":"2023-09-23T23:00:28.696029+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:00:28.696029+0000","last_undegraded":"2023-09-23T23:00:28.696029+0000","last_fullsized":"2023-09-23T23:00:28.696029+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"385'8760","reported_seq":13853,"reported_epoch":385,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:37.206632+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:00:37.206632+0000","last_peered":"2023-09-23T23:00:37.206632+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:00:37.206632+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:00:37.206632+0000","mapping_epoch":354,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216268800,"num_objects":3300,"num_object_clones":0,"num_object_copies":9900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2106,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8760,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2494,"num_bytes_recovered":163446784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1435},{"shards":"1,4,6","objects":1865}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":408,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696427+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:00:28.696427+0000","last_peered":"2023-09-23T23:00:28.696427+0000","last_clean":"2023-09-23T23:00:28.696427+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:00:28.696427+0000","last_undegraded":"2023-09-23T23:00:28.696427+0000","last_fullsized":"2023-09-23T23:00:28.696427+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":386,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.700393+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:00:28.700393+0000","last_peered":"2023-09-23T23:00:28.700393+0000","last_clean":"2023-09-23T23:00:28.700393+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:00:28.700393+0000","last_undegraded":"2023-09-23T23:00:28.700393+0000","last_fullsized":"2023-09-23T23:00:28.700393+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"385'8591","reported_seq":8681,"reported_epoch":385,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:36.971537+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:00:36.971537+0000","last_peered":"2023-09-23T23:00:36.971537+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:00:36.971537+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:00:36.971537+0000","mapping_epoch":353,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221315072,"num_objects":3377,"num_object_clones":0,"num_object_copies":10131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8591,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3377}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707593+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:00:28.707593+0000","last_peered":"2023-09-23T23:00:28.707593+0000","last_clean":"2023-09-23T23:00:28.707593+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:00:28.707593+0000","last_undegraded":"2023-09-23T23:00:28.707593+0000","last_fullsized":"2023-09-23T23:00:28.707593+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":288,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687920+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:00:34.687920+0000","last_peered":"2023-09-23T23:00:34.687920+0000","last_clean":"2023-09-23T23:00:34.687920+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:00:34.687920+0000","last_undegraded":"2023-09-23T23:00:34.687920+0000","last_fullsized":"2023-09-23T23:00:34.687920+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"385'8688","reported_seq":16064,"reported_epoch":385,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:00:37.206126+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:00:37.206126+0000","last_peered":"2023-09-23T23:00:37.206126+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:00:37.206126+0000","last_undegraded":"2023-09-23T23:00:37.206126+0000","last_fullsized":"2023-09-23T23:00:37.206126+0000","mapping_epoch":354,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":39,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8688,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3626,"num_bytes_recovered":237109248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":3349},{"shards":"0,4,6","objects":39}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":292,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:31.267587+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:00:31.267587+0000","last_peered":"2023-09-23T23:00:31.267587+0000","last_clean":"2023-09-23T23:00:31.267587+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:00:31.267587+0000","last_undegraded":"2023-09-23T23:00:31.267587+0000","last_fullsized":"2023-09-23T23:00:31.267587+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":295,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.686981+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:00:28.686981+0000","last_peered":"2023-09-23T23:00:28.686981+0000","last_clean":"2023-09-23T23:00:28.686981+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:00:28.686981+0000","last_undegraded":"2023-09-23T23:00:28.686981+0000","last_fullsized":"2023-09-23T23:00:28.686981+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.686358+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:00:28.686358+0000","last_peered":"2023-09-23T23:00:28.686358+0000","last_clean":"2023-09-23T23:00:28.686358+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:00:28.686358+0000","last_undegraded":"2023-09-23T23:00:28.686358+0000","last_fullsized":"2023-09-23T23:00:28.686358+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"385'8729","reported_seq":8831,"reported_epoch":385,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:00:34.688496+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:00:34.688496+0000","last_peered":"2023-09-23T23:00:34.688496+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:00:34.688496+0000","last_undegraded":"2023-09-23T23:00:34.688496+0000","last_fullsized":"2023-09-23T23:00:34.688496+0000","mapping_epoch":359,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222494720,"num_objects":3395,"num_object_clones":0,"num_object_copies":10185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3395,"num_objects_unfound":0,"num_objects_dirty":3395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8729,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3395}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":55,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680266+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:28.680266+0000","last_peered":"2023-09-23T23:00:28.680266+0000","last_clean":"2023-09-23T23:00:28.680266+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:28.680266+0000","last_undegraded":"2023-09-23T23:00:28.680266+0000","last_fullsized":"2023-09-23T23:00:28.680266+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":405,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.701167+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:00:28.701167+0000","last_peered":"2023-09-23T23:00:28.701167+0000","last_clean":"2023-09-23T23:00:28.701167+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:00:28.701167+0000","last_undegraded":"2023-09-23T23:00:28.701167+0000","last_fullsized":"2023-09-23T23:00:28.701167+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":277,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680338+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:00:28.680338+0000","last_peered":"2023-09-23T23:00:28.680338+0000","last_clean":"2023-09-23T23:00:28.680338+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:00:28.680338+0000","last_undegraded":"2023-09-23T23:00:28.680338+0000","last_fullsized":"2023-09-23T23:00:28.680338+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"385'8735","reported_seq":14938,"reported_epoch":385,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:36.151028+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T23:00:36.151028+0000","last_peered":"2023-09-23T23:00:36.151028+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:00:36.151028+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8235,"log_dups_size":0,"ondisk_log_size":8235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8735,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3053,"num_bytes_recovered":200081408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2811}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":352,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.681802+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:00:28.681802+0000","last_peered":"2023-09-23T23:00:28.681802+0000","last_clean":"2023-09-23T23:00:28.681802+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:00:28.681802+0000","last_undegraded":"2023-09-23T23:00:28.681802+0000","last_fullsized":"2023-09-23T23:00:28.681802+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":229,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.679946+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:00:28.679946+0000","last_peered":"2023-09-23T23:00:28.679946+0000","last_clean":"2023-09-23T23:00:28.679946+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:00:28.679946+0000","last_undegraded":"2023-09-23T23:00:28.679946+0000","last_fullsized":"2023-09-23T23:00:28.679946+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":295,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.689001+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:00:28.689001+0000","last_peered":"2023-09-23T23:00:28.689001+0000","last_clean":"2023-09-23T23:00:28.689001+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:00:28.689001+0000","last_undegraded":"2023-09-23T23:00:28.689001+0000","last_fullsized":"2023-09-23T23:00:28.689001+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"385'8884","reported_seq":10772,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:37.557780+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:00:37.557780+0000","last_peered":"2023-09-23T23:00:37.557780+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:00:37.557780+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8884,"log_dups_size":0,"ondisk_log_size":8884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218890240,"num_objects":3340,"num_object_clones":0,"num_object_copies":10020,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8884,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":1907},{"shards":"0,6","objects":-1641}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":368,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.686889+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:00:28.686889+0000","last_peered":"2023-09-23T23:00:28.686889+0000","last_clean":"2023-09-23T23:00:28.686889+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:00:28.686889+0000","last_undegraded":"2023-09-23T23:00:28.686889+0000","last_fullsized":"2023-09-23T23:00:28.686889+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":481,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:28.681585+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:28.681585+0000","last_peered":"2023-09-23T23:00:28.681585+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:28.681585+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":54,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707856+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:00:28.707856+0000","last_peered":"2023-09-23T23:00:28.707856+0000","last_clean":"2023-09-23T23:00:28.707856+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:00:28.707856+0000","last_undegraded":"2023-09-23T23:00:28.707856+0000","last_fullsized":"2023-09-23T23:00:28.707856+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":295,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696245+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:00:28.696245+0000","last_peered":"2023-09-23T23:00:28.696245+0000","last_clean":"2023-09-23T23:00:28.696245+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:00:28.696245+0000","last_undegraded":"2023-09-23T23:00:28.696245+0000","last_fullsized":"2023-09-23T23:00:28.696245+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"384'8791","reported_seq":8885,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:34.662615+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:00:34.662615+0000","last_peered":"2023-09-23T23:00:34.662615+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:00:34.662615+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"325'3300","ondisk_log_start":"325'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5491,"log_dups_size":0,"ondisk_log_size":5491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215810048,"num_objects":3293,"num_object_clones":0,"num_object_copies":9879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8791,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-214}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":375,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680445+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:28.680445+0000","last_peered":"2023-09-23T23:00:28.680445+0000","last_clean":"2023-09-23T23:00:28.680445+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:28.680445+0000","last_undegraded":"2023-09-23T23:00:28.680445+0000","last_fullsized":"2023-09-23T23:00:28.680445+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":346,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.681836+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:28.681836+0000","last_peered":"2023-09-23T23:00:28.681836+0000","last_clean":"2023-09-23T23:00:28.681836+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:28.681836+0000","last_undegraded":"2023-09-23T23:00:28.681836+0000","last_fullsized":"2023-09-23T23:00:28.681836+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":306,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.684189+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:00:28.684189+0000","last_peered":"2023-09-23T23:00:28.684189+0000","last_clean":"2023-09-23T23:00:28.684189+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:00:28.684189+0000","last_undegraded":"2023-09-23T23:00:28.684189+0000","last_fullsized":"2023-09-23T23:00:28.684189+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"385'8821","reported_seq":8930,"reported_epoch":385,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:36.998747+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:00:36.998747+0000","last_peered":"2023-09-23T23:00:36.998747+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:00:36.998747+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:00:36.998747+0000","mapping_epoch":354,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221642776,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3383,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8821,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3383}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":321,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.683374+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:28.683374+0000","last_peered":"2023-09-23T23:00:28.683374+0000","last_clean":"2023-09-23T23:00:28.683374+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:28.683374+0000","last_undegraded":"2023-09-23T23:00:28.683374+0000","last_fullsized":"2023-09-23T23:00:28.683374+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":229,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680410+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:00:28.680410+0000","last_peered":"2023-09-23T23:00:28.680410+0000","last_clean":"2023-09-23T23:00:28.680410+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:00:28.680410+0000","last_undegraded":"2023-09-23T23:00:28.680410+0000","last_fullsized":"2023-09-23T23:00:28.680410+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":198,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.686963+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:00:28.686963+0000","last_peered":"2023-09-23T23:00:28.686963+0000","last_clean":"2023-09-23T23:00:28.686963+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:00:28.686963+0000","last_undegraded":"2023-09-23T23:00:28.686963+0000","last_fullsized":"2023-09-23T23:00:28.686963+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"385'8825","reported_seq":13514,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:37.144379+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:00:37.144379+0000","last_peered":"2023-09-23T23:00:37.144379+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:00:37.144379+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217645056,"num_objects":3321,"num_object_clones":0,"num_object_copies":9963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8825,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2163,"num_bytes_recovered":141754368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":1936}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":78,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708549+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:00:28.708549+0000","last_peered":"2023-09-23T23:00:28.708549+0000","last_clean":"2023-09-23T23:00:28.708549+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:00:28.708549+0000","last_undegraded":"2023-09-23T23:00:28.708549+0000","last_fullsized":"2023-09-23T23:00:28.708549+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":93,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.700541+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:00:28.700541+0000","last_peered":"2023-09-23T23:00:28.700541+0000","last_clean":"2023-09-23T23:00:28.700541+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:00:28.700541+0000","last_undegraded":"2023-09-23T23:00:28.700541+0000","last_fullsized":"2023-09-23T23:00:28.700541+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707505+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:00:28.707505+0000","last_peered":"2023-09-23T23:00:28.707505+0000","last_clean":"2023-09-23T23:00:28.707505+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:00:28.707505+0000","last_undegraded":"2023-09-23T23:00:28.707505+0000","last_fullsized":"2023-09-23T23:00:28.707505+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"385'8786","reported_seq":9305,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:37.016902+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:00:37.016902+0000","last_peered":"2023-09-23T23:00:37.016902+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:00:37.016902+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215220224,"num_objects":3284,"num_object_clones":0,"num_object_copies":9852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3284,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8786,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-267}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":341,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.689856+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:00:34.689856+0000","last_peered":"2023-09-23T23:00:34.689856+0000","last_clean":"2023-09-23T23:00:34.689856+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:00:34.689856+0000","last_undegraded":"2023-09-23T23:00:34.689856+0000","last_fullsized":"2023-09-23T23:00:34.689856+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":288,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.684145+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:00:28.684145+0000","last_peered":"2023-09-23T23:00:28.684145+0000","last_clean":"2023-09-23T23:00:28.684145+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:00:28.684145+0000","last_undegraded":"2023-09-23T23:00:28.684145+0000","last_fullsized":"2023-09-23T23:00:28.684145+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":359,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696475+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:00:28.696475+0000","last_peered":"2023-09-23T23:00:28.696475+0000","last_clean":"2023-09-23T23:00:28.696475+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:00:28.696475+0000","last_undegraded":"2023-09-23T23:00:28.696475+0000","last_fullsized":"2023-09-23T23:00:28.696475+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":323,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687547+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:00:34.687547+0000","last_peered":"2023-09-23T23:00:34.687547+0000","last_clean":"2023-09-23T23:00:34.687547+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:00:34.687547+0000","last_undegraded":"2023-09-23T23:00:34.687547+0000","last_fullsized":"2023-09-23T23:00:34.687547+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":132,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.680074+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:00:28.680074+0000","last_peered":"2023-09-23T23:00:28.680074+0000","last_clean":"2023-09-23T23:00:28.680074+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:00:28.680074+0000","last_undegraded":"2023-09-23T23:00:28.680074+0000","last_fullsized":"2023-09-23T23:00:28.680074+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":194,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.691976+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:00:28.691976+0000","last_peered":"2023-09-23T23:00:28.691976+0000","last_clean":"2023-09-23T23:00:28.691976+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:00:28.691976+0000","last_undegraded":"2023-09-23T23:00:28.691976+0000","last_fullsized":"2023-09-23T23:00:28.691976+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":408,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.704348+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:00:28.704348+0000","last_peered":"2023-09-23T23:00:28.704348+0000","last_clean":"2023-09-23T23:00:28.704348+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:00:28.704348+0000","last_undegraded":"2023-09-23T23:00:28.704348+0000","last_fullsized":"2023-09-23T23:00:28.704348+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":208,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.696111+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:00:28.696111+0000","last_peered":"2023-09-23T23:00:28.696111+0000","last_clean":"2023-09-23T23:00:28.696111+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:00:28.696111+0000","last_undegraded":"2023-09-23T23:00:28.696111+0000","last_fullsized":"2023-09-23T23:00:28.696111+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":295,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.691638+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:00:28.691638+0000","last_peered":"2023-09-23T23:00:28.691638+0000","last_clean":"2023-09-23T23:00:28.691638+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:00:28.691638+0000","last_undegraded":"2023-09-23T23:00:28.691638+0000","last_fullsized":"2023-09-23T23:00:28.691638+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.708677+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:00:28.708677+0000","last_peered":"2023-09-23T23:00:28.708677+0000","last_clean":"2023-09-23T23:00:28.708677+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:00:28.708677+0000","last_undegraded":"2023-09-23T23:00:28.708677+0000","last_fullsized":"2023-09-23T23:00:28.708677+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.707615+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:00:28.707615+0000","last_peered":"2023-09-23T23:00:28.707615+0000","last_clean":"2023-09-23T23:00:28.707615+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:00:28.707615+0000","last_undegraded":"2023-09-23T23:00:28.707615+0000","last_fullsized":"2023-09-23T23:00:28.707615+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":327,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.683898+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:28.683898+0000","last_peered":"2023-09-23T23:00:28.683898+0000","last_clean":"2023-09-23T23:00:28.683898+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:28.683898+0000","last_undegraded":"2023-09-23T23:00:28.683898+0000","last_fullsized":"2023-09-23T23:00:28.683898+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":370,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-09-23T23:00:28.694831+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:00:28.694831+0000","last_peered":"2023-09-23T23:00:28.694831+0000","last_clean":"2023-09-23T23:00:28.694831+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:00:28.694831+0000","last_undegraded":"2023-09-23T23:00:28.694831+0000","last_fullsized":"2023-09-23T23:00:28.694831+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3509977112,"num_objects":53559,"num_object_clones":0,"num_object_copies":160677,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":29578,"num_objects_misplaced":30073,"num_objects_unfound":0,"num_objects_dirty":53559,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140039,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13680,"num_bytes_recovered":895287296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10667372544,"data_stored":10667360328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98639,"ondisk_log_size":98639,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430611,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626836,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253149,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2098016,"kb_used_data":1377624,"kb_used_omap":9,"kb_used_meta":720374,"kb_avail":91647136,"statfs":{"total":95995035648,"available":93846667264,"internally_reserved":0,"allocated":1410686976,"data_stored":1409237033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":737663977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.76900000000000002,"15min":0.82699999999999996},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80700000000000005,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.504,"15min":0.52000000000000002},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.93600000000000005,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.311},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.341,"5min":4.048,"15min":4.048},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.74299999999999999,"15min":2.153},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.827,"5min":12.997999999999999,"15min":87.787999999999997},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.83099999999999996,"15min":2.5960000000000001},"min":{"1min":0.29099999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.109,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.7850000000000001,"15min":2.6309999999999998},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5470000000000002,"5min":151.036,"15min":151.036},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.972,"15min":2.6419999999999999},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.488,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.39400000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":3.1960000000000002,"15min":2.391},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.6909999999999998,"5min":116.098,"15min":116.098},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.23,"5min":2.992,"15min":2.6890000000000001},"min":{"1min":0.378,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":7.431,"5min":116.08499999999999,"15min":116.08499999999999},"last":1.524}]},{"osd":6,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.649,"15min":2.2400000000000002},"min":{"1min":0.41599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5939999999999999,"5min":20.742000000000001,"15min":79.311000000000007},"last":4.5549999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.532,"15min":1.9319999999999999},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.313},"max":{"1min":1.857,"5min":20.614000000000001,"15min":79.328999999999994},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.1140000000000001,"15min":1.8899999999999999},"min":{"1min":0.46700000000000003,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":2.8039999999999998,"5min":5.8410000000000002,"15min":29.539000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.395,"15min":2.0459999999999998},"min":{"1min":0.45100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.7849999999999999,"5min":20.902999999999999,"15min":20.902999999999999},"last":0.44600000000000001}]}]},{"osd":0,"up_from":353,"seq":1516123455519,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2417884,"kb_used_data":2266580,"kb_used_omap":37,"kb_used_meta":151258,"kb_avail":91327268,"statfs":{"total":95995035648,"available":93519122432,"internally_reserved":0,"allocated":2320977920,"data_stored":2319408200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":154888354},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.73299999999999998,"15min":0.76400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.9319999999999999,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.0270000000000001,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":1.9059999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":0.55700000000000005}]}]},{"osd":2,"up_from":359,"seq":1541893259292,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":783212,"kb_used_data":677988,"kb_used_omap":83,"kb_used_meta":105132,"kb_avail":92961940,"statfs":{"total":95995035648,"available":95193026560,"internally_reserved":0,"allocated":694259712,"data_stored":692743369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":107655621},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.42799999999999999,"15min":0.379},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.1520000000000001,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.66300000000000003,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.2,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.23200000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.61699999999999999,"15min":0.64900000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.90500000000000003,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.1060000000000001,"15min":1.1379999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6269999999999998,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":1.038}]},{"osd":3,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.73499999999999999,"15min":0.71999999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7270000000000003,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":1.3979999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.86299999999999999,"15min":0.83199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.2140000000000004,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":1.014}]},{"osd":4,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":0.73999999999999999,"15min":0.623},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1159999999999997,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.7080000000000002,"5min":3.492,"15min":3.492},"last":1.2869999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.98899999999999999,"15min":0.96599999999999997},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.6029999999999998,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.101,"5min":0.76500000000000001,"15min":0.70899999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.0839999999999996,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.376}]},{"osd":6,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.046,"15min":0.995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.5679999999999996,"5min":5.5679999999999996,"15min":5.5679999999999996},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.365,"5min":0.75800000000000001,"15min":0.65700000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.1220000000000001,"15min":1.034},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":7.0499999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.014,"5min":1.1459999999999999,"15min":1.1679999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.8520000000000003,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.57499999999999996}]}]},{"osd":3,"up_from":94,"seq":403726926096,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534976,"kb_used_data":38520,"kb_used_omap":7,"kb_used_meta":496440,"kb_avail":93210176,"statfs":{"total":95995035648,"available":95447220224,"internally_reserved":0,"allocated":39444480,"data_stored":37904138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":508355176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.17299999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.224}]},{"osd":1,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.78300000000000003,"15min":2.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":3.355,"5min":16.596,"15min":123.866},"last":0.308},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95599999999999996,"15min":2.2429999999999999},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.222},"max":{"1min":1.746,"5min":16.649999999999999,"15min":122.265},"last":0.20999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.1339999999999999,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.76600000000000001,"15min":0.75},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":3.8740000000000001,"5min":3.8740000000000001,"15min":3.8740000000000001},"last":0.24299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":4.6769999999999996,"5min":1.798,"15min":2.0070000000000001},"min":{"1min":0.32600000000000001,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":87.046999999999997,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.34499999999999997},{"interface":"front","average":{"1min":4.6580000000000004,"5min":1.972,"15min":1.7430000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":87.066000000000003,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.29399999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2410000000000001,"15min":1.3839999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5910000000000002,"5min":10.260999999999999,"15min":12.127000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.2150000000000001,"15min":1.861},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3390000000000004,"5min":6.5510000000000002,"15min":11.951000000000001},"last":0.496}]},{"osd":6,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.5649999999999999,"15min":1.7090000000000001},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.2520000000000007,"5min":11.116,"15min":14.097},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.613,"5min":1.397,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.2810000000000006,"5min":10.859,"15min":12.887},"last":0.72499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:59:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.97899999999999998,"15min":2.1179999999999999},"min":{"1min":0.443,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1040000000000001,"5min":6.0270000000000001,"15min":91.757999999999996},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.92300000000000004,"15min":2.097},"min":{"1min":0.37,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":3.4820000000000002,"5min":6.468,"15min":91.709999999999994},"last":0.45900000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019293,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3459704,"kb_used_data":2548144,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":90285448,"statfs":{"total":95995035648,"available":92452298752,"internally_reserved":0,"allocated":2609299456,"data_stored":2608517067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":0.70599999999999996,"15min":0.55000000000000004},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.65300000000000002,"15min":0.61499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":1.389}]},{"osd":1,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":3.5110000000000001,"15min":3.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":2.77,"5min":98.846999999999994,"15min":132.304},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.302,"5min":3.6749999999999998,"15min":3.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":10.361000000000001,"5min":98.938999999999993,"15min":132.453},"last":1.4359999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.119,"15min":1.0529999999999999},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.1839999999999999,"5min":0.81100000000000005,"15min":0.749},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.9550000000000001,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":1.1579999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.4450000000000001,"15min":4.3140000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.32400000000000001},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.2110000000000001,"15min":4.3049999999999997},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":3.2490000000000001,"5min":12.805999999999999,"15min":167.05199999999999},"last":1.526}]},{"osd":5,"last update":"Sat Sep 23 23:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":3.0670000000000002,"15min":1.643},"min":{"1min":0.318,"5min":0.214,"15min":0.214},"max":{"1min":3.129,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.335,"5min":3.4940000000000002,"15min":1.78},"min":{"1min":0.318,"5min":0.252,"15min":0.248},"max":{"1min":3.7469999999999999,"5min":151.905,"15min":151.905},"last":0.29599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.585,"15min":1.1839999999999999},"min":{"1min":0.373,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":4.1269999999999998,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.155,"5min":1.371,"15min":1.105},"min":{"1min":0.372,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2130000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.32700000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.47,"15min":1.127},"min":{"1min":0.40899999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.045999999999999,"5min":12.84,"15min":12.84},"last":0.371},{"interface":"front","average":{"1min":1.615,"5min":1.5089999999999999,"15min":1.0740000000000001},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.227,"5min":10.587,"15min":10.587},"last":0.40300000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823067,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2491288,"kb_used_data":1591632,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":91253864,"statfs":{"total":95995035648,"available":93443956736,"internally_reserved":0,"allocated":1629831168,"data_stored":1629076567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.73599999999999999,"15min":0.64900000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":14.42,"5min":14.42,"15min":14.42},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.274,"15min":1.1459999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.496,"5min":14.496,"15min":14.496},"last":0.29299999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2150000000000001,"15min":2.2850000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.24199999999999999},"max":{"1min":3.21,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.748,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":6.7690000000000001,"5min":10.353,"15min":110.89},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.601,"5min":6.601,"15min":6.601},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7669999999999999,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.35699999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":1.304,"15min":4.9619999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29299999999999998},"max":{"1min":3.1400000000000001,"5min":8.6489999999999991,"15min":257.05000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.4850000000000001,"15min":4.9610000000000003},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.965,"5min":12.185,"15min":257.35000000000002},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":2.8580000000000001,"15min":1.5109999999999999},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8440000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.5710000000000002,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.524,"5min":114.908,"15min":114.908},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.345,"15min":1.1379999999999999},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":6.8150000000000004,"5min":11.121,"15min":12.615},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.2130000000000001,"15min":1.123},"min":{"1min":0.433,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.2719999999999998,"5min":12.137,"15min":12.74},"last":1.5509999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.1180000000000001,"15min":1.046},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.5980000000000008,"5min":12.23,"15min":12.23},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.627,"5min":1.4199999999999999,"15min":1.0940000000000001},"min":{"1min":0.45500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.8710000000000004,"5min":11.112,"15min":11.112},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2310213632,"data_stored":2310209560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1405747200,"data_stored":1405747200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":690159616,"data_stored":690159616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":34930688,"data_stored":34930688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2601390080,"data_stored":2601386008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1625620480,"data_stored":1625620480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1999310848,"data_stored":1999306776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:00:39.939 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:00:39.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:40.088 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:40.189 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:40.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:40.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:39 smithi151 ceph-mon[159210]: pgmap v1148: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 225 op/s; 29580/160749 objects degraded (18.401%); 30073/160749 objects misplaced (18.708%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T23:00:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:39 smithi151 ceph-mon[159210]: from='client.46523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:40.297 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:40.347 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:40.547 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:40.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:40.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:40.653 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:40.689 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:40.761 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:40.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:40.846 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:40.852 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:40.866 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:40.894 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:40.921 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 29580/160749 objects degraded (18.401%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:40.922 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 29580/160749 objects degraded (18.401%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:40.925 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:41.027 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:41.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:41.133 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:41.188 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 29580/160749 objects degraded (18.401%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:41.270 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:41.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:41.376 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:41.422 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:41.507 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:41.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:41.618 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:41.649 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:41.771 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:00:41.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:41 smithi052 ceph-mon[187743]: pgmap v1149: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 29524/159585 objects degraded (18.500%); 29907/159585 objects misplaced (18.740%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:41.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2457570826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:41.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:41 smithi052 ceph-mon[180979]: pgmap v1149: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 29524/159585 objects degraded (18.500%); 29907/159585 objects misplaced (18.740%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:41.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2457570826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:41 smithi151 ceph-mon[159210]: pgmap v1149: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 29524/159585 objects degraded (18.500%); 29907/159585 objects misplaced (18.740%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T23:00:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2457570826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:42.287 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:42.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:42.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:42.394 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:42.447 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2457570826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: osdmap e386: 8 total, 8 up, 6 in 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[187743]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2457570826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: osdmap e386: 8 total, 8 up, 6 in 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: failed to encode map e386 with expected crc 2023-09-23T23:00:42.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:42 smithi052 ceph-mon[180979]: failed to encode map e386 with expected crc 2023-09-23T23:00:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:00:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: failed to encode map e386 with expected crc 2023-09-23T23:00:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2457570826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: osdmap e386: 8 total, 8 up, 6 in 2023-09-23T23:00:43.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: failed to encode map e386 with expected crc 2023-09-23T23:00:43.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: failed to encode map e386 with expected crc 2023-09-23T23:00:43.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: failed to encode map e386 with expected crc 2023-09-23T23:00:43.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:42 smithi151 ceph-mon[159210]: failed to encode map e386 with expected crc 2023-09-23T23:00:43.626 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:43.627 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:00:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:00:43] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.33.4" 2023-09-23T23:00:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:43.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:43.730 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:43.775 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:43.887 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:43 smithi052 ceph-mon[180979]: pgmap v1151: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 284 op/s; 28918/157245 objects degraded (18.390%); 29782/157245 objects misplaced (18.940%); 5.3 MiB/s, 93 objects/s recovering 2023-09-23T23:00:43.887 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:43 smithi052 ceph-mon[187743]: pgmap v1151: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 284 op/s; 28918/157245 objects degraded (18.390%); 29782/157245 objects misplaced (18.940%); 5.3 MiB/s, 93 objects/s recovering 2023-09-23T23:00:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:43 smithi151 ceph-mon[159210]: pgmap v1151: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 284 op/s; 28918/157245 objects degraded (18.390%); 29782/157245 objects misplaced (18.940%); 5.3 MiB/s, 93 objects/s recovering 2023-09-23T23:00:44.260 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:44.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:44.361 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:44.366 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:44.390 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:44.416 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:44.418 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:44.443 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:44.471 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:44.484 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:44.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:44.498 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:44.525 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:44.553 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:44.556 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.556 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.556 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.556 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.556 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.556 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.557 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.557 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:00:44.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:44.628 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:44.685 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:44.685 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/457762399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/457762399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:45.006 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:45.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:45.111 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:45.114 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:45.114 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 28918/157002 objects degraded (18.419%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1577,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":386,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":52334,"data_bytes":3428718520,"bytes_used":11921334272,"bytes_avail":564048879616,"bytes_total":575970213888,"degraded_objects":28918,"degraded_total":157002,"degraded_ratio":0.18418873644921721,"misplaced_objects":29782,"misplaced_total":157002,"misplaced_ratio":0.18969185105922218,"recovering_objects_per_sec":75,"recovering_bytes_per_sec":4505107,"recovering_keys_per_sec":0,"num_objects_recovered":751,"num_bytes_recovered":44564480,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":238},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (4m)\n [======================......] (remaining: 60s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:00:45.152 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:45.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/457762399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:45.234 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:45.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:45.339 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:45.378 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:45.427 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:45.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:45.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:45.535 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:45.568 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:45.915 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:00:45.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:45 smithi052 ceph-mon[187743]: pgmap v1152: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 238 op/s; 28918/157002 objects degraded (18.419%); 29782/157002 objects misplaced (18.969%); 4.3 MiB/s, 75 objects/s recovering 2023-09-23T23:00:45.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1187609546' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:45.951 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 28918/157002 objects degraded (18.419%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:45.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:45 smithi052 ceph-mon[180979]: pgmap v1152: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 238 op/s; 28918/157002 objects degraded (18.419%); 29782/157002 objects misplaced (18.969%); 4.3 MiB/s, 75 objects/s recovering 2023-09-23T23:00:45.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1187609546' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:45.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 28918/157002 objects degraded (18.419%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:46.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:45 smithi151 ceph-mon[159210]: pgmap v1152: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 238 op/s; 28918/157002 objects degraded (18.419%); 29782/157002 objects misplaced (18.969%); 4.3 MiB/s, 75 objects/s recovering 2023-09-23T23:00:46.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1187609546' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:00:46.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 28918/157002 objects degraded (18.419%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:46.320 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:00:46.328 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:46.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:46.431 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:00:46.431 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:00:46.437 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:46.526 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:46.591 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:46.594 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:46.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:46.715 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:46.762 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:46.853 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:46.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:46.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:46.994 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:47.086 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:47.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:47.197 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:47.226 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:47.271 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:47.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:47.375 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:47.405 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:47.606 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:47.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:47.707 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:47.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:47.737 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:47.753 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:47.791 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:47.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:47.902 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:47.926 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:47 smithi052 ceph-mon[180979]: pgmap v1153: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 28649/155238 objects degraded (18.455%); 29625/155238 objects misplaced (19.084%); 4.3 MiB/s, 75 objects/s recovering 2023-09-23T23:00:47.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:47 smithi052 ceph-mon[187743]: pgmap v1153: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 28649/155238 objects degraded (18.455%); 29625/155238 objects misplaced (19.084%); 4.3 MiB/s, 75 objects/s recovering 2023-09-23T23:00:47.939 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:47.997 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:48.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:48.101 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:48.137 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:47 smithi151 ceph-mon[159210]: pgmap v1153: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 28649/155238 objects degraded (18.455%); 29625/155238 objects misplaced (19.084%); 4.3 MiB/s, 75 objects/s recovering 2023-09-23T23:00:48.138 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:48.209 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:48.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:48.329 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:48.352 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:00:48.361 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:48.415 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:48.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:48.526 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:48.559 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:49.076 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:49.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:49.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:49.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:49.216 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:49.266 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:49.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:49.377 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:49.419 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:49.467 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:49.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:49.573 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:49.607 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:49.756 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:00:49.758 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:49 smithi151 ceph-mon[159210]: pgmap v1154: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 28243/152958 objects degraded (18.465%); 29542/152958 objects misplaced (19.314%); 3.7 MiB/s, 78 objects/s recovering 2023-09-23T23:00:49.759 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2791366663' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:00:49.759 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:49.759 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:49.924 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[180979]: pgmap v1154: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 28243/152958 objects degraded (18.465%); 29542/152958 objects misplaced (19.314%); 3.7 MiB/s, 78 objects/s recovering 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2791366663' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[187743]: pgmap v1154: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 28243/152958 objects degraded (18.465%); 29542/152958 objects misplaced (19.314%); 3.7 MiB/s, 78 objects/s recovering 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2791366663' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:49.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:00:49.970 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:49.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:50.081 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:50.116 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:50.187 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:50.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:50.293 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:50.336 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:50.423 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:50.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:50.469 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:00:50.507 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:00:50.527 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:50.558 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:50.630 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:50.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:50.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:50.743 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:50.790 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: failed to encode map e387 with expected crc 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: failed to encode map e387 with expected crc 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2791366663' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: osdmap e387: 8 total, 8 up, 6 in 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 28243/152958 objects degraded (18.465%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:51.054 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: failed to encode map e387 with expected crc 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: failed to encode map e387 with expected crc 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2791366663' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: osdmap e387: 8 total, 8 up, 6 in 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 28243/152958 objects degraded (18.465%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:51.055 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:51.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: failed to encode map e387 with expected crc 2023-09-23T23:00:51.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: failed to encode map e387 with expected crc 2023-09-23T23:00:51.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2791366663' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:00:51.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: osdmap e387: 8 total, 8 up, 6 in 2023-09-23T23:00:51.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:00:51.056 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 28243/152958 objects degraded (18.465%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:51.307 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:51.575 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1155,"stamp":"2023-09-23T23:00:49.747105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3340375992,"num_objects":50986,"num_object_clones":0,"num_object_copies":152958,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":28243,"num_objects_misplaced":29542,"num_objects_unfound":0,"num_objects_dirty":50986,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":142896,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14499,"num_bytes_recovered":934362044,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99230,"ondisk_log_size":99230,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11440152,"kb_used_data":8159656,"kb_used_omap":137,"kb_used_meta":3280310,"kb_avail":551030760,"statfs":{"total":575970213888,"available":564255498240,"internally_reserved":0,"allocated":8355487744,"data_stored":8347835408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3359037624},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-144769024,"num_objects":-2209,"num_object_clones":0,"num_object_copies":-6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1281,"num_objects_misplaced":-365,"num_objects_unfound":0,"num_objects_dirty":-2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2209,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":784,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.171267"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":398,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694829+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:00:34.694829+0000","last_peered":"2023-09-23T23:00:34.694829+0000","last_clean":"2023-09-23T23:00:34.694829+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:00:34.694829+0000","last_undegraded":"2023-09-23T23:00:34.694829+0000","last_fullsized":"2023-09-23T23:00:34.694829+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":55,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706786+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:00:34.706786+0000","last_peered":"2023-09-23T23:00:34.706786+0000","last_clean":"2023-09-23T23:00:34.706786+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:00:34.706786+0000","last_undegraded":"2023-09-23T23:00:34.706786+0000","last_fullsized":"2023-09-23T23:00:34.706786+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705401+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:00:34.705401+0000","last_peered":"2023-09-23T23:00:34.705401+0000","last_clean":"2023-09-23T23:00:34.705401+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:00:34.705401+0000","last_undegraded":"2023-09-23T23:00:34.705401+0000","last_fullsized":"2023-09-23T23:00:34.705401+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":382,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706178+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:00:34.706178+0000","last_peered":"2023-09-23T23:00:34.706178+0000","last_clean":"2023-09-23T23:00:34.706178+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:00:34.706178+0000","last_undegraded":"2023-09-23T23:00:34.706178+0000","last_fullsized":"2023-09-23T23:00:34.706178+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":70,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705143+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:00:34.705143+0000","last_peered":"2023-09-23T23:00:34.705143+0000","last_clean":"2023-09-23T23:00:34.705143+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:00:34.705143+0000","last_undegraded":"2023-09-23T23:00:34.705143+0000","last_fullsized":"2023-09-23T23:00:34.705143+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":56,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687696+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:34.687696+0000","last_peered":"2023-09-23T23:00:34.687696+0000","last_clean":"2023-09-23T23:00:34.687696+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:34.687696+0000","last_undegraded":"2023-09-23T23:00:34.687696+0000","last_fullsized":"2023-09-23T23:00:34.687696+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":321,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.697232+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:00:34.697232+0000","last_peered":"2023-09-23T23:00:34.697232+0000","last_clean":"2023-09-23T23:00:34.697232+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:00:34.697232+0000","last_undegraded":"2023-09-23T23:00:34.697232+0000","last_fullsized":"2023-09-23T23:00:34.697232+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":64,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.704989+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:00:34.704989+0000","last_peered":"2023-09-23T23:00:34.704989+0000","last_clean":"2023-09-23T23:00:34.704989+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:00:34.704989+0000","last_undegraded":"2023-09-23T23:00:34.704989+0000","last_fullsized":"2023-09-23T23:00:34.704989+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":282,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.691562+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:00:34.691562+0000","last_peered":"2023-09-23T23:00:34.691562+0000","last_clean":"2023-09-23T23:00:34.691562+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:00:34.691562+0000","last_undegraded":"2023-09-23T23:00:34.691562+0000","last_fullsized":"2023-09-23T23:00:34.691562+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705281+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:00:34.705281+0000","last_peered":"2023-09-23T23:00:34.705281+0000","last_clean":"2023-09-23T23:00:34.705281+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:00:34.705281+0000","last_undegraded":"2023-09-23T23:00:34.705281+0000","last_fullsized":"2023-09-23T23:00:34.705281+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":75,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706743+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:00:34.706743+0000","last_peered":"2023-09-23T23:00:34.706743+0000","last_clean":"2023-09-23T23:00:34.706743+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:00:34.706743+0000","last_undegraded":"2023-09-23T23:00:34.706743+0000","last_fullsized":"2023-09-23T23:00:34.706743+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.704632+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:00:41.704632+0000","last_peered":"2023-09-23T23:00:41.704632+0000","last_clean":"2023-09-23T23:00:41.704632+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:00:41.704632+0000","last_undegraded":"2023-09-23T23:00:41.704632+0000","last_fullsized":"2023-09-23T23:00:41.704632+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":457,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695173+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:00:34.695173+0000","last_peered":"2023-09-23T23:00:34.695173+0000","last_clean":"2023-09-23T23:00:34.695173+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:00:34.695173+0000","last_undegraded":"2023-09-23T23:00:34.695173+0000","last_fullsized":"2023-09-23T23:00:34.695173+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":360,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695999+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:00:34.695999+0000","last_peered":"2023-09-23T23:00:34.695999+0000","last_clean":"2023-09-23T23:00:34.695999+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:00:34.695999+0000","last_undegraded":"2023-09-23T23:00:34.695999+0000","last_fullsized":"2023-09-23T23:00:34.695999+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":286,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.696110+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:00:34.696110+0000","last_peered":"2023-09-23T23:00:34.696110+0000","last_clean":"2023-09-23T23:00:34.696110+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:00:34.696110+0000","last_undegraded":"2023-09-23T23:00:34.696110+0000","last_fullsized":"2023-09-23T23:00:34.696110+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":66,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706379+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:00:34.706379+0000","last_peered":"2023-09-23T23:00:34.706379+0000","last_clean":"2023-09-23T23:00:34.706379+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:00:34.706379+0000","last_undegraded":"2023-09-23T23:00:34.706379+0000","last_fullsized":"2023-09-23T23:00:34.706379+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":375,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705674+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:00:34.705674+0000","last_peered":"2023-09-23T23:00:34.705674+0000","last_clean":"2023-09-23T23:00:34.705674+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:00:34.705674+0000","last_undegraded":"2023-09-23T23:00:34.705674+0000","last_fullsized":"2023-09-23T23:00:34.705674+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":286,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.691262+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:00:34.691262+0000","last_peered":"2023-09-23T23:00:34.691262+0000","last_clean":"2023-09-23T23:00:34.691262+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:00:34.691262+0000","last_undegraded":"2023-09-23T23:00:34.691262+0000","last_fullsized":"2023-09-23T23:00:34.691262+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":271,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694418+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:00:34.694418+0000","last_peered":"2023-09-23T23:00:34.694418+0000","last_clean":"2023-09-23T23:00:34.694418+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:00:34.694418+0000","last_undegraded":"2023-09-23T23:00:34.694418+0000","last_fullsized":"2023-09-23T23:00:34.694418+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706257+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:34.706257+0000","last_peered":"2023-09-23T23:00:34.706257+0000","last_clean":"2023-09-23T23:00:34.706257+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:34.706257+0000","last_undegraded":"2023-09-23T23:00:34.706257+0000","last_fullsized":"2023-09-23T23:00:34.706257+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":104,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.686945+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:00:34.686945+0000","last_peered":"2023-09-23T23:00:34.686945+0000","last_clean":"2023-09-23T23:00:34.686945+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:00:34.686945+0000","last_undegraded":"2023-09-23T23:00:34.686945+0000","last_fullsized":"2023-09-23T23:00:34.686945+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":79,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706973+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:00:34.706973+0000","last_peered":"2023-09-23T23:00:34.706973+0000","last_clean":"2023-09-23T23:00:34.706973+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:00:34.706973+0000","last_undegraded":"2023-09-23T23:00:34.706973+0000","last_fullsized":"2023-09-23T23:00:34.706973+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"386'8860","reported_seq":8950,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:44.430839+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:00:44.430839+0000","last_peered":"2023-09-23T23:00:44.430839+0000","last_clean":"2023-09-23T23:00:44.430839+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:00:44.430839+0000","last_undegraded":"2023-09-23T23:00:44.430839+0000","last_fullsized":"2023-09-23T23:00:44.430839+0000","mapping_epoch":353,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5460,"log_dups_size":0,"ondisk_log_size":5460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211419136,"num_objects":3226,"num_object_clones":0,"num_object_copies":9678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8860,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":270,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695717+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:00:34.695717+0000","last_peered":"2023-09-23T23:00:34.695717+0000","last_clean":"2023-09-23T23:00:34.695717+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:00:34.695717+0000","last_undegraded":"2023-09-23T23:00:34.695717+0000","last_fullsized":"2023-09-23T23:00:34.695717+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":256,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.690716+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:00:34.690716+0000","last_peered":"2023-09-23T23:00:34.690716+0000","last_clean":"2023-09-23T23:00:34.690716+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:00:34.690716+0000","last_undegraded":"2023-09-23T23:00:34.690716+0000","last_fullsized":"2023-09-23T23:00:34.690716+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"386'8764","reported_seq":11183,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:45.969761+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:00:45.969761+0000","last_peered":"2023-09-23T23:00:45.969761+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:00:45.969761+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8364,"log_dups_size":0,"ondisk_log_size":8364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208797696,"num_objects":3186,"num_object_clones":0,"num_object_copies":9558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3186,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8764,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-381}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":280,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695409+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:00:34.695409+0000","last_peered":"2023-09-23T23:00:34.695409+0000","last_clean":"2023-09-23T23:00:34.695409+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:00:34.695409+0000","last_undegraded":"2023-09-23T23:00:34.695409+0000","last_fullsized":"2023-09-23T23:00:34.695409+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":393,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694331+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:00:34.694331+0000","last_peered":"2023-09-23T23:00:34.694331+0000","last_clean":"2023-09-23T23:00:34.694331+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:00:34.694331+0000","last_undegraded":"2023-09-23T23:00:34.694331+0000","last_fullsized":"2023-09-23T23:00:34.694331+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"386'8893","reported_seq":8977,"reported_epoch":386,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:44.452108+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:00:44.452108+0000","last_peered":"2023-09-23T23:00:44.452108+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:00:44.452108+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:00:44.452108+0000","mapping_epoch":354,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208338944,"num_objects":3179,"num_object_clones":0,"num_object_copies":9537,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3179,"num_objects_unfound":0,"num_objects_dirty":3179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8893,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3179}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":137,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.691343+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:00:34.691343+0000","last_peered":"2023-09-23T23:00:34.691343+0000","last_clean":"2023-09-23T23:00:34.691343+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:00:34.691343+0000","last_undegraded":"2023-09-23T23:00:34.691343+0000","last_fullsized":"2023-09-23T23:00:34.691343+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":255,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.690868+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:00:34.690868+0000","last_peered":"2023-09-23T23:00:34.690868+0000","last_clean":"2023-09-23T23:00:34.690868+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:00:34.690868+0000","last_undegraded":"2023-09-23T23:00:34.690868+0000","last_fullsized":"2023-09-23T23:00:34.690868+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"386'8944","reported_seq":9214,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:44.463555+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:00:44.463555+0000","last_peered":"2023-09-23T23:00:44.463555+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:00:44.463555+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211681280,"num_objects":3230,"num_object_clones":0,"num_object_copies":9690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2713,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8944,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":516},{"shards":"0,5","objects":-380}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":55,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706302+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:00:34.706302+0000","last_peered":"2023-09-23T23:00:34.706302+0000","last_clean":"2023-09-23T23:00:34.706302+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:00:34.706302+0000","last_undegraded":"2023-09-23T23:00:34.706302+0000","last_fullsized":"2023-09-23T23:00:34.706302+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":285,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687384+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:00:34.687384+0000","last_peered":"2023-09-23T23:00:34.687384+0000","last_clean":"2023-09-23T23:00:34.687384+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:00:34.687384+0000","last_undegraded":"2023-09-23T23:00:34.687384+0000","last_fullsized":"2023-09-23T23:00:34.687384+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"386'8897","reported_seq":8988,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:46.896218+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:00:46.896218+0000","last_peered":"2023-09-23T23:00:46.896218+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:00:46.896218+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:00:46.896218+0000","mapping_epoch":353,"log_start":"326'3200","ondisk_log_start":"326'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8897,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3197}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":74,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706034+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:00:34.706034+0000","last_peered":"2023-09-23T23:00:34.706034+0000","last_clean":"2023-09-23T23:00:34.706034+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:00:34.706034+0000","last_undegraded":"2023-09-23T23:00:34.706034+0000","last_fullsized":"2023-09-23T23:00:34.706034+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":379,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695042+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:00:34.695042+0000","last_peered":"2023-09-23T23:00:34.695042+0000","last_clean":"2023-09-23T23:00:34.695042+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:00:34.695042+0000","last_undegraded":"2023-09-23T23:00:34.695042+0000","last_fullsized":"2023-09-23T23:00:34.695042+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"386'9023","reported_seq":9122,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:47.105313+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:00:47.105313+0000","last_peered":"2023-09-23T23:00:47.105313+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:00:47.105313+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208338944,"num_objects":3179,"num_object_clones":0,"num_object_copies":9537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3179,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9023,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-440}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":169,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694981+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:00:34.694981+0000","last_peered":"2023-09-23T23:00:34.694981+0000","last_clean":"2023-09-23T23:00:34.694981+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:00:34.694981+0000","last_undegraded":"2023-09-23T23:00:34.694981+0000","last_fullsized":"2023-09-23T23:00:34.694981+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":396,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694948+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:00:34.694948+0000","last_peered":"2023-09-23T23:00:34.694948+0000","last_clean":"2023-09-23T23:00:34.694948+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:00:34.694948+0000","last_undegraded":"2023-09-23T23:00:34.694948+0000","last_fullsized":"2023-09-23T23:00:34.694948+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"386'8915","reported_seq":14318,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:47.075073+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:00:47.075073+0000","last_peered":"2023-09-23T23:00:47.075073+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:00:47.075073+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:00:47.075073+0000","mapping_epoch":354,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5715,"log_dups_size":0,"ondisk_log_size":5715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206110720,"num_objects":3145,"num_object_clones":0,"num_object_copies":9435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8915,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2648,"num_bytes_recovered":173539328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":1711}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":409,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694118+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:00:34.694118+0000","last_peered":"2023-09-23T23:00:34.694118+0000","last_clean":"2023-09-23T23:00:34.694118+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:00:34.694118+0000","last_undegraded":"2023-09-23T23:00:34.694118+0000","last_fullsized":"2023-09-23T23:00:34.694118+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":387,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695477+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:00:34.695477+0000","last_peered":"2023-09-23T23:00:34.695477+0000","last_clean":"2023-09-23T23:00:34.695477+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:00:34.695477+0000","last_undegraded":"2023-09-23T23:00:34.695477+0000","last_fullsized":"2023-09-23T23:00:34.695477+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"386'8756","reported_seq":8847,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:47.161029+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:00:47.161029+0000","last_peered":"2023-09-23T23:00:47.161029+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:00:47.161029+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:00:47.161029+0000","mapping_epoch":353,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210501632,"num_objects":3212,"num_object_clones":0,"num_object_copies":9636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8756,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3212}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":55,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705391+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:00:34.705391+0000","last_peered":"2023-09-23T23:00:34.705391+0000","last_clean":"2023-09-23T23:00:34.705391+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:00:34.705391+0000","last_undegraded":"2023-09-23T23:00:34.705391+0000","last_fullsized":"2023-09-23T23:00:34.705391+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":289,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:36.269519+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:00:36.269519+0000","last_peered":"2023-09-23T23:00:36.269519+0000","last_clean":"2023-09-23T23:00:36.269519+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:00:36.269519+0000","last_undegraded":"2023-09-23T23:00:36.269519+0000","last_fullsized":"2023-09-23T23:00:36.269519+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"386'8859","reported_seq":16690,"reported_epoch":386,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:00:47.193995+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:00:47.193995+0000","last_peered":"2023-09-23T23:00:47.193995+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:00:47.193995+0000","last_undegraded":"2023-09-23T23:00:47.193995+0000","last_fullsized":"2023-09-23T23:00:47.193995+0000","mapping_epoch":354,"log_start":"325'3100","ondisk_log_start":"325'3100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210829312,"num_objects":3217,"num_object_clones":0,"num_object_copies":9651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8859,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3853,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":3217}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":293,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.686963+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:00:34.686963+0000","last_peered":"2023-09-23T23:00:34.686963+0000","last_clean":"2023-09-23T23:00:34.686963+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:00:34.686963+0000","last_undegraded":"2023-09-23T23:00:34.686963+0000","last_fullsized":"2023-09-23T23:00:34.686963+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":296,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.691242+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:00:34.691242+0000","last_peered":"2023-09-23T23:00:34.691242+0000","last_clean":"2023-09-23T23:00:34.691242+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:00:34.691242+0000","last_undegraded":"2023-09-23T23:00:34.691242+0000","last_fullsized":"2023-09-23T23:00:34.691242+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.690779+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:00:34.690779+0000","last_peered":"2023-09-23T23:00:34.690779+0000","last_clean":"2023-09-23T23:00:34.690779+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:00:34.690779+0000","last_undegraded":"2023-09-23T23:00:34.690779+0000","last_fullsized":"2023-09-23T23:00:34.690779+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"386'8902","reported_seq":9005,"reported_epoch":386,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:00:44.667475+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:00:44.667475+0000","last_peered":"2023-09-23T23:00:44.667475+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:00:44.667475+0000","last_undegraded":"2023-09-23T23:00:44.667475+0000","last_fullsized":"2023-09-23T23:00:44.667475+0000","mapping_epoch":359,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211156992,"num_objects":3222,"num_object_clones":0,"num_object_copies":9666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3222,"num_objects_unfound":0,"num_objects_dirty":3222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8902,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3222}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":56,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.688367+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:34.688367+0000","last_peered":"2023-09-23T23:00:34.688367+0000","last_clean":"2023-09-23T23:00:34.688367+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:34.688367+0000","last_undegraded":"2023-09-23T23:00:34.688367+0000","last_fullsized":"2023-09-23T23:00:34.688367+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":406,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.696194+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:00:34.696194+0000","last_peered":"2023-09-23T23:00:34.696194+0000","last_clean":"2023-09-23T23:00:34.696194+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:00:34.696194+0000","last_undegraded":"2023-09-23T23:00:34.696194+0000","last_fullsized":"2023-09-23T23:00:34.696194+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687029+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:00:34.687029+0000","last_peered":"2023-09-23T23:00:34.687029+0000","last_clean":"2023-09-23T23:00:34.687029+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:00:34.687029+0000","last_undegraded":"2023-09-23T23:00:34.687029+0000","last_fullsized":"2023-09-23T23:00:34.687029+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"386'8866","reported_seq":15536,"reported_epoch":386,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:46.165713+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T23:00:46.165713+0000","last_peered":"2023-09-23T23:00:46.165713+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:00:46.165713+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8366,"log_dups_size":0,"ondisk_log_size":8366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209584128,"num_objects":3198,"num_object_clones":0,"num_object_copies":9594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8866,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3286,"num_bytes_recovered":215351296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2913}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":353,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.686772+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:00:34.686772+0000","last_peered":"2023-09-23T23:00:34.686772+0000","last_clean":"2023-09-23T23:00:34.686772+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:00:34.686772+0000","last_undegraded":"2023-09-23T23:00:34.686772+0000","last_fullsized":"2023-09-23T23:00:34.686772+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":230,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.686513+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:00:34.686513+0000","last_peered":"2023-09-23T23:00:34.686513+0000","last_clean":"2023-09-23T23:00:34.686513+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:00:34.686513+0000","last_undegraded":"2023-09-23T23:00:34.686513+0000","last_fullsized":"2023-09-23T23:00:34.686513+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":296,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.697302+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:00:34.697302+0000","last_peered":"2023-09-23T23:00:34.697302+0000","last_clean":"2023-09-23T23:00:34.697302+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:00:34.697302+0000","last_undegraded":"2023-09-23T23:00:34.697302+0000","last_fullsized":"2023-09-23T23:00:34.697302+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"386'9055","reported_seq":10943,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:47.576570+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:00:47.576570+0000","last_peered":"2023-09-23T23:00:47.576570+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:00:47.576570+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":9055,"log_dups_size":0,"ondisk_log_size":9055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207683584,"num_objects":3169,"num_object_clones":0,"num_object_copies":9507,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9055,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":1736},{"shards":"0,6","objects":-1812}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":369,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.690939+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:00:34.690939+0000","last_peered":"2023-09-23T23:00:34.690939+0000","last_clean":"2023-09-23T23:00:34.690939+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:00:34.690939+0000","last_undegraded":"2023-09-23T23:00:34.690939+0000","last_fullsized":"2023-09-23T23:00:34.690939+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":482,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:34.688492+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:34.688492+0000","last_peered":"2023-09-23T23:00:34.688492+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:34.688492+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":55,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705828+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:00:34.705828+0000","last_peered":"2023-09-23T23:00:34.705828+0000","last_clean":"2023-09-23T23:00:34.705828+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:00:34.705828+0000","last_undegraded":"2023-09-23T23:00:34.705828+0000","last_fullsized":"2023-09-23T23:00:34.705828+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695159+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:00:34.695159+0000","last_peered":"2023-09-23T23:00:34.695159+0000","last_clean":"2023-09-23T23:00:34.695159+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:00:34.695159+0000","last_undegraded":"2023-09-23T23:00:34.695159+0000","last_fullsized":"2023-09-23T23:00:34.695159+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"386'8972","reported_seq":9067,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:44.641017+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:00:44.641017+0000","last_peered":"2023-09-23T23:00:44.641017+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:00:44.641017+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"326'3500","ondisk_log_start":"326'3500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203948032,"num_objects":3112,"num_object_clones":0,"num_object_copies":9336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8972,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-395}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":376,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687019+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:34.687019+0000","last_peered":"2023-09-23T23:00:34.687019+0000","last_clean":"2023-09-23T23:00:34.687019+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:34.687019+0000","last_undegraded":"2023-09-23T23:00:34.687019+0000","last_fullsized":"2023-09-23T23:00:34.687019+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":347,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.686880+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:34.686880+0000","last_peered":"2023-09-23T23:00:34.686880+0000","last_clean":"2023-09-23T23:00:34.686880+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:34.686880+0000","last_undegraded":"2023-09-23T23:00:34.686880+0000","last_fullsized":"2023-09-23T23:00:34.686880+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":307,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.688202+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:00:34.688202+0000","last_peered":"2023-09-23T23:00:34.688202+0000","last_clean":"2023-09-23T23:00:34.688202+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:00:34.688202+0000","last_undegraded":"2023-09-23T23:00:34.688202+0000","last_fullsized":"2023-09-23T23:00:34.688202+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"386'8990","reported_seq":9100,"reported_epoch":386,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:47.170350+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:00:47.170350+0000","last_peered":"2023-09-23T23:00:47.170350+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:00:47.170350+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:00:47.170350+0000","mapping_epoch":354,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210567192,"num_objects":3214,"num_object_clones":0,"num_object_copies":9642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3214,"num_objects_unfound":0,"num_objects_dirty":3214,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8990,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3214}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":322,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.688532+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:34.688532+0000","last_peered":"2023-09-23T23:00:34.688532+0000","last_clean":"2023-09-23T23:00:34.688532+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:34.688532+0000","last_undegraded":"2023-09-23T23:00:34.688532+0000","last_fullsized":"2023-09-23T23:00:34.688532+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":230,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687124+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:00:34.687124+0000","last_peered":"2023-09-23T23:00:34.687124+0000","last_clean":"2023-09-23T23:00:34.687124+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:00:34.687124+0000","last_undegraded":"2023-09-23T23:00:34.687124+0000","last_fullsized":"2023-09-23T23:00:34.687124+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":199,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.690947+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:00:34.690947+0000","last_peered":"2023-09-23T23:00:34.690947+0000","last_clean":"2023-09-23T23:00:34.690947+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:00:34.690947+0000","last_undegraded":"2023-09-23T23:00:34.690947+0000","last_fullsized":"2023-09-23T23:00:34.690947+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"386'8995","reported_seq":14195,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:47.089592+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:00:47.089592+0000","last_peered":"2023-09-23T23:00:47.089592+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:00:47.089592+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206503936,"num_objects":3151,"num_object_clones":0,"num_object_copies":9453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8995,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2333,"num_bytes_recovered":152895488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":1766}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":79,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.707158+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:00:34.707158+0000","last_peered":"2023-09-23T23:00:34.707158+0000","last_clean":"2023-09-23T23:00:34.707158+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:00:34.707158+0000","last_undegraded":"2023-09-23T23:00:34.707158+0000","last_fullsized":"2023-09-23T23:00:34.707158+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":94,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.697810+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:00:34.697810+0000","last_peered":"2023-09-23T23:00:34.697810+0000","last_clean":"2023-09-23T23:00:34.697810+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:00:34.697810+0000","last_undegraded":"2023-09-23T23:00:34.697810+0000","last_fullsized":"2023-09-23T23:00:34.697810+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705219+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:00:34.705219+0000","last_peered":"2023-09-23T23:00:34.705219+0000","last_clean":"2023-09-23T23:00:34.705219+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:00:34.705219+0000","last_undegraded":"2023-09-23T23:00:34.705219+0000","last_fullsized":"2023-09-23T23:00:34.705219+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"386'8945","reported_seq":9465,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:46.851097+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:00:46.851097+0000","last_peered":"2023-09-23T23:00:46.851097+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:00:46.851097+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"325'3200","ondisk_log_start":"325'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204800000,"num_objects":3125,"num_object_clones":0,"num_object_copies":9375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3125,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":3125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8945,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-426}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":342,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:36.269561+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:00:36.269561+0000","last_peered":"2023-09-23T23:00:36.269561+0000","last_clean":"2023-09-23T23:00:36.269561+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:00:36.269561+0000","last_undegraded":"2023-09-23T23:00:36.269561+0000","last_fullsized":"2023-09-23T23:00:36.269561+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":289,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.688126+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:00:34.688126+0000","last_peered":"2023-09-23T23:00:34.688126+0000","last_clean":"2023-09-23T23:00:34.688126+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:00:34.688126+0000","last_undegraded":"2023-09-23T23:00:34.688126+0000","last_fullsized":"2023-09-23T23:00:34.688126+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":360,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694179+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:00:34.694179+0000","last_peered":"2023-09-23T23:00:34.694179+0000","last_clean":"2023-09-23T23:00:34.694179+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:00:34.694179+0000","last_undegraded":"2023-09-23T23:00:34.694179+0000","last_fullsized":"2023-09-23T23:00:34.694179+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":324,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.779572+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:00:34.779572+0000","last_peered":"2023-09-23T23:00:34.779572+0000","last_clean":"2023-09-23T23:00:34.779572+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:00:34.779572+0000","last_undegraded":"2023-09-23T23:00:34.779572+0000","last_fullsized":"2023-09-23T23:00:34.779572+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.686775+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:00:34.686775+0000","last_peered":"2023-09-23T23:00:34.686775+0000","last_clean":"2023-09-23T23:00:34.686775+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:00:34.686775+0000","last_undegraded":"2023-09-23T23:00:34.686775+0000","last_fullsized":"2023-09-23T23:00:34.686775+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":195,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694957+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:00:34.694957+0000","last_peered":"2023-09-23T23:00:34.694957+0000","last_clean":"2023-09-23T23:00:34.694957+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:00:34.694957+0000","last_undegraded":"2023-09-23T23:00:34.694957+0000","last_fullsized":"2023-09-23T23:00:34.694957+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":409,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.696271+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:00:34.696271+0000","last_peered":"2023-09-23T23:00:34.696271+0000","last_clean":"2023-09-23T23:00:34.696271+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:00:34.696271+0000","last_undegraded":"2023-09-23T23:00:34.696271+0000","last_fullsized":"2023-09-23T23:00:34.696271+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":209,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.695076+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:00:34.695076+0000","last_peered":"2023-09-23T23:00:34.695076+0000","last_clean":"2023-09-23T23:00:34.695076+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:00:34.695076+0000","last_undegraded":"2023-09-23T23:00:34.695076+0000","last_fullsized":"2023-09-23T23:00:34.695076+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":296,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.694902+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:00:34.694902+0000","last_peered":"2023-09-23T23:00:34.694902+0000","last_clean":"2023-09-23T23:00:34.694902+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:00:34.694902+0000","last_undegraded":"2023-09-23T23:00:34.694902+0000","last_fullsized":"2023-09-23T23:00:34.694902+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.706337+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:00:34.706337+0000","last_peered":"2023-09-23T23:00:34.706337+0000","last_clean":"2023-09-23T23:00:34.706337+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:00:34.706337+0000","last_undegraded":"2023-09-23T23:00:34.706337+0000","last_fullsized":"2023-09-23T23:00:34.706337+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.705570+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:00:34.705570+0000","last_peered":"2023-09-23T23:00:34.705570+0000","last_clean":"2023-09-23T23:00:34.705570+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:00:34.705570+0000","last_undegraded":"2023-09-23T23:00:34.705570+0000","last_fullsized":"2023-09-23T23:00:34.705570+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":328,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.687956+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:34.687956+0000","last_peered":"2023-09-23T23:00:34.687956+0000","last_clean":"2023-09-23T23:00:34.687956+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:34.687956+0000","last_undegraded":"2023-09-23T23:00:34.687956+0000","last_fullsized":"2023-09-23T23:00:34.687956+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":371,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-09-23T23:00:34.697641+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:00:34.697641+0000","last_peered":"2023-09-23T23:00:34.697641+0000","last_clean":"2023-09-23T23:00:34.697641+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:00:34.697641+0000","last_undegraded":"2023-09-23T23:00:34.697641+0000","last_fullsized":"2023-09-23T23:00:34.697641+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3339780120,"num_objects":50962,"num_object_clones":0,"num_object_copies":152886,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":28241,"num_objects_misplaced":29542,"num_objects_unfound":0,"num_objects_dirty":50962,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142636,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14464,"num_bytes_recovered":934346752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10256265216,"data_stored":10256253000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99036,"ondisk_log_size":99036,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430613,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626838,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253151,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2041836,"kb_used_data":1317348,"kb_used_omap":9,"kb_used_meta":724470,"kb_avail":91703316,"statfs":{"total":95995035648,"available":93904195584,"internally_reserved":0,"allocated":1348964352,"data_stored":1347510439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":741858281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.76900000000000002,"15min":0.82699999999999996},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80700000000000005,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.504,"15min":0.52000000000000002},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.93600000000000005,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.341,"5min":4.048,"15min":4.048},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56100000000000005,"15min":1.9950000000000001},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.8580000000000001,"5min":3.141,"15min":87.787999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.71699999999999997,"15min":2.339},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.19500000000000001},"max":{"1min":0.95499999999999996,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.5}]},{"osd":4,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.6869999999999998,"15min":2.4529999999999998},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6669999999999998,"5min":151.036,"15min":151.036},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.581,"5min":2.7450000000000001,"15min":2.548},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":6.649,"5min":151.06899999999999,"15min":151.06899999999999},"last":2.0379999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.6219999999999999,"15min":2.319},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1720000000000002,"5min":116.098,"15min":116.098},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.5099999999999998,"15min":2.4809999999999999},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.9769999999999999,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.27,"15min":2.1480000000000001},"min":{"1min":0.40100000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":79.311000000000007},"last":2.0790000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.0820000000000001,"15min":1.8520000000000001},"min":{"1min":0.40999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.825,"5min":8.5370000000000008,"15min":79.328999999999994},"last":0.61499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.04,"15min":1.6799999999999999},"min":{"1min":0.46300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.581,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.177,"15min":1.8919999999999999},"min":{"1min":0.44600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9860000000000002,"5min":10.676,"15min":20.902999999999999},"last":2.1070000000000002}]}]},{"osd":0,"up_from":353,"seq":1516123455521,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2303484,"kb_used_data":2164468,"kb_used_omap":37,"kb_used_meta":138970,"kb_avail":91441668,"statfs":{"total":95995035648,"available":93636268032,"internally_reserved":0,"allocated":2216415232,"data_stored":2214830100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":142305442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.73299999999999998,"15min":0.76400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.9319999999999999,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.309},{"interface":"front","average":{"1min":0.621,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.0270000000000001,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.61899999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":2.3570000000000002},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":0.91500000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.71399999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.497},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":0.42699999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":0.52900000000000003}]}]},{"osd":2,"up_from":359,"seq":1541893259294,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":794300,"kb_used_data":684980,"kb_used_omap":83,"kb_used_meta":109228,"kb_avail":92950852,"statfs":{"total":95995035648,"available":95181672448,"internally_reserved":0,"allocated":701419520,"data_stored":699895471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":111849925},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.42799999999999999,"15min":0.379},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.1520000000000001,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.247},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.66300000000000003,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.2,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.26100000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.61699999999999999,"15min":0.64900000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.90500000000000003,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.1060000000000001,"15min":1.1379999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6269999999999998,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.73499999999999999,"15min":0.71999999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7270000000000003,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.86299999999999999,"15min":0.83199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.2140000000000004,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":0.73999999999999999,"15min":0.623},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1159999999999997,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.7080000000000002,"5min":3.492,"15min":3.492},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.98899999999999999,"15min":0.96599999999999997},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.6029999999999998,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.101,"5min":0.76500000000000001,"15min":0.70899999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.0839999999999996,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.046,"15min":0.995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.5679999999999996,"5min":5.5679999999999996,"15min":5.5679999999999996},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.365,"5min":0.75800000000000001,"15min":0.65700000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.1220000000000001,"15min":1.034},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":7.0499999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.621},{"interface":"front","average":{"1min":1.014,"5min":1.1459999999999999,"15min":1.1679999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.8520000000000003,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.97199999999999998}]}]},{"osd":3,"up_from":94,"seq":403726926098,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542156,"kb_used_data":37508,"kb_used_omap":7,"kb_used_meta":504632,"kb_avail":93202996,"statfs":{"total":95995035648,"available":95439867904,"internally_reserved":0,"allocated":38408192,"data_stored":36863880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":516743784},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.316}]},{"osd":1,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.63200000000000001,"15min":2.3639999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":123.866},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.73699999999999999,"15min":2.1549999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8010000000000002,"5min":6.8010000000000002,"15min":122.265},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.1339999999999999,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.496},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.76600000000000001,"15min":0.75},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":3.8740000000000001,"5min":3.8740000000000001,"15min":3.8740000000000001},"last":0.72299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":2.3519999999999999,"15min":2.0350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":64.406999999999996,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.67700000000000005},{"interface":"front","average":{"1min":4.008,"5min":2.5409999999999999,"15min":1.8999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":64.468000000000004,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.57199999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2290000000000001,"15min":1.3560000000000001},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1980000000000004,"5min":10.260999999999999,"15min":11.803000000000001},"last":3.149},{"interface":"front","average":{"1min":1.548,"5min":1.218,"15min":1.827},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.246,"5min":10.246,"15min":11.951000000000001},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.597,"15min":1.7150000000000001},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0229999999999997,"5min":11.116,"15min":14.097},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4039999999999999,"15min":1.956},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.79,"5min":10.859,"15min":12.887},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.89400000000000002,"15min":2.0499999999999998},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8449999999999998,"5min":4.8449999999999998,"15min":91.757999999999996},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.413,"5min":0.98099999999999998,"15min":1.9390000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":8.5830000000000002,"5min":8.5830000000000002,"15min":91.709999999999994},"last":0.754}]}]},{"osd":4,"up_from":30,"seq":128849019295,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3333764,"kb_used_data":2434492,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":90411388,"statfs":{"total":95995035648,"available":92581261312,"internally_reserved":0,"allocated":2492919808,"data_stored":2492133449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":0.70599999999999996,"15min":0.55000000000000004},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.65300000000000002,"15min":0.61499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":3.5110000000000001,"15min":3.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":2.77,"5min":98.846999999999994,"15min":132.304},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.302,"5min":3.6749999999999998,"15min":3.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":10.361000000000001,"5min":98.938999999999993,"15min":132.453},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.119,"15min":1.0529999999999999},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.1839999999999999,"5min":0.81100000000000005,"15min":0.749},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.9550000000000001,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.4450000000000001,"15min":4.3140000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.32400000000000001},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":164.453},"last":2.3639999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.2110000000000001,"15min":4.3049999999999997},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":3.2490000000000001,"5min":12.805999999999999,"15min":167.05199999999999},"last":7.0039999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":3.0670000000000002,"15min":1.643},"min":{"1min":0.318,"5min":0.214,"15min":0.214},"max":{"1min":3.129,"5min":148.61600000000001,"15min":148.61600000000001},"last":8.3439999999999994},{"interface":"front","average":{"1min":1.335,"5min":3.4940000000000002,"15min":1.78},"min":{"1min":0.318,"5min":0.252,"15min":0.248},"max":{"1min":3.7469999999999999,"5min":151.905,"15min":151.905},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.585,"15min":1.1839999999999999},"min":{"1min":0.373,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":4.1269999999999998,"5min":16.943000000000001,"15min":16.943000000000001},"last":1.274},{"interface":"front","average":{"1min":1.155,"5min":1.371,"15min":1.105},"min":{"1min":0.372,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2130000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.47,"15min":1.127},"min":{"1min":0.40899999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.045999999999999,"5min":12.84,"15min":12.84},"last":7.0869999999999997},{"interface":"front","average":{"1min":1.615,"5min":1.5089999999999999,"15min":1.0740000000000001},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.227,"5min":10.587,"15min":10.587},"last":0.55400000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823069,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2424612,"kb_used_data":1520860,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":91320540,"statfs":{"total":95995035648,"available":93512232960,"internally_reserved":0,"allocated":1557360640,"data_stored":1556602069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.73599999999999999,"15min":0.64900000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":14.42,"5min":14.42,"15min":14.42},"last":0.22800000000000001},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.274,"15min":1.1459999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.496,"5min":14.496,"15min":14.496},"last":0.374}]},{"osd":1,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.081,"15min":2.1909999999999998},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.24299999999999999},"max":{"1min":2.6040000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.857,"5min":1.6200000000000001,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":7.0739999999999998,"5min":10.353,"15min":110.89},"last":4.8230000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.601,"5min":6.601,"15min":6.601},"last":3.3730000000000002},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7669999999999999,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.216,"15min":4.7009999999999996},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2710000000000008,"5min":9.2710000000000008,"15min":257.05000000000001},"last":3.4140000000000001},{"interface":"front","average":{"1min":1.474,"5min":1.3859999999999999,"15min":4.7590000000000003},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.3399999999999999,"5min":12.185,"15min":257.35000000000002},"last":0.49299999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":2.8580000000000001,"15min":1.5109999999999999},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8440000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":4.7519999999999998},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.5710000000000002,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.524,"5min":114.908,"15min":114.908},"last":4.6070000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.512,"15min":1.2270000000000001},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.4630000000000001,"5min":11.121,"15min":12.615},"last":4.7930000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.2370000000000001,"15min":1.1519999999999999},"min":{"1min":0.41299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.4909999999999997,"5min":12.137,"15min":12.74},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.1180000000000001,"15min":1.046},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.5980000000000008,"5min":12.23,"15min":12.23},"last":1.3340000000000001},{"interface":"front","average":{"1min":1.627,"5min":1.4199999999999999,"15min":1.0940000000000001},"min":{"1min":0.45500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.8710000000000004,"5min":11.112,"15min":11.112},"last":5.4530000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2205028352,"data_stored":2205024280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1342767104,"data_stored":1342767104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":696516608,"data_stored":696516608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":33882112,"data_stored":33882112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2483752960,"data_stored":2483748888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1550123008,"data_stored":1550123008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1944195072,"data_stored":1944191000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:00:51.671 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:00:51.671 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:00:51.676 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:51.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:51.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:51.783 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:51.831 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:51.927 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:51.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:51.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:51 smithi052 ceph-mon[187743]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:51.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:51 smithi052 ceph-mon[187743]: pgmap v1156: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 28177/151800 objects degraded (18.562%); 29377/151800 objects misplaced (19.352%); 3.4 MiB/s, 75 objects/s recovering 2023-09-23T23:00:51.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:51 smithi052 ceph-mon[180979]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:51.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:51 smithi052 ceph-mon[180979]: pgmap v1156: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 28177/151800 objects degraded (18.562%); 29377/151800 objects misplaced (19.352%); 3.4 MiB/s, 75 objects/s recovering 2023-09-23T23:00:52.035 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:52.070 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:51 smithi151 ceph-mon[159210]: from='client.46547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:00:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:51 smithi151 ceph-mon[159210]: pgmap v1156: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 28177/151800 objects degraded (18.562%); 29377/151800 objects misplaced (19.352%); 3.4 MiB/s, 75 objects/s recovering 2023-09-23T23:00:52.387 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:52.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:52.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:52.491 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:52.543 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:52.564 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:00:52.932 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:52.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:52.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:53.036 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:00:53.077 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:53.259 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:00:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:53.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:00:53.344 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:00:53.368 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:53.438 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:53.640 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:00:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:00:53] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-09-23T23:00:53.659 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:53.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:53.770 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:53.835 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:53.850 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:53.925 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:53.927 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:53 smithi052 ceph-mon[180979]: pgmap v1157: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27650/149820 objects degraded (18.455%); 29325/149820 objects misplaced (19.573%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:53.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:53 smithi052 ceph-mon[187743]: pgmap v1157: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27650/149820 objects degraded (18.455%); 29325/149820 objects misplaced (19.573%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:53.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:53.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:54.029 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:00:54.080 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:54.153 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:54.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:53 smithi151 ceph-mon[159210]: pgmap v1157: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27650/149820 objects degraded (18.455%); 29325/149820 objects misplaced (19.573%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:54.314 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:00:54.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:00:54.342 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:00:54.380 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:00:54.406 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:00:54.415 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:54.433 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:00:54.464 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:54.468 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:00:54.503 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:00:54.538 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:00:54.572 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:00:54.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.573 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.574 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.574 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.574 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.574 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.574 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.574 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:00:54.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:00:54.661 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:54.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:54.772 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:54.837 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:55.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3082251876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[180979]: pgmap v1158: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27650/149571 objects degraded (18.486%); 29325/149571 objects misplaced (19.606%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3082251876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:55 smithi052 ceph-mon[187743]: pgmap v1158: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27650/149571 objects degraded (18.486%); 29325/149571 objects misplaced (19.606%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:55 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:55 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:00:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3082251876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:00:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:55 smithi151 ceph-mon[159210]: pgmap v1158: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27650/149571 objects degraded (18.486%); 29325/149571 objects misplaced (19.606%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:55.475 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:55.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:55.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:55.579 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:55.612 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:55.860 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:55.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:55.965 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:00:56.004 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:56.153 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:56 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 27650/149571 objects degraded (18.486%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:56.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:56 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 27650/149571 objects degraded (18.486%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:56.222 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:00:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:56.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:00:56.327 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:56.361 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:56 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 27650/149571 objects degraded (18.486%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:00:56.363 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:56.428 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:56.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:56.517 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:00:56.534 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:56.577 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:56.765 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:56.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:56.876 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:56.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:57.043 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:00:57.044 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:00:57.044 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:00:57.044 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:00:57.044 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.044 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114198 10.5 10:a53fd4ad:::benchmark_data_smithi052_461816_object17397:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.044 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.30503879099999, 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047289519, 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114198 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.045 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.046 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.046 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.046 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:00:57.046 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.046 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.046 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.047 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.047 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:00:57.047 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.047 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.047 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735120+0000", 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.048 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735128+0000", 2023-09-23T23:00:57.049 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6639999999999998e-06 2023-09-23T23:00:57.049 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.049 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.049 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.050 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735140+0000", 2023-09-23T23:00:57.050 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1888999999999999e-05 2023-09-23T23:00:57.050 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.050 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.050 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.051 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735151+0000", 2023-09-23T23:00:57.051 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T23:00:57.051 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.051 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.052 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.052 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736424+0000", 2023-09-23T23:00:57.052 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001272942 2023-09-23T23:00:57.053 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.053 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.053 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.053 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736451+0000", 2023-09-23T23:00:57.053 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7029000000000002e-05 2023-09-23T23:00:57.054 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.054 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.054 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.054 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772313+0000", 2023-09-23T23:00:57.054 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.035862128 2023-09-23T23:00:57.055 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.055 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.055 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.055 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772461+0000", 2023-09-23T23:00:57.055 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014821499999999999 2023-09-23T23:00:57.056 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.056 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.056 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.056 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772545+0000", 2023-09-23T23:00:57.056 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3183000000000006e-05 2023-09-23T23:00:57.057 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.057 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.057 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.057 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779124+0000", 2023-09-23T23:00:57.057 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065792929999999999 2023-09-23T23:00:57.058 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.058 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.058 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.058 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782207+0000", 2023-09-23T23:00:57.058 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030828169999999998 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782247+0000", 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0198999999999999e-05 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.059 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782304+0000", 2023-09-23T23:00:57.060 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7172000000000001e-05 2023-09-23T23:00:57.060 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.060 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.060 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.060 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782412+0000", 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107539 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114209 10.5 10:a8019dd6:::benchmark_data_smithi052_461816_object17408:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.30384433399999, 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.052215573, 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.061 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114209 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.062 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736316+0000", 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.063 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736321+0000", 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6640000000000003e-06 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736333+0000", 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1854e-05 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736343+0000", 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7399999999999999e-06 2023-09-23T23:00:57.064 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736479+0000", 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013588000000000001 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736486+0000", 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3150000000000003e-06 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774671+0000", 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0381846910000001 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774856+0000", 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000185179 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774979+0000", 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122843 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782160+0000", 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071812610000000004 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782451+0000", 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029058300000000002 2023-09-23T23:00:57.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788381+0000", 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059298370000000003 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788422+0000", 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1409999999999998e-05 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788532+0000", 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110103 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114217 10.5 10:a151ffa4:::benchmark_data_smithi052_461816_object17416:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.30381185300001, 2023-09-23T23:00:57.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068861074, 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114217 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:00:57.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736352+0000", 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9399999999999998e-06 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736358+0000", 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.072 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736363+0000", 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3199999999999999e-06 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737029+0000", 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000666032 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737038+0000", 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5490000000000006e-06 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779153+0000", 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0421153320000001 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779339+0000", 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000186016 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779477+0000", 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000137923 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782455+0000", 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029779519999999999 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790613+0000", 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081575689999999999 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805085+0000", 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014472021 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805126+0000", 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1619000000000003e-05 2023-09-23T23:00:57.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805210+0000", 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3728999999999997e-05 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.078 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114228 10.5 10:abf9b074:::benchmark_data_smithi052_461816_object17427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.303375813, 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068634885, 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.079 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114228 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.080 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736784+0000", 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736797+0000", 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2558000000000001e-05 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736807+0000", 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7669999999999998e-06 2023-09-23T23:00:57.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736815+0000", 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737073+0000", 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000257502 2023-09-23T23:00:57.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737079+0000", 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5180000000000001e-06 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782471+0000", 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0453921399999999 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782629+0000", 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015792200000000001 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782710+0000", 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0499000000000003e-05 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788710+0000", 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060006640000000002 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796703+0000", 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079926860000000006 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805261+0000", 2023-09-23T23:00:57.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085578570000000003 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805317+0000", 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5649000000000002e-05 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805420+0000", 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103467 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.088 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114233 10.5 10:adc2278f:::benchmark_data_smithi052_461816_object17432:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.30333055599999, 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068750884, 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.089 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114233 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736833+0000", 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0469999999999998e-06 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736840+0000", 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2130000000000002e-06 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736846+0000", 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1099999999999999e-06 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737097+0000", 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025043299999999999 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737103+0000", 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8150000000000002e-06 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785090+0000", 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047987419 2023-09-23T23:00:57.093 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785226+0000", 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013601299999999999 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785376+0000", 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000150407 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.094 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790542+0000", 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051658429999999998 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796752+0000", 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062099620000000003 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805468+0000", 2023-09-23T23:00:57.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0087155189999999997 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805509+0000", 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1789000000000003e-05 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805581+0000", 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1771000000000001e-05 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.096 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114234 10.5 10:a1bb5c32:::benchmark_data_smithi052_461816_object17433:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.30330993199999, 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068915523, 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.097 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114234 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:00:57.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736853+0000", 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3379999999999999e-06 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736859+0000", 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0499999999999997e-06 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736870+0000", 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1021e-05 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737132+0000", 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00026153199999999998 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737138+0000", 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9839999999999998e-06 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786800+0000", 2023-09-23T23:00:57.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.049662885 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786974+0000", 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017351000000000001 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.787107+0000", 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013272100000000001 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790552+0000", 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034448650000000001 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797758+0000", 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072065200000000001 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805623+0000", 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0078654350000000005 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805667+0000", 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3665000000000003e-05 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805766+0000", 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9365999999999997e-05 2023-09-23T23:00:57.104 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114235 10.5 10:a3d19bc6:::benchmark_data_smithi052_461816_object17434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.303286318, 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081414576, 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.105 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114235 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.106 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736874+0000", 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736877+0000", 2023-09-23T23:00:57.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8949999999999998e-06 2023-09-23T23:00:57.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736883+0000", 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3459999999999999e-06 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736891+0000", 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8050000000000007e-06 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737166+0000", 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027465399999999999 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737172+0000", 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.207e-06 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788737+0000", 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0515644399999999 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788877+0000", 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014082 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788968+0000", 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0167999999999995e-05 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792267+0000", 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003299256 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797797+0000", 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.005530274 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818146+0000", 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020349151999999999 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818185+0000", 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8575000000000003e-05 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818289+0000", 2023-09-23T23:00:57.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010435899999999999 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114248 10.5 10:a5b9c0ca:::benchmark_data_smithi052_461816_object17447:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.30326515799999, 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081611235, 2023-09-23T23:00:57.115 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114248 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736895+0000", 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736898+0000", 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4250000000000001e-06 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736904+0000", 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736909+0000", 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2310000000000004e-06 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737193+0000", 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000283885 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:00:57.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737199+0000", 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9610000000000002e-06 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790684+0000", 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.053484358 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790817+0000", 2023-09-23T23:00:57.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000133693 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790895+0000", 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8218000000000002e-05 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792358+0000", 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001462758 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797833+0000", 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054744759999999998 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818343+0000", 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020510347000000002 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818398+0000", 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4530999999999997e-05 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818507+0000", 2023-09-23T23:00:57.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010940100000000001 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:116534 10.6 10:62c2756c:::benchmark_data_smithi052_461816_object19733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 130.160630113, 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18822530200000001, 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.125 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 116534 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879530+0000", 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879535+0000", 2023-09-23T23:00:57.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879547+0000", 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1918e-05 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879555+0000", 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1459999999999996e-06 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879602+0000", 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6768999999999999e-05 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879810+0000", 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020844799999999999 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879951+0000", 2023-09-23T23:00:57.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014103400000000001 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.881979+0000", 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020274709999999999 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.883452+0000", 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001472985 2023-09-23T23:00:57.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067651+0000", 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18419902199999999 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067709+0000", 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8180000000000002e-05 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067756+0000", 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7275999999999999e-05 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119634 10.d e373/354 10:b8a57e34:::benchmark_data_smithi052_461816_object22833:head v 373'7541, mlcod=344'7206)", 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.258961+0000", 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 115.78120031500001, 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.119970613, 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.132 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258961+0000", 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258961+0000", 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258960+0000", 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258965+0000", 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8289999999999998e-06 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376011+0000", 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.117046202 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376028+0000", 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6413999999999999e-05 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376114+0000", 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6285000000000005e-05 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376141+0000", 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7178999999999998e-05 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.378888+0000", 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002747088 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.378931+0000", 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2837000000000003e-05 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:125541 10.8 10:17ca584a:::benchmark_data_smithi052_461816_object28740:head [delete] snapc 0=[] ondisk+write+known_if_redirected e376)", 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:27.772819+0000", 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 89.267341956999999, 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12880313700000001, 2023-09-23T23:00:57.137 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 125541 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772819+0000", 2023-09-23T23:00:57.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772819+0000", 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772818+0000", 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772823+0000", 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4949999999999999e-06 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772832+0000", 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4649999999999994e-06 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772837+0000", 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5539999999999997e-06 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772877+0000", 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9830000000000003e-05 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.772974+0000", 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6324999999999997e-05 2023-09-23T23:00:57.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.773067+0000", 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3523999999999994e-05 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.774662+0000", 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015949899999999999 2023-09-23T23:00:57.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900492+0000", 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12582966800000001 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900519+0000", 2023-09-23T23:00:57.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7359999999999999e-05 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900568+0000", 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8967000000000001e-05 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.901622+0000", 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010540700000000001 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.144 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.45953.0:125541 10.8 e376/360 ondisk, result = 0)", 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:27.776524+0000", 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 89.263636715000004, 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123971281, 2023-09-23T23:00:57.145 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.776524+0000", 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.776524+0000", 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.776523+0000", 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.776529+0000", 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1349999999999997e-06 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.776541+0000", 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1083e-05 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900408+0000", 2023-09-23T23:00:57.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12386771100000001 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900474+0000", 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6255999999999997e-05 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900483+0000", 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9449999999999999e-06 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900495+0000", 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2048999999999999e-05 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.150 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:125309 10.b e376/354 10:d740b7cc:::benchmark_data_smithi052_461816_object28508:head v 376'7900, mlcod=0'0)", 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:27.784505+0000", 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 89.255655430999994, 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122797846, 2023-09-23T23:00:57.151 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.784505+0000", 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.784505+0000", 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.784504+0000", 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.784514+0000", 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6010000000000001e-06 2023-09-23T23:00:57.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900420+0000", 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.115906195 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900430+0000", 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8709999999999992e-06 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900475+0000", 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5228999999999997e-05 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.900503+0000", 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8243999999999999e-05 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.907271+0000", 2023-09-23T23:00:57.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0067677930000000002 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:27.907303+0000", 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2067000000000003e-05 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138140 10.8 10:10bb8653:::benchmark_data_smithi052_461816_object41339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:00:57.157 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 29.427428025000001, 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.272336093, 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138140 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612731+0000", 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:57.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612741+0000", 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4800000000000007e-06 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612762+0000", 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1761000000000001e-05 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612772+0000", 2023-09-23T23:00:57.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6229999999999992e-06 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612837+0000", 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4789999999999995e-05 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613071+0000", 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023441400000000001 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613293+0000", 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022140800000000001 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.616187+0000", 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028945080000000001 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.618246+0000", 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020593949999999999 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883729+0000", 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265482459 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883802+0000", 2023-09-23T23:00:57.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3008000000000003e-05 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.885069+0000", 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012670579999999999 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138142 10.8 10:10a11927:::benchmark_data_smithi052_461816_object41341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:00:57.164 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 29.413336560000001, 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26049065999999998, 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138142 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626822+0000", 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626834+0000", 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626866+0000", 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1934999999999998e-05 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626881+0000", 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5491000000000001e-05 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626950+0000", 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8701000000000005e-05 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627110+0000", 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000160347 2023-09-23T23:00:57.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627220+0000", 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110009 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.629339+0000", 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021183280000000001 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.635341+0000", 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060026009999999998 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886154+0000", 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25081289200000001 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886210+0000", 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5321000000000001e-05 2023-09-23T23:00:57.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.887315+0000", 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001105396 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138146 10.6 10:6e738f12:::benchmark_data_smithi052_461816_object41345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.721322+0000", 2023-09-23T23:00:57.171 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 29.318838456999998, 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16252518799999999, 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138146 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721322+0000", 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721322+0000", 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721321+0000", 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:00:57.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721329+0000", 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9300000000000003e-06 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721345+0000", 2023-09-23T23:00:57.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6147999999999999e-05 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721360+0000", 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5622999999999999e-05 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721413+0000", 2023-09-23T23:00:57.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2743e-05 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721524+0000", 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110302 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721613+0000", 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9450999999999994e-05 2023-09-23T23:00:57.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.723428+0000", 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018152520000000001 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.724326+0000", 2023-09-23T23:00:57.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00089747799999999995 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883765+0000", 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15943891499999999 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883809+0000", 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4713999999999999e-05 2023-09-23T23:00:57.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883848+0000", 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8324000000000001e-05 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:140723 10.6 10:68c22ebc:::benchmark_data_smithi052_461816_object43922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.825137612999999, 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.195061394, 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:00:57.180 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140723 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215022+0000", 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:00:57.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215028+0000", 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215039+0000", 2023-09-23T23:00:57.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0998e-05 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215044+0000", 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6380000000000001e-06 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215089+0000", 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4929000000000003e-05 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215188+0000", 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8741999999999998e-05 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215268+0000", 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0049000000000006e-05 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:00:57.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217087+0000", 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018184539999999999 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217880+0000", 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079313599999999995 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:00:57.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410012+0000", 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19213219200000001 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410050+0000", 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8262000000000001e-05 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410085+0000", 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4588000000000002e-05 2023-09-23T23:00:57.188 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141836 10.2 e386/353 10:4baa1ec5:::benchmark_data_smithi052_461816_object45035:head v 386'8834, mlcod=344'7113)", 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:43.040925+0000", 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 13.999235465, 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12331816499999999, 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.189 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040925+0000", 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040925+0000", 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040925+0000", 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040931+0000", 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9479999999999996e-06 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040940+0000", 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4229999999999994e-06 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040945+0000", 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3539999999999999e-06 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040983+0000", 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7623999999999998e-05 2023-09-23T23:00:57.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.040999+0000", 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5582000000000001e-05 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164205+0000", 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12320643100000001 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164244+0000", 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8668000000000003e-05 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141837 10.d e386/354 10:bcbf0dcf:::benchmark_data_smithi052_461816_object45036:head v 386'8928, mlcod=344'7206)", 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:43.041277+0000", 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 13.998884085, 2023-09-23T23:00:57.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122970027, 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041277+0000", 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041277+0000", 2023-09-23T23:00:57.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041276+0000", 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041280+0000", 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1450000000000001e-06 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041286+0000", 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8780000000000003e-06 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041291+0000", 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6e-06 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041325+0000", 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4088999999999998e-05 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.041341+0000", 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5639000000000001e-05 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164195+0000", 2023-09-23T23:00:57.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122854479 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164247+0000", 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1699000000000001e-05 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141583 10.b e386/354 10:d4f63af0:::benchmark_data_smithi052_461816_object44782:head v 386'8974, mlcod=0'0)", 2023-09-23T23:00:57.199 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:43.052259+0000", 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 13.987901678, 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.121680917, 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.052259+0000", 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.052259+0000", 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.052258+0000", 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:00:57.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.052266+0000", 2023-09-23T23:00:57.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6340000000000001e-06 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164184+0000", 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11191794300000001 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164201+0000", 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7450999999999999e-05 2023-09-23T23:00:57.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164242+0000", 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0193e-05 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.164274+0000", 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2466999999999999e-05 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.173895+0000", 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0096210029999999995 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:43.173940+0000", 2023-09-23T23:00:57.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5108000000000002e-05 2023-09-23T23:00:57.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.209 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:00:57.209 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:00:57.209 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:00:57.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:57 smithi052 ceph-mon[187743]: pgmap v1159: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 246 op/s; 27376/147852 objects degraded (18.516%); 29167/147852 objects misplaced (19.727%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:57 smithi052 ceph-mon[180979]: pgmap v1159: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 246 op/s; 27376/147852 objects degraded (18.516%); 29167/147852 objects misplaced (19.727%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:57.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:57 smithi151 ceph-mon[159210]: pgmap v1159: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 246 op/s; 27376/147852 objects degraded (18.516%); 29167/147852 objects misplaced (19.727%); 3.3 MiB/s, 76 objects/s recovering 2023-09-23T23:00:57.838 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:57.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:57.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:57.942 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:57.984 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:58.069 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:58.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:58.081 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:00:58.090 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2456174159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:58.090 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:58 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:58.091 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:00:58.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2456174159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:58.096 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:58 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:58.179 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:00:58.214 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:58.270 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:00:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:58.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:00:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2456174159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:58.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:58 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:00:58.291 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:00:58.380 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:00:58.421 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:58.506 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:00:58.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:58.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:00:58.616 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:00:58.653 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:59.060 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:00:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:59.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:00:59.165 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:00:59.198 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:59.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:59 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:59.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:59 smithi151 ceph-mon[159210]: failed to encode map e388 with expected crc 2023-09-23T23:00:59.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:59.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:59 smithi151 ceph-mon[159210]: osdmap e388: 8 total, 8 up, 6 in 2023-09-23T23:00:59.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:59 smithi151 ceph-mon[159210]: failed to encode map e388 with expected crc 2023-09-23T23:00:59.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:00:59 smithi151 ceph-mon[159210]: pgmap v1161: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 274 op/s; 26945/145680 objects degraded (18.496%); 29096/145680 objects misplaced (19.973%); 3.9 MiB/s, 89 objects/s recovering 2023-09-23T23:00:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[180979]: failed to encode map e388 with expected crc 2023-09-23T23:00:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[180979]: osdmap e388: 8 total, 8 up, 6 in 2023-09-23T23:00:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[180979]: failed to encode map e388 with expected crc 2023-09-23T23:00:59.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[180979]: pgmap v1161: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 274 op/s; 26945/145680 objects degraded (18.496%); 29096/145680 objects misplaced (19.973%); 3.9 MiB/s, 89 objects/s recovering 2023-09-23T23:00:59.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:00:59.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[187743]: failed to encode map e388 with expected crc 2023-09-23T23:00:59.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:00:59.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[187743]: osdmap e388: 8 total, 8 up, 6 in 2023-09-23T23:00:59.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[187743]: failed to encode map e388 with expected crc 2023-09-23T23:00:59.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:00:59 smithi052 ceph-mon[187743]: pgmap v1161: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 274 op/s; 26945/145680 objects degraded (18.496%); 29096/145680 objects misplaced (19.973%); 3.9 MiB/s, 89 objects/s recovering 2023-09-23T23:00:59.609 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:00:59.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:59.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:00:59.715 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:00:59.757 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:00:59.840 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:00:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:00:59.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:00:59.955 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:00.002 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:00.003 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:00.116 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:00.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:00.201 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:00.221 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:00.249 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:00.318 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:00.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:00.426 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:00.462 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:00.496 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:00.502 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:01:00.527 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:00.531 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 26945/145680 objects degraded (18.496%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:00.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:00.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:00.545 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 26945/145680 objects degraded (18.496%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:00.545 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 26945/145680 objects degraded (18.496%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:00.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:00.671 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:00.731 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:00.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:00.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:00.841 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:00.877 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:01.503 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:01.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:01.612 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:01.625 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:01:01.626 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:01 smithi052 ceph-mon[187743]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:01.626 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:01 smithi052 ceph-mon[187743]: pgmap v1162: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 250 op/s; 26872/144273 objects degraded (18.626%); 28904/144273 objects misplaced (20.034%); 3.5 MiB/s, 78 objects/s recovering 2023-09-23T23:01:01.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:01 smithi052 ceph-mon[180979]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:01.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:01 smithi052 ceph-mon[180979]: pgmap v1162: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 250 op/s; 26872/144273 objects degraded (18.626%); 28904/144273 objects misplaced (20.034%); 3.5 MiB/s, 78 objects/s recovering 2023-09-23T23:01:01.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:01 smithi151 ceph-mon[159210]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:01.655 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:01 smithi151 ceph-mon[159210]: pgmap v1162: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 250 op/s; 26872/144273 objects degraded (18.626%); 28904/144273 objects misplaced (20.034%); 3.5 MiB/s, 78 objects/s recovering 2023-09-23T23:01:01.658 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:01.665 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1161,"stamp":"2023-09-23T23:00:58.582298+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3181385656,"num_objects":48560,"num_object_clones":0,"num_object_copies":145680,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":26945,"num_objects_misplaced":29096,"num_objects_unfound":0,"num_objects_dirty":48560,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":145322,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15289,"num_bytes_recovered":970865596,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99556,"ondisk_log_size":99556,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11127080,"kb_used_data":7838328,"kb_used_omap":137,"kb_used_meta":3288502,"kb_avail":551343832,"statfs":{"total":575970213888,"available":564576083968,"internally_reserved":0,"allocated":8026447872,"data_stored":8018764244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3367426232},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-158990336,"num_objects":-2426,"num_object_clones":0,"num_object_copies":-7278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1298,"num_objects_misplaced":-446,"num_objects_unfound":0,"num_objects_dirty":-2426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2426,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":790,"num_bytes_recovered":36503552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.835193"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":399,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781719+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:00:41.781719+0000","last_peered":"2023-09-23T23:00:41.781719+0000","last_clean":"2023-09-23T23:00:41.781719+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:00:41.781719+0000","last_undegraded":"2023-09-23T23:00:41.781719+0000","last_fullsized":"2023-09-23T23:00:41.781719+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":56,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.853094+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:00:41.853094+0000","last_peered":"2023-09-23T23:00:41.853094+0000","last_clean":"2023-09-23T23:00:41.853094+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:00:41.853094+0000","last_undegraded":"2023-09-23T23:00:41.853094+0000","last_fullsized":"2023-09-23T23:00:41.853094+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788840+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:00:41.788840+0000","last_peered":"2023-09-23T23:00:41.788840+0000","last_clean":"2023-09-23T23:00:41.788840+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:00:41.788840+0000","last_undegraded":"2023-09-23T23:00:41.788840+0000","last_fullsized":"2023-09-23T23:00:41.788840+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":383,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.789175+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:00:41.789175+0000","last_peered":"2023-09-23T23:00:41.789175+0000","last_clean":"2023-09-23T23:00:41.789175+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:00:41.789175+0000","last_undegraded":"2023-09-23T23:00:41.789175+0000","last_fullsized":"2023-09-23T23:00:41.789175+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":71,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788684+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:00:41.788684+0000","last_peered":"2023-09-23T23:00:41.788684+0000","last_clean":"2023-09-23T23:00:41.788684+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:00:41.788684+0000","last_undegraded":"2023-09-23T23:00:41.788684+0000","last_fullsized":"2023-09-23T23:00:41.788684+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":57,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.771170+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:41.771170+0000","last_peered":"2023-09-23T23:00:41.771170+0000","last_clean":"2023-09-23T23:00:41.771170+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:41.771170+0000","last_undegraded":"2023-09-23T23:00:41.771170+0000","last_fullsized":"2023-09-23T23:00:41.771170+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":322,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.776171+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:00:41.776171+0000","last_peered":"2023-09-23T23:00:41.776171+0000","last_clean":"2023-09-23T23:00:41.776171+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:00:41.776171+0000","last_undegraded":"2023-09-23T23:00:41.776171+0000","last_fullsized":"2023-09-23T23:00:41.776171+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":65,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788599+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:00:41.788599+0000","last_peered":"2023-09-23T23:00:41.788599+0000","last_clean":"2023-09-23T23:00:41.788599+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:00:41.788599+0000","last_undegraded":"2023-09-23T23:00:41.788599+0000","last_fullsized":"2023-09-23T23:00:41.788599+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":283,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.775163+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:00:41.775163+0000","last_peered":"2023-09-23T23:00:41.775163+0000","last_clean":"2023-09-23T23:00:41.775163+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:00:41.775163+0000","last_undegraded":"2023-09-23T23:00:41.775163+0000","last_fullsized":"2023-09-23T23:00:41.775163+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788903+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:00:41.788903+0000","last_peered":"2023-09-23T23:00:41.788903+0000","last_clean":"2023-09-23T23:00:41.788903+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:00:41.788903+0000","last_undegraded":"2023-09-23T23:00:41.788903+0000","last_fullsized":"2023-09-23T23:00:41.788903+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":76,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852489+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:00:41.852489+0000","last_peered":"2023-09-23T23:00:41.852489+0000","last_clean":"2023-09-23T23:00:41.852489+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:00:41.852489+0000","last_undegraded":"2023-09-23T23:00:41.852489+0000","last_fullsized":"2023-09-23T23:00:41.852489+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:46.281586+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:00:46.281586+0000","last_peered":"2023-09-23T23:00:46.281586+0000","last_clean":"2023-09-23T23:00:46.281586+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:00:46.281586+0000","last_undegraded":"2023-09-23T23:00:46.281586+0000","last_fullsized":"2023-09-23T23:00:46.281586+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":458,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.780235+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:00:41.780235+0000","last_peered":"2023-09-23T23:00:41.780235+0000","last_clean":"2023-09-23T23:00:41.780235+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:00:41.780235+0000","last_undegraded":"2023-09-23T23:00:41.780235+0000","last_fullsized":"2023-09-23T23:00:41.780235+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":361,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.783383+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:00:41.783383+0000","last_peered":"2023-09-23T23:00:41.783383+0000","last_clean":"2023-09-23T23:00:41.783383+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:00:41.783383+0000","last_undegraded":"2023-09-23T23:00:41.783383+0000","last_fullsized":"2023-09-23T23:00:41.783383+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":287,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.783529+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:00:41.783529+0000","last_peered":"2023-09-23T23:00:41.783529+0000","last_clean":"2023-09-23T23:00:41.783529+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:00:41.783529+0000","last_undegraded":"2023-09-23T23:00:41.783529+0000","last_fullsized":"2023-09-23T23:00:41.783529+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":67,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852615+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:00:41.852615+0000","last_peered":"2023-09-23T23:00:41.852615+0000","last_clean":"2023-09-23T23:00:41.852615+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:00:41.852615+0000","last_undegraded":"2023-09-23T23:00:41.852615+0000","last_fullsized":"2023-09-23T23:00:41.852615+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":376,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788732+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:00:41.788732+0000","last_peered":"2023-09-23T23:00:41.788732+0000","last_clean":"2023-09-23T23:00:41.788732+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:00:41.788732+0000","last_undegraded":"2023-09-23T23:00:41.788732+0000","last_fullsized":"2023-09-23T23:00:41.788732+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":287,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.774550+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:00:41.774550+0000","last_peered":"2023-09-23T23:00:41.774550+0000","last_clean":"2023-09-23T23:00:41.774550+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:00:41.774550+0000","last_undegraded":"2023-09-23T23:00:41.774550+0000","last_fullsized":"2023-09-23T23:00:41.774550+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":272,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.779991+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:00:41.779991+0000","last_peered":"2023-09-23T23:00:41.779991+0000","last_clean":"2023-09-23T23:00:41.779991+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:00:41.779991+0000","last_undegraded":"2023-09-23T23:00:41.779991+0000","last_fullsized":"2023-09-23T23:00:41.779991+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":400,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852254+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:41.852254+0000","last_peered":"2023-09-23T23:00:41.852254+0000","last_clean":"2023-09-23T23:00:41.852254+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:41.852254+0000","last_undegraded":"2023-09-23T23:00:41.852254+0000","last_fullsized":"2023-09-23T23:00:41.852254+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":105,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.711121+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:00:41.711121+0000","last_peered":"2023-09-23T23:00:41.711121+0000","last_clean":"2023-09-23T23:00:41.711121+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:00:41.711121+0000","last_undegraded":"2023-09-23T23:00:41.711121+0000","last_fullsized":"2023-09-23T23:00:41.711121+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852390+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:00:41.852390+0000","last_peered":"2023-09-23T23:00:41.852390+0000","last_clean":"2023-09-23T23:00:41.852390+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:00:41.852390+0000","last_undegraded":"2023-09-23T23:00:41.852390+0000","last_fullsized":"2023-09-23T23:00:41.852390+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"387'9010","reported_seq":9101,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T23:00:54.744223+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:00:54.744223+0000","last_peered":"2023-09-23T23:00:54.744223+0000","last_clean":"2023-09-23T23:00:54.744223+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:00:54.744223+0000","last_undegraded":"2023-09-23T23:00:54.744223+0000","last_fullsized":"2023-09-23T23:00:54.744223+0000","mapping_epoch":353,"log_start":"326'3500","ondisk_log_start":"326'3500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201588736,"num_objects":3076,"num_object_clones":0,"num_object_copies":9228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9010,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":271,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.782670+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:00:41.782670+0000","last_peered":"2023-09-23T23:00:41.782670+0000","last_clean":"2023-09-23T23:00:41.782670+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:00:41.782670+0000","last_undegraded":"2023-09-23T23:00:41.782670+0000","last_fullsized":"2023-09-23T23:00:41.782670+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.775446+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:00:41.775446+0000","last_peered":"2023-09-23T23:00:41.775446+0000","last_clean":"2023-09-23T23:00:41.775446+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:00:41.775446+0000","last_undegraded":"2023-09-23T23:00:41.775446+0000","last_fullsized":"2023-09-23T23:00:41.775446+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"387'8928","reported_seq":11348,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:56.020687+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:00:56.020687+0000","last_peered":"2023-09-23T23:00:56.020687+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:00:56.020687+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8528,"log_dups_size":0,"ondisk_log_size":8528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198049792,"num_objects":3022,"num_object_clones":0,"num_object_copies":9066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3022,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":3022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8928,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-545}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":281,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.780542+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:00:41.780542+0000","last_peered":"2023-09-23T23:00:41.780542+0000","last_clean":"2023-09-23T23:00:41.780542+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:00:41.780542+0000","last_undegraded":"2023-09-23T23:00:41.780542+0000","last_fullsized":"2023-09-23T23:00:41.780542+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":394,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.779908+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:00:41.779908+0000","last_peered":"2023-09-23T23:00:41.779908+0000","last_clean":"2023-09-23T23:00:41.779908+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:00:41.779908+0000","last_undegraded":"2023-09-23T23:00:41.779908+0000","last_fullsized":"2023-09-23T23:00:41.779908+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"387'9048","reported_seq":9133,"reported_epoch":387,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:54.762149+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:00:54.762149+0000","last_peered":"2023-09-23T23:00:54.762149+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:00:54.762149+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:00:54.762149+0000","mapping_epoch":354,"log_start":"326'3500","ondisk_log_start":"326'3500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198180864,"num_objects":3024,"num_object_clones":0,"num_object_copies":9072,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":3024,"num_objects_unfound":0,"num_objects_dirty":3024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9048,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3024}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":138,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.774656+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:00:41.774656+0000","last_peered":"2023-09-23T23:00:41.774656+0000","last_clean":"2023-09-23T23:00:41.774656+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:00:41.774656+0000","last_undegraded":"2023-09-23T23:00:41.774656+0000","last_fullsized":"2023-09-23T23:00:41.774656+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":256,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.775341+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:00:41.775341+0000","last_peered":"2023-09-23T23:00:41.775341+0000","last_clean":"2023-09-23T23:00:41.775341+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:00:41.775341+0000","last_undegraded":"2023-09-23T23:00:41.775341+0000","last_fullsized":"2023-09-23T23:00:41.775341+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"387'9093","reported_seq":9364,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:54.730752+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:00:54.730752+0000","last_peered":"2023-09-23T23:00:54.730752+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:00:54.730752+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201916416,"num_objects":3081,"num_object_clones":0,"num_object_copies":9243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2585,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":3081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9093,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":495},{"shards":"0,5","objects":-529}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":56,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852530+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:00:41.852530+0000","last_peered":"2023-09-23T23:00:41.852530+0000","last_clean":"2023-09-23T23:00:41.852530+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:00:41.852530+0000","last_undegraded":"2023-09-23T23:00:41.852530+0000","last_fullsized":"2023-09-23T23:00:41.852530+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":286,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.711377+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:00:41.711377+0000","last_peered":"2023-09-23T23:00:41.711377+0000","last_clean":"2023-09-23T23:00:41.711377+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:00:41.711377+0000","last_undegraded":"2023-09-23T23:00:41.711377+0000","last_fullsized":"2023-09-23T23:00:41.711377+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"387'9040","reported_seq":9132,"reported_epoch":387,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:57.196893+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:00:57.196893+0000","last_peered":"2023-09-23T23:00:57.196893+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:00:57.196893+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:00:57.196893+0000","mapping_epoch":353,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200146944,"num_objects":3054,"num_object_clones":0,"num_object_copies":9162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9040,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3054}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":75,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788900+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:00:41.788900+0000","last_peered":"2023-09-23T23:00:41.788900+0000","last_clean":"2023-09-23T23:00:41.788900+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:00:41.788900+0000","last_undegraded":"2023-09-23T23:00:41.788900+0000","last_fullsized":"2023-09-23T23:00:41.788900+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":380,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.780178+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:00:41.780178+0000","last_peered":"2023-09-23T23:00:41.780178+0000","last_clean":"2023-09-23T23:00:41.780178+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:00:41.780178+0000","last_undegraded":"2023-09-23T23:00:41.780178+0000","last_fullsized":"2023-09-23T23:00:41.780178+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"387'9173","reported_seq":9273,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:57.075872+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:00:57.075872+0000","last_peered":"2023-09-23T23:00:57.075872+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:00:57.075872+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"326'3500","ondisk_log_start":"326'3500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198508544,"num_objects":3029,"num_object_clones":0,"num_object_copies":9087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3029,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":3029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9173,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-590}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":170,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781879+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:00:41.781879+0000","last_peered":"2023-09-23T23:00:41.781879+0000","last_clean":"2023-09-23T23:00:41.781879+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:00:41.781879+0000","last_undegraded":"2023-09-23T23:00:41.781879+0000","last_fullsized":"2023-09-23T23:00:41.781879+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":397,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781009+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:00:41.781009+0000","last_peered":"2023-09-23T23:00:41.781009+0000","last_clean":"2023-09-23T23:00:41.781009+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:00:41.781009+0000","last_undegraded":"2023-09-23T23:00:41.781009+0000","last_fullsized":"2023-09-23T23:00:41.781009+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"387'9070","reported_seq":14786,"reported_epoch":387,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:57.208906+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:00:57.208906+0000","last_peered":"2023-09-23T23:00:57.208906+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:00:57.208906+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:00:57.208906+0000","mapping_epoch":354,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195952640,"num_objects":2990,"num_object_clones":0,"num_object_copies":8970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1796,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9070,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2804,"num_bytes_recovered":183762944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1435},{"shards":"1,4,6","objects":1555}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":410,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.779714+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:00:41.779714+0000","last_peered":"2023-09-23T23:00:41.779714+0000","last_clean":"2023-09-23T23:00:41.779714+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:00:41.779714+0000","last_undegraded":"2023-09-23T23:00:41.779714+0000","last_fullsized":"2023-09-23T23:00:41.779714+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":388,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.780616+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:00:41.780616+0000","last_peered":"2023-09-23T23:00:41.780616+0000","last_clean":"2023-09-23T23:00:41.780616+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:00:41.780616+0000","last_undegraded":"2023-09-23T23:00:41.780616+0000","last_fullsized":"2023-09-23T23:00:41.780616+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"387'8899","reported_seq":8991,"reported_epoch":387,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:00:57.043068+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:00:57.043068+0000","last_peered":"2023-09-23T23:00:57.043068+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:00:57.043068+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:00:57.043068+0000","mapping_epoch":353,"log_start":"326'3200","ondisk_log_start":"326'3200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201129984,"num_objects":3069,"num_object_clones":0,"num_object_copies":9207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8899,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":3069}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":56,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788840+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:00:41.788840+0000","last_peered":"2023-09-23T23:00:41.788840+0000","last_clean":"2023-09-23T23:00:41.788840+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:00:41.788840+0000","last_undegraded":"2023-09-23T23:00:41.788840+0000","last_fullsized":"2023-09-23T23:00:41.788840+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":290,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.704632+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:00:41.704632+0000","last_peered":"2023-09-23T23:00:41.704632+0000","last_clean":"2023-09-23T23:00:41.704632+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:00:41.704632+0000","last_undegraded":"2023-09-23T23:00:41.704632+0000","last_fullsized":"2023-09-23T23:00:41.704632+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"387'9006","reported_seq":17304,"reported_epoch":387,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:00:57.207603+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:00:57.207603+0000","last_peered":"2023-09-23T23:00:57.207603+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:00:57.207603+0000","last_undegraded":"2023-09-23T23:00:57.207603+0000","last_fullsized":"2023-09-23T23:00:57.207603+0000","mapping_epoch":354,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5706,"log_dups_size":0,"ondisk_log_size":5706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201195520,"num_objects":3070,"num_object_clones":0,"num_object_copies":9210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9006,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4086,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":3070}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":294,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.704939+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:00:41.704939+0000","last_peered":"2023-09-23T23:00:41.704939+0000","last_clean":"2023-09-23T23:00:41.704939+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:00:41.704939+0000","last_undegraded":"2023-09-23T23:00:41.704939+0000","last_fullsized":"2023-09-23T23:00:41.704939+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":297,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.774464+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:00:41.774464+0000","last_peered":"2023-09-23T23:00:41.774464+0000","last_clean":"2023-09-23T23:00:41.774464+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:00:41.774464+0000","last_undegraded":"2023-09-23T23:00:41.774464+0000","last_fullsized":"2023-09-23T23:00:41.774464+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.775200+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:00:41.775200+0000","last_peered":"2023-09-23T23:00:41.775200+0000","last_clean":"2023-09-23T23:00:41.775200+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:00:41.775200+0000","last_undegraded":"2023-09-23T23:00:41.775200+0000","last_fullsized":"2023-09-23T23:00:41.775200+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"387'9070","reported_seq":9174,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:00:54.670169+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:00:54.670169+0000","last_peered":"2023-09-23T23:00:54.670169+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:00:54.670169+0000","last_undegraded":"2023-09-23T23:00:54.670169+0000","last_fullsized":"2023-09-23T23:00:54.670169+0000","mapping_epoch":359,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200146944,"num_objects":3054,"num_object_clones":0,"num_object_copies":9162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3054,"num_objects_unfound":0,"num_objects_dirty":3054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9070,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":3054}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":57,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.711460+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:41.711460+0000","last_peered":"2023-09-23T23:00:41.711460+0000","last_clean":"2023-09-23T23:00:41.711460+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:41.711460+0000","last_undegraded":"2023-09-23T23:00:41.711460+0000","last_fullsized":"2023-09-23T23:00:41.711460+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":407,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.783601+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:00:41.783601+0000","last_peered":"2023-09-23T23:00:41.783601+0000","last_clean":"2023-09-23T23:00:41.783601+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:00:41.783601+0000","last_undegraded":"2023-09-23T23:00:41.783601+0000","last_fullsized":"2023-09-23T23:00:41.783601+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":279,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.704980+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:00:41.704980+0000","last_peered":"2023-09-23T23:00:41.704980+0000","last_clean":"2023-09-23T23:00:41.704980+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:00:41.704980+0000","last_undegraded":"2023-09-23T23:00:41.704980+0000","last_fullsized":"2023-09-23T23:00:41.704980+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"387'9025","reported_seq":16162,"reported_epoch":387,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:56.179118+0000","last_change":"2023-09-23T22:58:24.518054+0000","last_active":"2023-09-23T23:00:56.179118+0000","last_peered":"2023-09-23T23:00:56.179118+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:00:56.179118+0000","last_undegraded":"2023-09-23T22:58:24.276828+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8525,"log_dups_size":0,"ondisk_log_size":8525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199163904,"num_objects":3039,"num_object_clones":0,"num_object_copies":9117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9025,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3519,"num_bytes_recovered":230621184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2987}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":354,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.704456+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:00:41.704456+0000","last_peered":"2023-09-23T23:00:41.704456+0000","last_clean":"2023-09-23T23:00:41.704456+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:00:41.704456+0000","last_undegraded":"2023-09-23T23:00:41.704456+0000","last_fullsized":"2023-09-23T23:00:41.704456+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":231,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.704448+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:00:41.704448+0000","last_peered":"2023-09-23T23:00:41.704448+0000","last_clean":"2023-09-23T23:00:41.704448+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:00:41.704448+0000","last_undegraded":"2023-09-23T23:00:41.704448+0000","last_fullsized":"2023-09-23T23:00:41.704448+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":297,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.776237+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:00:41.776237+0000","last_peered":"2023-09-23T23:00:41.776237+0000","last_clean":"2023-09-23T23:00:41.776237+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:00:41.776237+0000","last_undegraded":"2023-09-23T23:00:41.776237+0000","last_fullsized":"2023-09-23T23:00:41.776237+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"387'9224","reported_seq":11114,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:57.698088+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:00:57.698088+0000","last_peered":"2023-09-23T23:00:57.698088+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:00:57.698088+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":9224,"log_dups_size":0,"ondisk_log_size":9224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608000,"num_objects":3000,"num_object_clones":0,"num_object_copies":9000,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9224,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":1567},{"shards":"0,6","objects":-1981}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":370,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.775690+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:00:41.775690+0000","last_peered":"2023-09-23T23:00:41.775690+0000","last_clean":"2023-09-23T23:00:41.775690+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:00:41.775690+0000","last_undegraded":"2023-09-23T23:00:41.775690+0000","last_fullsized":"2023-09-23T23:00:41.775690+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":483,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:41.711565+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:41.711565+0000","last_peered":"2023-09-23T23:00:41.711565+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:41.711565+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":56,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788847+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:00:41.788847+0000","last_peered":"2023-09-23T23:00:41.788847+0000","last_clean":"2023-09-23T23:00:41.788847+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:00:41.788847+0000","last_undegraded":"2023-09-23T23:00:41.788847+0000","last_fullsized":"2023-09-23T23:00:41.788847+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":297,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781254+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:00:41.781254+0000","last_peered":"2023-09-23T23:00:41.781254+0000","last_clean":"2023-09-23T23:00:41.781254+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:00:41.781254+0000","last_undegraded":"2023-09-23T23:00:41.781254+0000","last_fullsized":"2023-09-23T23:00:41.781254+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"387'9124","reported_seq":9220,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:54.557802+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:00:54.557802+0000","last_peered":"2023-09-23T23:00:54.557802+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:00:54.557802+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193986560,"num_objects":2960,"num_object_clones":0,"num_object_copies":8880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9124,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-547}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":377,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.711130+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:41.711130+0000","last_peered":"2023-09-23T23:00:41.711130+0000","last_clean":"2023-09-23T23:00:41.711130+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:41.711130+0000","last_undegraded":"2023-09-23T23:00:41.711130+0000","last_fullsized":"2023-09-23T23:00:41.711130+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":348,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.711199+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:41.711199+0000","last_peered":"2023-09-23T23:00:41.711199+0000","last_clean":"2023-09-23T23:00:41.711199+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:41.711199+0000","last_undegraded":"2023-09-23T23:00:41.711199+0000","last_fullsized":"2023-09-23T23:00:41.711199+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":308,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.770648+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:00:41.770648+0000","last_peered":"2023-09-23T23:00:41.770648+0000","last_clean":"2023-09-23T23:00:41.770648+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:00:41.770648+0000","last_undegraded":"2023-09-23T23:00:41.770648+0000","last_fullsized":"2023-09-23T23:00:41.770648+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"387'9113","reported_seq":9224,"reported_epoch":387,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:00:57.181342+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:00:57.181342+0000","last_peered":"2023-09-23T23:00:57.181342+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:00:57.181342+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:00:57.181342+0000","mapping_epoch":354,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202506264,"num_objects":3091,"num_object_clones":0,"num_object_copies":9273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3091,"num_objects_unfound":0,"num_objects_dirty":3091,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9113,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3091}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":323,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.771237+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:41.771237+0000","last_peered":"2023-09-23T23:00:41.771237+0000","last_clean":"2023-09-23T23:00:41.771237+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:41.771237+0000","last_undegraded":"2023-09-23T23:00:41.771237+0000","last_fullsized":"2023-09-23T23:00:41.771237+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":231,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.705021+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:00:41.705021+0000","last_peered":"2023-09-23T23:00:41.705021+0000","last_clean":"2023-09-23T23:00:41.705021+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:00:41.705021+0000","last_undegraded":"2023-09-23T23:00:41.705021+0000","last_fullsized":"2023-09-23T23:00:41.705021+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":200,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.775756+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:00:41.775756+0000","last_peered":"2023-09-23T23:00:41.775756+0000","last_clean":"2023-09-23T23:00:41.775756+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:00:41.775756+0000","last_undegraded":"2023-09-23T23:00:41.775756+0000","last_fullsized":"2023-09-23T23:00:41.775756+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"387'9163","reported_seq":14868,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:57.147692+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:00:57.147692+0000","last_peered":"2023-09-23T23:00:57.147692+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:00:57.147692+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"326'3500","ondisk_log_start":"326'3500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9163,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2501,"num_bytes_recovered":163905536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":1598}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852620+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:00:41.852620+0000","last_peered":"2023-09-23T23:00:41.852620+0000","last_clean":"2023-09-23T23:00:41.852620+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:00:41.852620+0000","last_undegraded":"2023-09-23T23:00:41.852620+0000","last_fullsized":"2023-09-23T23:00:41.852620+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":95,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.783988+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:00:41.783988+0000","last_peered":"2023-09-23T23:00:41.783988+0000","last_clean":"2023-09-23T23:00:41.783988+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:00:41.783988+0000","last_undegraded":"2023-09-23T23:00:41.783988+0000","last_fullsized":"2023-09-23T23:00:41.783988+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788745+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:00:41.788745+0000","last_peered":"2023-09-23T23:00:41.788745+0000","last_clean":"2023-09-23T23:00:41.788745+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:00:41.788745+0000","last_undegraded":"2023-09-23T23:00:41.788745+0000","last_fullsized":"2023-09-23T23:00:41.788745+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"387'9076","reported_seq":9597,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:57.193493+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:00:57.193493+0000","last_peered":"2023-09-23T23:00:57.193493+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:00:57.193493+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196214784,"num_objects":2994,"num_object_clones":0,"num_object_copies":8982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2994,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":2994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9076,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-557}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":343,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.706200+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:00:41.706200+0000","last_peered":"2023-09-23T23:00:41.706200+0000","last_clean":"2023-09-23T23:00:41.706200+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:00:41.706200+0000","last_undegraded":"2023-09-23T23:00:41.706200+0000","last_fullsized":"2023-09-23T23:00:41.706200+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":290,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.770580+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:00:41.770580+0000","last_peered":"2023-09-23T23:00:41.770580+0000","last_clean":"2023-09-23T23:00:41.770580+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:00:41.770580+0000","last_undegraded":"2023-09-23T23:00:41.770580+0000","last_fullsized":"2023-09-23T23:00:41.770580+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":361,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.779795+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:00:41.779795+0000","last_peered":"2023-09-23T23:00:41.779795+0000","last_clean":"2023-09-23T23:00:41.779795+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:00:41.779795+0000","last_undegraded":"2023-09-23T23:00:41.779795+0000","last_fullsized":"2023-09-23T23:00:41.779795+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":325,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.711582+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:00:41.711582+0000","last_peered":"2023-09-23T23:00:41.711582+0000","last_clean":"2023-09-23T23:00:41.711582+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:00:41.711582+0000","last_undegraded":"2023-09-23T23:00:41.711582+0000","last_fullsized":"2023-09-23T23:00:41.711582+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":135,"reported_epoch":387,"state":"active+clean","last_fresh":"2023-09-23T23:00:49.758282+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:00:49.758282+0000","last_peered":"2023-09-23T23:00:49.758282+0000","last_clean":"2023-09-23T23:00:49.758282+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:00:49.758282+0000","last_undegraded":"2023-09-23T23:00:49.758282+0000","last_fullsized":"2023-09-23T23:00:49.758282+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":196,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.780100+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:00:41.780100+0000","last_peered":"2023-09-23T23:00:41.780100+0000","last_clean":"2023-09-23T23:00:41.780100+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:00:41.780100+0000","last_undegraded":"2023-09-23T23:00:41.780100+0000","last_fullsized":"2023-09-23T23:00:41.780100+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":410,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.783670+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:00:41.783670+0000","last_peered":"2023-09-23T23:00:41.783670+0000","last_clean":"2023-09-23T23:00:41.783670+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:00:41.783670+0000","last_undegraded":"2023-09-23T23:00:41.783670+0000","last_fullsized":"2023-09-23T23:00:41.783670+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":210,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781098+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:00:41.781098+0000","last_peered":"2023-09-23T23:00:41.781098+0000","last_clean":"2023-09-23T23:00:41.781098+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:00:41.781098+0000","last_undegraded":"2023-09-23T23:00:41.781098+0000","last_fullsized":"2023-09-23T23:00:41.781098+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":297,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781799+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:00:41.781799+0000","last_peered":"2023-09-23T23:00:41.781799+0000","last_clean":"2023-09-23T23:00:41.781799+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:00:41.781799+0000","last_undegraded":"2023-09-23T23:00:41.781799+0000","last_fullsized":"2023-09-23T23:00:41.781799+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":80,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.852387+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:00:41.852387+0000","last_peered":"2023-09-23T23:00:41.852387+0000","last_clean":"2023-09-23T23:00:41.852387+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:00:41.852387+0000","last_undegraded":"2023-09-23T23:00:41.852387+0000","last_fullsized":"2023-09-23T23:00:41.852387+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.788981+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:00:41.788981+0000","last_peered":"2023-09-23T23:00:41.788981+0000","last_clean":"2023-09-23T23:00:41.788981+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:00:41.788981+0000","last_undegraded":"2023-09-23T23:00:41.788981+0000","last_fullsized":"2023-09-23T23:00:41.788981+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":329,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.771123+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:41.771123+0000","last_peered":"2023-09-23T23:00:41.771123+0000","last_clean":"2023-09-23T23:00:41.771123+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:41.771123+0000","last_undegraded":"2023-09-23T23:00:41.771123+0000","last_fullsized":"2023-09-23T23:00:41.771123+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":372,"reported_epoch":386,"state":"active+clean","last_fresh":"2023-09-23T23:00:41.781947+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:00:41.781947+0000","last_peered":"2023-09-23T23:00:41.781947+0000","last_clean":"2023-09-23T23:00:41.781947+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:00:41.781947+0000","last_undegraded":"2023-09-23T23:00:41.781947+0000","last_fullsized":"2023-09-23T23:00:41.781947+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3180789784,"num_objects":48536,"num_object_clones":0,"num_object_copies":145608,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":26943,"num_objects_misplaced":29096,"num_objects_unfound":0,"num_objects_dirty":48536,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":145062,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15254,"num_bytes_recovered":970850304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9872158720,"data_stored":9872146504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99362,"ondisk_log_size":99362,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430615,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626840,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253153,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969336,"kb_used_data":1257136,"kb_used_omap":9,"kb_used_meta":712182,"kb_avail":91775816,"statfs":{"total":95995035648,"available":93978435584,"internally_reserved":0,"allocated":1287307264,"data_stored":1285849381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":729275369},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.76900000000000002,"15min":0.82699999999999996},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80700000000000005,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.504,"15min":0.52000000000000002},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.93600000000000005,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":1.087},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.341,"5min":4.048,"15min":4.048},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56100000000000005,"15min":1.9950000000000001},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.8580000000000001,"5min":3.141,"15min":87.787999999999997},"last":1.125},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.71699999999999997,"15min":2.339},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.19500000000000001},"max":{"1min":0.95499999999999996,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.34100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.6869999999999998,"15min":2.4529999999999998},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6669999999999998,"5min":151.036,"15min":151.036},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.581,"5min":2.7450000000000001,"15min":2.548},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":6.649,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.6219999999999999,"15min":2.319},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1720000000000002,"5min":116.098,"15min":116.098},"last":1.1659999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.5099999999999998,"15min":2.4809999999999999},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.9769999999999999,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.27,"15min":2.1480000000000001},"min":{"1min":0.40100000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.0820000000000001,"15min":1.8520000000000001},"min":{"1min":0.40999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.825,"5min":8.5370000000000008,"15min":79.328999999999994},"last":1.1479999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.04,"15min":1.6799999999999999},"min":{"1min":0.46300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.581,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.177,"15min":1.8919999999999999},"min":{"1min":0.44600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9860000000000002,"5min":10.676,"15min":20.902999999999999},"last":0.60999999999999999}]}]},{"osd":0,"up_from":353,"seq":1516123455523,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2218444,"kb_used_data":2071172,"kb_used_omap":37,"kb_used_meta":147162,"kb_avail":91526708,"statfs":{"total":95995035648,"available":93723348992,"internally_reserved":0,"allocated":2120880128,"data_stored":2119287290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":150694050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.315},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.73299999999999998,"15min":0.76400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.9319999999999999,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.0270000000000001,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":0.66200000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.77400000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":0.58899999999999997}]}]},{"osd":2,"up_from":359,"seq":1541893259296,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":806924,"kb_used_data":689412,"kb_used_omap":83,"kb_used_meta":117420,"kb_avail":92938228,"statfs":{"total":95995035648,"available":95168745472,"internally_reserved":0,"allocated":705957888,"data_stored":704426133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":120238533},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.42799999999999999,"15min":0.379},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.1520000000000001,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.66300000000000003,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.2,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.61699999999999999,"15min":0.64900000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.90500000000000003,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.1060000000000001,"15min":1.1379999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6269999999999998,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":1.286}]},{"osd":3,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.73499999999999999,"15min":0.71999999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7270000000000003,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.86299999999999999,"15min":0.83199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.2140000000000004,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":1.5029999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":0.73999999999999999,"15min":0.623},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1159999999999997,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.7080000000000002,"5min":3.492,"15min":3.492},"last":0.379}]},{"osd":5,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.98899999999999999,"15min":0.96599999999999997},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.6029999999999998,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":1.321},{"interface":"front","average":{"1min":1.101,"5min":0.76500000000000001,"15min":0.70899999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.0839999999999996,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.046,"15min":0.995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.5679999999999996,"5min":5.5679999999999996,"15min":5.5679999999999996},"last":1.347},{"interface":"front","average":{"1min":1.365,"5min":0.75800000000000001,"15min":0.65700000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":3.1259999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.1220000000000001,"15min":1.034},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":7.0499999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":2.1299999999999999},{"interface":"front","average":{"1min":1.014,"5min":1.1459999999999999,"15min":1.1679999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.8520000000000003,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.44400000000000001}]}]},{"osd":3,"up_from":94,"seq":403726926100,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":544856,"kb_used_data":36112,"kb_used_omap":7,"kb_used_meta":508728,"kb_avail":93200296,"statfs":{"total":95995035648,"available":95437103104,"internally_reserved":0,"allocated":36978688,"data_stored":35430406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":520938088},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.252},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.63200000000000001,"15min":2.3639999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":123.866},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.73699999999999999,"15min":2.1549999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8010000000000002,"5min":6.8010000000000002,"15min":122.265},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.1339999999999999,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.76600000000000001,"15min":0.75},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":3.8740000000000001,"5min":3.8740000000000001,"15min":3.8740000000000001},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":2.3519999999999999,"15min":2.0350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":64.406999999999996,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":4.008,"5min":2.5409999999999999,"15min":1.8999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":64.468000000000004,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2290000000000001,"15min":1.3560000000000001},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1980000000000004,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.548,"5min":1.218,"15min":1.827},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.246,"5min":10.246,"15min":11.951000000000001},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.597,"15min":1.7150000000000001},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0229999999999997,"5min":11.116,"15min":14.097},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4039999999999999,"15min":1.956},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.79,"5min":10.859,"15min":12.887},"last":0.38800000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.89400000000000002,"15min":2.0499999999999998},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8449999999999998,"5min":4.8449999999999998,"15min":91.757999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.413,"5min":0.98099999999999998,"15min":1.9390000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":8.5830000000000002,"5min":8.5830000000000002,"15min":91.709999999999994},"last":0.46000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019297,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3242192,"kb_used_data":2334728,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":90502960,"statfs":{"total":95995035648,"available":92675031040,"internally_reserved":0,"allocated":2390761472,"data_stored":2389971143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":0.70599999999999996,"15min":0.55000000000000004},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.279,"5min":10.279,"15min":10.279},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.65300000000000002,"15min":0.61499999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":3.5110000000000001,"15min":3.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.24099999999999999},"max":{"1min":2.77,"5min":98.846999999999994,"15min":132.304},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.302,"5min":3.6749999999999998,"15min":3.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":10.361000000000001,"5min":98.938999999999993,"15min":132.453},"last":1.9139999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.119,"15min":1.0529999999999999},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.1839999999999999,"5min":0.81100000000000005,"15min":0.749},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.9550000000000001,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.4450000000000001,"15min":4.3140000000000001},"min":{"1min":0.41799999999999998,"5min":0.32700000000000001,"15min":0.32400000000000001},"max":{"1min":10.337999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.2110000000000001,"15min":4.3049999999999997},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":3.2490000000000001,"5min":12.805999999999999,"15min":167.05199999999999},"last":2.2869999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":3.0670000000000002,"15min":1.643},"min":{"1min":0.318,"5min":0.214,"15min":0.214},"max":{"1min":3.129,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.335,"5min":3.4940000000000002,"15min":1.78},"min":{"1min":0.318,"5min":0.252,"15min":0.248},"max":{"1min":3.7469999999999999,"5min":151.905,"15min":151.905},"last":3.1869999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.585,"15min":1.1839999999999999},"min":{"1min":0.373,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":4.1269999999999998,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.155,"5min":1.371,"15min":1.105},"min":{"1min":0.372,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2130000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.313}]},{"osd":7,"last update":"Sat Sep 23 23:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.47,"15min":1.127},"min":{"1min":0.40899999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.045999999999999,"5min":12.84,"15min":12.84},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.615,"5min":1.5089999999999999,"15min":1.0740000000000001},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.227,"5min":10.587,"15min":10.587},"last":0.76000000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823071,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2345328,"kb_used_data":1449768,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":91399824,"statfs":{"total":95995035648,"available":93593419776,"internally_reserved":0,"allocated":1484562432,"data_stored":1483799891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.73599999999999999,"15min":0.64900000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":14.42,"5min":14.42,"15min":14.42},"last":5.3449999999999998},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.274,"15min":1.1459999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.496,"5min":14.496,"15min":14.496},"last":7.3159999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.081,"15min":2.1909999999999998},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.24299999999999999},"max":{"1min":2.6040000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.857,"5min":1.6200000000000001,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":7.0739999999999998,"5min":10.353,"15min":110.89},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.601,"5min":6.601,"15min":6.601},"last":1.522},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7669999999999999,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.216,"15min":4.7009999999999996},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2710000000000008,"5min":9.2710000000000008,"15min":257.05000000000001},"last":1.5609999999999999},{"interface":"front","average":{"1min":1.474,"5min":1.3859999999999999,"15min":4.7590000000000003},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.3399999999999999,"5min":12.185,"15min":257.35000000000002},"last":0.58299999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":2.8580000000000001,"15min":1.5109999999999999},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8440000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.5710000000000002,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.524,"5min":114.908,"15min":114.908},"last":1.5920000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.512,"15min":1.2270000000000001},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.4630000000000001,"5min":11.121,"15min":12.615},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.246,"5min":1.2370000000000001,"15min":1.1519999999999999},"min":{"1min":0.41299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.4909999999999997,"5min":12.137,"15min":12.74},"last":0.71399999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.1180000000000001,"15min":1.046},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.5980000000000008,"5min":12.23,"15min":12.23},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.627,"5min":1.4199999999999999,"15min":1.0940000000000001},"min":{"1min":0.45500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.8710000000000004,"5min":11.112,"15min":11.112},"last":0.60599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2106134528,"data_stored":2106130456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1281228800,"data_stored":1281228800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":701628416,"data_stored":701628416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":32440320,"data_stored":32440320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2377977856,"data_stored":2377973784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1479671808,"data_stored":1479671808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1893076992,"data_stored":1893072920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:01:01.676 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:01:01.725 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:01.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:01.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:01.834 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:01.887 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:02.379 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:02.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:02.483 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:02.545 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:02.614 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:02.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:02.723 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:02.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:02 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:02.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:02 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:02 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:02.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:02 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:02.762 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:02.763 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:02 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:02.763 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:02 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:02.847 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:02.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:02.953 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:02.985 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:03.107 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:03.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:03.210 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:03.249 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:03.419 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:03.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:03.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[180979]: pgmap v1163: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 26264/141444 objects degraded (18.568%); 28818/141444 objects misplaced (20.374%); 3.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:01:03.525 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:03.526 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:01:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:01:03] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-09-23T23:01:03.533 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[187743]: pgmap v1163: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 26264/141444 objects degraded (18.568%); 28818/141444 objects misplaced (20.374%); 3.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:03.533 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:03.534 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:01:03.534 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:03.534 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:03.543 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:03.568 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:03.597 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:03.600 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:03.626 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:03.650 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:03.659 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:03.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:03.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:03.670 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:03.692 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:03.713 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:03.714 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.714 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.714 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.714 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.714 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.714 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.715 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.715 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:01:03.737 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:03.763 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:03.794 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:03 smithi151 ceph-mon[159210]: pgmap v1163: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 26264/141444 objects degraded (18.568%); 28818/141444 objects misplaced (20.374%); 3.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:03.794 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:01:03.794 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:01:03.794 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:03.796 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:03.814 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:03.814 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:03.860 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:03.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:03.966 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:04.003 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:04.065 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:04.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:04.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:04.173 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:04.215 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:04.277 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:04.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:04.386 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:04.431 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:04.684 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2610589134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:04.684 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2610589134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:04.909 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:04.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2610589134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:05.016 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:05.053 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:05.218 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:05.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:05.262 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:01:05.325 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:05.373 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:05.463 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:05.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:05.574 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:05.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:05.611 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:05.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:05 smithi151 ceph-mon[159210]: pgmap v1164: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 26264/141171 objects degraded (18.604%); 28818/141171 objects misplaced (20.414%); 3.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:05.612 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 26264/141444 objects degraded (18.568%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:05.615 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:05.717 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:05.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:05.823 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:05.852 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:05.852 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:05.852 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[187743]: pgmap v1164: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 26264/141171 objects degraded (18.604%); 28818/141171 objects misplaced (20.414%); 3.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:05.852 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 26264/141444 objects degraded (18.568%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:05.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:05.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:05.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[180979]: pgmap v1164: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 26264/141171 objects degraded (18.604%); 28818/141171 objects misplaced (20.414%); 3.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:05.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 26264/141444 objects degraded (18.568%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:05.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:05.993 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:05.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:06.099 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:06.138 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:06.487 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:06.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:06.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:06.592 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:06.634 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:06.750 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:06.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:06.863 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:06.912 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:06.982 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:01:06.992 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:07.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:07.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:07.102 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:07.127 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:07.183 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:07.187 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:07.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:07.292 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:07.341 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:07.409 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:07.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:07.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:07.530 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:07.582 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:07.648 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:07 smithi052 ceph-mon[180979]: pgmap v1165: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 26123/139527 objects degraded (18.723%); 28679/139527 objects misplaced (20.554%); 2.6 MiB/s, 83 objects/s recovering 2023-09-23T23:01:07.648 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/151182619' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:01:07.649 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:07 smithi052 ceph-mon[187743]: pgmap v1165: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 26123/139527 objects degraded (18.723%); 28679/139527 objects misplaced (20.554%); 2.6 MiB/s, 83 objects/s recovering 2023-09-23T23:01:07.649 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/151182619' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:01:07.676 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:01:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:07 smithi151 ceph-mon[159210]: pgmap v1165: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 26123/139527 objects degraded (18.723%); 28679/139527 objects misplaced (20.554%); 2.6 MiB/s, 83 objects/s recovering 2023-09-23T23:01:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/151182619' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:01:07.970 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:07.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:08.075 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:08.125 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:08.181 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:08.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:08.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:08.252 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 26123/139527 objects degraded (18.723%), 13 pgs degraded, 9 pgs undersized","count":22},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1600,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":389,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":46509,"data_bytes":3046971320,"bytes_used":11088158720,"bytes_avail":564882055168,"bytes_total":575970213888,"degraded_objects":26123,"degraded_total":139527,"degraded_ratio":0.18722541156908698,"misplaced_objects":28679,"misplaced_total":139527,"misplaced_ratio":0.20554444659456592,"recovering_objects_per_sec":83,"recovering_bytes_per_sec":2738273,"recovering_keys_per_sec":0,"num_objects_recovered":831,"num_bytes_recovered":27394048,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":277},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (5m)\n [======================......] (remaining: 65s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:01:08.287 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:08.335 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:08.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:08.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:08.523 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:08.560 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:08.624 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:08.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:08.651 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:08.651 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.651 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/151182619' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:01:08.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: osdmap e389: 8 total, 8 up, 6 in 2023-09-23T23:01:08.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2012991287' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/151182619' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: osdmap e389: 8 total, 8 up, 6 in 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.704 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2012991287' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:08.737 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:08.778 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:08.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:08.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:08.957 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:08.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:08.983 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/151182619' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: osdmap e389: 8 total, 8 up, 6 in 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: failed to encode map e389 with expected crc 2023-09-23T23:01:08.984 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2012991287' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:08.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:09.469 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:01:09.469 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:01:09.487 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:09.496 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:09.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:09.603 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:09.653 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:09.728 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:09.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:09 smithi052 ceph-mon[180979]: pgmap v1167: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 25698/137169 objects degraded (18.735%); 28597/137169 objects misplaced (20.848%); 2.5 MiB/s, 81 objects/s recovering 2023-09-23T23:01:09.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:09 smithi052 ceph-mon[187743]: pgmap v1167: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 25698/137169 objects degraded (18.735%); 28597/137169 objects misplaced (20.848%); 2.5 MiB/s, 81 objects/s recovering 2023-09-23T23:01:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:09.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:09.843 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:09.898 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:09.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:09 smithi151 ceph-mon[159210]: pgmap v1167: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 25698/137169 objects degraded (18.735%); 28597/137169 objects misplaced (20.848%); 2.5 MiB/s, 81 objects/s recovering 2023-09-23T23:01:09.984 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:09.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:09.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:10.000 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:10.001 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:10.096 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:10.163 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:10.241 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:10.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:10.351 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:10.408 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:10.502 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:10.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:10.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:10.609 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:10.657 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:10.747 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:10.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:10.858 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:10.894 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:10.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 25698/137169 objects degraded (18.735%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:10.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 25698/137169 objects degraded (18.735%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 25698/137169 objects degraded (18.735%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:11.203 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:11.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:11.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:11.309 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:11.339 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:11.452 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:11.470 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:11.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:11.484 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:01:11.521 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:01:11.580 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:11.632 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:11.672 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:11 smithi052 ceph-mon[180979]: pgmap v1168: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 25627/135855 objects degraded (18.863%); 28423/135855 objects misplaced (20.922%); 2.5 MiB/s, 81 objects/s recovering 2023-09-23T23:01:11.710 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:11.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:11.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:11.824 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:11.887 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:11.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:11 smithi151 ceph-mon[159210]: pgmap v1168: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 25627/135855 objects degraded (18.863%); 28423/135855 objects misplaced (20.922%); 2.5 MiB/s, 81 objects/s recovering 2023-09-23T23:01:11.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:11 smithi052 ceph-mon[187743]: pgmap v1168: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 25627/135855 objects degraded (18.863%); 28423/135855 objects misplaced (20.922%); 2.5 MiB/s, 81 objects/s recovering 2023-09-23T23:01:12.039 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:12.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:12.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:12.066 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:12.100 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:12.140 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:12.147 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:12.167 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:12.189 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:12.208 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:12.239 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:12.262 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:12.268 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:12.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:12.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:12.301 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:12.308 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.308 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.309 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.309 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.309 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.309 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.309 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.309 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:01:12.372 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:12.427 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:12.494 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:12.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:12.598 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:12.636 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:12.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:12.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:12.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:12.816 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:12.896 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:12.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3621428807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:12.943 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3621428807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:12.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3621428807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:13.209 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:13.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:13.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:13.317 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:13.364 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:13.463 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:13.468 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:01:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:01:13] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.33.4" 2023-09-23T23:01:13.507 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:13.523 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:13.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:13.636 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:13.681 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:13.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:13 smithi052 ceph-mon[187743]: pgmap v1169: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 25070/133005 objects degraded (18.849%); 28315/133005 objects misplaced (21.289%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:13.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:13 smithi052 ceph-mon[180979]: pgmap v1169: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 25070/133005 objects degraded (18.849%); 28315/133005 objects misplaced (21.289%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:13.787 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:13.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:13.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:13.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:13 smithi151 ceph-mon[159210]: pgmap v1169: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 25070/133005 objects degraded (18.849%); 28315/133005 objects misplaced (21.289%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:13.950 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:14.013 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:14.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:14.120 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:14.172 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:14.249 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:14.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:14.361 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:14.412 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:14.446 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:14.448 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:01:14.484 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:14.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:14.595 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:14.644 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:14.909 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:14.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:15.015 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:15.060 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:15.472 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:15.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:15.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:15.523 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:01:15.578 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:15.644 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:15 smithi052 ceph-mon[180979]: from='client.36504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:15 smithi052 ceph-mon[180979]: pgmap v1170: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 25070/132735 objects degraded (18.887%); 28315/132735 objects misplaced (21.332%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:15.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 25070/132735 objects degraded (18.887%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:15.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:15 smithi052 ceph-mon[187743]: from='client.36504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:15.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:15 smithi052 ceph-mon[187743]: pgmap v1170: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 25070/132735 objects degraded (18.887%); 28315/132735 objects misplaced (21.332%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:15.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 25070/132735 objects degraded (18.887%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:15 smithi151 ceph-mon[159210]: from='client.36504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:15 smithi151 ceph-mon[159210]: pgmap v1170: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 25070/132735 objects degraded (18.887%); 28315/132735 objects misplaced (21.332%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:15.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 25070/132735 objects degraded (18.887%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:16.271 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:16.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:16.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:16.379 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1169,"stamp":"2023-09-23T23:01:12.587634+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2904496056,"num_objects":44335,"num_object_clones":0,"num_object_copies":133005,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":25070,"num_objects_misplaced":28315,"num_objects_unfound":0,"num_objects_dirty":44335,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":149547,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16536,"num_bytes_recovered":1010252732,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100081,"ondisk_log_size":100081,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10570940,"kb_used_data":7254348,"kb_used_omap":137,"kb_used_meta":3316406,"kb_avail":551899972,"statfs":{"total":575970213888,"available":565145571328,"internally_reserved":0,"allocated":7428452352,"data_stored":7420710110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3395999928},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-184352768,"num_objects":-2813,"num_object_clones":0,"num_object_copies":-8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1194,"num_objects_misplaced":-503,"num_objects_unfound":0,"num_objects_dirty":-2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2813,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":832,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003342"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":401,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.086479+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:00:58.086479+0000","last_peered":"2023-09-23T23:00:58.086479+0000","last_clean":"2023-09-23T23:00:58.086479+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:00:58.086479+0000","last_undegraded":"2023-09-23T23:00:58.086479+0000","last_fullsized":"2023-09-23T23:00:58.086479+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":58,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095902+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:00:58.095902+0000","last_peered":"2023-09-23T23:00:58.095902+0000","last_clean":"2023-09-23T23:00:58.095902+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:00:58.095902+0000","last_undegraded":"2023-09-23T23:00:58.095902+0000","last_fullsized":"2023-09-23T23:00:58.095902+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.094919+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:00:58.094919+0000","last_peered":"2023-09-23T23:00:58.094919+0000","last_clean":"2023-09-23T23:00:58.094919+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:00:58.094919+0000","last_undegraded":"2023-09-23T23:00:58.094919+0000","last_fullsized":"2023-09-23T23:00:58.094919+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":385,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095285+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:00:58.095285+0000","last_peered":"2023-09-23T23:00:58.095285+0000","last_clean":"2023-09-23T23:00:58.095285+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:00:58.095285+0000","last_undegraded":"2023-09-23T23:00:58.095285+0000","last_fullsized":"2023-09-23T23:00:58.095285+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":73,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095332+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:00:58.095332+0000","last_peered":"2023-09-23T23:00:58.095332+0000","last_clean":"2023-09-23T23:00:58.095332+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:00:58.095332+0000","last_undegraded":"2023-09-23T23:00:58.095332+0000","last_fullsized":"2023-09-23T23:00:58.095332+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.073090+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:00:58.073090+0000","last_peered":"2023-09-23T23:00:58.073090+0000","last_clean":"2023-09-23T23:00:58.073090+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:00:58.073090+0000","last_undegraded":"2023-09-23T23:00:58.073090+0000","last_fullsized":"2023-09-23T23:00:58.073090+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":324,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.078724+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:00:58.078724+0000","last_peered":"2023-09-23T23:00:58.078724+0000","last_clean":"2023-09-23T23:00:58.078724+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:00:58.078724+0000","last_undegraded":"2023-09-23T23:00:58.078724+0000","last_fullsized":"2023-09-23T23:00:58.078724+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.094690+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:00:58.094690+0000","last_peered":"2023-09-23T23:00:58.094690+0000","last_clean":"2023-09-23T23:00:58.094690+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:00:58.094690+0000","last_undegraded":"2023-09-23T23:00:58.094690+0000","last_fullsized":"2023-09-23T23:00:58.094690+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":285,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.077819+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:00:58.077819+0000","last_peered":"2023-09-23T23:00:58.077819+0000","last_clean":"2023-09-23T23:00:58.077819+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:00:58.077819+0000","last_undegraded":"2023-09-23T23:00:58.077819+0000","last_fullsized":"2023-09-23T23:00:58.077819+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.094828+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:00:58.094828+0000","last_peered":"2023-09-23T23:00:58.094828+0000","last_clean":"2023-09-23T23:00:58.094828+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:00:58.094828+0000","last_undegraded":"2023-09-23T23:00:58.094828+0000","last_fullsized":"2023-09-23T23:00:58.094828+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":78,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095844+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:00:58.095844+0000","last_peered":"2023-09-23T23:00:58.095844+0000","last_clean":"2023-09-23T23:00:58.095844+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:00:58.095844+0000","last_undegraded":"2023-09-23T23:00:58.095844+0000","last_fullsized":"2023-09-23T23:00:58.095844+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":134,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.076207+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:00:58.076207+0000","last_peered":"2023-09-23T23:00:58.076207+0000","last_clean":"2023-09-23T23:00:58.076207+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:00:58.076207+0000","last_undegraded":"2023-09-23T23:00:58.076207+0000","last_fullsized":"2023-09-23T23:00:58.076207+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":460,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087178+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:00:58.087178+0000","last_peered":"2023-09-23T23:00:58.087178+0000","last_clean":"2023-09-23T23:00:58.087178+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:00:58.087178+0000","last_undegraded":"2023-09-23T23:00:58.087178+0000","last_fullsized":"2023-09-23T23:00:58.087178+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":363,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087938+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:00:58.087938+0000","last_peered":"2023-09-23T23:00:58.087938+0000","last_clean":"2023-09-23T23:00:58.087938+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:00:58.087938+0000","last_undegraded":"2023-09-23T23:00:58.087938+0000","last_fullsized":"2023-09-23T23:00:58.087938+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":289,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088026+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:00:58.088026+0000","last_peered":"2023-09-23T23:00:58.088026+0000","last_clean":"2023-09-23T23:00:58.088026+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:00:58.088026+0000","last_undegraded":"2023-09-23T23:00:58.088026+0000","last_fullsized":"2023-09-23T23:00:58.088026+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095401+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:00:58.095401+0000","last_peered":"2023-09-23T23:00:58.095401+0000","last_clean":"2023-09-23T23:00:58.095401+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:00:58.095401+0000","last_undegraded":"2023-09-23T23:00:58.095401+0000","last_fullsized":"2023-09-23T23:00:58.095401+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":378,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.094824+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:00:58.094824+0000","last_peered":"2023-09-23T23:00:58.094824+0000","last_clean":"2023-09-23T23:00:58.094824+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:00:58.094824+0000","last_undegraded":"2023-09-23T23:00:58.094824+0000","last_fullsized":"2023-09-23T23:00:58.094824+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":289,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.077987+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:00:58.077987+0000","last_peered":"2023-09-23T23:00:58.077987+0000","last_clean":"2023-09-23T23:00:58.077987+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:00:58.077987+0000","last_undegraded":"2023-09-23T23:00:58.077987+0000","last_fullsized":"2023-09-23T23:00:58.077987+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":274,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.090639+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:00:58.090639+0000","last_peered":"2023-09-23T23:00:58.090639+0000","last_clean":"2023-09-23T23:00:58.090639+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:00:58.090639+0000","last_undegraded":"2023-09-23T23:00:58.090639+0000","last_fullsized":"2023-09-23T23:00:58.090639+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":402,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095427+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:00:58.095427+0000","last_peered":"2023-09-23T23:00:58.095427+0000","last_clean":"2023-09-23T23:00:58.095427+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:00:58.095427+0000","last_undegraded":"2023-09-23T23:00:58.095427+0000","last_fullsized":"2023-09-23T23:00:58.095427+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":107,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.072721+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:00:58.072721+0000","last_peered":"2023-09-23T23:00:58.072721+0000","last_clean":"2023-09-23T23:00:58.072721+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:00:58.072721+0000","last_undegraded":"2023-09-23T23:00:58.072721+0000","last_fullsized":"2023-09-23T23:00:58.072721+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":82,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095819+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:00:58.095819+0000","last_peered":"2023-09-23T23:00:58.095819+0000","last_clean":"2023-09-23T23:00:58.095819+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:00:58.095819+0000","last_undegraded":"2023-09-23T23:00:58.095819+0000","last_fullsized":"2023-09-23T23:00:58.095819+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"389'9303","reported_seq":9396,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:09.742650+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:01:09.742650+0000","last_peered":"2023-09-23T23:01:09.742650+0000","last_clean":"2023-09-23T23:01:09.742650+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:01:09.742650+0000","last_undegraded":"2023-09-23T23:01:09.742650+0000","last_fullsized":"2023-09-23T23:01:09.742650+0000","mapping_epoch":353,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182386688,"num_objects":2783,"num_object_clones":0,"num_object_copies":8349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9303,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":273,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088407+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:00:58.088407+0000","last_peered":"2023-09-23T23:00:58.088407+0000","last_clean":"2023-09-23T23:00:58.088407+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:00:58.088407+0000","last_undegraded":"2023-09-23T23:00:58.088407+0000","last_fullsized":"2023-09-23T23:00:58.088407+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":259,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.077365+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:00:58.077365+0000","last_peered":"2023-09-23T23:00:58.077365+0000","last_clean":"2023-09-23T23:00:58.077365+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:00:58.077365+0000","last_undegraded":"2023-09-23T23:00:58.077365+0000","last_fullsized":"2023-09-23T23:00:58.077365+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"389'9188","reported_seq":11610,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:11.136062+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:01:11.136062+0000","last_peered":"2023-09-23T23:01:11.136062+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:01:11.136062+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'600","ondisk_log_start":"321'600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010432,"num_objects":2762,"num_object_clones":0,"num_object_copies":8286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2762,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":2762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9188,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-805}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":283,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.090601+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:00:58.090601+0000","last_peered":"2023-09-23T23:00:58.090601+0000","last_clean":"2023-09-23T23:00:58.090601+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:00:58.090601+0000","last_undegraded":"2023-09-23T23:00:58.090601+0000","last_fullsized":"2023-09-23T23:00:58.090601+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":396,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.090547+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:00:58.090547+0000","last_peered":"2023-09-23T23:00:58.090547+0000","last_clean":"2023-09-23T23:00:58.090547+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:00:58.090547+0000","last_undegraded":"2023-09-23T23:00:58.090547+0000","last_fullsized":"2023-09-23T23:00:58.090547+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"389'9302","reported_seq":9389,"reported_epoch":389,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:09.783513+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:01:09.783513+0000","last_peered":"2023-09-23T23:01:09.783513+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:01:09.783513+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:01:09.783513+0000","mapping_epoch":354,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181534720,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":2770,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9302,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2770}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":140,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.078076+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:00:58.078076+0000","last_peered":"2023-09-23T23:00:58.078076+0000","last_clean":"2023-09-23T23:00:58.078076+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:00:58.078076+0000","last_undegraded":"2023-09-23T23:00:58.078076+0000","last_fullsized":"2023-09-23T23:00:58.078076+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":258,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.077783+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:00:58.077783+0000","last_peered":"2023-09-23T23:00:58.077783+0000","last_clean":"2023-09-23T23:00:58.077783+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:00:58.077783+0000","last_undegraded":"2023-09-23T23:00:58.077783+0000","last_fullsized":"2023-09-23T23:00:58.077783+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"389'9336","reported_seq":9609,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:09.808701+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:01:09.808701+0000","last_peered":"2023-09-23T23:01:09.808701+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:01:09.808701+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5536,"log_dups_size":0,"ondisk_log_size":5536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185991168,"num_objects":2838,"num_object_clones":0,"num_object_copies":8514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2388,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":2838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9336,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":449},{"shards":"0,5","objects":-772}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":58,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095308+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:00:58.095308+0000","last_peered":"2023-09-23T23:00:58.095308+0000","last_clean":"2023-09-23T23:00:58.095308+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:00:58.095308+0000","last_undegraded":"2023-09-23T23:00:58.095308+0000","last_fullsized":"2023-09-23T23:00:58.095308+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":288,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.074265+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:00:58.074265+0000","last_peered":"2023-09-23T23:00:58.074265+0000","last_clean":"2023-09-23T23:00:58.074265+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:00:58.074265+0000","last_undegraded":"2023-09-23T23:00:58.074265+0000","last_fullsized":"2023-09-23T23:00:58.074265+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"389'9325","reported_seq":9419,"reported_epoch":389,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:12.125682+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:01:12.125682+0000","last_peered":"2023-09-23T23:01:12.125682+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:01:12.125682+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:01:12.125682+0000","mapping_epoch":353,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181469184,"num_objects":2769,"num_object_clones":0,"num_object_copies":8307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9325,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2769}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":77,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.094964+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:00:58.094964+0000","last_peered":"2023-09-23T23:00:58.094964+0000","last_clean":"2023-09-23T23:00:58.094964+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:00:58.094964+0000","last_undegraded":"2023-09-23T23:00:58.094964+0000","last_fullsized":"2023-09-23T23:00:58.094964+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":382,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087089+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:00:58.087089+0000","last_peered":"2023-09-23T23:00:58.087089+0000","last_clean":"2023-09-23T23:00:58.087089+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:00:58.087089+0000","last_undegraded":"2023-09-23T23:00:58.087089+0000","last_fullsized":"2023-09-23T23:00:58.087089+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"389'9434","reported_seq":9536,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:12.150730+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:01:12.150730+0000","last_peered":"2023-09-23T23:01:12.150730+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:01:12.150730+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"326'3700","ondisk_log_start":"326'3700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181403648,"num_objects":2768,"num_object_clones":0,"num_object_copies":8304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2768,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":2768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9434,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-851}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":172,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.086626+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:00:58.086626+0000","last_peered":"2023-09-23T23:00:58.086626+0000","last_clean":"2023-09-23T23:00:58.086626+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:00:58.086626+0000","last_undegraded":"2023-09-23T23:00:58.086626+0000","last_fullsized":"2023-09-23T23:00:58.086626+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":399,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087485+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:00:58.087485+0000","last_peered":"2023-09-23T23:00:58.087485+0000","last_clean":"2023-09-23T23:00:58.087485+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:00:58.087485+0000","last_undegraded":"2023-09-23T23:00:58.087485+0000","last_fullsized":"2023-09-23T23:00:58.087485+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"389'9340","reported_seq":15598,"reported_epoch":389,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:12.226151+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:01:12.226151+0000","last_peered":"2023-09-23T23:01:12.226151+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:01:12.226151+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:01:12.226151+0000","mapping_epoch":354,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178257920,"num_objects":2720,"num_object_clones":0,"num_object_copies":8160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1526,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9340,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3074,"num_bytes_recovered":201457664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1435},{"shards":"1,4,6","objects":1285}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":412,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088382+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:00:58.088382+0000","last_peered":"2023-09-23T23:00:58.088382+0000","last_clean":"2023-09-23T23:00:58.088382+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:00:58.088382+0000","last_undegraded":"2023-09-23T23:00:58.088382+0000","last_fullsized":"2023-09-23T23:00:58.088382+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":390,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.090670+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:00:58.090670+0000","last_peered":"2023-09-23T23:00:58.090670+0000","last_clean":"2023-09-23T23:00:58.090670+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:00:58.090670+0000","last_undegraded":"2023-09-23T23:00:58.090670+0000","last_fullsized":"2023-09-23T23:00:58.090670+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"389'9163","reported_seq":9257,"reported_epoch":389,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:12.090324+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:01:12.090324+0000","last_peered":"2023-09-23T23:01:12.090324+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:01:12.090324+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:01:12.090324+0000","mapping_epoch":353,"log_start":"326'3500","ondisk_log_start":"326'3500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183828480,"num_objects":2805,"num_object_clones":0,"num_object_copies":8415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9163,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2805}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":58,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095278+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:00:58.095278+0000","last_peered":"2023-09-23T23:00:58.095278+0000","last_clean":"2023-09-23T23:00:58.095278+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:00:58.095278+0000","last_undegraded":"2023-09-23T23:00:58.095278+0000","last_fullsized":"2023-09-23T23:00:58.095278+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":292,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.076661+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:00:58.076661+0000","last_peered":"2023-09-23T23:00:58.076661+0000","last_clean":"2023-09-23T23:00:58.076661+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:00:58.076661+0000","last_undegraded":"2023-09-23T23:00:58.076661+0000","last_fullsized":"2023-09-23T23:00:58.076661+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"389'9304","reported_seq":18301,"reported_epoch":389,"state":"active+recovering+remapped","last_fresh":"2023-09-23T23:01:12.222673+0000","last_change":"2023-09-23T22:58:08.009685+0000","last_active":"2023-09-23T23:01:12.222673+0000","last_peered":"2023-09-23T23:01:12.222673+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:01:12.222673+0000","last_undegraded":"2023-09-23T23:01:12.222673+0000","last_fullsized":"2023-09-23T23:01:12.222673+0000","mapping_epoch":354,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181665792,"num_objects":2772,"num_object_clones":0,"num_object_copies":8316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9304,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4435,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2772}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":296,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.075848+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:00:58.075848+0000","last_peered":"2023-09-23T23:00:58.075848+0000","last_clean":"2023-09-23T23:00:58.075848+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:00:58.075848+0000","last_undegraded":"2023-09-23T23:00:58.075848+0000","last_fullsized":"2023-09-23T23:00:58.075848+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":299,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.077909+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:00:58.077909+0000","last_peered":"2023-09-23T23:00:58.077909+0000","last_clean":"2023-09-23T23:00:58.077909+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:00:58.077909+0000","last_undegraded":"2023-09-23T23:00:58.077909+0000","last_fullsized":"2023-09-23T23:00:58.077909+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":112,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.077622+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:00:58.077622+0000","last_peered":"2023-09-23T23:00:58.077622+0000","last_clean":"2023-09-23T23:00:58.077622+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:00:58.077622+0000","last_undegraded":"2023-09-23T23:00:58.077622+0000","last_fullsized":"2023-09-23T23:00:58.077622+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"389'9321","reported_seq":9427,"reported_epoch":389,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:09.768037+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:01:09.768037+0000","last_peered":"2023-09-23T23:01:09.768037+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:01:09.768037+0000","last_undegraded":"2023-09-23T23:01:09.768037+0000","last_fullsized":"2023-09-23T23:01:09.768037+0000","mapping_epoch":359,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183697408,"num_objects":2803,"num_object_clones":0,"num_object_copies":8409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2803,"num_objects_unfound":0,"num_objects_dirty":2803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9321,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2803}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":59,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.072938+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:00:58.072938+0000","last_peered":"2023-09-23T23:00:58.072938+0000","last_clean":"2023-09-23T23:00:58.072938+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:00:58.072938+0000","last_undegraded":"2023-09-23T23:00:58.072938+0000","last_fullsized":"2023-09-23T23:00:58.072938+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":409,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088107+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:00:58.088107+0000","last_peered":"2023-09-23T23:00:58.088107+0000","last_clean":"2023-09-23T23:00:58.088107+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:00:58.088107+0000","last_undegraded":"2023-09-23T23:00:58.088107+0000","last_fullsized":"2023-09-23T23:00:58.088107+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":281,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.075887+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:00:58.075887+0000","last_peered":"2023-09-23T23:00:58.075887+0000","last_clean":"2023-09-23T23:00:58.075887+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:00:58.075887+0000","last_undegraded":"2023-09-23T23:00:58.075887+0000","last_fullsized":"2023-09-23T23:00:58.075887+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"389'9308","reported_seq":17145,"reported_epoch":389,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T23:01:11.183594+0000","last_change":"2023-09-23T23:00:58.445708+0000","last_active":"2023-09-23T23:01:11.183594+0000","last_peered":"2023-09-23T23:01:11.183594+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:01:11.183594+0000","last_undegraded":"2023-09-23T23:01:11.183594+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'700","ondisk_log_start":"321'700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8608,"log_dups_size":0,"ondisk_log_size":8608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180617216,"num_objects":2756,"num_object_clones":0,"num_object_copies":8268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9308,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3868,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2756}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":356,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.075479+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:00:58.075479+0000","last_peered":"2023-09-23T23:00:58.075479+0000","last_clean":"2023-09-23T23:00:58.075479+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:00:58.075479+0000","last_undegraded":"2023-09-23T23:00:58.075479+0000","last_fullsized":"2023-09-23T23:00:58.075479+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":233,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.075606+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:00:58.075606+0000","last_peered":"2023-09-23T23:00:58.075606+0000","last_clean":"2023-09-23T23:00:58.075606+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:00:58.075606+0000","last_undegraded":"2023-09-23T23:00:58.075606+0000","last_fullsized":"2023-09-23T23:00:58.075606+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":299,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.078795+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:00:58.078795+0000","last_peered":"2023-09-23T23:00:58.078795+0000","last_clean":"2023-09-23T23:00:58.078795+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:00:58.078795+0000","last_undegraded":"2023-09-23T23:00:58.078795+0000","last_fullsized":"2023-09-23T23:00:58.078795+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"389'9409","reported_seq":11300,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:07.771052+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:01:07.771052+0000","last_peered":"2023-09-23T23:01:07.771052+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:01:07.771052+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":9409,"log_dups_size":0,"ondisk_log_size":9409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184483840,"num_objects":2815,"num_object_clones":0,"num_object_copies":8445,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":2815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9409,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":1382},{"shards":"0,6","objects":-2166}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":372,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.078355+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:00:58.078355+0000","last_peered":"2023-09-23T23:00:58.078355+0000","last_clean":"2023-09-23T23:00:58.078355+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:00:58.078355+0000","last_undegraded":"2023-09-23T23:00:58.078355+0000","last_fullsized":"2023-09-23T23:00:58.078355+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":485,"reported_epoch":388,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:00:58.075504+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:00:58.075504+0000","last_peered":"2023-09-23T23:00:58.075504+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:00:58.075504+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":58,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095045+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:00:58.095045+0000","last_peered":"2023-09-23T23:00:58.095045+0000","last_clean":"2023-09-23T23:00:58.095045+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:00:58.095045+0000","last_undegraded":"2023-09-23T23:00:58.095045+0000","last_fullsized":"2023-09-23T23:00:58.095045+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":299,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087724+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:00:58.087724+0000","last_peered":"2023-09-23T23:00:58.087724+0000","last_clean":"2023-09-23T23:00:58.087724+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:00:58.087724+0000","last_undegraded":"2023-09-23T23:00:58.087724+0000","last_fullsized":"2023-09-23T23:00:58.087724+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"389'9370","reported_seq":9468,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:09.737963+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:01:09.737963+0000","last_peered":"2023-09-23T23:01:09.737963+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:01:09.737963+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"326'3900","ondisk_log_start":"326'3900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177864704,"num_objects":2714,"num_object_clones":0,"num_object_copies":8142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9370,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-793}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":379,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.072805+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:00:58.072805+0000","last_peered":"2023-09-23T23:00:58.072805+0000","last_clean":"2023-09-23T23:00:58.072805+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:00:58.072805+0000","last_undegraded":"2023-09-23T23:00:58.072805+0000","last_fullsized":"2023-09-23T23:00:58.072805+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":350,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.072765+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:00:58.072765+0000","last_peered":"2023-09-23T23:00:58.072765+0000","last_clean":"2023-09-23T23:00:58.072765+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:00:58.072765+0000","last_undegraded":"2023-09-23T23:00:58.072765+0000","last_fullsized":"2023-09-23T23:00:58.072765+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":310,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.073404+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:00:58.073404+0000","last_peered":"2023-09-23T23:00:58.073404+0000","last_clean":"2023-09-23T23:00:58.073404+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:00:58.073404+0000","last_undegraded":"2023-09-23T23:00:58.073404+0000","last_fullsized":"2023-09-23T23:00:58.073404+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"389'9389","reported_seq":9502,"reported_epoch":389,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:12.171220+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:01:12.171220+0000","last_peered":"2023-09-23T23:01:12.171220+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:01:12.171220+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:01:12.171220+0000","mapping_epoch":354,"log_start":"326'3700","ondisk_log_start":"326'3700","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184418328,"num_objects":2815,"num_object_clones":0,"num_object_copies":8445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2815,"num_objects_unfound":0,"num_objects_dirty":2815,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9389,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":2815}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":325,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.073244+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:00:58.073244+0000","last_peered":"2023-09-23T23:00:58.073244+0000","last_clean":"2023-09-23T23:00:58.073244+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:00:58.073244+0000","last_undegraded":"2023-09-23T23:00:58.073244+0000","last_fullsized":"2023-09-23T23:00:58.073244+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":233,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.075943+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:00:58.075943+0000","last_peered":"2023-09-23T23:00:58.075943+0000","last_clean":"2023-09-23T23:00:58.075943+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:00:58.075943+0000","last_undegraded":"2023-09-23T23:00:58.075943+0000","last_fullsized":"2023-09-23T23:00:58.075943+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":202,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.078426+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:00:58.078426+0000","last_peered":"2023-09-23T23:00:58.078426+0000","last_clean":"2023-09-23T23:00:58.078426+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:00:58.078426+0000","last_undegraded":"2023-09-23T23:00:58.078426+0000","last_fullsized":"2023-09-23T23:00:58.078426+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"389'9442","reported_seq":15986,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:12.200169+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:01:12.200169+0000","last_peered":"2023-09-23T23:01:12.200169+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:01:12.200169+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"326'3700","ondisk_log_start":"326'3700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5742,"log_dups_size":0,"ondisk_log_size":5742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177209344,"num_objects":2704,"num_object_clones":0,"num_object_copies":8112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9442,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2780,"num_bytes_recovered":182190080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":1319}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":82,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.096035+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:00:58.096035+0000","last_peered":"2023-09-23T23:00:58.096035+0000","last_clean":"2023-09-23T23:00:58.096035+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:00:58.096035+0000","last_undegraded":"2023-09-23T23:00:58.096035+0000","last_fullsized":"2023-09-23T23:00:58.096035+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":97,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.092103+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:00:58.092103+0000","last_peered":"2023-09-23T23:00:58.092103+0000","last_clean":"2023-09-23T23:00:58.092103+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:00:58.092103+0000","last_undegraded":"2023-09-23T23:00:58.092103+0000","last_fullsized":"2023-09-23T23:00:58.092103+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":73,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095137+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:00:58.095137+0000","last_peered":"2023-09-23T23:00:58.095137+0000","last_clean":"2023-09-23T23:00:58.095137+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:00:58.095137+0000","last_undegraded":"2023-09-23T23:00:58.095137+0000","last_fullsized":"2023-09-23T23:00:58.095137+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"389'9353","reported_seq":9876,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:12.143793+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:01:12.143793+0000","last_peered":"2023-09-23T23:01:12.143793+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:01:12.143793+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"326'3600","ondisk_log_start":"326'3600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5753,"log_dups_size":0,"ondisk_log_size":5753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178061312,"num_objects":2717,"num_object_clones":0,"num_object_copies":8151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2717,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":2717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9353,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-834}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":345,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.076900+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:00:58.076900+0000","last_peered":"2023-09-23T23:00:58.076900+0000","last_clean":"2023-09-23T23:00:58.076900+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:00:58.076900+0000","last_undegraded":"2023-09-23T23:00:58.076900+0000","last_fullsized":"2023-09-23T23:00:58.076900+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":292,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.073331+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:00:58.073331+0000","last_peered":"2023-09-23T23:00:58.073331+0000","last_clean":"2023-09-23T23:00:58.073331+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:00:58.073331+0000","last_undegraded":"2023-09-23T23:00:58.073331+0000","last_fullsized":"2023-09-23T23:00:58.073331+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":363,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088459+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:00:58.088459+0000","last_peered":"2023-09-23T23:00:58.088459+0000","last_clean":"2023-09-23T23:00:58.088459+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:00:58.088459+0000","last_undegraded":"2023-09-23T23:00:58.088459+0000","last_fullsized":"2023-09-23T23:00:58.088459+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":327,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.074505+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:00:58.074505+0000","last_peered":"2023-09-23T23:00:58.074505+0000","last_clean":"2023-09-23T23:00:58.074505+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:00:58.074505+0000","last_undegraded":"2023-09-23T23:00:58.074505+0000","last_fullsized":"2023-09-23T23:00:58.074505+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.075732+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:00:58.075732+0000","last_peered":"2023-09-23T23:00:58.075732+0000","last_clean":"2023-09-23T23:00:58.075732+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:00:58.075732+0000","last_undegraded":"2023-09-23T23:00:58.075732+0000","last_fullsized":"2023-09-23T23:00:58.075732+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":198,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087013+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:00:58.087013+0000","last_peered":"2023-09-23T23:00:58.087013+0000","last_clean":"2023-09-23T23:00:58.087013+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:00:58.087013+0000","last_undegraded":"2023-09-23T23:00:58.087013+0000","last_fullsized":"2023-09-23T23:00:58.087013+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":412,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088182+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:00:58.088182+0000","last_peered":"2023-09-23T23:00:58.088182+0000","last_clean":"2023-09-23T23:00:58.088182+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:00:58.088182+0000","last_undegraded":"2023-09-23T23:00:58.088182+0000","last_fullsized":"2023-09-23T23:00:58.088182+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":212,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.087570+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:00:58.087570+0000","last_peered":"2023-09-23T23:00:58.087570+0000","last_clean":"2023-09-23T23:00:58.087570+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:00:58.087570+0000","last_undegraded":"2023-09-23T23:00:58.087570+0000","last_fullsized":"2023-09-23T23:00:58.087570+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":299,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.086549+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:00:58.086549+0000","last_peered":"2023-09-23T23:00:58.086549+0000","last_clean":"2023-09-23T23:00:58.086549+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:00:58.086549+0000","last_undegraded":"2023-09-23T23:00:58.086549+0000","last_fullsized":"2023-09-23T23:00:58.086549+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":82,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.095574+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:00:58.095574+0000","last_peered":"2023-09-23T23:00:58.095574+0000","last_clean":"2023-09-23T23:00:58.095574+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:00:58.095574+0000","last_undegraded":"2023-09-23T23:00:58.095574+0000","last_fullsized":"2023-09-23T23:00:58.095574+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":85,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.094998+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:00:58.094998+0000","last_peered":"2023-09-23T23:00:58.094998+0000","last_clean":"2023-09-23T23:00:58.094998+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:00:58.094998+0000","last_undegraded":"2023-09-23T23:00:58.094998+0000","last_fullsized":"2023-09-23T23:00:58.094998+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":331,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.073577+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:00:58.073577+0000","last_peered":"2023-09-23T23:00:58.073577+0000","last_clean":"2023-09-23T23:00:58.073577+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:00:58.073577+0000","last_undegraded":"2023-09-23T23:00:58.073577+0000","last_fullsized":"2023-09-23T23:00:58.073577+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":374,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T23:00:58.088361+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:00:58.088361+0000","last_peered":"2023-09-23T23:00:58.088361+0000","last_clean":"2023-09-23T23:00:58.088361+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:00:58.088361+0000","last_undegraded":"2023-09-23T23:00:58.088361+0000","last_fullsized":"2023-09-23T23:00:58.088361+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2903900184,"num_objects":44311,"num_object_clones":0,"num_object_copies":132933,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":25068,"num_objects_misplaced":28315,"num_objects_unfound":0,"num_objects_dirty":44311,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149287,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16501,"num_bytes_recovered":1010237440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9210703872,"data_stored":9210691656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99887,"ondisk_log_size":99887,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430618,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626842,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253155,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1909508,"kb_used_data":1189116,"kb_used_omap":9,"kb_used_meta":720374,"kb_avail":91835644,"statfs":{"total":95995035648,"available":94039699456,"internally_reserved":0,"allocated":1217654784,"data_stored":1216192931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":737663977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36399999999999999,"15min":0.35599999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.7849999999999999,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":0.378},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.76900000000000002,"15min":0.82699999999999996},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80700000000000005,"5min":5.5960000000000001,"15min":5.5960000000000001},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.504,"15min":0.52000000000000002},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.93600000000000005,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.341,"5min":4.048,"15min":4.048},"last":0.437}]},{"osd":3,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56100000000000005,"15min":1.9950000000000001},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.8580000000000001,"5min":3.141,"15min":87.787999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.71699999999999997,"15min":2.339},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.19500000000000001},"max":{"1min":0.95499999999999996,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.6869999999999998,"15min":2.4529999999999998},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6669999999999998,"5min":151.036,"15min":151.036},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.581,"5min":2.7450000000000001,"15min":2.548},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":6.649,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.6219999999999999,"15min":2.319},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1720000000000002,"5min":116.098,"15min":116.098},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.5099999999999998,"15min":2.4809999999999999},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.9769999999999999,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.27,"15min":2.1480000000000001},"min":{"1min":0.40100000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.0820000000000001,"15min":1.8520000000000001},"min":{"1min":0.40999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.825,"5min":8.5370000000000008,"15min":79.328999999999994},"last":0.65500000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.04,"15min":1.6799999999999999},"min":{"1min":0.46300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.581,"5min":5.8410000000000002,"15min":11.481999999999999},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.177,"15min":1.8919999999999999},"min":{"1min":0.44600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9860000000000002,"5min":10.676,"15min":20.902999999999999},"last":0.501}]}]},{"osd":0,"up_from":353,"seq":1516123455526,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2066924,"kb_used_data":1905764,"kb_used_omap":37,"kb_used_meta":161114,"kb_avail":91678228,"statfs":{"total":95995035648,"available":93878505472,"internally_reserved":0,"allocated":1951502336,"data_stored":1949894086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":164980898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.754,"5min":2.754,"15min":2.754},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.73299999999999998,"15min":0.76400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.9319999999999999,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.0270000000000001,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.504,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.96,"5min":3.96,"15min":3.96},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.365,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.82999999999999996,"15min":0.84999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.877,"5min":0.70999999999999996,"15min":0.68200000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":0.80500000000000005,"15min":0.72099999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":8.5950000000000006,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.94499999999999995,"15min":0.97999999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.1320000000000001,"5min":3.3330000000000002,"15min":3.3330000000000002},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.84499999999999997,"15min":0.83799999999999997},"min":{"1min":0.41799999999999998,"5min":0.379,"15min":0.379},"max":{"1min":3.0859999999999999,"5min":3.0920000000000001,"15min":3.0920000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.76500000000000001,"15min":0.73699999999999999},"min":{"1min":0.442,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.1139999999999999,"5min":3.1139999999999999,"15min":3.1139999999999999},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:00:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2549999999999999,"15min":1.2869999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":5.4779999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.232,"5min":1.2689999999999999,"15min":1.2749999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":5.5019999999999998,"5min":5.5019999999999998,"15min":5.5019999999999998},"last":0.59499999999999997}]}]},{"osd":2,"up_from":359,"seq":1541893259299,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":787948,"kb_used_data":656484,"kb_used_omap":83,"kb_used_meta":131372,"kb_avail":92957204,"statfs":{"total":95995035648,"available":95188176896,"internally_reserved":0,"allocated":672239616,"data_stored":670692449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":134525381},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.42799999999999999,"15min":0.379},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.1520000000000001,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.245},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.66300000000000003,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.2,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.61699999999999999,"15min":0.64900000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.90500000000000003,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.1060000000000001,"15min":1.1379999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6269999999999998,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.73499999999999999,"15min":0.71999999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7270000000000003,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.86299999999999999,"15min":0.83199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.2140000000000004,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":0.73999999999999999,"15min":0.623},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1159999999999997,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":1.748},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.7080000000000002,"5min":3.492,"15min":3.492},"last":1.964}]},{"osd":5,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.98899999999999999,"15min":0.96599999999999997},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.6029999999999998,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.101,"5min":0.76500000000000001,"15min":0.70899999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.0839999999999996,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.046,"15min":0.995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.5679999999999996,"5min":5.5679999999999996,"15min":5.5679999999999996},"last":8.9990000000000006},{"interface":"front","average":{"1min":1.365,"5min":0.75800000000000001,"15min":0.65700000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.1220000000000001,"15min":1.034},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":7.0499999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.501},{"interface":"front","average":{"1min":1.014,"5min":1.1459999999999999,"15min":1.1679999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.8520000000000003,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":1.524}]}]},{"osd":3,"up_from":94,"seq":403726926103,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529648,"kb_used_data":33192,"kb_used_omap":7,"kb_used_meta":496440,"kb_avail":93215504,"statfs":{"total":95995035648,"available":95452676096,"internally_reserved":0,"allocated":33988608,"data_stored":32432386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":508355176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40899999999999997,"15min":0.40699999999999997},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.63200000000000001,"15min":2.3639999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":123.866},"last":1.121},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.73699999999999999,"15min":2.1549999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8010000000000002,"5min":6.8010000000000002,"15min":122.265},"last":1.145}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.1339999999999999,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.76600000000000001,"15min":0.75},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":3.8740000000000001,"5min":3.8740000000000001,"15min":3.8740000000000001},"last":0.95399999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":2.3519999999999999,"15min":2.0350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":64.406999999999996,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":4.008,"5min":2.5409999999999999,"15min":1.8999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":64.468000000000004,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2290000000000001,"15min":1.3560000000000001},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1980000000000004,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.548,"5min":1.218,"15min":1.827},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.246,"5min":10.246,"15min":11.951000000000001},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.597,"15min":1.7150000000000001},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0229999999999997,"5min":11.116,"15min":14.097},"last":0.622},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4039999999999999,"15min":1.956},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.79,"5min":10.859,"15min":12.887},"last":0.46600000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.89400000000000002,"15min":2.0499999999999998},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8449999999999998,"5min":4.8449999999999998,"15min":91.757999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.413,"5min":0.98099999999999998,"15min":1.9390000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":8.5830000000000002,"5min":8.5830000000000002,"15min":91.709999999999994},"last":0.63600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019300,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3042856,"kb_used_data":2139488,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":90702296,"statfs":{"total":95995035648,"available":92879151104,"internally_reserved":0,"allocated":2190835712,"data_stored":2190037443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.54400000000000004}]},{"osd":1,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.4380000000000002,"15min":3.5270000000000001},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":3.081,"5min":87.325000000000003,"15min":132.304},"last":3.5089999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.6320000000000001,"15min":3.4849999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.253},"max":{"1min":3.6120000000000001,"5min":87.572000000000003,"15min":132.453},"last":3.8159999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.119,"15min":1.0529999999999999},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":2.9350000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":0.81100000000000005,"15min":0.749},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.9550000000000001,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.484,"15min":3.9009999999999998},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32400000000000001},"max":{"1min":4.0830000000000002,"5min":10.337999999999999,"15min":164.453},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.784,"5min":1.4410000000000001,"15min":3.8929999999999998},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0039999999999996,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":3.0670000000000002,"15min":1.643},"min":{"1min":0.318,"5min":0.214,"15min":0.214},"max":{"1min":3.129,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.335,"5min":3.4940000000000002,"15min":1.78},"min":{"1min":0.318,"5min":0.252,"15min":0.248},"max":{"1min":3.7469999999999999,"5min":151.905,"15min":151.905},"last":0.27200000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.655,"15min":1.181},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":3.5680000000000001,"5min":16.943000000000001,"15min":16.943000000000001},"last":1.9890000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.4379999999999999,"15min":1.113},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.1859999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 23:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.47,"15min":1.127},"min":{"1min":0.40899999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.045999999999999,"5min":12.84,"15min":12.84},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.615,"5min":1.5089999999999999,"15min":1.0740000000000001},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.227,"5min":10.587,"15min":10.587},"last":0.42199999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823074,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2234056,"kb_used_data":1330304,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":91511096,"statfs":{"total":95995035648,"available":93707362304,"internally_reserved":0,"allocated":1362231296,"data_stored":1361460815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.878,"15min":0.69599999999999995},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.5709999999999997,"5min":14.42,"15min":14.42},"last":1.2190000000000001},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2430000000000001,"15min":1.135},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3159999999999998,"5min":14.496,"15min":14.496},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.081,"15min":2.1909999999999998},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.24299999999999999},"max":{"1min":2.6040000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.857,"5min":1.6200000000000001,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":7.0739999999999998,"5min":10.353,"15min":110.89},"last":7.1219999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.601,"5min":6.601,"15min":6.601},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7669999999999999,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.59299999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.216,"15min":4.7009999999999996},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2710000000000008,"5min":9.2710000000000008,"15min":257.05000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.474,"5min":1.3859999999999999,"15min":4.7590000000000003},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.3399999999999999,"5min":12.185,"15min":257.35000000000002},"last":2.5750000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":2.8580000000000001,"15min":1.5109999999999999},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8440000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":7.0380000000000003},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.5710000000000002,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.524,"5min":114.908,"15min":114.908},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.512,"15min":1.2270000000000001},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.4630000000000001,"5min":11.121,"15min":12.615},"last":7.0780000000000003},{"interface":"front","average":{"1min":1.246,"5min":1.2370000000000001,"15min":1.1519999999999999},"min":{"1min":0.41299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.4909999999999997,"5min":12.137,"15min":12.74},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":1.1180000000000001,"15min":1.046},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.5980000000000008,"5min":12.23,"15min":12.23},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.627,"5min":1.4199999999999999,"15min":1.0940000000000001},"min":{"1min":0.45500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":6.8710000000000004,"5min":11.112,"15min":11.112},"last":7.1020000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1933709312,"data_stored":1933705240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1211498496,"data_stored":1211498496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":666632192,"data_stored":666632192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":29360128,"data_stored":29360128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2177044480,"data_stored":2177040408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1356922880,"data_stored":1356922880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1835536384,"data_stored":1835532312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:01:16.386 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:16.426 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:16.572 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:16.583 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:16.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:16.692 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:16.758 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:17.172 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119492 10.9 e372/360 10:9e63863c:::benchmark_data_smithi052_461816_object22691:head v 372'7491, mlcod=326'3298)", 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:00.614037+0000", 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 136.550490774, 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036220987000000003, 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.173 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614037+0000", 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614037+0000", 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614036+0000", 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614041+0000", 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0259999999999998e-06 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614048+0000", 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6710000000000002e-06 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614058+0000", 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9340000000000001e-06 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.615825+0000", 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017668009999999999 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.615843+0000", 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.872e-05 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650230+0000", 2023-09-23T23:01:17.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.034386247000000002 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650258+0000", 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8113999999999999e-05 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119487 10.e e372/360 10:76db422f:::benchmark_data_smithi052_461816_object22686:head v 372'7460, mlcod=0'0)", 2023-09-23T23:01:17.178 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:00.618065+0000", 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 136.54646250499999, 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.037691571, 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618065+0000", 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618065+0000", 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618050+0000", 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999852 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618077+0000", 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6390999999999999e-05 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618091+0000", 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4443e-05 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618099+0000", 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9920000000000007e-06 2023-09-23T23:01:17.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618149+0000", 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9818000000000003e-05 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650207+0000", 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.032058193999999998 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.655710+0000", 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055033679999999998 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.655757+0000", 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6239000000000002e-05 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119496 10.f e372/360 10:f4ed15ba:::benchmark_data_smithi052_461816_object22695:head v 372'7463, mlcod=323'2484)", 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:00.625660+0000", 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 136.53886734599999, 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.035401146000000001, 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625660+0000", 2023-09-23T23:01:17.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625660+0000", 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625659+0000", 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625672+0000", 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.346e-05 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650169+0000", 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.024496401000000001 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650184+0000", 2023-09-23T23:01:17.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5075000000000001e-05 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650234+0000", 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0618000000000002e-05 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650259+0000", 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4414e-05 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.661021+0000", 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010762177 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.661061+0000", 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0342000000000001e-05 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119633 10.e e373/360 10:72cd2dfa:::benchmark_data_smithi052_461816_object22832:head v 373'7472, mlcod=0'0)", 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.278330+0000", 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 135.88619774700001, 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10197167, 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278330+0000", 2023-09-23T23:01:17.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278330+0000", 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278328+0000", 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:01:17.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278343+0000", 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.449e-05 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278364+0000", 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1593000000000001e-05 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.374216+0000", 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095851378000000001 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.374273+0000", 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7846000000000001e-05 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.374310+0000", 2023-09-23T23:01:17.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7002000000000002e-05 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.380258+0000", 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059473010000000003 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.380302+0000", 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3859999999999997e-05 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.193 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(10.9 373/360 [PushOp(10:9072a8b6:::benchmark_data_smithi052_461816_object53040:head, version: 326'3315, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:9072a8b6:::benchmark_data_smithi052_461816_object53040:head@326'3315, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 135.85945936600001, 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070204256000000007, 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:17.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305148+0000", 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0175999999999999e-05 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371739+0000", 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.066591289999999997 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371773+0000", 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3893000000000002e-05 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371871+0000", 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8084000000000005e-05 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371884+0000", 2023-09-23T23:01:17.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3312999999999999e-05 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.375273+0000", 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033882019999999999 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:140907 10.3 e385/359 10:c6e00ab6:::benchmark_data_smithi052_461816_object44106:head v 385'8883, mlcod=323'2482)", 2023-09-23T23:01:17.198 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:39.907906+0000", 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 37.256621660999997, 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.024217079999999998, 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.907906+0000", 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.907906+0000", 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.907905+0000", 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.907915+0000", 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0019e-05 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.907936+0000", 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1430999999999999e-05 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.907946+0000", 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2769999999999994e-06 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.910731+0000", 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00278532 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.910760+0000", 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9365000000000001e-05 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.932075+0000", 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.021314763 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.932123+0000", 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7824999999999997e-05 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.203 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141425 10.e e385/360 10:7f34fb6d:::benchmark_data_smithi052_461816_object44624:head v 385'8905, mlcod=0'0)", 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.704109+0000", 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.460418222999998, 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071707641000000003, 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704109+0000", 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704109+0000", 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704108+0000", 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704116+0000", 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8259999999999993e-06 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704129+0000", 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2669e-05 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704138+0000", 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6289999999999999e-06 2023-09-23T23:01:17.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704193+0000", 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5735000000000001e-05 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704225+0000", 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1414000000000001e-05 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775742+0000", 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071517739999999996 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775817+0000", 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4647999999999994e-05 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141427 10.3 e385/359 10:c1b733eb:::benchmark_data_smithi052_461816_object44626:head v 385'8926, mlcod=323'2482)", 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.710101+0000", 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.454426451000003, 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071061101000000002, 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710101+0000", 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710101+0000", 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710100+0000", 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710108+0000", 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2479999999999996e-06 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775731+0000", 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065622335000000004 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775744+0000", 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3657e-05 2023-09-23T23:01:17.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775807+0000", 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2572999999999999e-05 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775836+0000", 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8844999999999999e-05 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.781134+0000", 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052976380000000003 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.781162+0000", 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8767000000000001e-05 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.213 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(10.9 385/360 [PushOp(10:94cabeba:::benchmark_data_smithi052_461816_object90470:head, version: 335'5644, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:94cabeba:::benchmark_data_smithi052_461816_object90470:head@335'5644, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.736654+0000", 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.427873794999996, 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.042611151, 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736654+0000", 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736654+0000", 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736652+0000", 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736725+0000", 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3126000000000004e-05 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775722+0000", 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.038996958999999998 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775747+0000", 2023-09-23T23:01:17.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4967e-05 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775836+0000", 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8986000000000005e-05 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775850+0000", 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4197e-05 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.779265+0000", 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003415169 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:144557 10.7 e387/360 10:ee112f9e:::benchmark_data_smithi052_461816_object47756:head v 387'8892, mlcod=323'2408)", 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:54.514258+0000", 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 22.650270023000001, 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.023700332000000001, 2023-09-23T23:01:17.218 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.514258+0000", 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.514258+0000", 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.514257+0000", 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.514263+0000", 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3619999999999997e-06 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.514271+0000", 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0059999999999992e-06 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.514276+0000", 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7209999999999999e-06 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.522980+0000", 2023-09-23T23:01:17.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0087039720000000008 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.523010+0000", 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9828999999999999e-05 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.537917+0000", 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014906964999999999 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.537958+0000", 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0887999999999999e-05 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:144659 10.7 e387/360 10:e8fde3cc:::benchmark_data_smithi052_461816_object47858:head v 387'8901, mlcod=323'2408)", 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:54.835692+0000", 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 22.328835841, 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022973260999999998, 2023-09-23T23:01:17.223 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835692+0000", 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835692+0000", 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835691+0000", 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835701+0000", 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0611999999999999e-05 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835722+0000", 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1029000000000001e-05 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835731+0000", 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5019999999999992e-06 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835788+0000", 2023-09-23T23:01:17.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7008e-05 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.835819+0000", 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1644999999999999e-05 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.858606+0000", 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022786224000000001 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:54.858665+0000", 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9457999999999999e-05 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:145924 10.7 e388/360 10:eaa10b09:::benchmark_data_smithi052_461816_object49123:head v 388'8980, mlcod=323'2408)", 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.178877+0000", 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.985650844999999, 2023-09-23T23:01:17.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.044936373000000002, 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178877+0000", 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178877+0000", 2023-09-23T23:01:17.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178875+0000", 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178884+0000", 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0469999999999999e-06 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178896+0000", 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1557e-05 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178905+0000", 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5900000000000008e-06 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178950+0000", 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5021999999999997e-05 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178970+0000", 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0276e-05 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223766+0000", 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.044796509999999998 2023-09-23T23:01:17.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223813+0000", 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6841000000000001e-05 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGRecoveryDelete(10.9 e388,360 10:96ac8436:::benchmark_data_smithi052_461816_object202:head,336'6050)", 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.182567+0000", 2023-09-23T23:01:17.233 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.981960223000002, 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.037664665, 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182567+0000", 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182567+0000", 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182566+0000", 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182573+0000", 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0400000000000004e-06 2023-09-23T23:01:17.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220077+0000", 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.037503309999999998 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220094+0000", 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6864999999999999e-05 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220154+0000", 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0449999999999999e-05 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220167+0000", 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3030000000000001e-05 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220232+0000", 2023-09-23T23:01:17.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4976999999999998e-05 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:145930 10.f e388/360 10:fcfdf602:::benchmark_data_smithi052_461816_object49129:head v 388'9116, mlcod=323'2484)", 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.184108+0000", 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.980419586, 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.039735528999999999, 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.238 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184108+0000", 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184108+0000", 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184107+0000", 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184120+0000", 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2561000000000001e-05 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217204+0000", 2023-09-23T23:01:17.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.033084163 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217214+0000", 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0376e-05 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217253+0000", 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9258999999999998e-05 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217283+0000", 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9300000000000001e-05 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223816+0000", 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065333190000000001 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223844+0000", 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7546e-05 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:145931 10.a e388/359 10:5f2e1c5d:::benchmark_data_smithi052_461816_object49130:head v 388'9151, mlcod=388'9149)", 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.186834+0000", 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.977694086, 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.031626363999999997, 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.243 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186834+0000", 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186834+0000", 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186833+0000", 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186844+0000", 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1157e-05 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216039+0000", 2023-09-23T23:01:17.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.029194931 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216064+0000", 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5401999999999998e-05 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216197+0000", 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013290700000000001 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216219+0000", 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2169000000000001e-05 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.218413+0000", 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021937879999999999 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.218460+0000", 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7030999999999997e-05 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:149648 10.5 e389/359 10:a0b7041b:::benchmark_data_smithi052_461816_object52847:head v 389'9381, mlcod=323'2477)", 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:12.147143+0000", 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 5.0173842090000003, 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022799353000000001, 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.248 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147143+0000", 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147143+0000", 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147142+0000", 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147151+0000", 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6929999999999996e-06 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147166+0000", 2023-09-23T23:01:17.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5038000000000001e-05 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147174+0000", 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4919999999999993e-06 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147221+0000", 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6876999999999999e-05 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.147248+0000", 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7064e-05 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.169909+0000", 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022660961 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:12.169943+0000", 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3393999999999999e-05 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:150167 10.7 e389/360 10:e6ed0eb3:::benchmark_data_smithi052_461816_object53366:head v 389'9238, mlcod=323'2408)", 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:13.518183+0000", 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 3.6463442710000002, 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.027834406999999999, 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.253 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518183+0000", 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518183+0000", 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518182+0000", 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518193+0000", 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0730000000000001e-05 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518219+0000", 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5919e-05 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518228+0000", 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5249999999999994e-06 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518295+0000", 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6520999999999995e-05 2023-09-23T23:01:17.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.518321+0000", 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5732999999999999e-05 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.545973+0000", 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.027652421999999999 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:13.546018+0000", 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4703000000000002e-05 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:150935 10.5 e389/359 10:a2940dfc:::benchmark_data_smithi052_461816_object54134:head v 389'9471, mlcod=323'2477)", 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:15.961173+0000", 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 1.2033549889999999, 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.026725597, 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.258 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.961173+0000", 2023-09-23T23:01:17.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.961173+0000", 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.961172+0000", 2023-09-23T23:01:17.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.961181+0000", 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3850000000000008e-06 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.961194+0000", 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3213999999999999e-05 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.961202+0000", 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7600000000000002e-06 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.967118+0000", 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059160289999999997 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.967135+0000", 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7044000000000001e-05 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.987861+0000", 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020726222999999998 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.987898+0000", 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6798999999999998e-05 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.264 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151030 10.f e389/360 10:fe8e1dd7:::benchmark_data_smithi052_461816_object54229:head v 389'9440, mlcod=323'2484)", 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 1.0267382679999999, 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.034341280000000002, 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.265 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:01:17.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137794+0000", 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9699999999999998e-06 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137800+0000", 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5379999999999999e-06 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137806+0000", 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7939999999999999e-06 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137854+0000", 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8021e-05 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137879+0000", 2023-09-23T23:01:17.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4388000000000001e-05 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.172083+0000", 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.034204076999999999 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.172131+0000", 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7913e-05 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.269 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151032 10.f e389/360 10:f94955ab:::benchmark_data_smithi052_461816_object54231:head v 389'9441, mlcod=323'2484)", 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:16.145824+0000", 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 1.0187040000000001, 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.026340435999999998, 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145824+0000", 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145824+0000", 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145823+0000", 2023-09-23T23:01:17.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145828+0000", 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.899e-06 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145835+0000", 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4749999999999998e-06 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145839+0000", 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5340000000000001e-06 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145899+0000", 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9969999999999997e-05 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.145922+0000", 2023-09-23T23:01:17.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2498e-05 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.172137+0000", 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.026215859000000001 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.172164+0000", 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6616999999999999e-05 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.274 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.275 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.275 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:17.275 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:17.275 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:01:17.712 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:01:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:17 smithi052 ceph-mon[187743]: pgmap v1171: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 300 op/s; 24878/130515 objects degraded (19.061%); 28106/130515 objects misplaced (21.535%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/99570469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:17 smithi052 ceph-mon[180979]: pgmap v1171: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 300 op/s; 24878/130515 objects degraded (19.061%); 28106/130515 objects misplaced (21.535%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/99570469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:17 smithi151 ceph-mon[159210]: pgmap v1171: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.7 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 300 op/s; 24878/130515 objects degraded (19.061%); 28106/130515 objects misplaced (21.535%); 2.3 MiB/s, 83 objects/s recovering 2023-09-23T23:01:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/99570469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:18.266 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:18.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:18.370 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:18.425 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:18.497 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:18.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:18.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:18.518 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:01:18.605 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:18.656 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:18.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:01:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[187743]: failed to encode map e390 with expected crc 2023-09-23T23:01:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/99570469' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[187743]: osdmap e390: 8 total, 8 up, 6 in 2023-09-23T23:01:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[187743]: failed to encode map e390 with expected crc 2023-09-23T23:01:18.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[187743]: failed to encode map e390 with expected crc 2023-09-23T23:01:18.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:01:18.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[180979]: failed to encode map e390 with expected crc 2023-09-23T23:01:18.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/99570469' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:18.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[180979]: osdmap e390: 8 total, 8 up, 6 in 2023-09-23T23:01:18.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[180979]: failed to encode map e390 with expected crc 2023-09-23T23:01:18.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:18 smithi052 ceph-mon[180979]: failed to encode map e390 with expected crc 2023-09-23T23:01:19.032 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:19.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:19.138 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:19.175 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:19.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:18 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:01:19.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:18 smithi151 ceph-mon[159210]: failed to encode map e390 with expected crc 2023-09-23T23:01:19.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/99570469' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:19.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:18 smithi151 ceph-mon[159210]: osdmap e390: 8 total, 8 up, 6 in 2023-09-23T23:01:19.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:18 smithi151 ceph-mon[159210]: failed to encode map e390 with expected crc 2023-09-23T23:01:19.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:18 smithi151 ceph-mon[159210]: failed to encode map e390 with expected crc 2023-09-23T23:01:19.314 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:19.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:19.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:19.387 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:01:19.420 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:19.521 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:19.617 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:19.621 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:19.632 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:19.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:19.655 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:19.715 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:19.740 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:19.750 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:19.779 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:19.798 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:19.810 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:19.844 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:19.871 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:19.873 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.873 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.873 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.874 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.874 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.874 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.874 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.874 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:01:19.903 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[187743]: failed to encode map e390 with expected crc 2023-09-23T23:01:19.903 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[187743]: pgmap v1173: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 24339/127539 objects degraded (19.084%); 27999/127539 objects misplaced (21.953%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T23:01:19.903 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3424631046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[180979]: failed to encode map e390 with expected crc 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[180979]: pgmap v1173: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 24339/127539 objects degraded (19.084%); 27999/127539 objects misplaced (21.953%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:19.904 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3424631046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:19.937 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:19.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:19.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:20.047 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:20.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:19 smithi151 ceph-mon[159210]: failed to encode map e390 with expected crc 2023-09-23T23:01:20.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:19 smithi151 ceph-mon[159210]: pgmap v1173: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 24339/127539 objects degraded (19.084%); 27999/127539 objects misplaced (21.953%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T23:01:20.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:20.080 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:20.080 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3424631046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:20.081 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:20.146 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:20.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:20.254 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:20.282 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:20.325 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:20.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:20.431 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:20.462 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:20.818 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:20.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:20.923 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:20.956 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 24339/127539 objects degraded (19.084%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:20.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 24339/127539 objects degraded (19.084%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 24339/127539 objects degraded (19.084%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:21.216 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:01:21.314 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:21.337 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:21.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:21.445 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:21.524 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:21.647 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:21.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:21.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:21.751 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:21.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:21 smithi052 ceph-mon[180979]: pgmap v1174: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 24221/125631 objects degraded (19.279%); 27745/125631 objects misplaced (22.085%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T23:01:21.808 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:21 smithi052 ceph-mon[187743]: pgmap v1174: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 24221/125631 objects degraded (19.279%); 27745/125631 objects misplaced (22.085%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T23:01:21.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:22.068 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:22.068 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:21 smithi151 ceph-mon[159210]: pgmap v1174: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 24221/125631 objects degraded (19.279%); 27745/125631 objects misplaced (22.085%); 2.5 MiB/s, 86 objects/s recovering 2023-09-23T23:01:22.955 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:22.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:22.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:23.061 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:23.111 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:23.266 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:23.280 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:23.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:23.296 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:01:23.298 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:01:23.390 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:23.467 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:23.540 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:01:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:01:23] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-09-23T23:01:23.669 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:23.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:23.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:23.776 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:23.813 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:23.857 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:23 smithi052 ceph-mon[187743]: pgmap v1175: 89 pgs: 2 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 356 op/s; 24274/122301 objects degraded (19.848%); 30118/122301 objects misplaced (24.626%); 2.6 MiB/s, 71 objects/s recovering 2023-09-23T23:01:23.857 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:23 smithi052 ceph-mon[180979]: pgmap v1175: 89 pgs: 2 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 356 op/s; 24274/122301 objects degraded (19.848%); 30118/122301 objects misplaced (24.626%); 2.6 MiB/s, 71 objects/s recovering 2023-09-23T23:01:23.870 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:23.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:23.978 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:24.007 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:23 smithi151 ceph-mon[159210]: pgmap v1175: 89 pgs: 2 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 356 op/s; 24274/122301 objects degraded (19.848%); 30118/122301 objects misplaced (24.626%); 2.6 MiB/s, 71 objects/s recovering 2023-09-23T23:01:24.011 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:24.085 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:24.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:24.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:24.194 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:24.196 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:24.196 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:01:24.226 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:24.272 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:24.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:24.376 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:24.417 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:24.690 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:24.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:24.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:24.795 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:24.839 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:24.897 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:24.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:25.008 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:25.009 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:25.044 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:25.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:25.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:25.212 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:25.262 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:25.279 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1175,"stamp":"2023-09-23T23:01:22.590748+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2670663608,"num_objects":40767,"num_object_clones":0,"num_object_copies":122301,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":24274,"num_objects_misplaced":30118,"num_objects_unfound":0,"num_objects_dirty":40767,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":153115,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17254,"num_bytes_recovered":1037319100,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100249,"ondisk_log_size":100249,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10082640,"kb_used_data":6745504,"kb_used_omap":137,"kb_used_meta":3336886,"kb_avail":552388272,"statfs":{"total":575970213888,"available":565645590528,"internally_reserved":0,"allocated":6907396096,"data_stored":6899618826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3416971448},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-233832448,"num_objects":-3568,"num_object_clones":0,"num_object_copies":-10704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-796,"num_objects_misplaced":1803,"num_objects_unfound":0,"num_objects_dirty":-3568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":718,"num_bytes_recovered":27066368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003113"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":402,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.689320+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:01:07.689320+0000","last_peered":"2023-09-23T23:01:07.689320+0000","last_clean":"2023-09-23T23:01:07.689320+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:01:07.689320+0000","last_undegraded":"2023-09-23T23:01:07.689320+0000","last_fullsized":"2023-09-23T23:01:07.689320+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":59,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.704199+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:01:07.704199+0000","last_peered":"2023-09-23T23:01:07.704199+0000","last_clean":"2023-09-23T23:01:07.704199+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:01:07.704199+0000","last_undegraded":"2023-09-23T23:01:07.704199+0000","last_fullsized":"2023-09-23T23:01:07.704199+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":58,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701471+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:01:07.701471+0000","last_peered":"2023-09-23T23:01:07.701471+0000","last_clean":"2023-09-23T23:01:07.701471+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:01:07.701471+0000","last_undegraded":"2023-09-23T23:01:07.701471+0000","last_fullsized":"2023-09-23T23:01:07.701471+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":386,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701691+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:01:07.701691+0000","last_peered":"2023-09-23T23:01:07.701691+0000","last_clean":"2023-09-23T23:01:07.701691+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:01:07.701691+0000","last_undegraded":"2023-09-23T23:01:07.701691+0000","last_fullsized":"2023-09-23T23:01:07.701691+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":74,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701151+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:01:07.701151+0000","last_peered":"2023-09-23T23:01:07.701151+0000","last_clean":"2023-09-23T23:01:07.701151+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:01:07.701151+0000","last_undegraded":"2023-09-23T23:01:07.701151+0000","last_fullsized":"2023-09-23T23:01:07.701151+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.666571+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:01:07.666571+0000","last_peered":"2023-09-23T23:01:07.666571+0000","last_clean":"2023-09-23T23:01:07.666571+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:01:07.666571+0000","last_undegraded":"2023-09-23T23:01:07.666571+0000","last_fullsized":"2023-09-23T23:01:07.666571+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":325,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.673056+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:01:07.673056+0000","last_peered":"2023-09-23T23:01:07.673056+0000","last_clean":"2023-09-23T23:01:07.673056+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:01:07.673056+0000","last_undegraded":"2023-09-23T23:01:07.673056+0000","last_fullsized":"2023-09-23T23:01:07.673056+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":68,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701111+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:01:07.701111+0000","last_peered":"2023-09-23T23:01:07.701111+0000","last_clean":"2023-09-23T23:01:07.701111+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:01:07.701111+0000","last_undegraded":"2023-09-23T23:01:07.701111+0000","last_fullsized":"2023-09-23T23:01:07.701111+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":286,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.668446+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:01:07.668446+0000","last_peered":"2023-09-23T23:01:07.668446+0000","last_clean":"2023-09-23T23:01:07.668446+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:01:07.668446+0000","last_undegraded":"2023-09-23T23:01:07.668446+0000","last_fullsized":"2023-09-23T23:01:07.668446+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":73,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701339+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:01:07.701339+0000","last_peered":"2023-09-23T23:01:07.701339+0000","last_clean":"2023-09-23T23:01:07.701339+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:01:07.701339+0000","last_undegraded":"2023-09-23T23:01:07.701339+0000","last_fullsized":"2023-09-23T23:01:07.701339+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":79,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.704118+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:01:07.704118+0000","last_peered":"2023-09-23T23:01:07.704118+0000","last_clean":"2023-09-23T23:01:07.704118+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:01:07.704118+0000","last_undegraded":"2023-09-23T23:01:07.704118+0000","last_fullsized":"2023-09-23T23:01:07.704118+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":135,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.670324+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:01:07.670324+0000","last_peered":"2023-09-23T23:01:07.670324+0000","last_clean":"2023-09-23T23:01:07.670324+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:01:07.670324+0000","last_undegraded":"2023-09-23T23:01:07.670324+0000","last_fullsized":"2023-09-23T23:01:07.670324+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":461,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690196+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:01:07.690196+0000","last_peered":"2023-09-23T23:01:07.690196+0000","last_clean":"2023-09-23T23:01:07.690196+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:01:07.690196+0000","last_undegraded":"2023-09-23T23:01:07.690196+0000","last_fullsized":"2023-09-23T23:01:07.690196+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":364,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.691630+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:01:07.691630+0000","last_peered":"2023-09-23T23:01:07.691630+0000","last_clean":"2023-09-23T23:01:07.691630+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:01:07.691630+0000","last_undegraded":"2023-09-23T23:01:07.691630+0000","last_fullsized":"2023-09-23T23:01:07.691630+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":290,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.691722+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:01:07.691722+0000","last_peered":"2023-09-23T23:01:07.691722+0000","last_clean":"2023-09-23T23:01:07.691722+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:01:07.691722+0000","last_undegraded":"2023-09-23T23:01:07.691722+0000","last_fullsized":"2023-09-23T23:01:07.691722+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":70,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.702524+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:01:07.702524+0000","last_peered":"2023-09-23T23:01:07.702524+0000","last_clean":"2023-09-23T23:01:07.702524+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:01:07.702524+0000","last_undegraded":"2023-09-23T23:01:07.702524+0000","last_fullsized":"2023-09-23T23:01:07.702524+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":379,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701301+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:01:07.701301+0000","last_peered":"2023-09-23T23:01:07.701301+0000","last_clean":"2023-09-23T23:01:07.701301+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:01:07.701301+0000","last_undegraded":"2023-09-23T23:01:07.701301+0000","last_fullsized":"2023-09-23T23:01:07.701301+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":290,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.668031+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:01:07.668031+0000","last_peered":"2023-09-23T23:01:07.668031+0000","last_clean":"2023-09-23T23:01:07.668031+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:01:07.668031+0000","last_undegraded":"2023-09-23T23:01:07.668031+0000","last_fullsized":"2023-09-23T23:01:07.668031+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":275,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690180+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:01:07.690180+0000","last_peered":"2023-09-23T23:01:07.690180+0000","last_clean":"2023-09-23T23:01:07.690180+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:01:07.690180+0000","last_undegraded":"2023-09-23T23:01:07.690180+0000","last_fullsized":"2023-09-23T23:01:07.690180+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":403,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701816+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:01:07.701816+0000","last_peered":"2023-09-23T23:01:07.701816+0000","last_clean":"2023-09-23T23:01:07.701816+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:01:07.701816+0000","last_undegraded":"2023-09-23T23:01:07.701816+0000","last_fullsized":"2023-09-23T23:01:07.701816+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":108,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.665395+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:01:07.665395+0000","last_peered":"2023-09-23T23:01:07.665395+0000","last_clean":"2023-09-23T23:01:07.665395+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:01:07.665395+0000","last_undegraded":"2023-09-23T23:01:07.665395+0000","last_fullsized":"2023-09-23T23:01:07.665395+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.702746+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:01:07.702746+0000","last_peered":"2023-09-23T23:01:07.702746+0000","last_clean":"2023-09-23T23:01:07.702746+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:01:07.702746+0000","last_undegraded":"2023-09-23T23:01:07.702746+0000","last_fullsized":"2023-09-23T23:01:07.702746+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"390'9533","reported_seq":9627,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:19.752481+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:01:19.752481+0000","last_peered":"2023-09-23T23:01:19.752481+0000","last_clean":"2023-09-23T23:01:19.752481+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:01:19.752481+0000","last_undegraded":"2023-09-23T23:01:19.752481+0000","last_fullsized":"2023-09-23T23:01:19.752481+0000","mapping_epoch":353,"log_start":"327'4000","ondisk_log_start":"327'4000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5533,"log_dups_size":0,"ondisk_log_size":5533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9533,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":274,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.695742+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:01:07.695742+0000","last_peered":"2023-09-23T23:01:07.695742+0000","last_clean":"2023-09-23T23:01:07.695742+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:01:07.695742+0000","last_undegraded":"2023-09-23T23:01:07.695742+0000","last_fullsized":"2023-09-23T23:01:07.695742+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":260,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.667394+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:01:07.667394+0000","last_peered":"2023-09-23T23:01:07.667394+0000","last_clean":"2023-09-23T23:01:07.667394+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:01:07.667394+0000","last_undegraded":"2023-09-23T23:01:07.667394+0000","last_fullsized":"2023-09-23T23:01:07.667394+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"390'9385","reported_seq":11808,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:21.192627+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:01:21.192627+0000","last_peered":"2023-09-23T23:01:21.192627+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:01:21.192627+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'800","ondisk_log_start":"321'800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8585,"log_dups_size":0,"ondisk_log_size":8585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168099840,"num_objects":2565,"num_object_clones":0,"num_object_copies":7695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2565,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":2565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9385,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-1002}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":284,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690393+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:01:07.690393+0000","last_peered":"2023-09-23T23:01:07.690393+0000","last_clean":"2023-09-23T23:01:07.690393+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:01:07.690393+0000","last_undegraded":"2023-09-23T23:01:07.690393+0000","last_fullsized":"2023-09-23T23:01:07.690393+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":397,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690110+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:01:07.690110+0000","last_peered":"2023-09-23T23:01:07.690110+0000","last_clean":"2023-09-23T23:01:07.690110+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:01:07.690110+0000","last_undegraded":"2023-09-23T23:01:07.690110+0000","last_fullsized":"2023-09-23T23:01:07.690110+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"390'9538","reported_seq":9626,"reported_epoch":390,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:19.788380+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:01:19.788380+0000","last_peered":"2023-09-23T23:01:19.788380+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:01:19.788380+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:01:19.788380+0000","mapping_epoch":354,"log_start":"327'4000","ondisk_log_start":"327'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166068224,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":2534,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9538,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2534}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":141,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.668119+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:01:07.668119+0000","last_peered":"2023-09-23T23:01:07.668119+0000","last_clean":"2023-09-23T23:01:07.668119+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:01:07.668119+0000","last_undegraded":"2023-09-23T23:01:07.668119+0000","last_fullsized":"2023-09-23T23:01:07.668119+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":259,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.667783+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:01:07.667783+0000","last_peered":"2023-09-23T23:01:07.667783+0000","last_clean":"2023-09-23T23:01:07.667783+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:01:07.667783+0000","last_undegraded":"2023-09-23T23:01:07.667783+0000","last_fullsized":"2023-09-23T23:01:07.667783+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"390'9580","reported_seq":9854,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:19.847175+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:01:19.847175+0000","last_peered":"2023-09-23T23:01:19.847175+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:01:19.847175+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"327'4100","ondisk_log_start":"327'4100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170000384,"num_objects":2594,"num_object_clones":0,"num_object_copies":7782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2179,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":2594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9580,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":414},{"shards":"0,5","objects":-1016}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.702448+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:01:07.702448+0000","last_peered":"2023-09-23T23:01:07.702448+0000","last_clean":"2023-09-23T23:01:07.702448+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:01:07.702448+0000","last_undegraded":"2023-09-23T23:01:07.702448+0000","last_fullsized":"2023-09-23T23:01:07.702448+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":289,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.665777+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:01:07.665777+0000","last_peered":"2023-09-23T23:01:07.665777+0000","last_clean":"2023-09-23T23:01:07.665777+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:01:07.665777+0000","last_undegraded":"2023-09-23T23:01:07.665777+0000","last_fullsized":"2023-09-23T23:01:07.665777+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"390'9552","reported_seq":9647,"reported_epoch":390,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:22.083205+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:01:22.083205+0000","last_peered":"2023-09-23T23:01:22.083205+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:01:22.083205+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:01:22.083205+0000","mapping_epoch":353,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166592512,"num_objects":2542,"num_object_clones":0,"num_object_copies":7626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9552,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2542}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":78,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701557+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:01:07.701557+0000","last_peered":"2023-09-23T23:01:07.701557+0000","last_clean":"2023-09-23T23:01:07.701557+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:01:07.701557+0000","last_undegraded":"2023-09-23T23:01:07.701557+0000","last_fullsized":"2023-09-23T23:01:07.701557+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":383,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690105+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:01:07.690105+0000","last_peered":"2023-09-23T23:01:07.690105+0000","last_clean":"2023-09-23T23:01:07.690105+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:01:07.690105+0000","last_undegraded":"2023-09-23T23:01:07.690105+0000","last_fullsized":"2023-09-23T23:01:07.690105+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"390'9662","reported_seq":9765,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:22.057196+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:01:22.057196+0000","last_peered":"2023-09-23T23:01:22.057196+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:01:22.057196+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"327'4000","ondisk_log_start":"327'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166461440,"num_objects":2540,"num_object_clones":0,"num_object_copies":7620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2540,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9662,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-1079}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.689450+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:01:07.689450+0000","last_peered":"2023-09-23T23:01:07.689450+0000","last_clean":"2023-09-23T23:01:07.689450+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:01:07.689450+0000","last_undegraded":"2023-09-23T23:01:07.689450+0000","last_fullsized":"2023-09-23T23:01:07.689450+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":400,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690400+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:01:07.690400+0000","last_peered":"2023-09-23T23:01:07.690400+0000","last_clean":"2023-09-23T23:01:07.690400+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:01:07.690400+0000","last_undegraded":"2023-09-23T23:01:07.690400+0000","last_fullsized":"2023-09-23T23:01:07.690400+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"390'9535","reported_seq":16182,"reported_epoch":390,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:22.183894+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:01:22.183894+0000","last_peered":"2023-09-23T23:01:22.183894+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:01:22.183894+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:01:22.183894+0000","mapping_epoch":354,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1332,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9535,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3268,"num_bytes_recovered":214171648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":1091}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":413,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.689918+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:01:07.689918+0000","last_peered":"2023-09-23T23:01:07.689918+0000","last_clean":"2023-09-23T23:01:07.689918+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:01:07.689918+0000","last_undegraded":"2023-09-23T23:01:07.689918+0000","last_fullsized":"2023-09-23T23:01:07.689918+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":391,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690460+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:01:07.690460+0000","last_peered":"2023-09-23T23:01:07.690460+0000","last_clean":"2023-09-23T23:01:07.690460+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:01:07.690460+0000","last_undegraded":"2023-09-23T23:01:07.690460+0000","last_fullsized":"2023-09-23T23:01:07.690460+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"390'9382","reported_seq":9477,"reported_epoch":390,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:22.108021+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:01:22.108021+0000","last_peered":"2023-09-23T23:01:22.108021+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:01:22.108021+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:01:22.108021+0000","mapping_epoch":353,"log_start":"326'3700","ondisk_log_start":"326'3700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169476096,"num_objects":2586,"num_object_clones":0,"num_object_copies":7758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9382,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2586}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":59,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701308+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:01:07.701308+0000","last_peered":"2023-09-23T23:01:07.701308+0000","last_clean":"2023-09-23T23:01:07.701308+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:01:07.701308+0000","last_undegraded":"2023-09-23T23:01:07.701308+0000","last_fullsized":"2023-09-23T23:01:07.701308+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":293,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.670250+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:01:07.670250+0000","last_peered":"2023-09-23T23:01:07.670250+0000","last_clean":"2023-09-23T23:01:07.670250+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:01:07.670250+0000","last_undegraded":"2023-09-23T23:01:07.670250+0000","last_fullsized":"2023-09-23T23:01:07.670250+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"390'9553","reported_seq":18834,"reported_epoch":390,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:22.217553+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:01:22.217553+0000","last_peered":"2023-09-23T23:01:22.217553+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:01:22.217553+0000","last_undegraded":"2023-09-23T23:01:22.217553+0000","last_fullsized":"2023-09-23T23:01:22.217553+0000","mapping_epoch":354,"log_start":"326'3800","ondisk_log_start":"326'3800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5753,"log_dups_size":0,"ondisk_log_size":5753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165347328,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2523,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9553,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":2523}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":297,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.670075+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:01:07.670075+0000","last_peered":"2023-09-23T23:01:07.670075+0000","last_clean":"2023-09-23T23:01:07.670075+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:01:07.670075+0000","last_undegraded":"2023-09-23T23:01:07.670075+0000","last_fullsized":"2023-09-23T23:01:07.670075+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":300,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.667954+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:01:07.667954+0000","last_peered":"2023-09-23T23:01:07.667954+0000","last_clean":"2023-09-23T23:01:07.667954+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:01:07.667954+0000","last_undegraded":"2023-09-23T23:01:07.667954+0000","last_fullsized":"2023-09-23T23:01:07.667954+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.667624+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:01:07.667624+0000","last_peered":"2023-09-23T23:01:07.667624+0000","last_clean":"2023-09-23T23:01:07.667624+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:01:07.667624+0000","last_undegraded":"2023-09-23T23:01:07.667624+0000","last_fullsized":"2023-09-23T23:01:07.667624+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"390'9548","reported_seq":9655,"reported_epoch":390,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:19.781821+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:01:19.781821+0000","last_peered":"2023-09-23T23:01:19.781821+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:01:19.781821+0000","last_undegraded":"2023-09-23T23:01:19.781821+0000","last_fullsized":"2023-09-23T23:01:19.781821+0000","mapping_epoch":359,"log_start":"327'4000","ondisk_log_start":"327'4000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168820736,"num_objects":2576,"num_object_clones":0,"num_object_copies":7728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2576,"num_objects_unfound":0,"num_objects_dirty":2576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9548,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2576}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":60,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.666520+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:01:07.666520+0000","last_peered":"2023-09-23T23:01:07.666520+0000","last_clean":"2023-09-23T23:01:07.666520+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:01:07.666520+0000","last_undegraded":"2023-09-23T23:01:07.666520+0000","last_fullsized":"2023-09-23T23:01:07.666520+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":410,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.691800+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:01:07.691800+0000","last_peered":"2023-09-23T23:01:07.691800+0000","last_clean":"2023-09-23T23:01:07.691800+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:01:07.691800+0000","last_undegraded":"2023-09-23T23:01:07.691800+0000","last_fullsized":"2023-09-23T23:01:07.691800+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":282,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.670151+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:01:07.670151+0000","last_peered":"2023-09-23T23:01:07.670151+0000","last_clean":"2023-09-23T23:01:07.670151+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:01:07.670151+0000","last_undegraded":"2023-09-23T23:01:07.670151+0000","last_fullsized":"2023-09-23T23:01:07.670151+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"390'9525","reported_seq":17695,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:21.165857+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:01:21.165857+0000","last_peered":"2023-09-23T23:01:21.165857+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:01:21.165857+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'900","ondisk_log_start":"321'900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8625,"log_dups_size":0,"ondisk_log_size":8625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166395904,"num_objects":2539,"num_object_clones":0,"num_object_copies":7617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9525,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2539}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":357,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.669746+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:01:07.669746+0000","last_peered":"2023-09-23T23:01:07.669746+0000","last_clean":"2023-09-23T23:01:07.669746+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:01:07.669746+0000","last_undegraded":"2023-09-23T23:01:07.669746+0000","last_fullsized":"2023-09-23T23:01:07.669746+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":234,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.669709+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:01:07.669709+0000","last_peered":"2023-09-23T23:01:07.669709+0000","last_clean":"2023-09-23T23:01:07.669709+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:01:07.669709+0000","last_undegraded":"2023-09-23T23:01:07.669709+0000","last_fullsized":"2023-09-23T23:01:07.669709+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":300,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.673129+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:01:07.673129+0000","last_peered":"2023-09-23T23:01:07.673129+0000","last_clean":"2023-09-23T23:01:07.673129+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:01:07.673129+0000","last_undegraded":"2023-09-23T23:01:07.673129+0000","last_fullsized":"2023-09-23T23:01:07.673129+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"390'9596","reported_seq":11488,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:17.801485+0000","last_change":"2023-09-23T22:58:34.203580+0000","last_active":"2023-09-23T23:01:17.801485+0000","last_peered":"2023-09-23T23:01:17.801485+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:01:17.801485+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":9596,"log_dups_size":0,"ondisk_log_size":9596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172228608,"num_objects":2628,"num_object_clones":0,"num_object_copies":7884,"num_objects_missing_on_primary":2561,"num_objects_missing":2561,"num_objects_degraded":2562,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":2628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9596,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":58720256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"0,1,6","objects":1195},{"shards":"0,6","objects":-2353}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":373,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.667771+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:01:07.667771+0000","last_peered":"2023-09-23T23:01:07.667771+0000","last_clean":"2023-09-23T23:01:07.667771+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:01:07.667771+0000","last_undegraded":"2023-09-23T23:01:07.667771+0000","last_fullsized":"2023-09-23T23:01:07.667771+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":486,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:07.668337+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:01:07.668337+0000","last_peered":"2023-09-23T23:01:07.668337+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:01:07.668337+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":59,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701498+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:01:07.701498+0000","last_peered":"2023-09-23T23:01:07.701498+0000","last_clean":"2023-09-23T23:01:07.701498+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:01:07.701498+0000","last_undegraded":"2023-09-23T23:01:07.701498+0000","last_fullsized":"2023-09-23T23:01:07.701498+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":300,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690639+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:01:07.690639+0000","last_peered":"2023-09-23T23:01:07.690639+0000","last_clean":"2023-09-23T23:01:07.690639+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:01:07.690639+0000","last_undegraded":"2023-09-23T23:01:07.690639+0000","last_fullsized":"2023-09-23T23:01:07.690639+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"390'9595","reported_seq":9694,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:19.876871+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:01:19.876871+0000","last_peered":"2023-09-23T23:01:19.876871+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:01:19.876871+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"327'4100","ondisk_log_start":"327'4100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5495,"log_dups_size":0,"ondisk_log_size":5495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9595,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1018}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":380,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.665493+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:01:07.665493+0000","last_peered":"2023-09-23T23:01:07.665493+0000","last_clean":"2023-09-23T23:01:07.665493+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:01:07.665493+0000","last_undegraded":"2023-09-23T23:01:07.665493+0000","last_fullsized":"2023-09-23T23:01:07.665493+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":351,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.665447+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:01:07.665447+0000","last_peered":"2023-09-23T23:01:07.665447+0000","last_clean":"2023-09-23T23:01:07.665447+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:01:07.665447+0000","last_undegraded":"2023-09-23T23:01:07.665447+0000","last_fullsized":"2023-09-23T23:01:07.665447+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":311,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.666090+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:01:07.666090+0000","last_peered":"2023-09-23T23:01:07.666090+0000","last_clean":"2023-09-23T23:01:07.666090+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:01:07.666090+0000","last_undegraded":"2023-09-23T23:01:07.666090+0000","last_fullsized":"2023-09-23T23:01:07.666090+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"390'9646","reported_seq":9760,"reported_epoch":390,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:22.229062+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:01:22.229062+0000","last_peered":"2023-09-23T23:01:22.229062+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:01:22.229062+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:01:22.229062+0000","mapping_epoch":354,"log_start":"326'3900","ondisk_log_start":"326'3900","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167575576,"num_objects":2558,"num_object_clones":0,"num_object_copies":7674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2558,"num_objects_unfound":0,"num_objects_dirty":2558,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9646,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":2558}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":326,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.666035+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:01:07.666035+0000","last_peered":"2023-09-23T23:01:07.666035+0000","last_clean":"2023-09-23T23:01:07.666035+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:01:07.666035+0000","last_undegraded":"2023-09-23T23:01:07.666035+0000","last_fullsized":"2023-09-23T23:01:07.666035+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":234,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.670230+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:01:07.670230+0000","last_peered":"2023-09-23T23:01:07.670230+0000","last_clean":"2023-09-23T23:01:07.670230+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:01:07.670230+0000","last_undegraded":"2023-09-23T23:01:07.670230+0000","last_fullsized":"2023-09-23T23:01:07.670230+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":203,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.667848+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:01:07.667848+0000","last_peered":"2023-09-23T23:01:07.667848+0000","last_clean":"2023-09-23T23:01:07.667848+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:01:07.667848+0000","last_undegraded":"2023-09-23T23:01:07.667848+0000","last_fullsized":"2023-09-23T23:01:07.667848+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"390'9661","reported_seq":16864,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:22.213549+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:01:22.213549+0000","last_peered":"2023-09-23T23:01:22.213549+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:01:22.213549+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"326'4000","ondisk_log_start":"326'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5257,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9661,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2999,"num_bytes_recovered":196542464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":1100}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":83,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.702910+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:01:07.702910+0000","last_peered":"2023-09-23T23:01:07.702910+0000","last_clean":"2023-09-23T23:01:07.702910+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:01:07.702910+0000","last_undegraded":"2023-09-23T23:01:07.702910+0000","last_fullsized":"2023-09-23T23:01:07.702910+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.691403+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:01:07.691403+0000","last_peered":"2023-09-23T23:01:07.691403+0000","last_clean":"2023-09-23T23:01:07.691403+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:01:07.691403+0000","last_undegraded":"2023-09-23T23:01:07.691403+0000","last_fullsized":"2023-09-23T23:01:07.691403+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":74,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701210+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:01:07.701210+0000","last_peered":"2023-09-23T23:01:07.701210+0000","last_clean":"2023-09-23T23:01:07.701210+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:01:07.701210+0000","last_undegraded":"2023-09-23T23:01:07.701210+0000","last_fullsized":"2023-09-23T23:01:07.701210+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"390'9564","reported_seq":10088,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:22.079452+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:01:22.079452+0000","last_peered":"2023-09-23T23:01:22.079452+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:01:22.079452+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"326'3900","ondisk_log_start":"326'3900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5664,"log_dups_size":0,"ondisk_log_size":5664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164233216,"num_objects":2506,"num_object_clones":0,"num_object_copies":7518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2506,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":2506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9564,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-1045}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":346,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.670513+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:01:07.670513+0000","last_peered":"2023-09-23T23:01:07.670513+0000","last_clean":"2023-09-23T23:01:07.670513+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:01:07.670513+0000","last_undegraded":"2023-09-23T23:01:07.670513+0000","last_fullsized":"2023-09-23T23:01:07.670513+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":293,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.665985+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:01:07.665985+0000","last_peered":"2023-09-23T23:01:07.665985+0000","last_clean":"2023-09-23T23:01:07.665985+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:01:07.665985+0000","last_undegraded":"2023-09-23T23:01:07.665985+0000","last_fullsized":"2023-09-23T23:01:07.665985+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":364,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.689990+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:01:07.689990+0000","last_peered":"2023-09-23T23:01:07.689990+0000","last_clean":"2023-09-23T23:01:07.689990+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:01:07.689990+0000","last_undegraded":"2023-09-23T23:01:07.689990+0000","last_fullsized":"2023-09-23T23:01:07.689990+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":328,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.669435+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:01:07.669435+0000","last_peered":"2023-09-23T23:01:07.669435+0000","last_clean":"2023-09-23T23:01:07.669435+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:01:07.669435+0000","last_undegraded":"2023-09-23T23:01:07.669435+0000","last_fullsized":"2023-09-23T23:01:07.669435+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":138,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.669861+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:01:07.669861+0000","last_peered":"2023-09-23T23:01:07.669861+0000","last_clean":"2023-09-23T23:01:07.669861+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:01:07.669861+0000","last_undegraded":"2023-09-23T23:01:07.669861+0000","last_fullsized":"2023-09-23T23:01:07.669861+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":199,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690022+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:01:07.690022+0000","last_peered":"2023-09-23T23:01:07.690022+0000","last_clean":"2023-09-23T23:01:07.690022+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:01:07.690022+0000","last_undegraded":"2023-09-23T23:01:07.690022+0000","last_fullsized":"2023-09-23T23:01:07.690022+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":413,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.691872+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:01:07.691872+0000","last_peered":"2023-09-23T23:01:07.691872+0000","last_clean":"2023-09-23T23:01:07.691872+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:01:07.691872+0000","last_undegraded":"2023-09-23T23:01:07.691872+0000","last_fullsized":"2023-09-23T23:01:07.691872+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":213,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.690491+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:01:07.690491+0000","last_peered":"2023-09-23T23:01:07.690491+0000","last_clean":"2023-09-23T23:01:07.690491+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:01:07.690491+0000","last_undegraded":"2023-09-23T23:01:07.690491+0000","last_fullsized":"2023-09-23T23:01:07.690491+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":300,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.689384+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:01:07.689384+0000","last_peered":"2023-09-23T23:01:07.689384+0000","last_clean":"2023-09-23T23:01:07.689384+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:01:07.689384+0000","last_undegraded":"2023-09-23T23:01:07.689384+0000","last_fullsized":"2023-09-23T23:01:07.689384+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":83,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701924+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:01:07.701924+0000","last_peered":"2023-09-23T23:01:07.701924+0000","last_clean":"2023-09-23T23:01:07.701924+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:01:07.701924+0000","last_undegraded":"2023-09-23T23:01:07.701924+0000","last_fullsized":"2023-09-23T23:01:07.701924+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":86,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.701599+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:01:07.701599+0000","last_peered":"2023-09-23T23:01:07.701599+0000","last_clean":"2023-09-23T23:01:07.701599+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:01:07.701599+0000","last_undegraded":"2023-09-23T23:01:07.701599+0000","last_fullsized":"2023-09-23T23:01:07.701599+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":332,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.666960+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:01:07.666960+0000","last_peered":"2023-09-23T23:01:07.666960+0000","last_clean":"2023-09-23T23:01:07.666960+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:01:07.666960+0000","last_undegraded":"2023-09-23T23:01:07.666960+0000","last_fullsized":"2023-09-23T23:01:07.666960+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":375,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T23:01:07.689127+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:01:07.689127+0000","last_peered":"2023-09-23T23:01:07.689127+0000","last_clean":"2023-09-23T23:01:07.689127+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:01:07.689127+0000","last_undegraded":"2023-09-23T23:01:07.689127+0000","last_fullsized":"2023-09-23T23:01:07.689127+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2670067736,"num_objects":40743,"num_object_clones":0,"num_object_copies":122229,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":24272,"num_objects_misplaced":30118,"num_objects_unfound":0,"num_objects_dirty":40743,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152855,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17219,"num_bytes_recovered":1037303808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8619503616,"data_stored":8619491400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100055,"ondisk_log_size":100055,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430620,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626844,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253157,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1834000,"kb_used_data":1109512,"kb_used_omap":9,"kb_used_meta":724470,"kb_avail":91911152,"statfs":{"total":95995035648,"available":94117019648,"internally_reserved":0,"allocated":1136140288,"data_stored":1134674465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":741858281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8250000000000002,"5min":1.258,"15min":0.65400000000000003},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":83.664000000000001,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":5.0039999999999996,"5min":1.5980000000000001,"15min":1.103},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":83.700999999999993,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.34699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.504,"15min":0.52000000000000002},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.93600000000000005,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.341,"5min":4.048,"15min":4.048},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56100000000000005,"15min":1.9950000000000001},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.8580000000000001,"5min":3.141,"15min":87.787999999999997},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.71699999999999997,"15min":2.339},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.19500000000000001},"max":{"1min":0.95499999999999996,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.56799999999999995}]},{"osd":4,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.6869999999999998,"15min":2.4529999999999998},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6669999999999998,"5min":151.036,"15min":151.036},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.581,"5min":2.7450000000000001,"15min":2.548},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":6.649,"5min":151.06899999999999,"15min":151.06899999999999},"last":1.204}]},{"osd":5,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.6219999999999999,"15min":2.319},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1720000000000002,"5min":116.098,"15min":116.098},"last":2.2440000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.5099999999999998,"15min":2.4809999999999999},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.9769999999999999,"5min":116.08499999999999,"15min":116.08499999999999},"last":2.1459999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.27,"15min":2.1480000000000001},"min":{"1min":0.40100000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.437},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.0820000000000001,"15min":1.8520000000000001},"min":{"1min":0.40999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.825,"5min":8.5370000000000008,"15min":79.328999999999994},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.04,"15min":1.6799999999999999},"min":{"1min":0.46300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.581,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.177,"15min":1.8919999999999999},"min":{"1min":0.44600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9860000000000002,"5min":10.676,"15min":20.902999999999999},"last":0.47399999999999998}]}]},{"osd":0,"up_from":353,"seq":1516123455528,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1932348,"kb_used_data":1758900,"kb_used_omap":37,"kb_used_meta":173402,"kb_avail":91812804,"statfs":{"total":95995035648,"available":94016311296,"internally_reserved":0,"allocated":1801113600,"data_stored":1799497644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":177563810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.59199999999999997,"15min":0.55700000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.7629999999999999,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64000000000000001,"15min":0.66100000000000003},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.8650000000000002,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":1.135}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.73299999999999998,"15min":0.76400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":2.9319999999999999,"5min":4.7000000000000002,"15min":4.7000000000000002},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.0270000000000001,"5min":3.2280000000000002,"15min":3.2280000000000002},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.623,"15min":0.51000000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.76800000000000002,"15min":0.86199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6499999999999999,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.878}]},{"osd":4,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":12.855,"5min":3.2290000000000001,"15min":1.649},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":183.148,"5min":183.148,"15min":183.148},"last":0.35399999999999998},{"interface":"front","average":{"1min":13.087999999999999,"5min":3.194,"15min":1.51},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":183.39500000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":1.2010000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.86799999999999999,"15min":0.74199999999999999},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4089999999999998,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.282,"5min":1.0009999999999999,"15min":0.999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.774,"5min":7.774,"15min":7.774},"last":1.1160000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.92900000000000005,"15min":0.86599999999999999},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.7309999999999999,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":1.6910000000000001},{"interface":"front","average":{"1min":1.242,"5min":0.86799999999999999,"15min":0.77200000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":6.375,"5min":6.375,"15min":6.375},"last":1.1020000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.159,"15min":1.2549999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.8439999999999999,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":2.5070000000000001},{"interface":"front","average":{"1min":1.008,"5min":1.2150000000000001,"15min":1.2569999999999999},"min":{"1min":0.49299999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":6.343,"5min":6.343,"15min":6.343},"last":2.4950000000000001}]}]},{"osd":2,"up_from":359,"seq":1541893259301,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":755132,"kb_used_data":631860,"kb_used_omap":83,"kb_used_meta":123180,"kb_avail":92990020,"statfs":{"total":95995035648,"available":95221780480,"internally_reserved":0,"allocated":647024640,"data_stored":645469767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":126136773},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.42799999999999999,"15min":0.379},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.1520000000000001,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.309},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.66300000000000003,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.2,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":2.0190000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.61699999999999999,"15min":0.64900000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.90500000000000003,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.1060000000000001,"15min":1.1379999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6269999999999998,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.498}]},{"osd":3,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.73499999999999999,"15min":0.71999999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7270000000000003,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":1.885},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.86299999999999999,"15min":0.83199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.2140000000000004,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":0.73999999999999999,"15min":0.623},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1159999999999997,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.7080000000000002,"5min":3.492,"15min":3.492},"last":0.32100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.98899999999999999,"15min":0.96599999999999997},"min":{"1min":0.36299999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.6029999999999998,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.101,"5min":0.76500000000000001,"15min":0.70899999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.0839999999999996,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.046,"15min":0.995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.5679999999999996,"5min":5.5679999999999996,"15min":5.5679999999999996},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.365,"5min":0.75800000000000001,"15min":0.65700000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":1.2929999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:00:28 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.1220000000000001,"15min":1.034},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":7.0499999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":2.173},{"interface":"front","average":{"1min":1.014,"5min":1.1459999999999999,"15min":1.1679999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.8520000000000003,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.53300000000000003}]}]},{"osd":3,"up_from":94,"seq":403726926105,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":535488,"kb_used_data":30840,"kb_used_omap":7,"kb_used_meta":504632,"kb_avail":93209664,"statfs":{"total":95995035648,"available":95446695936,"internally_reserved":0,"allocated":31580160,"data_stored":30016232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":516743784},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":1.5640000000000001,"15min":0.73399999999999999},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":122.289,"5min":122.289,"15min":122.289},"last":0.60699999999999998},{"interface":"front","average":{"1min":6.1609999999999996,"5min":1.5600000000000001,"15min":0.79100000000000004},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":0.19800000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.63200000000000001,"15min":2.3639999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":123.866},"last":0.75},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.73699999999999999,"15min":2.1549999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8010000000000002,"5min":6.8010000000000002,"15min":122.265},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:00:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.67800000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.1339999999999999,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.76600000000000001,"15min":0.75},"min":{"1min":0.24399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":3.8740000000000001,"5min":3.8740000000000001,"15min":3.8740000000000001},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":2.3519999999999999,"15min":2.0350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":64.406999999999996,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.78900000000000003},{"interface":"front","average":{"1min":4.008,"5min":2.5409999999999999,"15min":1.8999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":64.468000000000004,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2290000000000001,"15min":1.3560000000000001},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1980000000000004,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.548,"5min":1.218,"15min":1.827},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.246,"5min":10.246,"15min":11.951000000000001},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.597,"15min":1.7150000000000001},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0229999999999997,"5min":11.116,"15min":14.097},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4039999999999999,"15min":1.956},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.79,"5min":10.859,"15min":12.887},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.89400000000000002,"15min":2.0499999999999998},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8449999999999998,"5min":4.8449999999999998,"15min":91.757999999999996},"last":0.505},{"interface":"front","average":{"1min":1.413,"5min":0.98099999999999998,"15min":1.9390000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":8.5830000000000002,"5min":8.5830000000000002,"15min":91.709999999999994},"last":0.81999999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019302,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2897588,"kb_used_data":1981868,"kb_used_omap":0,"kb_used_meta":915648,"kb_avail":90847564,"statfs":{"total":95995035648,"available":93027905536,"internally_reserved":0,"allocated":2029432832,"data_stored":2028630593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":937623552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":2.0710000000000002},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":4.3869999999999996}]},{"osd":1,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.4380000000000002,"15min":3.5270000000000001},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":3.081,"5min":87.325000000000003,"15min":132.304},"last":4.4509999999999996},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.6320000000000001,"15min":3.4849999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.253},"max":{"1min":3.6120000000000001,"5min":87.572000000000003,"15min":132.453},"last":2.1219999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.119,"15min":1.0529999999999999},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":2.0960000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":0.81100000000000005,"15min":0.749},"min":{"1min":0.378,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.9550000000000001,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":4.4770000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.484,"15min":3.9009999999999998},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32400000000000001},"max":{"1min":4.0830000000000002,"5min":10.337999999999999,"15min":164.453},"last":2.145},{"interface":"front","average":{"1min":1.784,"5min":1.4410000000000001,"15min":3.8929999999999998},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0039999999999996,"5min":12.805999999999999,"15min":167.05199999999999},"last":4.5039999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":3.0670000000000002,"15min":1.643},"min":{"1min":0.318,"5min":0.214,"15min":0.214},"max":{"1min":3.129,"5min":148.61600000000001,"15min":148.61600000000001},"last":4.5289999999999999},{"interface":"front","average":{"1min":1.335,"5min":3.4940000000000002,"15min":1.78},"min":{"1min":0.318,"5min":0.252,"15min":0.248},"max":{"1min":3.7469999999999999,"5min":151.905,"15min":151.905},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.655,"15min":1.181},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":3.5680000000000001,"5min":16.943000000000001,"15min":16.943000000000001},"last":2.4950000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.4379999999999999,"15min":1.113},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.1859999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":4.423}]},{"osd":7,"last update":"Sat Sep 23 23:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.47,"15min":1.127},"min":{"1min":0.40899999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.045999999999999,"5min":12.84,"15min":12.84},"last":4.5549999999999997},{"interface":"front","average":{"1min":1.615,"5min":1.5089999999999999,"15min":1.0740000000000001},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.227,"5min":10.587,"15min":10.587},"last":2.0379999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823076,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128084,"kb_used_data":1232524,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":91617068,"statfs":{"total":95995035648,"available":93815877632,"internally_reserved":0,"allocated":1262104576,"data_stored":1261330125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.878,"15min":0.69599999999999995},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.5709999999999997,"5min":14.42,"15min":14.42},"last":2.1699999999999999},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2430000000000001,"15min":1.135},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3159999999999998,"5min":14.496,"15min":14.496},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.081,"15min":2.1909999999999998},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.24299999999999999},"max":{"1min":2.6040000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.314},{"interface":"front","average":{"1min":1.857,"5min":1.6200000000000001,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":7.0739999999999998,"5min":10.353,"15min":110.89},"last":1.3740000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.82999999999999996,"15min":0.81000000000000005},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.601,"5min":6.601,"15min":6.601},"last":2.1459999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7669999999999999,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.34399999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.216,"15min":4.7009999999999996},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2710000000000008,"5min":9.2710000000000008,"15min":257.05000000000001},"last":2.1899999999999999},{"interface":"front","average":{"1min":1.474,"5min":1.3859999999999999,"15min":4.7590000000000003},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.3399999999999999,"5min":12.185,"15min":257.35000000000002},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":2.8580000000000001,"15min":1.5109999999999999},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8440000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":1.417},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.5710000000000002,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.524,"5min":114.908,"15min":114.908},"last":2.21}]},{"osd":6,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.512,"15min":1.2270000000000001},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.4630000000000001,"5min":11.121,"15min":12.615},"last":1.4430000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.2370000000000001,"15min":1.1519999999999999},"min":{"1min":0.41299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.4909999999999997,"5min":12.137,"15min":12.74},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.2350000000000001,"15min":1.0629999999999999},"min":{"1min":0.45000000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.8370000000000002,"5min":12.23,"15min":12.23},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.5830000000000002,"5min":1.7490000000000001,"15min":1.2170000000000001},"min":{"1min":0.378,"5min":0.31,"15min":0.31},"max":{"1min":7.6719999999999997,"5min":11.112,"15min":11.112},"last":1.4690000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1781665792,"data_stored":1781661720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1129840640,"data_stored":1129840640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":641662976,"data_stored":641662976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":26935296,"data_stored":26935296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2013728768,"data_stored":2013724696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1255800832,"data_stored":1255800832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1769869312,"data_stored":1769865240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:01:25.286 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:01:25.302 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:25.325 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:25.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:25.439 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:25.477 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:25.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:25 smithi052 ceph-mon[180979]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:25.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:25 smithi052 ceph-mon[180979]: pgmap v1176: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 24166/122001 objects degraded (19.808%); 32444/122001 objects misplaced (26.593%); 2.6 MiB/s, 71 objects/s recovering 2023-09-23T23:01:25.829 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 24166/122001 objects degraded (19.808%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:25.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:25 smithi052 ceph-mon[187743]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:25.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:25 smithi052 ceph-mon[187743]: pgmap v1176: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 24166/122001 objects degraded (19.808%); 32444/122001 objects misplaced (26.593%); 2.6 MiB/s, 71 objects/s recovering 2023-09-23T23:01:25.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 24166/122001 objects degraded (19.808%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:26.063 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:26.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:26.167 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:26.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:25 smithi151 ceph-mon[159210]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:26.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:25 smithi151 ceph-mon[159210]: pgmap v1176: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 24166/122001 objects degraded (19.808%); 32444/122001 objects misplaced (26.593%); 2.6 MiB/s, 71 objects/s recovering 2023-09-23T23:01:26.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 24166/122001 objects degraded (19.808%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:26.207 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:26.741 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:01:26.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1963411823' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:26.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:26.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1963411823' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:26.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1963411823' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:27.381 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:27.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:27.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:27.535 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:27.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: pgmap v1177: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.4 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 23942/119625 objects degraded (20.014%); 32234/119625 objects misplaced (26.946%); 2.6 MiB/s, 60 objects/s recovering 2023-09-23T23:01:27.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:27.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:01:27.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: osdmap e391: 8 total, 8 up, 6 in 2023-09-23T23:01:27.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.741 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[180979]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.910 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:27.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:27.924 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:27.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: pgmap v1177: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.4 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 23942/119625 objects degraded (20.014%); 32234/119625 objects misplaced (26.946%); 2.6 MiB/s, 60 objects/s recovering 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: osdmap e391: 8 total, 8 up, 6 in 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: failed to encode map e391 with expected crc 2023-09-23T23:01:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:27 smithi052 ceph-mon[187743]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.017 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:28.079 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: pgmap v1177: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.4 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 23942/119625 objects degraded (20.014%); 32234/119625 objects misplaced (26.946%); 2.6 MiB/s, 60 objects/s recovering 2023-09-23T23:01:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:01:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: osdmap e391: 8 total, 8 up, 6 in 2023-09-23T23:01:28.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:27 smithi151 ceph-mon[159210]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.542 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:28.543 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:28.603 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:28.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:28.708 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:28.725 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:28.753 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:28.754 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:28.777 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:28.844 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:28.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:28.914 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:28 smithi052 ceph-mon[180979]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.915 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:28 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4186998521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:28.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:28 smithi052 ceph-mon[187743]: failed to encode map e391 with expected crc 2023-09-23T23:01:28.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:28 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4186998521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:28.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:28.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:28.920 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:28.921 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:28.921 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:28.922 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.922 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.922 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.923 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.923 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.923 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.924 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:01:28.924 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:01:29.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:29.044 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:28 smithi151 ceph-mon[159210]: failed to encode map e391 with expected crc 2023-09-23T23:01:29.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:28 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4186998521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:29.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:29 smithi052 ceph-mon[180979]: pgmap v1179: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.4 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 23506/116703 objects degraded (20.142%); 32011/116703 objects misplaced (27.429%); 2.7 MiB/s, 51 objects/s recovering 2023-09-23T23:01:29.832 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:29 smithi052 ceph-mon[187743]: pgmap v1179: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.4 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 23506/116703 objects degraded (20.142%); 32011/116703 objects misplaced (27.429%); 2.7 MiB/s, 51 objects/s recovering 2023-09-23T23:01:30.000 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:01:30.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:29 smithi151 ceph-mon[159210]: pgmap v1179: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.4 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 23506/116703 objects degraded (20.142%); 32011/116703 objects misplaced (27.429%); 2.7 MiB/s, 51 objects/s recovering 2023-09-23T23:01:30.286 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:30.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:30.388 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:01:30.392 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:30.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:30.574 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:30.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:30.685 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:30.725 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:30.741 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:30.843 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:30.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:30.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:30.947 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:30.979 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:31.097 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:31.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:31.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:31.205 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:31.246 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:31.455 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:31.459 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:31 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 23506/116703 objects degraded (20.142%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:31.459 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:31 smithi052 ceph-mon[180979]: pgmap v1180: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 23409/115071 objects degraded (20.343%); 31780/115071 objects misplaced (27.618%); 2.7 MiB/s, 51 objects/s recovering 2023-09-23T23:01:31.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:31.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:31.460 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:31 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 23506/116703 objects degraded (20.142%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:31.460 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:31 smithi052 ceph-mon[187743]: pgmap v1180: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 23409/115071 objects degraded (20.343%); 31780/115071 objects misplaced (27.618%); 2.7 MiB/s, 51 objects/s recovering 2023-09-23T23:01:31.465 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:31.465 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23409/115071 objects degraded (20.343%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1624,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":391,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":38357,"data_bytes":2512721848,"bytes_used":9933680640,"bytes_avail":566036533248,"bytes_total":575970213888,"degraded_objects":23409,"degraded_total":115071,"degraded_ratio":0.20343092525484266,"misplaced_objects":31780,"misplaced_total":115071,"misplaced_ratio":0.27617731661322142,"recovering_objects_per_sec":51,"recovering_bytes_per_sec":2869506,"recovering_keys_per_sec":0,"num_objects_recovered":511,"num_bytes_recovered":28704768,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":351},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (5m)\n [======================......] (remaining: 71s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:01:31.561 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:31.608 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:31 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 23506/116703 objects degraded (20.142%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:31.608 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:31 smithi151 ceph-mon[159210]: pgmap v1180: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 23409/115071 objects degraded (20.343%); 31780/115071 objects misplaced (27.618%); 2.7 MiB/s, 51 objects/s recovering 2023-09-23T23:01:31.615 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:31.674 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:31.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:31.784 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:31.822 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:32.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:01:32.474 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1384884010' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:32.475 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1384884010' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:32.517 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:32.552 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:32.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:32.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:32.636 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:01:32.636 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:01:32.662 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:32.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1384884010' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:32.689 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:32.742 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:32.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:32.851 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:32.901 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:33.217 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:33.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:33.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:33.324 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:33.354 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:33.387 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:33 smithi052 ceph-mon[187743]: pgmap v1181: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 22744/111978 objects degraded (20.311%); 31569/111978 objects misplaced (28.192%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:01:33.387 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:33 smithi052 ceph-mon[180979]: pgmap v1181: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 22744/111978 objects degraded (20.311%); 31569/111978 objects misplaced (28.192%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:01:33.387 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:01:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:01:33] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-09-23T23:01:33.391 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:33.520 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:33.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:33.624 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:33.650 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:33 smithi151 ceph-mon[159210]: pgmap v1181: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 22744/111978 objects degraded (20.311%); 31569/111978 objects misplaced (28.192%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T23:01:33.653 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:33.699 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:33.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:33.808 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:33.839 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:34.076 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:34.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:34.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:34.135 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:34.180 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:34.208 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:34.261 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:34.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:34.302 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:01:34.368 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:34.395 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4086324313' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:34.395 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4086324313' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:34.412 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:34.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4086324313' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:35.378 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:35.378 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: failed to encode map e392 with expected crc 2023-09-23T23:01:35.379 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4086324313' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:35.379 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: osdmap e392: 8 total, 8 up, 6 in 2023-09-23T23:01:35.379 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:35.379 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:35.379 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: pgmap v1183: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 22854/111648 objects degraded (20.470%); 31569/111648 objects misplaced (28.275%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T23:01:35.379 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 22744/111978 objects degraded (20.311%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:35.380 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: failed to encode map e392 with expected crc 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4086324313' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: osdmap e392: 8 total, 8 up, 6 in 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: pgmap v1183: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 22854/111648 objects degraded (20.470%); 31569/111648 objects misplaced (28.275%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T23:01:35.381 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 22744/111978 objects degraded (20.311%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:35.435 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:35.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:35.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:35.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:35.571 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: failed to encode map e392 with expected crc 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4086324313' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: osdmap e392: 8 total, 8 up, 6 in 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:35.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: pgmap v1183: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 22854/111648 objects degraded (20.470%); 31569/111648 objects misplaced (28.275%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T23:01:35.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 22744/111978 objects degraded (20.311%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:36.153 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:01:36.153 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:01:36.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:01:36.153 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:01:36.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.153 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110891 9.3 9:c27b0003:::benchmark_data_smithi052_402718_object21250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.055634+0000", 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 494.93590189499997, 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25767365199999998, 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110891 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.154 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055634+0000", 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055633+0000", 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055637+0000", 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.126e-06 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055645+0000", 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9850000000000006e-06 2023-09-23T23:01:36.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055650+0000", 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4990000000000002e-06 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055758+0000", 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010737 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.055921+0000", 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163437 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.056098+0000", 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000176563 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.059728+0000", 2023-09-23T23:01:36.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036304570000000001 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.062797+0000", 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00306851 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312384+0000", 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.249587273 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312438+0000", 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4354999999999997e-05 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313307+0000", 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00086921899999999996 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110899 9.3 9:c9f9c7b0:::benchmark_data_smithi052_402718_object21258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063297+0000", 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 494.92823858499997, 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29968218299999999, 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110899 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063297+0000", 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063299+0000", 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5469999999999999e-06 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063306+0000", 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6889999999999998e-06 2023-09-23T23:01:36.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063312+0000", 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6489999999999996e-06 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066300+0000", 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029886909999999999 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066461+0000", 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016098600000000001 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.066620+0000", 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015845099999999999 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312307+0000", 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24568741599999999 2023-09-23T23:01:36.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312349+0000", 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1686e-05 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362127+0000", 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049778356000000003 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362181+0000", 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3421999999999998e-05 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.362979+0000", 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079871 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.167 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110900 9.3 9:cfcfdec5:::benchmark_data_smithi052_402718_object21259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063317+0000", 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 494.92821850799999, 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.300308513, 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110900 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063317+0000", 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063319+0000", 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.525e-06 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063325+0000", 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5999999999999997e-06 2023-09-23T23:01:36.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063329+0000", 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5170000000000001e-06 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069300+0000", 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059708189999999996 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069454+0000", 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015409899999999999 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.069610+0000", 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000155332 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.312314+0000", 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24270393900000001 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.313359+0000", 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010457520000000001 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363082+0000", 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.049722865999999998 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363133+0000", 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0757999999999998e-05 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363626+0000", 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000492659 2023-09-23T23:01:36.174 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110905 9.3 9:cdd24a36:::benchmark_data_smithi052_402718_object21264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.064202+0000", 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 494.92733386899999, 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29997646300000003, 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:36.175 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110905 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064202+0000", 2023-09-23T23:01:36.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064201+0000", 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064206+0000", 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7419999999999997e-06 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064217+0000", 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.094e-05 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.064224+0000", 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2529999999999998e-06 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072314+0000", 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0080899820000000008 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072465+0000", 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015051600000000001 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.309128+0000", 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23666314399999999 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.314036+0000", 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049078869999999997 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.320379+0000", 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00634324 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:36.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363720+0000", 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.043340956999999999 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.363771+0000", 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1734999999999998e-05 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.364178+0000", 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00040673700000000001 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.181 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44617 10.2 e325/317 10:42f612af:::benchmark_data_smithi052_461816_object44616:head v 325'2757, mlcod=325'2755)", 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.230235+0000", 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 339.76130103700001, 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.097530256999999995, 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230234+0000", 2023-09-23T23:01:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230284+0000", 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0538999999999998e-05 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230294+0000", 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2580000000000001e-06 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230301+0000", 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0979999999999996e-06 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230348+0000", 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6904999999999999e-05 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230370+0000", 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2036999999999999e-05 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.231896+0000", 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015261840000000001 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327765+0000", 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095868996999999997 2023-09-23T23:01:36.186 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44626 10.2 e325/317 10:4fdb9bab:::benchmark_data_smithi052_461816_object44625:head v 325'2758, mlcod=325'2755)", 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.234607+0000", 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 339.75692906400002, 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096288367, 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234606+0000", 2023-09-23T23:01:36.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234684+0000", 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8349999999999994e-05 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327708+0000", 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.093023764999999994 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327737+0000", 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9223000000000001e-05 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327860+0000", 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122982 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327916+0000", 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5405999999999998e-05 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330864+0000", 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002948492 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330895+0000", 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0898999999999997e-05 2023-09-23T23:01:36.191 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44632 10.b e325/317 10:d9603e28:::benchmark_data_smithi052_461816_object44631:head v 325'2853, mlcod=325'2851)", 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.237428+0000", 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 339.75410748600001, 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096280273999999999, 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237427+0000", 2023-09-23T23:01:36.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237481+0000", 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3566000000000002e-05 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327715+0000", 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.090234522999999997 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327736+0000", 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0746000000000001e-05 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327869+0000", 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013274200000000001 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327926+0000", 2023-09-23T23:01:36.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7645999999999997e-05 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333664+0000", 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057375569999999999 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333708+0000", 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4566e-05 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.196 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71910 10.b e327/317 10:da00891c:::benchmark_data_smithi052_461816_object71909:head v 327'4562, mlcod=327'4560)", 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.838808+0000", 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 318.15272789900001, 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070775292000000004, 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838807+0000", 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838881+0000", 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3671999999999995e-05 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838927+0000", 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6146999999999997e-05 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838934+0000", 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3769999999999998e-06 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838979+0000", 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.494e-05 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838998+0000", 2023-09-23T23:01:36.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8782000000000001e-05 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909541+0000", 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070543146000000001 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909583+0000", 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1967e-05 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.201 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71913 10.1 e327/317 10:868eb1a2:::benchmark_data_smithi052_461816_object71912:head v 327'4350, mlcod=327'4347)", 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.840731+0000", 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 318.15080457099998, 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.066217576, 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840730+0000", 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840777+0000", 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6655e-05 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061592056999999999 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902407+0000", 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7666000000000002e-05 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902538+0000", 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000131793 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902555+0000", 2023-09-23T23:01:36.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6888000000000002e-05 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906899+0000", 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043436869999999997 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906949+0000", 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9629999999999997e-05 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.206 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71918 10.b e327/317 10:d9e3a463:::benchmark_data_smithi052_461816_object71917:head v 327'4563, mlcod=327'4560)", 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.844519+0000", 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 318.14701657199998, 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074161447000000005, 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844518+0000", 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844600+0000", 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2000000000000001e-05 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.057768742999999997 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902406+0000", 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7252999999999997e-05 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.904983+0000", 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025770709999999998 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.905005+0000", 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2028e-05 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918640+0000", 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01363467 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918681+0000", 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0522000000000003e-05 2023-09-23T23:01:36.211 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104103 10.c e340/329 10:3a122c6e:::benchmark_data_smithi052_461816_object7302:head v 340'6509, mlcod=0'0)", 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.450864+0000", 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 261.54067158700002, 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083293366999999993, 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450864+0000", 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450864+0000", 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450862+0000", 2023-09-23T23:01:36.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450879+0000", 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7056000000000001e-05 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450910+0000", 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0960000000000002e-05 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450929+0000", 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.853e-05 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450997+0000", 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8225000000000005e-05 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451014+0000", 2023-09-23T23:01:36.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6988999999999999e-05 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534114+0000", 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083099808999999997 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534157+0000", 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3625000000000003e-05 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.216 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104104 10.c e340/329 10:3017b5bb:::benchmark_data_smithi052_461816_object7303:head v 340'6510, mlcod=0'0)", 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.451247+0000", 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 261.54028892700001, 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.088953937999999996, 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451247+0000", 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451247+0000", 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451246+0000", 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451251+0000", 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3050000000000003e-06 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451258+0000", 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9550000000000003e-06 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451263+0000", 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1240000000000004e-06 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457336+0000", 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060733820000000004 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457354+0000", 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7779999999999999e-05 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540161+0000", 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.082807243000000003 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540201+0000", 2023-09-23T23:01:36.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9532e-05 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104107 10.4 e340/317 10:2627ee68:::benchmark_data_smithi052_461816_object7306:head v 340'6525, mlcod=340'6521)", 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.451379+0000", 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 261.54015696900001, 2023-09-23T23:01:36.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.082812793999999995, 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451379+0000", 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451379+0000", 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451378+0000", 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451384+0000", 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4760000000000003e-06 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451394+0000", 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9410000000000002e-06 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451400+0000", 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.015e-06 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457323+0000", 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059230339999999998 2023-09-23T23:01:36.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457338+0000", 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4263e-05 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534163+0000", 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.076825685000000005 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534191+0000", 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8124999999999999e-05 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104106 10.c e340/329 10:3b7c91e7:::benchmark_data_smithi052_461816_object7305:head v 340'6511, mlcod=0'0)", 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.457021+0000", 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 261.53451431500002, 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083208802999999998, 2023-09-23T23:01:36.227 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457021+0000", 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457021+0000", 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457020+0000", 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457029+0000", 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2579999999999995e-06 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457043+0000", 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4239e-05 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457063+0000", 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9656999999999998e-05 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.463087+0000", 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060247390000000003 2023-09-23T23:01:36.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.463117+0000", 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9611000000000002e-05 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540206+0000", 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.077089550000000007 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540230+0000", 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3666999999999999e-05 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104110 10.c e340/329 10:3405d415:::benchmark_data_smithi052_461816_object7309:head v 340'6512, mlcod=0'0)", 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.460318+0000", 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 261.53121803200003, 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.079936283999999996, 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.232 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460318+0000", 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460318+0000", 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460316+0000", 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460327+0000", 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0478e-05 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460337+0000", 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5440000000000004e-06 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460342+0000", 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2249999999999999e-06 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.520173+0000", 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.059831670000000003 2023-09-23T23:01:36.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.520196+0000", 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2121e-05 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540232+0000", 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020036824000000002 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540254+0000", 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1539000000000002e-05 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104853 10.c e340/329 10:3b1f5b6f:::benchmark_data_smithi052_461816_object8052:head v 340'6559, mlcod=0'0)", 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 260.06695437399998, 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080972484999999997, 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.237 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924585+0000", 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0080000000000004e-06 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924591+0000", 2023-09-23T23:01:36.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.942e-06 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924595+0000", 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4329999999999997e-06 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924640+0000", 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4134999999999997e-05 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924664+0000", 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4360999999999999e-05 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:16.005514+0000", 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080849657000000005 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:16.005554+0000", 2023-09-23T23:01:36.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0201999999999999e-05 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104855 10.4 e340/317 10:2c6443b8:::benchmark_data_smithi052_461816_object8054:head v 340'6568, mlcod=340'6566)", 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924876+0000", 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 260.066659813, 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.075044514000000007, 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.242 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924876+0000", 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924876+0000", 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924875+0000", 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924881+0000", 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.57e-06 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924890+0000", 2023-09-23T23:01:36.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1330000000000005e-06 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924896+0000", 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924934+0000", 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7849000000000003e-05 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924952+0000", 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8175000000000001e-05 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.999878+0000", 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074926648999999998 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.999920+0000", 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1924e-05 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104856 10.2 e340/317 10:4d1d554a:::benchmark_data_smithi052_461816_object8055:head v 340'6546, mlcod=340'6544)", 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 260.06656249100001, 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070607910999999995, 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.247 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924978+0000", 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9599999999999999e-06 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924986+0000", 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924992+0000", 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5219999999999998e-06 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.986453+0000", 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061461233999999997 2023-09-23T23:01:36.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.986477+0000", 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3586000000000001e-05 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.995543+0000", 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090668009999999993 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.995581+0000", 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7732999999999999e-05 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113501 10.4 e344/343 10:2471dee2:::benchmark_data_smithi052_461816_object16700:head v 344'7088, mlcod=344'7086)", 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.300805+0000", 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.69073040000001, 2023-09-23T23:01:36.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.114329005, 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300805+0000", 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300805+0000", 2023-09-23T23:01:36.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300804+0000", 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300813+0000", 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.312e-06 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300829+0000", 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5485000000000002e-05 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300839+0000", 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0029e-05 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300894+0000", 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5551999999999999e-05 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405126+0000", 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.104231453 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.415005+0000", 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098791069999999998 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.415134+0000", 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012936600000000001 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113493 10.c e344/343 10:3a2fa64b:::benchmark_data_smithi052_461816_object16692:head v 344'7088, mlcod=0'0)", 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.302714+0000", 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 243.68882209, 2023-09-23T23:01:36.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.115107467, 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302714+0000", 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302714+0000", 2023-09-23T23:01:36.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302713+0000", 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302720+0000", 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9609999999999999e-06 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405115+0000", 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10239490699999999 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405128+0000", 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3213e-05 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:36.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405167+0000", 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8720000000000002e-05 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405189+0000", 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2401000000000001e-05 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.417788+0000", 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012598971 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:36.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.417821+0000", 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3126000000000001e-05 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:36.262 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:01:36.600 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:36.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:36.704 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:36.739 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:36.786 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:36.790 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:36 smithi151 ceph-mon[159210]: failed to encode map e392 with expected crc 2023-09-23T23:01:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:36.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:36.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:36 smithi052 ceph-mon[180979]: failed to encode map e392 with expected crc 2023-09-23T23:01:36.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:36 smithi052 ceph-mon[187743]: failed to encode map e392 with expected crc 2023-09-23T23:01:36.893 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:36.936 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:37.143 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:37.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:37.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:37.187 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:01:37.206 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:01:37.250 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:37.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:37.601 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:37.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:37.705 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:37.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:37 smithi052 ceph-mon[187743]: pgmap v1184: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 22692/109800 objects degraded (20.667%); 31398/109800 objects misplaced (28.596%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T23:01:37.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:37 smithi052 ceph-mon[180979]: pgmap v1184: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 22692/109800 objects degraded (20.667%); 31398/109800 objects misplaced (28.596%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T23:01:37.761 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:37.777 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:37.803 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:37.836 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:37.865 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:37.893 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:37.918 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:37.926 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:37.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:37.931 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:37 smithi151 ceph-mon[159210]: pgmap v1184: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 22692/109800 objects degraded (20.667%); 31398/109800 objects misplaced (28.596%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T23:01:37.967 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:37.968 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:38.014 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:38.015 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:38.016 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:01:38.022 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.022 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.022 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.023 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.023 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.023 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.023 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.023 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:01:38.030 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:38.078 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:38.427 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:38.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:38.534 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:38.564 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3372430487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:38.564 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3372430487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:38.565 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:38.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3372430487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:39.592 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:39.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:39.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:39.696 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:39.742 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:39.809 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:39.811 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:39 smithi052 ceph-mon[180979]: from='client.36543 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:39.812 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:39 smithi052 ceph-mon[180979]: pgmap v1185: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 22350/107337 objects degraded (20.822%); 31226/107337 objects misplaced (29.092%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:01:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:39.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:39.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:39 smithi052 ceph-mon[187743]: from='client.36543 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:39.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:39 smithi052 ceph-mon[187743]: pgmap v1185: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 22350/107337 objects degraded (20.822%); 31226/107337 objects misplaced (29.092%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:01:39.913 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:39.943 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:39.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:39 smithi151 ceph-mon[159210]: from='client.36543 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:39.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:39 smithi151 ceph-mon[159210]: pgmap v1185: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 312 op/s; 22350/107337 objects degraded (20.822%); 31226/107337 objects misplaced (29.092%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:01:40.528 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:40.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:40.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:40.636 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:40.663 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:40.710 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:40.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:40.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:40.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 22350/107337 objects degraded (20.822%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:40.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 22350/107337 objects degraded (20.822%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:40.820 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:40.848 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1184,"stamp":"2023-09-23T23:01:36.595594+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2397575096,"num_objects":36600,"num_object_clones":0,"num_object_copies":109800,"num_objects_missing_on_primary":2555,"num_objects_missing":2555,"num_objects_degraded":22692,"num_objects_misplaced":31398,"num_objects_unfound":0,"num_objects_dirty":36600,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":157282,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17709,"num_bytes_recovered":1067137980,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100416,"ondisk_log_size":100416,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9445972,"kb_used_data":6112932,"kb_used_omap":137,"kb_used_meta":3332790,"kb_avail":553024940,"statfs":{"total":575970213888,"available":566297538560,"internally_reserved":0,"allocated":6259642368,"data_stored":6251814190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3412777144},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-214630400,"num_objects":-3275,"num_object_clones":0,"num_object_copies":-9825,"num_objects_missing_on_primary":-5,"num_objects_missing":-5,"num_objects_degraded":-1250,"num_objects_misplaced":-836,"num_objects_unfound":0,"num_objects_dirty":-3275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":453,"num_bytes_recovered":29687808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.863509"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":403,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715419+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:01:17.715419+0000","last_peered":"2023-09-23T23:01:17.715419+0000","last_clean":"2023-09-23T23:01:17.715419+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:01:17.715419+0000","last_undegraded":"2023-09-23T23:01:17.715419+0000","last_fullsized":"2023-09-23T23:01:17.715419+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757745+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:01:26.757745+0000","last_peered":"2023-09-23T23:01:26.757745+0000","last_clean":"2023-09-23T23:01:26.757745+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:01:26.757745+0000","last_undegraded":"2023-09-23T23:01:26.757745+0000","last_fullsized":"2023-09-23T23:01:26.757745+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":60,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756670+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:01:26.756670+0000","last_peered":"2023-09-23T23:01:26.756670+0000","last_clean":"2023-09-23T23:01:26.756670+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:01:26.756670+0000","last_undegraded":"2023-09-23T23:01:26.756670+0000","last_fullsized":"2023-09-23T23:01:26.756670+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":389,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322422+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:01:34.322422+0000","last_peered":"2023-09-23T23:01:34.322422+0000","last_clean":"2023-09-23T23:01:34.322422+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:01:34.322422+0000","last_undegraded":"2023-09-23T23:01:34.322422+0000","last_fullsized":"2023-09-23T23:01:34.322422+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":76,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.755940+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:01:26.755940+0000","last_peered":"2023-09-23T23:01:26.755940+0000","last_clean":"2023-09-23T23:01:26.755940+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:01:26.755940+0000","last_undegraded":"2023-09-23T23:01:26.755940+0000","last_fullsized":"2023-09-23T23:01:26.755940+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.737314+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:01:26.737314+0000","last_peered":"2023-09-23T23:01:26.737314+0000","last_clean":"2023-09-23T23:01:26.737314+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:01:26.737314+0000","last_undegraded":"2023-09-23T23:01:26.737314+0000","last_fullsized":"2023-09-23T23:01:26.737314+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":327,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.758411+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:01:26.758411+0000","last_peered":"2023-09-23T23:01:26.758411+0000","last_clean":"2023-09-23T23:01:26.758411+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:01:26.758411+0000","last_undegraded":"2023-09-23T23:01:26.758411+0000","last_fullsized":"2023-09-23T23:01:26.758411+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":70,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756366+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:01:26.756366+0000","last_peered":"2023-09-23T23:01:26.756366+0000","last_clean":"2023-09-23T23:01:26.756366+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:01:26.756366+0000","last_undegraded":"2023-09-23T23:01:26.756366+0000","last_fullsized":"2023-09-23T23:01:26.756366+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":288,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.755331+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:01:26.755331+0000","last_peered":"2023-09-23T23:01:26.755331+0000","last_clean":"2023-09-23T23:01:26.755331+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:01:26.755331+0000","last_undegraded":"2023-09-23T23:01:26.755331+0000","last_fullsized":"2023-09-23T23:01:26.755331+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.755850+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:01:26.755850+0000","last_peered":"2023-09-23T23:01:26.755850+0000","last_clean":"2023-09-23T23:01:26.755850+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:01:26.755850+0000","last_undegraded":"2023-09-23T23:01:26.755850+0000","last_fullsized":"2023-09-23T23:01:26.755850+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":81,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756209+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:01:26.756209+0000","last_peered":"2023-09-23T23:01:26.756209+0000","last_clean":"2023-09-23T23:01:26.756209+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:01:26.756209+0000","last_undegraded":"2023-09-23T23:01:26.756209+0000","last_fullsized":"2023-09-23T23:01:26.756209+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.751315+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:01:26.751315+0000","last_peered":"2023-09-23T23:01:26.751315+0000","last_clean":"2023-09-23T23:01:26.751315+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:01:26.751315+0000","last_undegraded":"2023-09-23T23:01:26.751315+0000","last_fullsized":"2023-09-23T23:01:26.751315+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":462,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716280+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:01:17.716280+0000","last_peered":"2023-09-23T23:01:17.716280+0000","last_clean":"2023-09-23T23:01:17.716280+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:01:17.716280+0000","last_undegraded":"2023-09-23T23:01:17.716280+0000","last_fullsized":"2023-09-23T23:01:17.716280+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":365,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716456+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:01:17.716456+0000","last_peered":"2023-09-23T23:01:17.716456+0000","last_clean":"2023-09-23T23:01:17.716456+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:01:17.716456+0000","last_undegraded":"2023-09-23T23:01:17.716456+0000","last_fullsized":"2023-09-23T23:01:17.716456+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":291,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716552+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:01:17.716552+0000","last_peered":"2023-09-23T23:01:17.716552+0000","last_clean":"2023-09-23T23:01:17.716552+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:01:17.716552+0000","last_undegraded":"2023-09-23T23:01:17.716552+0000","last_fullsized":"2023-09-23T23:01:17.716552+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757194+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:01:26.757194+0000","last_peered":"2023-09-23T23:01:26.757194+0000","last_clean":"2023-09-23T23:01:26.757194+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:01:26.757194+0000","last_undegraded":"2023-09-23T23:01:26.757194+0000","last_fullsized":"2023-09-23T23:01:26.757194+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":381,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756154+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:01:26.756154+0000","last_peered":"2023-09-23T23:01:26.756154+0000","last_clean":"2023-09-23T23:01:26.756154+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:01:26.756154+0000","last_undegraded":"2023-09-23T23:01:26.756154+0000","last_fullsized":"2023-09-23T23:01:26.756154+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":292,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757941+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:01:26.757941+0000","last_peered":"2023-09-23T23:01:26.757941+0000","last_clean":"2023-09-23T23:01:26.757941+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:01:26.757941+0000","last_undegraded":"2023-09-23T23:01:26.757941+0000","last_fullsized":"2023-09-23T23:01:26.757941+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":276,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715265+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:01:17.715265+0000","last_peered":"2023-09-23T23:01:17.715265+0000","last_clean":"2023-09-23T23:01:17.715265+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:01:17.715265+0000","last_undegraded":"2023-09-23T23:01:17.715265+0000","last_fullsized":"2023-09-23T23:01:17.715265+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":405,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757090+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:01:26.757090+0000","last_peered":"2023-09-23T23:01:26.757090+0000","last_clean":"2023-09-23T23:01:26.757090+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:01:26.757090+0000","last_undegraded":"2023-09-23T23:01:26.757090+0000","last_fullsized":"2023-09-23T23:01:26.757090+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":110,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.738113+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:01:26.738113+0000","last_peered":"2023-09-23T23:01:26.738113+0000","last_clean":"2023-09-23T23:01:26.738113+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:01:26.738113+0000","last_undegraded":"2023-09-23T23:01:26.738113+0000","last_fullsized":"2023-09-23T23:01:26.738113+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757129+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:01:26.757129+0000","last_peered":"2023-09-23T23:01:26.757129+0000","last_clean":"2023-09-23T23:01:26.757129+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:01:26.757129+0000","last_undegraded":"2023-09-23T23:01:26.757129+0000","last_fullsized":"2023-09-23T23:01:26.757129+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"392'9819","reported_seq":9915,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.890903+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:01:34.890903+0000","last_peered":"2023-09-23T23:01:34.890903+0000","last_clean":"2023-09-23T23:01:34.890903+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:01:34.890903+0000","last_undegraded":"2023-09-23T23:01:34.890903+0000","last_fullsized":"2023-09-23T23:01:34.890903+0000","mapping_epoch":353,"log_start":"327'4300","ondisk_log_start":"327'4300","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148570112,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9819,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":275,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716942+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:01:17.716942+0000","last_peered":"2023-09-23T23:01:17.716942+0000","last_clean":"2023-09-23T23:01:17.716942+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:01:17.716942+0000","last_undegraded":"2023-09-23T23:01:17.716942+0000","last_fullsized":"2023-09-23T23:01:17.716942+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":262,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.754622+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:01:26.754622+0000","last_peered":"2023-09-23T23:01:26.754622+0000","last_clean":"2023-09-23T23:01:26.754622+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:01:26.754622+0000","last_undegraded":"2023-09-23T23:01:26.754622+0000","last_fullsized":"2023-09-23T23:01:26.754622+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"392'9696","reported_seq":12121,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:36.188915+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:01:36.188915+0000","last_peered":"2023-09-23T23:01:36.188915+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:01:36.188915+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'1100","ondisk_log_start":"321'1100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8596,"log_dups_size":0,"ondisk_log_size":8596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2254,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9696,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-1313}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":285,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715504+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:01:17.715504+0000","last_peered":"2023-09-23T23:01:17.715504+0000","last_clean":"2023-09-23T23:01:17.715504+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:01:17.715504+0000","last_undegraded":"2023-09-23T23:01:17.715504+0000","last_fullsized":"2023-09-23T23:01:17.715504+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":398,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715184+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:01:17.715184+0000","last_peered":"2023-09-23T23:01:17.715184+0000","last_clean":"2023-09-23T23:01:17.715184+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:01:17.715184+0000","last_undegraded":"2023-09-23T23:01:17.715184+0000","last_fullsized":"2023-09-23T23:01:17.715184+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"392'9821","reported_seq":9911,"reported_epoch":392,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:34.902164+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:01:34.902164+0000","last_peered":"2023-09-23T23:01:34.902164+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:01:34.902164+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:01:34.902164+0000","mapping_epoch":354,"log_start":"327'4300","ondisk_log_start":"327'4300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147521536,"num_objects":2251,"num_object_clones":0,"num_object_copies":6753,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":2251,"num_objects_unfound":0,"num_objects_dirty":2251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9821,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2251}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":143,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.758043+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:01:26.758043+0000","last_peered":"2023-09-23T23:01:26.758043+0000","last_clean":"2023-09-23T23:01:26.758043+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:01:26.758043+0000","last_undegraded":"2023-09-23T23:01:26.758043+0000","last_fullsized":"2023-09-23T23:01:26.758043+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":261,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.755259+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:01:26.755259+0000","last_peered":"2023-09-23T23:01:26.755259+0000","last_clean":"2023-09-23T23:01:26.755259+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:01:26.755259+0000","last_undegraded":"2023-09-23T23:01:26.755259+0000","last_fullsized":"2023-09-23T23:01:26.755259+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"392'9901","reported_seq":10177,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:34.923912+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:01:34.923912+0000","last_peered":"2023-09-23T23:01:34.923912+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:01:34.923912+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"327'4400","ondisk_log_start":"327'4400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148963328,"num_objects":2273,"num_object_clones":0,"num_object_copies":6819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1905,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":2273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9901,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":367},{"shards":"0,5","objects":-1337}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757197+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:01:26.757197+0000","last_peered":"2023-09-23T23:01:26.757197+0000","last_clean":"2023-09-23T23:01:26.757197+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:01:26.757197+0000","last_undegraded":"2023-09-23T23:01:26.757197+0000","last_fullsized":"2023-09-23T23:01:26.757197+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":291,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.740620+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:01:26.740620+0000","last_peered":"2023-09-23T23:01:26.740620+0000","last_clean":"2023-09-23T23:01:26.740620+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:01:26.740620+0000","last_undegraded":"2023-09-23T23:01:26.740620+0000","last_fullsized":"2023-09-23T23:01:26.740620+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"391'9771","reported_seq":9867,"reported_epoch":391,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:32.055960+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:01:32.055960+0000","last_peered":"2023-09-23T23:01:32.055960+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:01:32.055960+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:01:32.055960+0000","mapping_epoch":353,"log_start":"327'4100","ondisk_log_start":"327'4100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9771,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2323}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":80,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756197+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:01:26.756197+0000","last_peered":"2023-09-23T23:01:26.756197+0000","last_clean":"2023-09-23T23:01:26.756197+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:01:26.756197+0000","last_undegraded":"2023-09-23T23:01:26.756197+0000","last_fullsized":"2023-09-23T23:01:26.756197+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":384,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716209+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:01:17.716209+0000","last_peered":"2023-09-23T23:01:17.716209+0000","last_clean":"2023-09-23T23:01:17.716209+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:01:17.716209+0000","last_undegraded":"2023-09-23T23:01:17.716209+0000","last_fullsized":"2023-09-23T23:01:17.716209+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"391'9877","reported_seq":9981,"reported_epoch":391,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:32.109127+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:01:32.109127+0000","last_peered":"2023-09-23T23:01:32.109127+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:01:32.109127+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"327'4200","ondisk_log_start":"327'4200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152371200,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2325,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9877,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-1294}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":174,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715582+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:01:17.715582+0000","last_peered":"2023-09-23T23:01:17.715582+0000","last_clean":"2023-09-23T23:01:17.715582+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:01:17.715582+0000","last_undegraded":"2023-09-23T23:01:17.715582+0000","last_fullsized":"2023-09-23T23:01:17.715582+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":401,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715555+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:01:17.715555+0000","last_peered":"2023-09-23T23:01:17.715555+0000","last_clean":"2023-09-23T23:01:17.715555+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:01:17.715555+0000","last_undegraded":"2023-09-23T23:01:17.715555+0000","last_fullsized":"2023-09-23T23:01:17.715555+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"391'9739","reported_seq":16795,"reported_epoch":391,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:32.231019+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:01:32.231019+0000","last_peered":"2023-09-23T23:01:32.231019+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:01:32.231019+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:01:32.231019+0000","mapping_epoch":354,"log_start":"327'4000","ondisk_log_start":"327'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152109056,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9739,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3472,"num_bytes_recovered":227540992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":887}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":414,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715092+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:01:17.715092+0000","last_peered":"2023-09-23T23:01:17.715092+0000","last_clean":"2023-09-23T23:01:17.715092+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:01:17.715092+0000","last_undegraded":"2023-09-23T23:01:17.715092+0000","last_fullsized":"2023-09-23T23:01:17.715092+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":392,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715582+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:01:17.715582+0000","last_peered":"2023-09-23T23:01:17.715582+0000","last_clean":"2023-09-23T23:01:17.715582+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:01:17.715582+0000","last_undegraded":"2023-09-23T23:01:17.715582+0000","last_fullsized":"2023-09-23T23:01:17.715582+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"391'9589","reported_seq":9685,"reported_epoch":391,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:32.036513+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:01:32.036513+0000","last_peered":"2023-09-23T23:01:32.036513+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:01:32.036513+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:01:32.036513+0000","mapping_epoch":353,"log_start":"327'3900","ondisk_log_start":"327'3900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155910144,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9589,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2379}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":61,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756503+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:01:26.756503+0000","last_peered":"2023-09-23T23:01:26.756503+0000","last_clean":"2023-09-23T23:01:26.756503+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:01:26.756503+0000","last_undegraded":"2023-09-23T23:01:26.756503+0000","last_fullsized":"2023-09-23T23:01:26.756503+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":295,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750157+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:01:26.750157+0000","last_peered":"2023-09-23T23:01:26.750157+0000","last_clean":"2023-09-23T23:01:26.750157+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:01:26.750157+0000","last_undegraded":"2023-09-23T23:01:26.750157+0000","last_fullsized":"2023-09-23T23:01:26.750157+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"391'9759","reported_seq":19041,"reported_epoch":391,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:32.085752+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:01:32.085752+0000","last_peered":"2023-09-23T23:01:32.085752+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:01:32.085752+0000","last_undegraded":"2023-09-23T23:01:32.085752+0000","last_fullsized":"2023-09-23T23:01:32.085752+0000","mapping_epoch":354,"log_start":"327'4000","ondisk_log_start":"327'4000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2317,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9759,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":2317}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":299,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750248+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:01:26.750248+0000","last_peered":"2023-09-23T23:01:26.750248+0000","last_clean":"2023-09-23T23:01:26.750248+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:01:26.750248+0000","last_undegraded":"2023-09-23T23:01:26.750248+0000","last_fullsized":"2023-09-23T23:01:26.750248+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.758093+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:01:26.758093+0000","last_peered":"2023-09-23T23:01:26.758093+0000","last_clean":"2023-09-23T23:01:26.758093+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:01:26.758093+0000","last_undegraded":"2023-09-23T23:01:26.758093+0000","last_fullsized":"2023-09-23T23:01:26.758093+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.755110+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:01:26.755110+0000","last_peered":"2023-09-23T23:01:26.755110+0000","last_clean":"2023-09-23T23:01:26.755110+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:01:26.755110+0000","last_undegraded":"2023-09-23T23:01:26.755110+0000","last_fullsized":"2023-09-23T23:01:26.755110+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"392'9877","reported_seq":9986,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:34.923414+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:01:34.923414+0000","last_peered":"2023-09-23T23:01:34.923414+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:01:34.923414+0000","last_undegraded":"2023-09-23T23:01:34.923414+0000","last_fullsized":"2023-09-23T23:01:34.923414+0000","mapping_epoch":359,"log_start":"327'4400","ondisk_log_start":"327'4400","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5477,"log_dups_size":0,"ondisk_log_size":5477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147259392,"num_objects":2247,"num_object_clones":0,"num_object_copies":6741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2247,"num_objects_unfound":0,"num_objects_dirty":2247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9877,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2247}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":62,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.736812+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:01:26.736812+0000","last_peered":"2023-09-23T23:01:26.736812+0000","last_clean":"2023-09-23T23:01:26.736812+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:01:26.736812+0000","last_undegraded":"2023-09-23T23:01:26.736812+0000","last_fullsized":"2023-09-23T23:01:26.736812+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":411,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716638+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:01:17.716638+0000","last_peered":"2023-09-23T23:01:17.716638+0000","last_clean":"2023-09-23T23:01:17.716638+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:01:17.716638+0000","last_undegraded":"2023-09-23T23:01:17.716638+0000","last_fullsized":"2023-09-23T23:01:17.716638+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":284,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750319+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:01:26.750319+0000","last_peered":"2023-09-23T23:01:26.750319+0000","last_clean":"2023-09-23T23:01:26.750319+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:01:26.750319+0000","last_undegraded":"2023-09-23T23:01:26.750319+0000","last_fullsized":"2023-09-23T23:01:26.750319+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"392'9813","reported_seq":17985,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:36.001269+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:01:36.001269+0000","last_peered":"2023-09-23T23:01:36.001269+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:01:36.001269+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'1200","ondisk_log_start":"321'1200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8613,"log_dups_size":0,"ondisk_log_size":8613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147521536,"num_objects":2251,"num_object_clones":0,"num_object_copies":6753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9813,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2251}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":359,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750178+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:01:26.750178+0000","last_peered":"2023-09-23T23:01:26.750178+0000","last_clean":"2023-09-23T23:01:26.750178+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:01:26.750178+0000","last_undegraded":"2023-09-23T23:01:26.750178+0000","last_fullsized":"2023-09-23T23:01:26.750178+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":236,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.749954+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:01:26.749954+0000","last_peered":"2023-09-23T23:01:26.749954+0000","last_clean":"2023-09-23T23:01:26.749954+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:01:26.749954+0000","last_undegraded":"2023-09-23T23:01:26.749954+0000","last_fullsized":"2023-09-23T23:01:26.749954+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":302,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.758478+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:01:26.758478+0000","last_peered":"2023-09-23T23:01:26.758478+0000","last_clean":"2023-09-23T23:01:26.758478+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:01:26.758478+0000","last_undegraded":"2023-09-23T23:01:26.758478+0000","last_fullsized":"2023-09-23T23:01:26.758478+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"391'9909","reported_seq":11820,"reported_epoch":391,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:32.871003+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:01:32.871003+0000","last_peered":"2023-09-23T23:01:32.871003+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:01:32.871003+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":9909,"log_dups_size":0,"ondisk_log_size":9909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151715840,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":2553,"num_objects_missing":2553,"num_objects_degraded":2667,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":904,"num_bytes_recovered":59244544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1424},{"shards":"0,1,6","objects":882},{"shards":"0,6","objects":-2658}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":375,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.754893+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:01:26.754893+0000","last_peered":"2023-09-23T23:01:26.754893+0000","last_clean":"2023-09-23T23:01:26.754893+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:01:26.754893+0000","last_undegraded":"2023-09-23T23:01:26.754893+0000","last_fullsized":"2023-09-23T23:01:26.754893+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":488,"reported_epoch":391,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:26.740728+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:01:26.740728+0000","last_peered":"2023-09-23T23:01:26.740728+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:01:26.740728+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321953+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:01:34.321953+0000","last_peered":"2023-09-23T23:01:34.321953+0000","last_clean":"2023-09-23T23:01:34.321953+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:01:34.321953+0000","last_undegraded":"2023-09-23T23:01:34.321953+0000","last_fullsized":"2023-09-23T23:01:34.321953+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":301,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715791+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:01:17.715791+0000","last_peered":"2023-09-23T23:01:17.715791+0000","last_clean":"2023-09-23T23:01:17.715791+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:01:17.715791+0000","last_undegraded":"2023-09-23T23:01:17.715791+0000","last_fullsized":"2023-09-23T23:01:17.715791+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"392'9912","reported_seq":10013,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:34.882444+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:01:34.882444+0000","last_peered":"2023-09-23T23:01:34.882444+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:01:34.882444+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"327'4400","ondisk_log_start":"327'4400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9912,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1335}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":382,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.740309+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:01:26.740309+0000","last_peered":"2023-09-23T23:01:26.740309+0000","last_clean":"2023-09-23T23:01:26.740309+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:01:26.740309+0000","last_undegraded":"2023-09-23T23:01:26.740309+0000","last_fullsized":"2023-09-23T23:01:26.740309+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":353,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.736522+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:01:26.736522+0000","last_peered":"2023-09-23T23:01:26.736522+0000","last_clean":"2023-09-23T23:01:26.736522+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:01:26.736522+0000","last_undegraded":"2023-09-23T23:01:26.736522+0000","last_fullsized":"2023-09-23T23:01:26.736522+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":313,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.737014+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:01:26.737014+0000","last_peered":"2023-09-23T23:01:26.737014+0000","last_clean":"2023-09-23T23:01:26.737014+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:01:26.737014+0000","last_undegraded":"2023-09-23T23:01:26.737014+0000","last_fullsized":"2023-09-23T23:01:26.737014+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"391'9874","reported_seq":9989,"reported_epoch":391,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:32.023180+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:01:32.023180+0000","last_peered":"2023-09-23T23:01:32.023180+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:01:32.023180+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:01:32.023180+0000","mapping_epoch":354,"log_start":"327'4200","ondisk_log_start":"327'4200","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152633368,"num_objects":2330,"num_object_clones":0,"num_object_copies":6990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2330,"num_objects_unfound":0,"num_objects_dirty":2330,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9874,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":2330}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":328,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.737155+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:01:26.737155+0000","last_peered":"2023-09-23T23:01:26.737155+0000","last_clean":"2023-09-23T23:01:26.737155+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:01:26.737155+0000","last_undegraded":"2023-09-23T23:01:26.737155+0000","last_fullsized":"2023-09-23T23:01:26.737155+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":236,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750394+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:01:26.750394+0000","last_peered":"2023-09-23T23:01:26.750394+0000","last_clean":"2023-09-23T23:01:26.750394+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:01:26.750394+0000","last_undegraded":"2023-09-23T23:01:26.750394+0000","last_fullsized":"2023-09-23T23:01:26.750394+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":205,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.754965+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:01:26.754965+0000","last_peered":"2023-09-23T23:01:26.754965+0000","last_clean":"2023-09-23T23:01:26.754965+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:01:26.754965+0000","last_undegraded":"2023-09-23T23:01:26.754965+0000","last_fullsized":"2023-09-23T23:01:26.754965+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"391'9904","reported_seq":17836,"reported_epoch":391,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:32.204555+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:01:32.204555+0000","last_peered":"2023-09-23T23:01:32.204555+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:01:32.204555+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"327'4200","ondisk_log_start":"327'4200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146931712,"num_objects":2242,"num_object_clones":0,"num_object_copies":6726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9904,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":857}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":85,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.757218+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:01:26.757218+0000","last_peered":"2023-09-23T23:01:26.757218+0000","last_clean":"2023-09-23T23:01:26.757218+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:01:26.757218+0000","last_undegraded":"2023-09-23T23:01:26.757218+0000","last_fullsized":"2023-09-23T23:01:26.757218+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":99,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716539+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:01:17.716539+0000","last_peered":"2023-09-23T23:01:17.716539+0000","last_clean":"2023-09-23T23:01:17.716539+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:01:17.716539+0000","last_undegraded":"2023-09-23T23:01:17.716539+0000","last_fullsized":"2023-09-23T23:01:17.716539+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.755844+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:01:26.755844+0000","last_peered":"2023-09-23T23:01:26.755844+0000","last_clean":"2023-09-23T23:01:26.755844+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:01:26.755844+0000","last_undegraded":"2023-09-23T23:01:26.755844+0000","last_fullsized":"2023-09-23T23:01:26.755844+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"391'9761","reported_seq":10286,"reported_epoch":391,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:32.175148+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:01:32.175148+0000","last_peered":"2023-09-23T23:01:32.175148+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:01:32.175148+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"327'4100","ondisk_log_start":"327'4100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151322624,"num_objects":2309,"num_object_clones":0,"num_object_copies":6927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2309,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":2309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9761,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-1242}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":348,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750511+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:01:26.750511+0000","last_peered":"2023-09-23T23:01:26.750511+0000","last_clean":"2023-09-23T23:01:26.750511+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:01:26.750511+0000","last_undegraded":"2023-09-23T23:01:26.750511+0000","last_fullsized":"2023-09-23T23:01:26.750511+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":295,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.736940+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:01:26.736940+0000","last_peered":"2023-09-23T23:01:26.736940+0000","last_clean":"2023-09-23T23:01:26.736940+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:01:26.736940+0000","last_undegraded":"2023-09-23T23:01:26.736940+0000","last_fullsized":"2023-09-23T23:01:26.736940+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":365,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715126+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:01:17.715126+0000","last_peered":"2023-09-23T23:01:17.715126+0000","last_clean":"2023-09-23T23:01:17.715126+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:01:17.715126+0000","last_undegraded":"2023-09-23T23:01:17.715126+0000","last_fullsized":"2023-09-23T23:01:17.715126+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":330,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.740747+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:01:26.740747+0000","last_peered":"2023-09-23T23:01:26.740747+0000","last_clean":"2023-09-23T23:01:26.740747+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:01:26.740747+0000","last_undegraded":"2023-09-23T23:01:26.740747+0000","last_fullsized":"2023-09-23T23:01:26.740747+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":140,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.750021+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:01:26.750021+0000","last_peered":"2023-09-23T23:01:26.750021+0000","last_clean":"2023-09-23T23:01:26.750021+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:01:26.750021+0000","last_undegraded":"2023-09-23T23:01:26.750021+0000","last_fullsized":"2023-09-23T23:01:26.750021+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":200,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716148+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:01:17.716148+0000","last_peered":"2023-09-23T23:01:17.716148+0000","last_clean":"2023-09-23T23:01:17.716148+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:01:17.716148+0000","last_undegraded":"2023-09-23T23:01:17.716148+0000","last_fullsized":"2023-09-23T23:01:17.716148+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":414,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.716723+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:01:17.716723+0000","last_peered":"2023-09-23T23:01:17.716723+0000","last_clean":"2023-09-23T23:01:17.716723+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:01:17.716723+0000","last_undegraded":"2023-09-23T23:01:17.716723+0000","last_fullsized":"2023-09-23T23:01:17.716723+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":214,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715652+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:01:17.715652+0000","last_peered":"2023-09-23T23:01:17.715652+0000","last_clean":"2023-09-23T23:01:17.715652+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:01:17.715652+0000","last_undegraded":"2023-09-23T23:01:17.715652+0000","last_fullsized":"2023-09-23T23:01:17.715652+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":301,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715499+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:01:17.715499+0000","last_peered":"2023-09-23T23:01:17.715499+0000","last_clean":"2023-09-23T23:01:17.715499+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:01:17.715499+0000","last_undegraded":"2023-09-23T23:01:17.715499+0000","last_fullsized":"2023-09-23T23:01:17.715499+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":86,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:31.252591+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:01:31.252591+0000","last_peered":"2023-09-23T23:01:31.252591+0000","last_clean":"2023-09-23T23:01:31.252591+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:01:31.252591+0000","last_undegraded":"2023-09-23T23:01:31.252591+0000","last_fullsized":"2023-09-23T23:01:31.252591+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":88,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.756759+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:01:26.756759+0000","last_peered":"2023-09-23T23:01:26.756759+0000","last_clean":"2023-09-23T23:01:26.756759+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:01:26.756759+0000","last_undegraded":"2023-09-23T23:01:26.756759+0000","last_fullsized":"2023-09-23T23:01:26.756759+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":334,"reported_epoch":391,"state":"active+clean","last_fresh":"2023-09-23T23:01:26.737455+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:01:26.737455+0000","last_peered":"2023-09-23T23:01:26.737455+0000","last_clean":"2023-09-23T23:01:26.737455+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:01:26.737455+0000","last_undegraded":"2023-09-23T23:01:26.737455+0000","last_fullsized":"2023-09-23T23:01:26.737455+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":376,"reported_epoch":390,"state":"active+clean","last_fresh":"2023-09-23T23:01:17.715068+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:01:17.715068+0000","last_peered":"2023-09-23T23:01:17.715068+0000","last_clean":"2023-09-23T23:01:17.715068+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:01:17.715068+0000","last_undegraded":"2023-09-23T23:01:17.715068+0000","last_fullsized":"2023-09-23T23:01:17.715068+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2396979224,"num_objects":36576,"num_object_clones":0,"num_object_copies":109728,"num_objects_missing_on_primary":2555,"num_objects_missing":2555,"num_objects_degraded":22690,"num_objects_misplaced":31398,"num_objects_unfound":0,"num_objects_dirty":36576,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157022,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17674,"num_bytes_recovered":1067122688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7872000000,"data_stored":7871987784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100222,"ondisk_log_size":100222,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430623,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626847,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253160,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1705196,"kb_used_data":988900,"kb_used_omap":9,"kb_used_meta":716278,"kb_avail":92039956,"statfs":{"total":95995035648,"available":94248914944,"internally_reserved":0,"allocated":1012633600,"data_stored":1011156101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":733469673},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8250000000000002,"5min":1.258,"15min":0.65400000000000003},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":83.664000000000001,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.247},{"interface":"front","average":{"1min":5.0039999999999996,"5min":1.5980000000000001,"15min":1.103},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":83.700999999999993,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.58799999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.48699999999999999,"15min":0.51400000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.087,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.64200000000000002,"15min":0.69799999999999995},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.74,"5min":4.048,"15min":4.048},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.56100000000000005,"15min":1.9950000000000001},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.8580000000000001,"5min":3.141,"15min":87.787999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.71699999999999997,"15min":2.339},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.19500000000000001},"max":{"1min":0.95499999999999996,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.6869999999999998,"15min":2.4529999999999998},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6669999999999998,"5min":151.036,"15min":151.036},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.581,"5min":2.7450000000000001,"15min":2.548},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":6.649,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.32900000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.6219999999999999,"15min":2.319},"min":{"1min":0.35199999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1720000000000002,"5min":116.098,"15min":116.098},"last":0.374},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.5099999999999998,"15min":2.4809999999999999},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.9769999999999999,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.27,"15min":2.1480000000000001},"min":{"1min":0.40100000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.0820000000000001,"15min":1.8520000000000001},"min":{"1min":0.40999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.825,"5min":8.5370000000000008,"15min":79.328999999999994},"last":0.432}]},{"osd":7,"last update":"Sat Sep 23 23:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.04,"15min":1.6799999999999999},"min":{"1min":0.46300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.581,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.177,"15min":1.8919999999999999},"min":{"1min":0.44600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9860000000000002,"5min":10.676,"15min":20.902999999999999},"last":0.44600000000000001}]}]},{"osd":0,"up_from":353,"seq":1516123455531,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1726988,"kb_used_data":1557572,"kb_used_omap":37,"kb_used_meta":169306,"kb_avail":92018164,"statfs":{"total":95995035648,"available":94226599936,"internally_reserved":0,"allocated":1594953728,"data_stored":1593330066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":173369506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.59199999999999997,"15min":0.55700000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.7629999999999999,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.623},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64000000000000001,"15min":0.66100000000000003},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.8650000000000002,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":1.9099999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":0.86399999999999999,"15min":0.80800000000000005},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.012,"5min":0.72499999999999998,"15min":0.68500000000000005},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"last":2.0819999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.623,"15min":0.51000000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.76800000000000002,"15min":0.86199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6499999999999999,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":2.2429999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":12.855,"5min":3.2290000000000001,"15min":1.649},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":183.148,"5min":183.148,"15min":183.148},"last":1.155},{"interface":"front","average":{"1min":13.087999999999999,"5min":3.194,"15min":1.51},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":183.39500000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":1.2969999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.86799999999999999,"15min":0.74199999999999999},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4089999999999998,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.282,"5min":1.0009999999999999,"15min":0.999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.774,"5min":7.774,"15min":7.774},"last":1.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.92900000000000005,"15min":0.86599999999999999},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.7309999999999999,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":1.333},{"interface":"front","average":{"1min":1.242,"5min":0.86799999999999999,"15min":0.77200000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":6.375,"5min":6.375,"15min":6.375},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.159,"15min":1.2549999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.8439999999999999,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":1.258},{"interface":"front","average":{"1min":1.008,"5min":1.2150000000000001,"15min":1.2569999999999999},"min":{"1min":0.49299999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":6.343,"5min":6.343,"15min":6.343},"last":0.71999999999999997}]}]},{"osd":2,"up_from":359,"seq":1541893259304,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":749144,"kb_used_data":613584,"kb_used_omap":83,"kb_used_meta":135468,"kb_avail":92996008,"statfs":{"total":95995035648,"available":95227912192,"internally_reserved":0,"allocated":628310016,"data_stored":626743467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":138719685},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.35099999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.49099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":1.254}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.443},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.61699999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926108,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":524504,"kb_used_data":28048,"kb_used_omap":7,"kb_used_meta":496440,"kb_avail":93220648,"statfs":{"total":95995035648,"available":95457943552,"internally_reserved":0,"allocated":28721152,"data_stored":27149284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":508355176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":1.5640000000000001,"15min":0.73399999999999999},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":122.289,"5min":122.289,"15min":122.289},"last":0.255},{"interface":"front","average":{"1min":6.1609999999999996,"5min":1.5600000000000001,"15min":0.79100000000000004},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.63200000000000001,"15min":2.3639999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":123.866},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.73699999999999999,"15min":2.1549999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8010000000000002,"5min":6.8010000000000002,"15min":122.265},"last":0.39500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.63,"15min":0.68899999999999995},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.7,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.307},{"interface":"front","average":{"1min":1.857,"5min":0.98899999999999999,"15min":0.82499999999999996},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.223},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":1.135}]},{"osd":4,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":2.3519999999999999,"15min":2.0350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":64.406999999999996,"5min":87.046999999999997,"15min":87.046999999999997},"last":1.212},{"interface":"front","average":{"1min":4.008,"5min":2.5409999999999999,"15min":1.8999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":64.468000000000004,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2290000000000001,"15min":1.3560000000000001},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1980000000000004,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.548,"5min":1.218,"15min":1.827},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.246,"5min":10.246,"15min":11.951000000000001},"last":0.32500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.597,"15min":1.7150000000000001},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0229999999999997,"5min":11.116,"15min":14.097},"last":0.436},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4039999999999999,"15min":1.956},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.79,"5min":10.859,"15min":12.887},"last":0.34100000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.89400000000000002,"15min":2.0499999999999998},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8449999999999998,"5min":4.8449999999999998,"15min":91.757999999999996},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.413,"5min":0.98099999999999998,"15min":1.9390000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":8.5830000000000002,"5min":8.5830000000000002,"15min":91.709999999999994},"last":1.1930000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019304,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2741120,"kb_used_data":1833656,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91004032,"statfs":{"total":95995035648,"available":93188128768,"internally_reserved":0,"allocated":1877663744,"data_stored":1876857535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.4380000000000002,"15min":3.5270000000000001},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":3.081,"5min":87.325000000000003,"15min":132.304},"last":0.317},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.6320000000000001,"15min":3.4849999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.253},"max":{"1min":3.6120000000000001,"5min":87.572000000000003,"15min":132.453},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:01:28 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.0549999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4870000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.76200000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4770000000000003,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.484,"15min":3.9009999999999998},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32400000000000001},"max":{"1min":4.0830000000000002,"5min":10.337999999999999,"15min":164.453},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.784,"5min":1.4410000000000001,"15min":3.8929999999999998},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0039999999999996,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.35099999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.655,"15min":1.181},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":3.5680000000000001,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.4379999999999999,"15min":1.113},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.1859999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.5700000000000001,"15min":1.169},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0869999999999997,"5min":12.84,"15min":12.84},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.585,"15min":1.117},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4199999999999999,"5min":10.587,"15min":10.587},"last":0.48899999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823079,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1999020,"kb_used_data":1091172,"kb_used_omap":0,"kb_used_meta":907840,"kb_avail":91746132,"statfs":{"total":95995035648,"available":93948039168,"internally_reserved":0,"allocated":1117360128,"data_stored":1116577737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929628160},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.878,"15min":0.69599999999999995},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.5709999999999997,"5min":14.42,"15min":14.42},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2430000000000001,"15min":1.135},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3159999999999998,"5min":14.496,"15min":14.496},"last":0.60299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.081,"15min":2.1909999999999998},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.24299999999999999},"max":{"1min":2.6040000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.857,"5min":1.6200000000000001,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":7.0739999999999998,"5min":10.353,"15min":110.89},"last":1.282}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.67200000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.216,"15min":4.7009999999999996},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2710000000000008,"5min":9.2710000000000008,"15min":257.05000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.474,"5min":1.3859999999999999,"15min":4.7590000000000003},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.3399999999999999,"5min":12.185,"15min":257.35000000000002},"last":0.628}]},{"osd":4,"last update":"Sat Sep 23 23:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":2.8580000000000001,"15min":1.5109999999999999},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8440000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":1.25},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.5710000000000002,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.524,"5min":114.908,"15min":114.908},"last":0.318}]},{"osd":6,"last update":"Sat Sep 23 23:00:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.512,"15min":1.2270000000000001},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.4630000000000001,"5min":11.121,"15min":12.615},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.2370000000000001,"15min":1.1519999999999999},"min":{"1min":0.41299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.4909999999999997,"5min":12.137,"15min":12.74},"last":1.03}]},{"osd":7,"last update":"Sat Sep 23 23:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.2350000000000001,"15min":1.0629999999999999},"min":{"1min":0.45000000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.8370000000000002,"5min":12.23,"15min":12.23},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.5830000000000002,"5min":1.7490000000000001,"15min":1.2170000000000001},"min":{"1min":0.378,"5min":0.31,"15min":0.31},"max":{"1min":7.6719999999999997,"5min":11.112,"15min":11.112},"last":0.58099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1580470272,"data_stored":1580466200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1006698496,"data_stored":1006698496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":622919680,"data_stored":622919680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":23855104,"data_stored":23855104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1862930432,"data_stored":1862926360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1112276992,"data_stored":1112276992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1662849024,"data_stored":1662844952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:01:40.861 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:40.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 22350/107337 objects degraded (20.822%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:41.078 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:41.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:41.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:41.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:41.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:41.339 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:41.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:41.444 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:41.593 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:41 smithi052 ceph-mon[180979]: pgmap v1186: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 307 op/s; 22275/105855 objects degraded (21.043%); 31032/105855 objects misplaced (29.316%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:01:41.593 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:41 smithi052 ceph-mon[187743]: pgmap v1186: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 307 op/s; 22275/105855 objects degraded (21.043%); 31032/105855 objects misplaced (29.316%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:01:41.595 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:41.725 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:41.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:41.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:41.829 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:41.829 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:41.831 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:41.872 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:41 smithi151 ceph-mon[159210]: pgmap v1186: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 307 op/s; 22275/105855 objects degraded (21.043%); 31032/105855 objects misplaced (29.316%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:01:42.080 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:42.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:42.109 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:01:42.185 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:42.214 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:42.263 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:42.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:42.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:42.367 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:42.422 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:42.572 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:01:42.652 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:42.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:42.680 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/169362506' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:01:42.681 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/169362506' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:01:42.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:01:42.760 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:42.803 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/169362506' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:01:42.806 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:42.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:42.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:42.969 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:43.028 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:43.326 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:43.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:43.432 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:43.490 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:43.513 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:43.561 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:43.575 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:43.585 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:01:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:01:43] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.33.4" 2023-09-23T23:01:43.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:43.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[187743]: failed to encode map e393 with expected crc 2023-09-23T23:01:43.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/169362506' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:01:43.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[187743]: osdmap e393: 8 total, 8 up, 6 in 2023-09-23T23:01:43.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[187743]: failed to encode map e393 with expected crc 2023-09-23T23:01:43.586 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[187743]: pgmap v1188: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 355 op/s; 21645/102783 objects degraded (21.059%); 30849/102783 objects misplaced (30.014%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:01:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:43.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:43.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:43.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[180979]: failed to encode map e393 with expected crc 2023-09-23T23:01:43.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/169362506' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:01:43.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[180979]: osdmap e393: 8 total, 8 up, 6 in 2023-09-23T23:01:43.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[180979]: failed to encode map e393 with expected crc 2023-09-23T23:01:43.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:43 smithi052 ceph-mon[180979]: pgmap v1188: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 355 op/s; 21645/102783 objects degraded (21.059%); 30849/102783 objects misplaced (30.014%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:01:43.688 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:43.731 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:43.812 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:43.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:43.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:43.856 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:01:43.917 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:43 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:01:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:43 smithi151 ceph-mon[159210]: failed to encode map e393 with expected crc 2023-09-23T23:01:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/169362506' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:01:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:43 smithi151 ceph-mon[159210]: osdmap e393: 8 total, 8 up, 6 in 2023-09-23T23:01:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:43 smithi151 ceph-mon[159210]: failed to encode map e393 with expected crc 2023-09-23T23:01:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:43 smithi151 ceph-mon[159210]: pgmap v1188: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 355 op/s; 21645/102783 objects degraded (21.059%); 30849/102783 objects misplaced (30.014%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:01:43.972 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:44.041 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:44.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:44.148 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:44.187 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:44.296 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:44.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:44.399 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:44.426 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:44.798 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:44.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:44.903 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:44.937 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:44.993 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:44.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:45.100 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:45.127 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:45.361 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:45.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:45.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:45.431 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:01:45.466 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:45.534 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:45.696 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:45.704 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:45 smithi052 ceph-mon[180979]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 21754/102456 objects degraded (21.233%); 30849/102456 objects misplaced (30.110%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:01:45.704 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 21754/102456 objects degraded (21.233%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:45.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:45.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:45 smithi052 ceph-mon[187743]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 21754/102456 objects degraded (21.233%); 30849/102456 objects misplaced (30.110%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:01:45.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 21754/102456 objects degraded (21.233%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:45.806 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:45.839 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:45.901 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:45.902 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:45 smithi151 ceph-mon[159210]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 21754/102456 objects degraded (21.233%); 30849/102456 objects misplaced (30.110%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:01:45.903 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 21754/102456 objects degraded (21.233%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:45.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:45.910 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:45.923 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:45.923 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:46.005 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:46.080 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:46.152 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:46.157 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:46.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:46.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:46.265 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:46.292 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:46.336 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:46.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:46.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:46.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:46.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:46.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:46.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:46.354 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.355 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:01:46.442 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:46.483 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:46.540 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:46.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:46.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:46.647 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:46.683 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:46.745 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:46.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:46.857 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:46.885 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2744410160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:46.888 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:46.917 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:01:46.917 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2744410160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:46.920 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2744410160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:46.954 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:46.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:46.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:47.060 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:47.108 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:47.745 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:47.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:47.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:47.850 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:47.878 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:47.930 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:47.934 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:47 smithi151 ceph-mon[159210]: pgmap v1190: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 21532/100125 objects degraded (21.505%); 30640/100125 objects misplaced (30.602%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:01:47.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:47.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:47.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:47 smithi052 ceph-mon[187743]: pgmap v1190: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 21532/100125 objects degraded (21.505%); 30640/100125 objects misplaced (30.602%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:01:47.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:47 smithi052 ceph-mon[180979]: pgmap v1190: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 21532/100125 objects degraded (21.505%); 30640/100125 objects misplaced (30.602%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T23:01:48.035 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:48.078 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:48.154 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:48.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:48.265 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:48.298 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:48.362 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:48.367 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:48.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:48.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:48.444 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:48.473 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:48.523 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:48.574 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:48.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:48.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:48.685 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:48.728 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:49.197 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:49.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:49.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:49.291 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:49.293 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:01:49.303 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:49.330 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:49.392 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:49.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:49.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:49.500 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:49.545 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:49.665 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:49 smithi052 ceph-mon[180979]: pgmap v1191: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 21113/97104 objects degraded (21.743%); 30398/97104 objects misplaced (31.305%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:01:49.665 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:49.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:49 smithi151 ceph-mon[159210]: pgmap v1191: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 21113/97104 objects degraded (21.743%); 30398/97104 objects misplaced (31.305%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:01:49.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:49.800 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:49.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:49.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:49.908 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:49.922 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:49 smithi052 ceph-mon[187743]: pgmap v1191: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 21113/97104 objects degraded (21.743%); 30398/97104 objects misplaced (31.305%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:01:49.923 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:01:49.944 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:50.004 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:50.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:50.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:50.113 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:50.153 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:50.227 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:01:50.233 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:50.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:50.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:50.346 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:50.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:50.442 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:50.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:50.551 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:50.580 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:50.625 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:50.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:50.733 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:50.768 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:50 smithi151 ceph-mon[159210]: from='client.36564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 21113/97104 objects degraded (21.743%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:50 smithi052 ceph-mon[187743]: from='client.36564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 21113/97104 objects degraded (21.743%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:50 smithi052 ceph-mon[180979]: from='client.36564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:01:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 21113/97104 objects degraded (21.743%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:50.996 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:51.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:51.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:51.101 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:51.137 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:51.144 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1191,"stamp":"2023-09-23T23:01:48.599485+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2120226744,"num_objects":32368,"num_object_clones":0,"num_object_copies":97104,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":21113,"num_objects_misplaced":30398,"num_objects_unfound":0,"num_objects_dirty":32368,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":161514,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18322,"num_bytes_recovered":1107311548,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100648,"ondisk_log_size":100648,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8866640,"kb_used_data":5545952,"kb_used_omap":137,"kb_used_meta":3320502,"kb_avail":553604272,"statfs":{"total":575970213888,"available":566890774528,"internally_reserved":0,"allocated":5679054848,"data_stored":5671175996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3400194232},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-223543296,"num_objects":-3411,"num_object_clones":0,"num_object_copies":-10233,"num_objects_missing_on_primary":-12,"num_objects_missing":-12,"num_objects_degraded":-1237,"num_objects_misplaced":-828,"num_objects_unfound":0,"num_objects_dirty":-3411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3411,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27918336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003083"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":405,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313472+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:01:34.313472+0000","last_peered":"2023-09-23T23:01:34.313472+0000","last_clean":"2023-09-23T23:01:34.313472+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:01:34.313472+0000","last_undegraded":"2023-09-23T23:01:34.313472+0000","last_fullsized":"2023-09-23T23:01:34.313472+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322661+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:01:34.322661+0000","last_peered":"2023-09-23T23:01:34.322661+0000","last_clean":"2023-09-23T23:01:34.322661+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:01:34.322661+0000","last_undegraded":"2023-09-23T23:01:34.322661+0000","last_fullsized":"2023-09-23T23:01:34.322661+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":61,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321607+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:01:34.321607+0000","last_peered":"2023-09-23T23:01:34.321607+0000","last_clean":"2023-09-23T23:01:34.321607+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:01:34.321607+0000","last_undegraded":"2023-09-23T23:01:34.321607+0000","last_fullsized":"2023-09-23T23:01:34.321607+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":390,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:36.254006+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:01:36.254006+0000","last_peered":"2023-09-23T23:01:36.254006+0000","last_clean":"2023-09-23T23:01:36.254006+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:01:36.254006+0000","last_undegraded":"2023-09-23T23:01:36.254006+0000","last_fullsized":"2023-09-23T23:01:36.254006+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":77,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321487+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:01:34.321487+0000","last_peered":"2023-09-23T23:01:34.321487+0000","last_clean":"2023-09-23T23:01:34.321487+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:01:34.321487+0000","last_undegraded":"2023-09-23T23:01:34.321487+0000","last_fullsized":"2023-09-23T23:01:34.321487+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":63,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299543+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:01:34.299543+0000","last_peered":"2023-09-23T23:01:34.299543+0000","last_clean":"2023-09-23T23:01:34.299543+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:01:34.299543+0000","last_undegraded":"2023-09-23T23:01:34.299543+0000","last_fullsized":"2023-09-23T23:01:34.299543+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":328,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322029+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:01:34.322029+0000","last_peered":"2023-09-23T23:01:34.322029+0000","last_clean":"2023-09-23T23:01:34.322029+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:01:34.322029+0000","last_undegraded":"2023-09-23T23:01:34.322029+0000","last_fullsized":"2023-09-23T23:01:34.322029+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321342+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:01:34.321342+0000","last_peered":"2023-09-23T23:01:34.321342+0000","last_clean":"2023-09-23T23:01:34.321342+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:01:34.321342+0000","last_undegraded":"2023-09-23T23:01:34.321342+0000","last_fullsized":"2023-09-23T23:01:34.321342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321877+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:01:34.321877+0000","last_peered":"2023-09-23T23:01:34.321877+0000","last_clean":"2023-09-23T23:01:34.321877+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:01:34.321877+0000","last_undegraded":"2023-09-23T23:01:34.321877+0000","last_fullsized":"2023-09-23T23:01:34.321877+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":76,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321467+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:01:34.321467+0000","last_peered":"2023-09-23T23:01:34.321467+0000","last_clean":"2023-09-23T23:01:34.321467+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:01:34.321467+0000","last_undegraded":"2023-09-23T23:01:34.321467+0000","last_fullsized":"2023-09-23T23:01:34.321467+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":82,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322615+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:01:34.322615+0000","last_peered":"2023-09-23T23:01:34.322615+0000","last_clean":"2023-09-23T23:01:34.322615+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:01:34.322615+0000","last_undegraded":"2023-09-23T23:01:34.322615+0000","last_fullsized":"2023-09-23T23:01:34.322615+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":138,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321536+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:01:34.321536+0000","last_peered":"2023-09-23T23:01:34.321536+0000","last_clean":"2023-09-23T23:01:34.321536+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:01:34.321536+0000","last_undegraded":"2023-09-23T23:01:34.321536+0000","last_fullsized":"2023-09-23T23:01:34.321536+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":464,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.314104+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:01:34.314104+0000","last_peered":"2023-09-23T23:01:34.314104+0000","last_clean":"2023-09-23T23:01:34.314104+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:01:34.314104+0000","last_undegraded":"2023-09-23T23:01:34.314104+0000","last_fullsized":"2023-09-23T23:01:34.314104+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":367,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.316778+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:01:34.316778+0000","last_peered":"2023-09-23T23:01:34.316778+0000","last_clean":"2023-09-23T23:01:34.316778+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:01:34.316778+0000","last_undegraded":"2023-09-23T23:01:34.316778+0000","last_fullsized":"2023-09-23T23:01:34.316778+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":293,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.316868+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:01:34.316868+0000","last_peered":"2023-09-23T23:01:34.316868+0000","last_clean":"2023-09-23T23:01:34.316868+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:01:34.316868+0000","last_undegraded":"2023-09-23T23:01:34.316868+0000","last_fullsized":"2023-09-23T23:01:34.316868+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322324+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:01:34.322324+0000","last_peered":"2023-09-23T23:01:34.322324+0000","last_clean":"2023-09-23T23:01:34.322324+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:01:34.322324+0000","last_undegraded":"2023-09-23T23:01:34.322324+0000","last_fullsized":"2023-09-23T23:01:34.322324+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":382,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321795+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:01:34.321795+0000","last_peered":"2023-09-23T23:01:34.321795+0000","last_clean":"2023-09-23T23:01:34.321795+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:01:34.321795+0000","last_undegraded":"2023-09-23T23:01:34.321795+0000","last_fullsized":"2023-09-23T23:01:34.321795+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":293,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321821+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:01:34.321821+0000","last_peered":"2023-09-23T23:01:34.321821+0000","last_clean":"2023-09-23T23:01:34.321821+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:01:34.321821+0000","last_undegraded":"2023-09-23T23:01:34.321821+0000","last_fullsized":"2023-09-23T23:01:34.321821+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":278,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313243+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:01:34.313243+0000","last_peered":"2023-09-23T23:01:34.313243+0000","last_clean":"2023-09-23T23:01:34.313243+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:01:34.313243+0000","last_undegraded":"2023-09-23T23:01:34.313243+0000","last_fullsized":"2023-09-23T23:01:34.313243+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":406,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322496+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:01:34.322496+0000","last_peered":"2023-09-23T23:01:34.322496+0000","last_clean":"2023-09-23T23:01:34.322496+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:01:34.322496+0000","last_undegraded":"2023-09-23T23:01:34.322496+0000","last_fullsized":"2023-09-23T23:01:34.322496+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":111,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.298375+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:01:34.298375+0000","last_peered":"2023-09-23T23:01:34.298375+0000","last_clean":"2023-09-23T23:01:34.298375+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:01:34.298375+0000","last_undegraded":"2023-09-23T23:01:34.298375+0000","last_fullsized":"2023-09-23T23:01:34.298375+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322875+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:01:34.322875+0000","last_peered":"2023-09-23T23:01:34.322875+0000","last_clean":"2023-09-23T23:01:34.322875+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:01:34.322875+0000","last_undegraded":"2023-09-23T23:01:34.322875+0000","last_fullsized":"2023-09-23T23:01:34.322875+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"393'10030","reported_seq":10127,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:44.844891+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:01:44.844891+0000","last_peered":"2023-09-23T23:01:44.844891+0000","last_clean":"2023-09-23T23:01:44.844891+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:01:44.844891+0000","last_undegraded":"2023-09-23T23:01:44.844891+0000","last_fullsized":"2023-09-23T23:01:44.844891+0000","mapping_epoch":353,"log_start":"327'4500","ondisk_log_start":"327'4500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134742016,"num_objects":2056,"num_object_clones":0,"num_object_copies":6168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10030,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":277,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.316683+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:01:34.316683+0000","last_peered":"2023-09-23T23:01:34.316683+0000","last_clean":"2023-09-23T23:01:34.316683+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:01:34.316683+0000","last_undegraded":"2023-09-23T23:01:34.316683+0000","last_fullsized":"2023-09-23T23:01:34.316683+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":263,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321240+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:01:34.321240+0000","last_peered":"2023-09-23T23:01:34.321240+0000","last_clean":"2023-09-23T23:01:34.321240+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:01:34.321240+0000","last_undegraded":"2023-09-23T23:01:34.321240+0000","last_fullsized":"2023-09-23T23:01:34.321240+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"393'9926","reported_seq":12352,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:46.158139+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:01:46.158139+0000","last_peered":"2023-09-23T23:01:46.158139+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:01:46.158139+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"321'1300","ondisk_log_start":"321'1300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132644864,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2024,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9926,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-1543}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":287,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.311807+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:01:34.311807+0000","last_peered":"2023-09-23T23:01:34.311807+0000","last_clean":"2023-09-23T23:01:34.311807+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:01:34.311807+0000","last_undegraded":"2023-09-23T23:01:34.311807+0000","last_fullsized":"2023-09-23T23:01:34.311807+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":400,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313169+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:01:34.313169+0000","last_peered":"2023-09-23T23:01:34.313169+0000","last_clean":"2023-09-23T23:01:34.313169+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:01:34.313169+0000","last_undegraded":"2023-09-23T23:01:34.313169+0000","last_fullsized":"2023-09-23T23:01:34.313169+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"393'10019","reported_seq":10110,"reported_epoch":393,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:44.900550+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:01:44.900550+0000","last_peered":"2023-09-23T23:01:44.900550+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:01:44.900550+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:01:44.900550+0000","mapping_epoch":354,"log_start":"327'4500","ondisk_log_start":"327'4500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134545408,"num_objects":2053,"num_object_clones":0,"num_object_copies":6159,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":2053,"num_objects_unfound":0,"num_objects_dirty":2053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10019,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2053}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":144,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321906+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:01:34.321906+0000","last_peered":"2023-09-23T23:01:34.321906+0000","last_clean":"2023-09-23T23:01:34.321906+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:01:34.321906+0000","last_undegraded":"2023-09-23T23:01:34.321906+0000","last_fullsized":"2023-09-23T23:01:34.321906+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":262,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321506+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:01:34.321506+0000","last_peered":"2023-09-23T23:01:34.321506+0000","last_clean":"2023-09-23T23:01:34.321506+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:01:34.321506+0000","last_undegraded":"2023-09-23T23:01:34.321506+0000","last_fullsized":"2023-09-23T23:01:34.321506+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"393'10112","reported_seq":10389,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:44.939629+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:01:44.939629+0000","last_peered":"2023-09-23T23:01:44.939629+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:01:44.939629+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"327'4600","ondisk_log_start":"327'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135135232,"num_objects":2062,"num_object_clones":0,"num_object_copies":6186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1726,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":2062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10112,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":335},{"shards":"0,5","objects":-1548}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":62,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322328+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:01:34.322328+0000","last_peered":"2023-09-23T23:01:34.322328+0000","last_clean":"2023-09-23T23:01:34.322328+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:01:34.322328+0000","last_undegraded":"2023-09-23T23:01:34.322328+0000","last_fullsized":"2023-09-23T23:01:34.322328+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":292,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299315+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:01:34.299315+0000","last_peered":"2023-09-23T23:01:34.299315+0000","last_clean":"2023-09-23T23:01:34.299315+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:01:34.299315+0000","last_undegraded":"2023-09-23T23:01:34.299315+0000","last_fullsized":"2023-09-23T23:01:34.299315+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"393'10076","reported_seq":10174,"reported_epoch":393,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:47.129210+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:01:47.129210+0000","last_peered":"2023-09-23T23:01:47.129210+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:01:47.129210+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:01:47.129210+0000","mapping_epoch":353,"log_start":"327'4400","ondisk_log_start":"327'4400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132251648,"num_objects":2018,"num_object_clones":0,"num_object_copies":6054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10076,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2018}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":81,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322028+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:01:34.322028+0000","last_peered":"2023-09-23T23:01:34.322028+0000","last_clean":"2023-09-23T23:01:34.322028+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:01:34.322028+0000","last_undegraded":"2023-09-23T23:01:34.322028+0000","last_fullsized":"2023-09-23T23:01:34.322028+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":386,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.314021+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:01:34.314021+0000","last_peered":"2023-09-23T23:01:34.314021+0000","last_clean":"2023-09-23T23:01:34.314021+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:01:34.314021+0000","last_undegraded":"2023-09-23T23:01:34.314021+0000","last_fullsized":"2023-09-23T23:01:34.314021+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"393'10190","reported_seq":10296,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:47.137255+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:01:47.137255+0000","last_peered":"2023-09-23T23:01:47.137255+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:01:47.137255+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"327'4500","ondisk_log_start":"327'4500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131858432,"num_objects":2012,"num_object_clones":0,"num_object_copies":6036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2012,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":2012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10190,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-1607}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":176,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313774+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:01:34.313774+0000","last_peered":"2023-09-23T23:01:34.313774+0000","last_clean":"2023-09-23T23:01:34.313774+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:01:34.313774+0000","last_undegraded":"2023-09-23T23:01:34.313774+0000","last_fullsized":"2023-09-23T23:01:34.313774+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":403,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.314204+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:01:34.314204+0000","last_peered":"2023-09-23T23:01:34.314204+0000","last_clean":"2023-09-23T23:01:34.314204+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:01:34.314204+0000","last_undegraded":"2023-09-23T23:01:34.314204+0000","last_fullsized":"2023-09-23T23:01:34.314204+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"393'10043","reported_seq":17709,"reported_epoch":393,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:47.148194+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:01:47.148194+0000","last_peered":"2023-09-23T23:01:47.148194+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:01:47.148194+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:01:47.148194+0000","mapping_epoch":354,"log_start":"327'4300","ondisk_log_start":"327'4300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":824,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10043,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3776,"num_bytes_recovered":247463936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":583}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":416,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.312978+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:01:34.312978+0000","last_peered":"2023-09-23T23:01:34.312978+0000","last_clean":"2023-09-23T23:01:34.312978+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:01:34.312978+0000","last_undegraded":"2023-09-23T23:01:34.312978+0000","last_fullsized":"2023-09-23T23:01:34.312978+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":394,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.311884+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:01:34.311884+0000","last_peered":"2023-09-23T23:01:34.311884+0000","last_clean":"2023-09-23T23:01:34.311884+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:01:34.311884+0000","last_undegraded":"2023-09-23T23:01:34.311884+0000","last_fullsized":"2023-09-23T23:01:34.311884+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"393'9884","reported_seq":9982,"reported_epoch":393,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:47.133464+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:01:47.133464+0000","last_peered":"2023-09-23T23:01:47.133464+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:01:47.133464+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:01:47.133464+0000","mapping_epoch":353,"log_start":"327'4200","ondisk_log_start":"327'4200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9884,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":2084}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574358+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:01:42.574358+0000","last_peered":"2023-09-23T23:01:42.574358+0000","last_clean":"2023-09-23T23:01:42.574358+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:01:42.574358+0000","last_undegraded":"2023-09-23T23:01:42.574358+0000","last_fullsized":"2023-09-23T23:01:42.574358+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":296,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321613+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:01:34.321613+0000","last_peered":"2023-09-23T23:01:34.321613+0000","last_clean":"2023-09-23T23:01:34.321613+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:01:34.321613+0000","last_undegraded":"2023-09-23T23:01:34.321613+0000","last_fullsized":"2023-09-23T23:01:34.321613+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"393'10050","reported_seq":19334,"reported_epoch":393,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:47.228962+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:01:47.228962+0000","last_peered":"2023-09-23T23:01:47.228962+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:01:47.228962+0000","last_undegraded":"2023-09-23T23:01:47.228962+0000","last_fullsized":"2023-09-23T23:01:47.228962+0000","mapping_epoch":354,"log_start":"327'4300","ondisk_log_start":"327'4300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132775936,"num_objects":2026,"num_object_clones":0,"num_object_copies":6078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2026,"num_objects_unfound":0,"num_objects_dirty":2026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10050,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":2026}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":300,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322376+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:01:34.322376+0000","last_peered":"2023-09-23T23:01:34.322376+0000","last_clean":"2023-09-23T23:01:34.322376+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:01:34.322376+0000","last_undegraded":"2023-09-23T23:01:34.322376+0000","last_fullsized":"2023-09-23T23:01:34.322376+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":303,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321742+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:01:34.321742+0000","last_peered":"2023-09-23T23:01:34.321742+0000","last_clean":"2023-09-23T23:01:34.321742+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:01:34.321742+0000","last_undegraded":"2023-09-23T23:01:34.321742+0000","last_fullsized":"2023-09-23T23:01:34.321742+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321406+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:01:34.321406+0000","last_peered":"2023-09-23T23:01:34.321406+0000","last_clean":"2023-09-23T23:01:34.321406+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:01:34.321406+0000","last_undegraded":"2023-09-23T23:01:34.321406+0000","last_fullsized":"2023-09-23T23:01:34.321406+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"393'10082","reported_seq":10192,"reported_epoch":393,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:44.822802+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:01:44.822802+0000","last_peered":"2023-09-23T23:01:44.822802+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:01:44.822802+0000","last_undegraded":"2023-09-23T23:01:44.822802+0000","last_fullsized":"2023-09-23T23:01:44.822802+0000","mapping_epoch":359,"log_start":"327'4600","ondisk_log_start":"327'4600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5482,"log_dups_size":0,"ondisk_log_size":5482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133824512,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2042,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10082,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2042}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299379+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:01:34.299379+0000","last_peered":"2023-09-23T23:01:34.299379+0000","last_clean":"2023-09-23T23:01:34.299379+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:01:34.299379+0000","last_undegraded":"2023-09-23T23:01:34.299379+0000","last_fullsized":"2023-09-23T23:01:34.299379+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":413,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.316944+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:01:34.316944+0000","last_peered":"2023-09-23T23:01:34.316944+0000","last_clean":"2023-09-23T23:01:34.316944+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:01:34.316944+0000","last_undegraded":"2023-09-23T23:01:34.316944+0000","last_fullsized":"2023-09-23T23:01:34.316944+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":285,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322452+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:01:34.322452+0000","last_peered":"2023-09-23T23:01:34.322452+0000","last_clean":"2023-09-23T23:01:34.322452+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:01:34.322452+0000","last_undegraded":"2023-09-23T23:01:34.322452+0000","last_fullsized":"2023-09-23T23:01:34.322452+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"393'10026","reported_seq":18199,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:46.175993+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:01:46.175993+0000","last_peered":"2023-09-23T23:01:46.175993+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:01:46.175993+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"321'1400","ondisk_log_start":"321'1400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133562368,"num_objects":2038,"num_object_clones":0,"num_object_copies":6114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10026,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2038}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":360,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321791+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:01:34.321791+0000","last_peered":"2023-09-23T23:01:34.321791+0000","last_clean":"2023-09-23T23:01:34.321791+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:01:34.321791+0000","last_undegraded":"2023-09-23T23:01:34.321791+0000","last_fullsized":"2023-09-23T23:01:34.321791+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":237,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321285+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:01:34.321285+0000","last_peered":"2023-09-23T23:01:34.321285+0000","last_clean":"2023-09-23T23:01:34.321285+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:01:34.321285+0000","last_undegraded":"2023-09-23T23:01:34.321285+0000","last_fullsized":"2023-09-23T23:01:34.321285+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":303,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322100+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:01:34.322100+0000","last_peered":"2023-09-23T23:01:34.322100+0000","last_clean":"2023-09-23T23:01:34.322100+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:01:34.322100+0000","last_undegraded":"2023-09-23T23:01:34.322100+0000","last_fullsized":"2023-09-23T23:01:34.322100+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"393'10245","reported_seq":12192,"reported_epoch":393,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:47.906304+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:01:47.906304+0000","last_peered":"2023-09-23T23:01:47.906304+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:01:47.906304+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'200","ondisk_log_start":"321'200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":2536,"num_objects_missing":2536,"num_objects_degraded":3003,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10245,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":921,"num_bytes_recovered":60358656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1407},{"shards":"0,1,6","objects":546},{"shards":"0,6","objects":-2977}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":376,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321457+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:01:34.321457+0000","last_peered":"2023-09-23T23:01:34.321457+0000","last_clean":"2023-09-23T23:01:34.321457+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:01:34.321457+0000","last_undegraded":"2023-09-23T23:01:34.321457+0000","last_fullsized":"2023-09-23T23:01:34.321457+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":489,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:34.298765+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:01:34.298765+0000","last_peered":"2023-09-23T23:01:34.298765+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:01:34.298765+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:36.253915+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:01:36.253915+0000","last_peered":"2023-09-23T23:01:36.253915+0000","last_clean":"2023-09-23T23:01:36.253915+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:01:36.253915+0000","last_undegraded":"2023-09-23T23:01:36.253915+0000","last_fullsized":"2023-09-23T23:01:36.253915+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":303,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.314433+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:01:34.314433+0000","last_peered":"2023-09-23T23:01:34.314433+0000","last_clean":"2023-09-23T23:01:34.314433+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:01:34.314433+0000","last_undegraded":"2023-09-23T23:01:34.314433+0000","last_fullsized":"2023-09-23T23:01:34.314433+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"393'10129","reported_seq":10231,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:44.906783+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:01:44.906783+0000","last_peered":"2023-09-23T23:01:44.906783+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:01:44.906783+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"327'4600","ondisk_log_start":"327'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10129,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1552}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":383,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.298366+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:01:34.298366+0000","last_peered":"2023-09-23T23:01:34.298366+0000","last_clean":"2023-09-23T23:01:34.298366+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:01:34.298366+0000","last_undegraded":"2023-09-23T23:01:34.298366+0000","last_fullsized":"2023-09-23T23:01:34.298366+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":354,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.298453+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:01:34.298453+0000","last_peered":"2023-09-23T23:01:34.298453+0000","last_clean":"2023-09-23T23:01:34.298453+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:01:34.298453+0000","last_undegraded":"2023-09-23T23:01:34.298453+0000","last_fullsized":"2023-09-23T23:01:34.298453+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":314,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299749+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:01:34.299749+0000","last_peered":"2023-09-23T23:01:34.299749+0000","last_clean":"2023-09-23T23:01:34.299749+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:01:34.299749+0000","last_undegraded":"2023-09-23T23:01:34.299749+0000","last_fullsized":"2023-09-23T23:01:34.299749+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"393'10180","reported_seq":10297,"reported_epoch":393,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:46.976301+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:01:46.976301+0000","last_peered":"2023-09-23T23:01:46.976301+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:01:46.976301+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:01:46.976301+0000","mapping_epoch":354,"log_start":"327'4500","ondisk_log_start":"327'4500","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579352,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2024,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10180,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":2024}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":329,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.300369+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:01:34.300369+0000","last_peered":"2023-09-23T23:01:34.300369+0000","last_clean":"2023-09-23T23:01:34.300369+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:01:34.300369+0000","last_undegraded":"2023-09-23T23:01:34.300369+0000","last_fullsized":"2023-09-23T23:01:34.300369+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":237,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322546+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:01:34.322546+0000","last_peered":"2023-09-23T23:01:34.322546+0000","last_clean":"2023-09-23T23:01:34.322546+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:01:34.322546+0000","last_undegraded":"2023-09-23T23:01:34.322546+0000","last_fullsized":"2023-09-23T23:01:34.322546+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":206,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321531+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:01:34.321531+0000","last_peered":"2023-09-23T23:01:34.321531+0000","last_clean":"2023-09-23T23:01:34.321531+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:01:34.321531+0000","last_undegraded":"2023-09-23T23:01:34.321531+0000","last_fullsized":"2023-09-23T23:01:34.321531+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"393'10196","reported_seq":19006,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:47.136965+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:01:47.136965+0000","last_peered":"2023-09-23T23:01:47.136965+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:01:47.136965+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"327'4500","ondisk_log_start":"327'4500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127795200,"num_objects":1950,"num_object_clones":0,"num_object_copies":5850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4187,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10196,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3534,"num_bytes_recovered":231604224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":565}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":87,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.575554+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:01:42.575554+0000","last_peered":"2023-09-23T23:01:42.575554+0000","last_clean":"2023-09-23T23:01:42.575554+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:01:42.575554+0000","last_undegraded":"2023-09-23T23:01:42.575554+0000","last_fullsized":"2023-09-23T23:01:42.575554+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":101,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313620+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:01:34.313620+0000","last_peered":"2023-09-23T23:01:34.313620+0000","last_clean":"2023-09-23T23:01:34.313620+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:01:34.313620+0000","last_undegraded":"2023-09-23T23:01:34.313620+0000","last_fullsized":"2023-09-23T23:01:34.313620+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":78,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321563+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:01:34.321563+0000","last_peered":"2023-09-23T23:01:34.321563+0000","last_clean":"2023-09-23T23:01:34.321563+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:01:34.321563+0000","last_undegraded":"2023-09-23T23:01:34.321563+0000","last_fullsized":"2023-09-23T23:01:34.321563+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"393'10066","reported_seq":10593,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:47.083501+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:01:47.083501+0000","last_peered":"2023-09-23T23:01:47.083501+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:01:47.083501+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"327'4400","ondisk_log_start":"327'4400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10066,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-1547}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":349,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321802+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:01:34.321802+0000","last_peered":"2023-09-23T23:01:34.321802+0000","last_clean":"2023-09-23T23:01:34.321802+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:01:34.321802+0000","last_undegraded":"2023-09-23T23:01:34.321802+0000","last_fullsized":"2023-09-23T23:01:34.321802+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":296,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299706+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:01:34.299706+0000","last_peered":"2023-09-23T23:01:34.299706+0000","last_clean":"2023-09-23T23:01:34.299706+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:01:34.299706+0000","last_undegraded":"2023-09-23T23:01:34.299706+0000","last_fullsized":"2023-09-23T23:01:34.299706+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":367,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313049+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:01:34.313049+0000","last_peered":"2023-09-23T23:01:34.313049+0000","last_clean":"2023-09-23T23:01:34.313049+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:01:34.313049+0000","last_undegraded":"2023-09-23T23:01:34.313049+0000","last_fullsized":"2023-09-23T23:01:34.313049+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":331,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299614+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:01:34.299614+0000","last_peered":"2023-09-23T23:01:34.299614+0000","last_clean":"2023-09-23T23:01:34.299614+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:01:34.299614+0000","last_undegraded":"2023-09-23T23:01:34.299614+0000","last_fullsized":"2023-09-23T23:01:34.299614+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":141,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322165+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:01:34.322165+0000","last_peered":"2023-09-23T23:01:34.322165+0000","last_clean":"2023-09-23T23:01:34.322165+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:01:34.322165+0000","last_undegraded":"2023-09-23T23:01:34.322165+0000","last_fullsized":"2023-09-23T23:01:34.322165+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":202,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313950+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:01:34.313950+0000","last_peered":"2023-09-23T23:01:34.313950+0000","last_clean":"2023-09-23T23:01:34.313950+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:01:34.313950+0000","last_undegraded":"2023-09-23T23:01:34.313950+0000","last_fullsized":"2023-09-23T23:01:34.313950+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":416,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.317011+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:01:34.317011+0000","last_peered":"2023-09-23T23:01:34.317011+0000","last_clean":"2023-09-23T23:01:34.317011+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:01:34.317011+0000","last_undegraded":"2023-09-23T23:01:34.317011+0000","last_fullsized":"2023-09-23T23:01:34.317011+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":216,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.314290+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:01:34.314290+0000","last_peered":"2023-09-23T23:01:34.314290+0000","last_clean":"2023-09-23T23:01:34.314290+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:01:34.314290+0000","last_undegraded":"2023-09-23T23:01:34.314290+0000","last_fullsized":"2023-09-23T23:01:34.314290+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":303,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.313690+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:01:34.313690+0000","last_peered":"2023-09-23T23:01:34.313690+0000","last_clean":"2023-09-23T23:01:34.313690+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:01:34.313690+0000","last_undegraded":"2023-09-23T23:01:34.313690+0000","last_fullsized":"2023-09-23T23:01:34.313690+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":87,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.322604+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:01:34.322604+0000","last_peered":"2023-09-23T23:01:34.322604+0000","last_clean":"2023-09-23T23:01:34.322604+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:01:34.322604+0000","last_undegraded":"2023-09-23T23:01:34.322604+0000","last_fullsized":"2023-09-23T23:01:34.322604+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":89,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.321734+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:01:34.321734+0000","last_peered":"2023-09-23T23:01:34.321734+0000","last_clean":"2023-09-23T23:01:34.321734+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:01:34.321734+0000","last_undegraded":"2023-09-23T23:01:34.321734+0000","last_fullsized":"2023-09-23T23:01:34.321734+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":335,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.299951+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:01:34.299951+0000","last_peered":"2023-09-23T23:01:34.299951+0000","last_clean":"2023-09-23T23:01:34.299951+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:01:34.299951+0000","last_undegraded":"2023-09-23T23:01:34.299951+0000","last_fullsized":"2023-09-23T23:01:34.299951+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":378,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T23:01:34.312776+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:01:34.312776+0000","last_peered":"2023-09-23T23:01:34.312776+0000","last_clean":"2023-09-23T23:01:34.312776+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:01:34.312776+0000","last_undegraded":"2023-09-23T23:01:34.312776+0000","last_fullsized":"2023-09-23T23:01:34.312776+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2119630872,"num_objects":32344,"num_object_clones":0,"num_object_copies":97032,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":21111,"num_objects_misplaced":30398,"num_objects_unfound":0,"num_objects_dirty":32344,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161254,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18287,"num_bytes_recovered":1107296256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7181316096,"data_stored":7181303880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100454,"ondisk_log_size":100454,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430625,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626850,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253163,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595784,"kb_used_data":871296,"kb_used_omap":9,"kb_used_meta":724470,"kb_avail":92149368,"statfs":{"total":95995035648,"available":94360952832,"internally_reserved":0,"allocated":892207104,"data_stored":890717929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":741858281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8250000000000002,"5min":1.258,"15min":0.65400000000000003},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":83.664000000000001,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.24299999999999999},{"interface":"front","average":{"1min":5.0039999999999996,"5min":1.5980000000000001,"15min":1.103},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":83.700999999999993,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.48699999999999999,"15min":0.51400000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.087,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.64200000000000002,"15min":0.69799999999999995},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.74,"5min":4.048,"15min":4.048},"last":4.1310000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":4.0570000000000004},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":3.694}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":4.2000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":3.819},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":4.5389999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":4.1669999999999998},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":3.7869999999999999}]}]},{"osd":0,"up_from":353,"seq":1516123455533,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1590956,"kb_used_data":1425700,"kb_used_omap":37,"kb_used_meta":165210,"kb_avail":92154196,"statfs":{"total":95995035648,"available":94365896704,"internally_reserved":0,"allocated":1459916800,"data_stored":1458277726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":169175202},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.59199999999999997,"15min":0.55700000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.7629999999999999,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64000000000000001,"15min":0.66100000000000003},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.8650000000000002,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":0.86399999999999999,"15min":0.80800000000000005},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.012,"5min":0.72499999999999998,"15min":0.68500000000000005},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"last":0.376}]},{"osd":3,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.623,"15min":0.51000000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.76800000000000002,"15min":0.86199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6499999999999999,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":12.855,"5min":3.2290000000000001,"15min":1.649},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":183.148,"5min":183.148,"15min":183.148},"last":0.49299999999999999},{"interface":"front","average":{"1min":13.087999999999999,"5min":3.194,"15min":1.51},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":183.39500000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.36099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.86799999999999999,"15min":0.74199999999999999},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4089999999999998,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.282,"5min":1.0009999999999999,"15min":0.999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.774,"5min":7.774,"15min":7.774},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.92900000000000005,"15min":0.86599999999999999},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.7309999999999999,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.242,"5min":0.86799999999999999,"15min":0.77200000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":6.375,"5min":6.375,"15min":6.375},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.159,"15min":1.2549999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.8439999999999999,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.008,"5min":1.2150000000000001,"15min":1.2569999999999999},"min":{"1min":0.49299999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":6.343,"5min":6.343,"15min":6.343},"last":2.1429999999999998}]}]},{"osd":2,"up_from":359,"seq":1541893259306,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":727336,"kb_used_data":599968,"kb_used_omap":83,"kb_used_meta":127276,"kb_avail":93017816,"statfs":{"total":95995035648,"available":95250243584,"internally_reserved":0,"allocated":614367232,"data_stored":612792977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":130331077},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.36899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.253},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.27100000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.434},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":0.373}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.53000000000000003}]}]},{"osd":3,"up_from":94,"seq":403726926110,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":526564,"kb_used_data":26012,"kb_used_omap":7,"kb_used_meta":500536,"kb_avail":93218588,"statfs":{"total":95995035648,"available":95455834112,"internally_reserved":0,"allocated":26636288,"data_stored":25060450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":512549480},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":1.5640000000000001,"15min":0.73399999999999999},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":122.289,"5min":122.289,"15min":122.289},"last":37.249000000000002},{"interface":"front","average":{"1min":6.1609999999999996,"5min":1.5600000000000001,"15min":0.79100000000000004},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":35.991}]},{"osd":1,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.63200000000000001,"15min":2.3639999999999999},"min":{"1min":0.25,"5min":0.187,"15min":0.187},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":123.866},"last":0.25},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.73699999999999999,"15min":2.1549999999999998},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8010000000000002,"5min":6.8010000000000002,"15min":122.265},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.63,"15min":0.68899999999999995},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.7,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.857,"5min":0.98899999999999999,"15min":0.82499999999999996},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.223},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.27400000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":2.3519999999999999,"15min":2.0350000000000001},"min":{"1min":0.30199999999999999,"5min":0.30099999999999999,"15min":0.27300000000000002},"max":{"1min":64.406999999999996,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":4.008,"5min":2.5409999999999999,"15min":1.8999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":64.468000000000004,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.32500000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2290000000000001,"15min":1.3560000000000001},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1980000000000004,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.548,"5min":1.218,"15min":1.827},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.246,"5min":10.246,"15min":11.951000000000001},"last":0.40999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.597,"15min":1.7150000000000001},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0229999999999997,"5min":11.116,"15min":14.097},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4039999999999999,"15min":1.956},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.79,"5min":10.859,"15min":12.887},"last":0.441}]},{"osd":7,"last update":"Sat Sep 23 23:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.89400000000000002,"15min":2.0499999999999998},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.8449999999999998,"5min":4.8449999999999998,"15min":91.757999999999996},"last":0.379},{"interface":"front","average":{"1min":1.413,"5min":0.98099999999999998,"15min":1.9390000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":8.5830000000000002,"5min":8.5830000000000002,"15min":91.709999999999994},"last":0.45300000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019307,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2531992,"kb_used_data":1624528,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91213160,"statfs":{"total":95995035648,"available":93402275840,"internally_reserved":0,"allocated":1663516672,"data_stored":1662702523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":54.350000000000001},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":55.566000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.4380000000000002,"15min":3.5270000000000001},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":3.081,"5min":87.325000000000003,"15min":132.304},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.6320000000000001,"15min":3.4849999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.253},"max":{"1min":3.6120000000000001,"5min":87.572000000000003,"15min":132.453},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:28 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.0549999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4870000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.76200000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4770000000000003,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.72299999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.484,"15min":3.9009999999999998},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32400000000000001},"max":{"1min":4.0830000000000002,"5min":10.337999999999999,"15min":164.453},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.4410000000000001,"15min":3.8929999999999998},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0039999999999996,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.69299999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.748},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.655,"15min":1.181},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":3.5680000000000001,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.4379999999999999,"15min":1.113},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.1859999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.77000000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.5700000000000001,"15min":1.169},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0869999999999997,"5min":12.84,"15min":12.84},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.585,"15min":1.117},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4199999999999999,"5min":10.587,"15min":10.587},"last":0.53400000000000003}]}]},{"osd":5,"up_from":36,"seq":154618823081,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1894008,"kb_used_data":998448,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":91851144,"statfs":{"total":95995035648,"available":94055571456,"internally_reserved":0,"allocated":1022410752,"data_stored":1021624391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.878,"15min":0.69599999999999995},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.5709999999999997,"5min":14.42,"15min":14.42},"last":42.155000000000001},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2430000000000001,"15min":1.135},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3159999999999998,"5min":14.496,"15min":14.496},"last":43.378999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.312},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.39400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.2350000000000001,"15min":1.0629999999999999},"min":{"1min":0.45000000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.8370000000000002,"5min":12.23,"15min":12.23},"last":0.432},{"interface":"front","average":{"1min":2.5830000000000002,"5min":1.7490000000000001,"15min":1.2170000000000001},"min":{"1min":0.378,"5min":0.31,"15min":0.31},"max":{"1min":7.6719999999999997,"5min":11.112,"15min":11.112},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1443500032,"data_stored":1443495960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":885981184,"data_stored":885981184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":608960512,"data_stored":608960512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":21889024,"data_stored":21889024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1645613056,"data_stored":1645608984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1015742464,"data_stored":1015742464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1559629824,"data_stored":1559625752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:01:51.151 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:01:51.203 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:51.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:51.315 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:51.411 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:51.674 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:51 smithi052 ceph-mon[187743]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 21024/95391 objects degraded (22.040%); 30140/95391 objects misplaced (31.596%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:01:51.674 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:51 smithi052 ceph-mon[180979]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 21024/95391 objects degraded (22.040%); 30140/95391 objects misplaced (31.596%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:01:51.721 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:51.729 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:51.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:51.836 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:51.858 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:01:51.858 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:01:51.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59534 9.a 9:56ae8077:::benchmark_data_smithi052_402718_object59533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.094302+0000", 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 582.75467621500002, 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20214791800000001, 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.859 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59534 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T23:01:51.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094302+0000", 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094300+0000", 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094331+0000", 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0930999999999999e-05 2023-09-23T23:01:51.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094353+0000", 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2340999999999998e-05 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094362+0000", 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2519999999999999e-06 2023-09-23T23:01:51.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094415+0000", 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2997000000000001e-05 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094637+0000", 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000222208 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.094844+0000", 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000207631 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097382+0000", 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025371360000000002 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.097400+0000", 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8714000000000001e-05 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.098411+0000", 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001011131 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294679+0000", 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19626732399999999 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.294738+0000", 2023-09-23T23:01:51.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8801000000000002e-05 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296450+0000", 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001712618 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:59542 9.a 9:51ad1323:::benchmark_data_smithi052_402718_object59541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-09-23T23:01:51.868 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:52:09.099256+0000", 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 582.74972202499998, 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19894724899999999, 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 59542 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099256+0000", 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099254+0000", 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099282+0000", 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7971999999999999e-05 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099301+0000", 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8593e-05 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099311+0000", 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0074999999999999e-05 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099389+0000", 2023-09-23T23:01:51.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7689999999999996e-05 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099595+0000", 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020635099999999999 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.099809+0000", 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021421 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.102469+0000", 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002660314 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104681+0000", 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022113889999999998 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.104831+0000", 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015015699999999999 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296512+0000", 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19168089399999999 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.296580+0000", 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8103999999999997e-05 2023-09-23T23:01:51.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:52:09.298204+0000", 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016236810000000001 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:98500 9.a 9:5e200aba:::benchmark_data_smithi052_402718_object8859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-09-23T23:01:51.876 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:02.967496+0000", 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 528.88148257399996, 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22489546499999999, 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 98500 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T23:01:51.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967496+0000", 2023-09-23T23:01:51.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967494+0000", 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967502+0000", 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6190000000000002e-06 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967516+0000", 2023-09-23T23:01:51.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4651000000000001e-05 2023-09-23T23:01:51.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.967525+0000", 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6279999999999994e-06 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968657+0000", 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011323209999999999 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968810+0000", 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015326000000000001 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.968949+0000", 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000138429 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.974764+0000", 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058149839999999996 2023-09-23T23:01:51.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.975363+0000", 2023-09-23T23:01:51.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00059912599999999996 2023-09-23T23:01:51.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:02.977959+0000", 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025958399999999999 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190744+0000", 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.212785264 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.190803+0000", 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8777000000000003e-05 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:03.192391+0000", 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015884969999999999 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.887 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110895 9.a 9:5225ae0d:::benchmark_data_smithi052_402718_object21254:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063579+0000", 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 510.78539961000001, 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25354543899999998, 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:51.888 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110895 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063579+0000", 2023-09-23T23:01:51.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063577+0000", 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063584+0000", 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1860000000000003e-06 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063611+0000", 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6279e-05 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063621+0000", 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8179999999999999e-06 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063692+0000", 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1106000000000007e-05 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063841+0000", 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014895899999999999 2023-09-23T23:01:51.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063967+0000", 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012629000000000001 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.071444+0000", 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0074768090000000001 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072687+0000", 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00124325 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072764+0000", 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7496999999999993e-05 2023-09-23T23:01:51.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315667+0000", 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24290224499999999 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.315734+0000", 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7041999999999999e-05 2023-09-23T23:01:51.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.317124+0000", 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001390575 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.896 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45593.0:110898 9.a 9:538cf70b:::benchmark_data_smithi052_402718_object21257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:53:21.063646+0000", 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 510.785332514, 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25922483400000002, 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45593", 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3951755560", 2023-09-23T23:01:51.897 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 110898 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063646+0000", 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063645+0000", 2023-09-23T23:01:51.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063650+0000", 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6210000000000003e-06 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063660+0000", 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8379999999999997e-06 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.063667+0000", 2023-09-23T23:01:51.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9879999999999998e-06 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067524+0000", 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003857624 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067678+0000", 2023-09-23T23:01:51.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015363900000000001 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.067821+0000", 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000143096 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.072775+0000", 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049535079999999997 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.073473+0000", 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00069833799999999995 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.074443+0000", 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00097013000000000004 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321598+0000", 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.24715498 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.321672+0000", 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4229999999999999e-05 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:53:21.322871+0000", 2023-09-23T23:01:51.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001198483 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113718 10.f e344/343 10:f7ba7a2d:::benchmark_data_smithi052_461816_object16917:head v 344'7098, mlcod=0'0)", 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.801170+0000", 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 259.047808385, 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017963037000000001, 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.906 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801170+0000", 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801170+0000", 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801169+0000", 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801179+0000", 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8689999999999999e-06 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801196+0000", 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6954999999999999e-05 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801206+0000", 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0044000000000001e-05 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801266+0000", 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0256e-05 2023-09-23T23:01:51.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801291+0000", 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5378999999999999e-05 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.819085+0000", 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017794152000000001 2023-09-23T23:01:51.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.819133+0000", 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7685999999999997e-05 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119634 10.d e373/354 10:b8a57e34:::benchmark_data_smithi052_461816_object22833:head v 373'7541, mlcod=344'7206)", 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:01:51.911 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 170.58999828099999, 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11753303800000001, 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258986+0000", 2023-09-23T23:01:51.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9780000000000002e-06 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371535+0000", 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.112549314 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371552+0000", 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7370000000000001e-05 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371649+0000", 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6151999999999997e-05 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371672+0000", 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3422e-05 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376456+0000", 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047836850000000002 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376513+0000", 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7488999999999998e-05 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141431 10.c e386/354 10:3edfacc6:::benchmark_data_smithi052_461816_object44630:head v 386'8756, mlcod=338'6097)", 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.787594+0000", 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 70.061384872000005, 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.064963310999999996, 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787594+0000", 2023-09-23T23:01:51.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787594+0000", 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787593+0000", 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787599+0000", 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4389999999999999e-06 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787620+0000", 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0981000000000001e-05 2023-09-23T23:01:51.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787629+0000", 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5459999999999997e-06 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787683+0000", 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.41e-05 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787706+0000", 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2459e-05 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852517+0000", 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.064811553999999993 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852557+0000", 2023-09-23T23:01:51.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9502000000000002e-05 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141433 10.d e386/354 10:b300ffcd:::benchmark_data_smithi052_461816_object44632:head v 386'8896, mlcod=344'7206)", 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 70.056844209000005, 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063758645000000003, 2023-09-23T23:01:51.922 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792139+0000", 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.857e-06 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852291+0000", 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.060151688000000002 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852300+0000", 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.132e-06 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.853137+0000", 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083692499999999997 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.853157+0000", 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9819000000000001e-05 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855854+0000", 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026973240000000001 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855893+0000", 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8544000000000002e-05 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141434 10.5 e386/359 10:ac9ab671:::benchmark_data_smithi052_461816_object44633:head v 386'8906, mlcod=323'2477)", 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:01:51.928 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 70.033048057000002, 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.039674503, 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815935+0000", 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1019999999999996e-06 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852242+0000", 2023-09-23T23:01:51.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036307219000000002 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852253+0000", 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1071e-05 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852543+0000", 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00028993500000000001 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852567+0000", 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3705000000000001e-05 2023-09-23T23:01:51.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855565+0000", 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029977020000000001 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855605+0000", 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0085e-05 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:146903 10.f e388/360 10:ff31cbf4:::benchmark_data_smithi052_461816_object50102:head v 388'9186, mlcod=323'2484)", 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:02.433106+0000", 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 49.415872733, 2023-09-23T23:01:51.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015857768000000001, 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433106+0000", 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433106+0000", 2023-09-23T23:01:51.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433105+0000", 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433110+0000", 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1499999999999998e-06 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433121+0000", 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0342e-05 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433126+0000", 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4079999999999997e-06 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.437133+0000", 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040065090000000001 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.437167+0000", 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4368e-05 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.448916+0000", 2023-09-23T23:01:51.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011748827 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.448964+0000", 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7559000000000003e-05 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:150994 10.7 e389/360 10:ef49469c:::benchmark_data_smithi052_461816_object54193:head v 389'9284, mlcod=323'2408)", 2023-09-23T23:01:51.940 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:16.053798+0000", 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.795180821999999, 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015287967, 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053798+0000", 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053798+0000", 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053797+0000", 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053806+0000", 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5589999999999994e-06 2023-09-23T23:01:51.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053826+0000", 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9704999999999999e-05 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053835+0000", 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7420000000000003e-06 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053896+0000", 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1141000000000003e-05 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053923+0000", 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.707e-05 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.069025+0000", 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015102546 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.069086+0000", 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0239999999999999e-05 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151618 10.4 e390/354 10:21546296:::benchmark_data_smithi052_461816_object54817:head v 390'9478, mlcod=344'7124)", 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 34.099301443999998, 2023-09-23T23:01:51.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065726571999999997, 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:01:51.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749682+0000", 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9110000000000001e-06 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749708+0000", 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6335000000000001e-05 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749717+0000", 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0129999999999999e-06 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749774+0000", 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7306000000000003e-05 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749799+0000", 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4269000000000001e-05 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815362+0000", 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065563124 2023-09-23T23:01:51.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815404+0000", 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.18e-05 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151620 10.5 e390/359 10:a897b5ac:::benchmark_data_smithi052_461816_object54819:head v 390'9515, mlcod=323'2477)", 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.802451+0000", 2023-09-23T23:01:51.953 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 34.046527546, 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017011258000000001, 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802451+0000", 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802451+0000", 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802450+0000", 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802457+0000", 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5320000000000003e-06 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815234+0000", 2023-09-23T23:01:51.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012777011 2023-09-23T23:01:51.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815243+0000", 2023-09-23T23:01:51.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1640000000000007e-06 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815317+0000", 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4259999999999997e-05 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815342+0000", 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4425000000000001e-05 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.819411+0000", 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00406909 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.819462+0000", 2023-09-23T23:01:51.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1418000000000001e-05 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:152376 10.4 e390/354 10:20a7f37c:::benchmark_data_smithi052_461816_object55575:head v 390'9534, mlcod=344'7124)", 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:19.623703+0000", 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 32.225275056000001, 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017693904999999999, 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.960 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623703+0000", 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623703+0000", 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623702+0000", 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623713+0000", 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0216e-05 2023-09-23T23:01:51.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623735+0000", 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2589e-05 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623745+0000", 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4730000000000006e-06 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623814+0000", 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9177000000000005e-05 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623837+0000", 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3247e-05 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.641367+0000", 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017529718999999999 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.641397+0000", 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0599000000000003e-05 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:157581 10.7 e392/360 10:e5c44f0b:::benchmark_data_smithi052_461816_object60780:head v 392'9678, mlcod=323'2408)", 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:35.636443+0000", 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 16.212535390999999, 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013346139, 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636443+0000", 2023-09-23T23:01:51.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636443+0000", 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636442+0000", 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636452+0000", 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7350000000000008e-06 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636464+0000", 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2279999999999999e-05 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636470+0000", 2023-09-23T23:01:51.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0990000000000004e-06 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636510+0000", 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0105999999999999e-05 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.636538+0000", 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7997e-05 2023-09-23T23:01:51.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.649738+0000", 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013199845 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:35.649789+0000", 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1008999999999999e-05 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.970 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:158299 10.3 e392/359 10:c936ed6f:::benchmark_data_smithi052_461816_object61498:head v 392'9973, mlcod=323'2482)", 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:37.824797+0000", 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.024181384, 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011803895999999999, 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824797+0000", 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824797+0000", 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824796+0000", 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:51.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824805+0000", 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5480000000000002e-06 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824825+0000", 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0129e-05 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824838+0000", 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3263e-05 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824925+0000", 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7000000000000001e-05 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.824953+0000", 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7996000000000001e-05 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.836564+0000", 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011610996 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:37.836601+0000", 2023-09-23T23:01:51.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6927000000000001e-05 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:162995 10.f e393/360 10:ff5f2db6:::benchmark_data_smithi052_461816_object66194:head v 393'10158, mlcod=323'2484)", 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:51.231179+0000", 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.61779936199999996, 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0121003, 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.976 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231179+0000", 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231179+0000", 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231178+0000", 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231186+0000", 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1359999999999997e-06 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231198+0000", 2023-09-23T23:01:51.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2174999999999999e-05 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231206+0000", 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6040000000000004e-06 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231267+0000", 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0458000000000003e-05 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.231289+0000", 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2121999999999999e-05 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.243228+0000", 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011939294 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.243279+0000", 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1474999999999998e-05 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:163118 10.7 e393/360 10:ed4c418d:::benchmark_data_smithi052_461816_object66317:head v 393'10040, mlcod=323'2408)", 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:51.495929+0000", 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.35304929600000001, 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013701269, 2023-09-23T23:01:51.981 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.495929+0000", 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.495929+0000", 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.495928+0000", 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.495938+0000", 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7340000000000003e-06 2023-09-23T23:01:51.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.495962+0000", 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4213e-05 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.495972+0000", 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8579999999999995e-06 2023-09-23T23:01:51.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.496048+0000", 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6150999999999996e-05 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.496092+0000", 2023-09-23T23:01:51.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3498000000000003e-05 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.509562+0000", 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013470027000000001 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.509630+0000", 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8873000000000003e-05 2023-09-23T23:01:51.986 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:163207 10.4 e393/354 10:22cd7b83:::benchmark_data_smithi052_461816_object66406:head v 393'10188, mlcod=344'7124)", 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:51.692889+0000", 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.15608935800000001, 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011723290000000001, 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:01:51.987 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.692889+0000", 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.692889+0000", 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.692887+0000", 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.692903+0000", 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5438e-05 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:01:51.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.692926+0000", 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3861000000000001e-05 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.692938+0000", 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1942e-05 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.696693+0000", 2023-09-23T23:01:51.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003754917 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.696715+0000", 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1599000000000001e-05 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.704567+0000", 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0078520980000000001 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:01:51.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:51.704612+0000", 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5354e-05 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:01:51.992 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:01:51.993 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:51.993 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:51 smithi151 ceph-mon[159210]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 21024/95391 objects degraded (22.040%); 30140/95391 objects misplaced (31.596%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T23:01:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:51.994 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:51.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:52.095 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:52.129 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:52.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:52.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:52.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:52.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:52.339 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:52.693 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:01:52.748 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:52.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:52.834 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:01:52.856 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:52.904 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:52.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3093475327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:52.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:52 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3093475327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:52.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:52 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3093475327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:52.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:52 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:01:53.314 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:53.385 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:53.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:53.463 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:01:53.491 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:53.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:01:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:01:53] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.33.4" 2023-09-23T23:01:53.528 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 20322/91854 objects degraded (22.124%); 29893/91854 objects misplaced (32.544%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: osdmap e394: 8 total, 8 up, 6 in 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[180979]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 20322/91854 objects degraded (22.124%); 29893/91854 objects misplaced (32.544%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: osdmap e394: 8 total, 8 up, 6 in 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.810 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:53 smithi052 ceph-mon[187743]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 362 op/s; 20322/91854 objects degraded (22.124%); 29893/91854 objects misplaced (32.544%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: osdmap e394: 8 total, 8 up, 6 in 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: failed to encode map e394 with expected crc 2023-09-23T23:01:53.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:53 smithi151 ceph-mon[159210]: failed to encode map e394 with expected crc 2023-09-23T23:01:54.153 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:54.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:54.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:54.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:54.316 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:54.407 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:54.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:54.512 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:54.541 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:54.915 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:54.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:55.020 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:55.078 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:55.178 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:55.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:55.289 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:55.315 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:55.362 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:55.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:55.380 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:01:55.398 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:01:55.417 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:01:55.437 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:01:55.455 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:01:55.466 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:55.522 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:55.522 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:01:55.554 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:01:55.592 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:01:55.594 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.594 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.594 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.594 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.595 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.595 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.595 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.595 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:01:55.685 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:55 smithi151 ceph-mon[159210]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 364 op/s; 20436/91512 objects degraded (22.331%); 29893/91512 objects misplaced (32.666%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:55.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 20436/91512 objects degraded (22.331%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:55.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1882560482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:55.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:55 smithi052 ceph-mon[180979]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 364 op/s; 20436/91512 objects degraded (22.331%); 29893/91512 objects misplaced (32.666%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:55.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 20436/91512 objects degraded (22.331%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:55.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1882560482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:55.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:55 smithi052 ceph-mon[187743]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 364 op/s; 20436/91512 objects degraded (22.331%); 29893/91512 objects misplaced (32.666%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:55.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 20436/91512 objects degraded (22.331%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:01:55.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1882560482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:01:55.812 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:01:55.851 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:55.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:55.967 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:01:55.996 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:56.107 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:01:56.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:56.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:01:56.181 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:01:56.181 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 20436/91512 objects degraded (22.331%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1648,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":394,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":30504,"data_bytes":1998067640,"bytes_used":8824373248,"bytes_avail":567145840640,"bytes_total":575970213888,"degraded_objects":20436,"degraded_total":91512,"degraded_ratio":0.22331497508523473,"misplaced_objects":29893,"misplaced_total":91512,"misplaced_ratio":0.32665661334032697,"recovering_objects_per_sec":44,"recovering_bytes_per_sec":2928429,"recovering_keys_per_sec":0,"num_objects_recovered":447,"num_bytes_recovered":29294592,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":364},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (5m)\n [======================......] (remaining: 75s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:01:56.227 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:56.268 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:56.523 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:56.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:56.623 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:01:56.623 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:01:56.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:56.673 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:56.797 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4209896574' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:56.798 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4209896574' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4209896574' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:01:57.058 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:01:57.058 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:01:57.063 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:57.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:57.155 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:57.168 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:57.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:57.344 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:57.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:57.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:57.485 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:57.585 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:01:57.598 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:57.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:57.705 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:57.750 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:57 smithi151 ceph-mon[159210]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.8 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 20195/88806 objects degraded (22.741%); 29652/88806 objects misplaced (33.390%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:57.755 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:57.848 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:57.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:57.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:57.962 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:01:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:57 smithi052 ceph-mon[180979]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.8 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 20195/88806 objects degraded (22.741%); 29652/88806 objects misplaced (33.390%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:57.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:57 smithi052 ceph-mon[187743]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.8 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 20195/88806 objects degraded (22.741%); 29652/88806 objects misplaced (33.390%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T23:01:58.000 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:58.078 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:01:58.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:58.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:01:58.144 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:01:58.183 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:58.224 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:58.317 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:58.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:01:58.406 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:01:58.428 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:01:58.489 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:58.607 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:01:58.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:58.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:01:58.718 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:01:58.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:58.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:01:58.901 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:01:58.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:58.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:01:59.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:01:59.058 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:59.227 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:01:59.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:59.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:01:59.334 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:01:59.389 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:59.467 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:01:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:59.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:01:59.571 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:01:59.599 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:59.686 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:01:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:59.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:01:59.721 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:01:59.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:01:59.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:59 smithi151 ceph-mon[159210]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 19701/85572 objects degraded (23.023%); 29403/85572 objects misplaced (34.361%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:01:59.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/707827780' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:59.872 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:01:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:59.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:01:59.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:59 smithi052 ceph-mon[180979]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 19701/85572 objects degraded (23.023%); 29403/85572 objects misplaced (34.361%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:01:59.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/707827780' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:59.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:01:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:59.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:59 smithi052 ceph-mon[187743]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 19701/85572 objects degraded (23.023%); 29403/85572 objects misplaced (34.361%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T23:01:59.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/707827780' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:59.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:01:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:01:59.962 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:01:59.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:01:59.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:00.068 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:00.124 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:00.182 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:00.183 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: osdmap e395: 8 total, 8 up, 6 in 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 19701/85572 objects degraded (23.023%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:00.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: osdmap e395: 8 total, 8 up, 6 in 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: failed to encode map e395 with expected crc 2023-09-23T23:02:00.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 19701/85572 objects degraded (23.023%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:01.034 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:01.039 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:01.048 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:01.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:01.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:01.066 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:01.070 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: osdmap e395: 8 total, 8 up, 6 in 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: failed to encode map e395 with expected crc 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: failed to encode map e395 with expected crc 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: failed to encode map e395 with expected crc 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: failed to encode map e395 with expected crc 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: failed to encode map e395 with expected crc 2023-09-23T23:02:01.071 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 19701/85572 objects degraded (23.023%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:01.103 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:02:01.157 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:01.208 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:01.581 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:01.726 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:01 smithi052 ceph-mon[187743]: pgmap v1199: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 337 op/s; 19621/83751 objects degraded (23.428%); 29147/83751 objects misplaced (34.802%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:02:01.797 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:01.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:01.904 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:01.956 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1199,"stamp":"2023-09-23T23:02:00.603845+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1828526008,"num_objects":27917,"num_object_clones":0,"num_object_copies":83751,"num_objects_missing_on_primary":2524,"num_objects_missing":2524,"num_objects_degraded":19621,"num_objects_misplaced":29147,"num_objects_unfound":0,"num_objects_dirty":27917,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":165965,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18821,"num_bytes_recovered":1140014012,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100799,"ondisk_log_size":100799,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8255520,"kb_used_data":4929072,"kb_used_omap":137,"kb_used_meta":3326262,"kb_avail":554215392,"statfs":{"total":575970213888,"available":567516561408,"internally_reserved":0,"allocated":5047369728,"data_stored":5039459584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3406092472},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-177012736,"num_objects":-2701,"num_object_clones":0,"num_object_copies":-8103,"num_objects_missing_on_primary":-14,"num_objects_missing":-14,"num_objects_degraded":-701,"num_objects_misplaced":-746,"num_objects_unfound":0,"num_objects_dirty":-2701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2701,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":265,"num_bytes_recovered":17367040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003137"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":406,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567898+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:01:42.567898+0000","last_peered":"2023-09-23T23:01:42.567898+0000","last_clean":"2023-09-23T23:01:42.567898+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:01:42.567898+0000","last_undegraded":"2023-09-23T23:01:42.567898+0000","last_fullsized":"2023-09-23T23:01:42.567898+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.575263+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:01:42.575263+0000","last_peered":"2023-09-23T23:01:42.575263+0000","last_clean":"2023-09-23T23:01:42.575263+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:01:42.575263+0000","last_undegraded":"2023-09-23T23:01:42.575263+0000","last_fullsized":"2023-09-23T23:01:42.575263+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":62,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574216+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:01:42.574216+0000","last_peered":"2023-09-23T23:01:42.574216+0000","last_clean":"2023-09-23T23:01:42.574216+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:01:42.574216+0000","last_undegraded":"2023-09-23T23:01:42.574216+0000","last_fullsized":"2023-09-23T23:01:42.574216+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":391,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574622+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:01:42.574622+0000","last_peered":"2023-09-23T23:01:42.574622+0000","last_clean":"2023-09-23T23:01:42.574622+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:01:42.574622+0000","last_undegraded":"2023-09-23T23:01:42.574622+0000","last_fullsized":"2023-09-23T23:01:42.574622+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574138+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:01:42.574138+0000","last_peered":"2023-09-23T23:01:42.574138+0000","last_clean":"2023-09-23T23:01:42.574138+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:01:42.574138+0000","last_undegraded":"2023-09-23T23:01:42.574138+0000","last_fullsized":"2023-09-23T23:01:42.574138+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681132+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:01:52.681132+0000","last_peered":"2023-09-23T23:01:52.681132+0000","last_clean":"2023-09-23T23:01:52.681132+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:01:52.681132+0000","last_undegraded":"2023-09-23T23:01:52.681132+0000","last_fullsized":"2023-09-23T23:01:52.681132+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":329,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.573873+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:01:42.573873+0000","last_peered":"2023-09-23T23:01:42.573873+0000","last_clean":"2023-09-23T23:01:42.573873+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:01:42.573873+0000","last_undegraded":"2023-09-23T23:01:42.573873+0000","last_fullsized":"2023-09-23T23:01:42.573873+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.573968+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:01:42.573968+0000","last_peered":"2023-09-23T23:01:42.573968+0000","last_clean":"2023-09-23T23:01:42.573968+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:01:42.573968+0000","last_undegraded":"2023-09-23T23:01:42.573968+0000","last_fullsized":"2023-09-23T23:01:42.573968+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":290,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.572383+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:01:42.572383+0000","last_peered":"2023-09-23T23:01:42.572383+0000","last_clean":"2023-09-23T23:01:42.572383+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:01:42.572383+0000","last_undegraded":"2023-09-23T23:01:42.572383+0000","last_fullsized":"2023-09-23T23:01:42.572383+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574075+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:01:42.574075+0000","last_peered":"2023-09-23T23:01:42.574075+0000","last_clean":"2023-09-23T23:01:42.574075+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:01:42.574075+0000","last_undegraded":"2023-09-23T23:01:42.574075+0000","last_fullsized":"2023-09-23T23:01:42.574075+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":83,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.575265+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:01:42.575265+0000","last_peered":"2023-09-23T23:01:42.575265+0000","last_clean":"2023-09-23T23:01:42.575265+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:01:42.575265+0000","last_undegraded":"2023-09-23T23:01:42.575265+0000","last_fullsized":"2023-09-23T23:01:42.575265+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":139,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.560313+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:01:42.560313+0000","last_peered":"2023-09-23T23:01:42.560313+0000","last_clean":"2023-09-23T23:01:42.560313+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:01:42.560313+0000","last_undegraded":"2023-09-23T23:01:42.560313+0000","last_fullsized":"2023-09-23T23:01:42.560313+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":465,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567887+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:01:42.567887+0000","last_peered":"2023-09-23T23:01:42.567887+0000","last_clean":"2023-09-23T23:01:42.567887+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:01:42.567887+0000","last_undegraded":"2023-09-23T23:01:42.567887+0000","last_fullsized":"2023-09-23T23:01:42.567887+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":368,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568161+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:01:42.568161+0000","last_peered":"2023-09-23T23:01:42.568161+0000","last_clean":"2023-09-23T23:01:42.568161+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:01:42.568161+0000","last_undegraded":"2023-09-23T23:01:42.568161+0000","last_fullsized":"2023-09-23T23:01:42.568161+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":294,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568254+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:01:42.568254+0000","last_peered":"2023-09-23T23:01:42.568254+0000","last_clean":"2023-09-23T23:01:42.568254+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:01:42.568254+0000","last_undegraded":"2023-09-23T23:01:42.568254+0000","last_fullsized":"2023-09-23T23:01:42.568254+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":74,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.575310+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:01:42.575310+0000","last_peered":"2023-09-23T23:01:42.575310+0000","last_clean":"2023-09-23T23:01:42.575310+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:01:42.575310+0000","last_undegraded":"2023-09-23T23:01:42.575310+0000","last_fullsized":"2023-09-23T23:01:42.575310+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":383,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574355+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:01:42.574355+0000","last_peered":"2023-09-23T23:01:42.574355+0000","last_clean":"2023-09-23T23:01:42.574355+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:01:42.574355+0000","last_undegraded":"2023-09-23T23:01:42.574355+0000","last_fullsized":"2023-09-23T23:01:42.574355+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":294,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.571971+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:01:42.571971+0000","last_peered":"2023-09-23T23:01:42.571971+0000","last_clean":"2023-09-23T23:01:42.571971+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:01:42.571971+0000","last_undegraded":"2023-09-23T23:01:42.571971+0000","last_fullsized":"2023-09-23T23:01:42.571971+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":279,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567711+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:01:42.567711+0000","last_peered":"2023-09-23T23:01:42.567711+0000","last_clean":"2023-09-23T23:01:42.567711+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:01:42.567711+0000","last_undegraded":"2023-09-23T23:01:42.567711+0000","last_fullsized":"2023-09-23T23:01:42.567711+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":407,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574754+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:01:42.574754+0000","last_peered":"2023-09-23T23:01:42.574754+0000","last_clean":"2023-09-23T23:01:42.574754+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:01:42.574754+0000","last_undegraded":"2023-09-23T23:01:42.574754+0000","last_fullsized":"2023-09-23T23:01:42.574754+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":113,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680699+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:01:52.680699+0000","last_peered":"2023-09-23T23:01:52.680699+0000","last_clean":"2023-09-23T23:01:52.680699+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:01:52.680699+0000","last_undegraded":"2023-09-23T23:01:52.680699+0000","last_fullsized":"2023-09-23T23:01:52.680699+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.575437+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:01:42.575437+0000","last_peered":"2023-09-23T23:01:42.575437+0000","last_clean":"2023-09-23T23:01:42.575437+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:01:42.575437+0000","last_undegraded":"2023-09-23T23:01:42.575437+0000","last_fullsized":"2023-09-23T23:01:42.575437+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"395'10392","reported_seq":10491,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.944478+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:01:59.944478+0000","last_peered":"2023-09-23T23:01:59.944478+0000","last_clean":"2023-09-23T23:01:59.944478+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:01:59.944478+0000","last_undegraded":"2023-09-23T23:01:59.944478+0000","last_fullsized":"2023-09-23T23:01:59.944478+0000","mapping_epoch":353,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5492,"log_dups_size":0,"ondisk_log_size":5492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111017984,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10392,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":278,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568624+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:01:42.568624+0000","last_peered":"2023-09-23T23:01:42.568624+0000","last_clean":"2023-09-23T23:01:42.568624+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:01:42.568624+0000","last_undegraded":"2023-09-23T23:01:42.568624+0000","last_fullsized":"2023-09-23T23:01:42.568624+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":264,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568703+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:01:42.568703+0000","last_peered":"2023-09-23T23:01:42.568703+0000","last_clean":"2023-09-23T23:01:42.568703+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:01:42.568703+0000","last_undegraded":"2023-09-23T23:01:42.568703+0000","last_fullsized":"2023-09-23T23:01:42.568703+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"394'10171","reported_seq":12598,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:56.178619+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:01:56.178619+0000","last_peered":"2023-09-23T23:01:56.178619+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:01:56.178619+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"322'1500","ondisk_log_start":"322'1500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8671,"log_dups_size":0,"ondisk_log_size":8671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116588544,"num_objects":1779,"num_object_clones":0,"num_object_copies":5337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1779,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":1779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10171,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-1788}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":288,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568983+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:01:42.568983+0000","last_peered":"2023-09-23T23:01:42.568983+0000","last_clean":"2023-09-23T23:01:42.568983+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:01:42.568983+0000","last_undegraded":"2023-09-23T23:01:42.568983+0000","last_fullsized":"2023-09-23T23:01:42.568983+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":401,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567627+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:01:42.567627+0000","last_peered":"2023-09-23T23:01:42.567627+0000","last_clean":"2023-09-23T23:01:42.567627+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:01:42.567627+0000","last_undegraded":"2023-09-23T23:01:42.567627+0000","last_fullsized":"2023-09-23T23:01:42.567627+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"395'10404","reported_seq":10497,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:59.855637+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:01:59.855637+0000","last_peered":"2023-09-23T23:01:59.855637+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:01:59.855637+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:01:59.855637+0000","mapping_epoch":354,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5504,"log_dups_size":0,"ondisk_log_size":5504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109314048,"num_objects":1668,"num_object_clones":0,"num_object_copies":5004,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":1668,"num_objects_unfound":0,"num_objects_dirty":1668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10404,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1668}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":145,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.572020+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:01:42.572020+0000","last_peered":"2023-09-23T23:01:42.572020+0000","last_clean":"2023-09-23T23:01:42.572020+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:01:42.572020+0000","last_undegraded":"2023-09-23T23:01:42.572020+0000","last_fullsized":"2023-09-23T23:01:42.572020+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":263,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.571460+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:01:42.571460+0000","last_peered":"2023-09-23T23:01:42.571460+0000","last_clean":"2023-09-23T23:01:42.571460+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:01:42.571460+0000","last_undegraded":"2023-09-23T23:01:42.571460+0000","last_fullsized":"2023-09-23T23:01:42.571460+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"395'10444","reported_seq":10723,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:59.919372+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:01:59.919372+0000","last_peered":"2023-09-23T23:01:59.919372+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:01:59.919372+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113377280,"num_objects":1730,"num_object_clones":0,"num_object_copies":5190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1454,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":1730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10444,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":276},{"shards":"0,5","objects":-1880}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":64,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696676+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:01:52.696676+0000","last_peered":"2023-09-23T23:01:52.696676+0000","last_clean":"2023-09-23T23:01:52.696676+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:01:52.696676+0000","last_undegraded":"2023-09-23T23:01:52.696676+0000","last_fullsized":"2023-09-23T23:01:52.696676+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":294,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680862+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:01:52.680862+0000","last_peered":"2023-09-23T23:01:52.680862+0000","last_clean":"2023-09-23T23:01:52.680862+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:01:52.680862+0000","last_undegraded":"2023-09-23T23:01:52.680862+0000","last_fullsized":"2023-09-23T23:01:52.680862+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"394'10309","reported_seq":10407,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:57.243433+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:01:57.243433+0000","last_peered":"2023-09-23T23:01:57.243433+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:01:57.243433+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:01:57.243433+0000","mapping_epoch":353,"log_start":"331'4600","ondisk_log_start":"331'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116981760,"num_objects":1785,"num_object_clones":0,"num_object_copies":5355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10309,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1785}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":83,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695772+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:01:52.695772+0000","last_peered":"2023-09-23T23:01:52.695772+0000","last_clean":"2023-09-23T23:01:52.695772+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:01:52.695772+0000","last_undegraded":"2023-09-23T23:01:52.695772+0000","last_fullsized":"2023-09-23T23:01:52.695772+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":387,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567805+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:01:42.567805+0000","last_peered":"2023-09-23T23:01:42.567805+0000","last_clean":"2023-09-23T23:01:42.567805+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:01:42.567805+0000","last_undegraded":"2023-09-23T23:01:42.567805+0000","last_fullsized":"2023-09-23T23:01:42.567805+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"394'10423","reported_seq":10530,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:57.194434+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:01:57.194434+0000","last_peered":"2023-09-23T23:01:57.194434+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:01:57.194434+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"331'4700","ondisk_log_start":"331'4700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116588544,"num_objects":1779,"num_object_clones":0,"num_object_copies":5337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1779,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":1779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10423,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-1840}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":177,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568056+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:01:42.568056+0000","last_peered":"2023-09-23T23:01:42.568056+0000","last_clean":"2023-09-23T23:01:42.568056+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:01:42.568056+0000","last_undegraded":"2023-09-23T23:01:42.568056+0000","last_fullsized":"2023-09-23T23:01:42.568056+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":404,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567602+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:01:42.567602+0000","last_peered":"2023-09-23T23:01:42.567602+0000","last_clean":"2023-09-23T23:01:42.567602+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:01:42.567602+0000","last_undegraded":"2023-09-23T23:01:42.567602+0000","last_fullsized":"2023-09-23T23:01:42.567602+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"394'10288","reported_seq":18445,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:57.200270+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:01:57.200270+0000","last_peered":"2023-09-23T23:01:57.200270+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:01:57.200270+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:01:57.200270+0000","mapping_epoch":354,"log_start":"330'4600","ondisk_log_start":"330'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116129792,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10288,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4021,"num_bytes_recovered":263520256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":338}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":417,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567405+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:01:42.567405+0000","last_peered":"2023-09-23T23:01:42.567405+0000","last_clean":"2023-09-23T23:01:42.567405+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:01:42.567405+0000","last_undegraded":"2023-09-23T23:01:42.567405+0000","last_fullsized":"2023-09-23T23:01:42.567405+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":395,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.570647+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:01:42.570647+0000","last_peered":"2023-09-23T23:01:42.570647+0000","last_clean":"2023-09-23T23:01:42.570647+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:01:42.570647+0000","last_undegraded":"2023-09-23T23:01:42.570647+0000","last_fullsized":"2023-09-23T23:01:42.570647+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"394'10129","reported_seq":10228,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:01:57.000590+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:01:57.000590+0000","last_peered":"2023-09-23T23:01:57.000590+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:01:57.000590+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:01:57.000590+0000","mapping_epoch":353,"log_start":"327'4400","ondisk_log_start":"327'4400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120520704,"num_objects":1839,"num_object_clones":0,"num_object_copies":5517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10129,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1839}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:46.255457+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:01:46.255457+0000","last_peered":"2023-09-23T23:01:46.255457+0000","last_clean":"2023-09-23T23:01:46.255457+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:01:46.255457+0000","last_undegraded":"2023-09-23T23:01:46.255457+0000","last_fullsized":"2023-09-23T23:01:46.255457+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":297,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.562315+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:01:42.562315+0000","last_peered":"2023-09-23T23:01:42.562315+0000","last_clean":"2023-09-23T23:01:42.562315+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:01:42.562315+0000","last_undegraded":"2023-09-23T23:01:42.562315+0000","last_fullsized":"2023-09-23T23:01:42.562315+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"394'10287","reported_seq":19572,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:57.000589+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:01:57.000589+0000","last_peered":"2023-09-23T23:01:57.000589+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:01:57.000589+0000","last_undegraded":"2023-09-23T23:01:57.000589+0000","last_fullsized":"2023-09-23T23:01:57.000589+0000","mapping_epoch":354,"log_start":"331'4600","ondisk_log_start":"331'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117243904,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1789,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10287,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1789}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":301,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.561212+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:01:42.561212+0000","last_peered":"2023-09-23T23:01:42.561212+0000","last_clean":"2023-09-23T23:01:42.561212+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:01:42.561212+0000","last_undegraded":"2023-09-23T23:01:42.561212+0000","last_fullsized":"2023-09-23T23:01:42.561212+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.571930+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:01:42.571930+0000","last_peered":"2023-09-23T23:01:42.571930+0000","last_clean":"2023-09-23T23:01:42.571930+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:01:42.571930+0000","last_undegraded":"2023-09-23T23:01:42.571930+0000","last_fullsized":"2023-09-23T23:01:42.571930+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568568+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:01:42.568568+0000","last_peered":"2023-09-23T23:01:42.568568+0000","last_clean":"2023-09-23T23:01:42.568568+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:01:42.568568+0000","last_undegraded":"2023-09-23T23:01:42.568568+0000","last_fullsized":"2023-09-23T23:01:42.568568+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"395'10452","reported_seq":10564,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:01:59.876097+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:01:59.876097+0000","last_peered":"2023-09-23T23:01:59.876097+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:01:59.876097+0000","last_undegraded":"2023-09-23T23:01:59.876097+0000","last_fullsized":"2023-09-23T23:01:59.876097+0000","mapping_epoch":359,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109576192,"num_objects":1672,"num_object_clones":0,"num_object_copies":5016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1672,"num_objects_unfound":0,"num_objects_dirty":1672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10452,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1672}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681795+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:01:52.681795+0000","last_peered":"2023-09-23T23:01:52.681795+0000","last_clean":"2023-09-23T23:01:52.681795+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:01:52.681795+0000","last_undegraded":"2023-09-23T23:01:52.681795+0000","last_fullsized":"2023-09-23T23:01:52.681795+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":414,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568333+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:01:42.568333+0000","last_peered":"2023-09-23T23:01:42.568333+0000","last_clean":"2023-09-23T23:01:42.568333+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:01:42.568333+0000","last_undegraded":"2023-09-23T23:01:42.568333+0000","last_fullsized":"2023-09-23T23:01:42.568333+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":286,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.561280+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:01:42.561280+0000","last_peered":"2023-09-23T23:01:42.561280+0000","last_clean":"2023-09-23T23:01:42.561280+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:01:42.561280+0000","last_undegraded":"2023-09-23T23:01:42.561280+0000","last_fullsized":"2023-09-23T23:01:42.561280+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"394'10273","reported_seq":18447,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:56.129149+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:01:56.129149+0000","last_peered":"2023-09-23T23:01:56.129149+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:01:56.129149+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"322'1700","ondisk_log_start":"322'1700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8573,"log_dups_size":0,"ondisk_log_size":8573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117374976,"num_objects":1791,"num_object_clones":0,"num_object_copies":5373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10273,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1791}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":361,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.559916+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:01:42.559916+0000","last_peered":"2023-09-23T23:01:42.559916+0000","last_clean":"2023-09-23T23:01:42.559916+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:01:42.559916+0000","last_undegraded":"2023-09-23T23:01:42.559916+0000","last_fullsized":"2023-09-23T23:01:42.559916+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":238,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.560150+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:01:42.560150+0000","last_peered":"2023-09-23T23:01:42.560150+0000","last_clean":"2023-09-23T23:01:42.560150+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:01:42.560150+0000","last_undegraded":"2023-09-23T23:01:42.560150+0000","last_fullsized":"2023-09-23T23:01:42.560150+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":304,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.573939+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:01:42.573939+0000","last_peered":"2023-09-23T23:01:42.573939+0000","last_clean":"2023-09-23T23:01:42.573939+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:01:42.573939+0000","last_undegraded":"2023-09-23T23:01:42.573939+0000","last_fullsized":"2023-09-23T23:01:42.573939+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"394'10470","reported_seq":12444,"reported_epoch":394,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:58.030947+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:01:58.030947+0000","last_peered":"2023-09-23T23:01:58.030947+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:01:58.030947+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'400","ondisk_log_start":"321'400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":2522,"num_objects_missing":2522,"num_objects_degraded":3228,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10470,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":935,"num_bytes_recovered":61276160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1393},{"shards":"0,1,6","objects":321},{"shards":"0,6","objects":-3188}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":377,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.571801+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:01:42.571801+0000","last_peered":"2023-09-23T23:01:42.571801+0000","last_clean":"2023-09-23T23:01:42.571801+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:01:42.571801+0000","last_undegraded":"2023-09-23T23:01:42.571801+0000","last_fullsized":"2023-09-23T23:01:42.571801+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":491,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:52.682380+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:01:52.682380+0000","last_peered":"2023-09-23T23:01:52.682380+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:01:52.682380+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574477+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:01:42.574477+0000","last_peered":"2023-09-23T23:01:42.574477+0000","last_clean":"2023-09-23T23:01:42.574477+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:01:42.574477+0000","last_undegraded":"2023-09-23T23:01:42.574477+0000","last_fullsized":"2023-09-23T23:01:42.574477+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":304,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567809+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:01:42.567809+0000","last_peered":"2023-09-23T23:01:42.567809+0000","last_clean":"2023-09-23T23:01:42.567809+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:01:42.567809+0000","last_undegraded":"2023-09-23T23:01:42.567809+0000","last_fullsized":"2023-09-23T23:01:42.567809+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"395'10480","reported_seq":10584,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:59.924335+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:01:59.924335+0000","last_peered":"2023-09-23T23:01:59.924335+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:01:59.924335+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"332'5000","ondisk_log_start":"332'5000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10480,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1903}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":385,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680693+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:01:52.680693+0000","last_peered":"2023-09-23T23:01:52.680693+0000","last_clean":"2023-09-23T23:01:52.680693+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:01:52.680693+0000","last_undegraded":"2023-09-23T23:01:52.680693+0000","last_fullsized":"2023-09-23T23:01:52.680693+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":356,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680811+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:01:52.680811+0000","last_peered":"2023-09-23T23:01:52.680811+0000","last_clean":"2023-09-23T23:01:52.680811+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:01:52.680811+0000","last_undegraded":"2023-09-23T23:01:52.680811+0000","last_fullsized":"2023-09-23T23:01:52.680811+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":316,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681775+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:01:52.681775+0000","last_peered":"2023-09-23T23:01:52.681775+0000","last_clean":"2023-09-23T23:01:52.681775+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:01:52.681775+0000","last_undegraded":"2023-09-23T23:01:52.681775+0000","last_fullsized":"2023-09-23T23:01:52.681775+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"394'10439","reported_seq":10557,"reported_epoch":394,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:01:57.005468+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:01:57.005468+0000","last_peered":"2023-09-23T23:01:57.005468+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:01:57.005468+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:01:57.005468+0000","mapping_epoch":354,"log_start":"331'4700","ondisk_log_start":"331'4700","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115605528,"num_objects":1765,"num_object_clones":0,"num_object_copies":5295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1765,"num_objects_unfound":0,"num_objects_dirty":1765,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10439,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1765}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":331,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681222+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:01:52.681222+0000","last_peered":"2023-09-23T23:01:52.681222+0000","last_clean":"2023-09-23T23:01:52.681222+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:01:52.681222+0000","last_undegraded":"2023-09-23T23:01:52.681222+0000","last_fullsized":"2023-09-23T23:01:52.681222+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":238,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.561350+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:01:42.561350+0000","last_peered":"2023-09-23T23:01:42.561350+0000","last_clean":"2023-09-23T23:01:42.561350+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:01:42.561350+0000","last_undegraded":"2023-09-23T23:01:42.561350+0000","last_fullsized":"2023-09-23T23:01:42.561350+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.571842+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:01:42.571842+0000","last_peered":"2023-09-23T23:01:42.571842+0000","last_clean":"2023-09-23T23:01:42.571842+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:01:42.571842+0000","last_undegraded":"2023-09-23T23:01:42.571842+0000","last_fullsized":"2023-09-23T23:01:42.571842+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"394'10436","reported_seq":19967,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:57.175245+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:01:57.175245+0000","last_peered":"2023-09-23T23:01:57.175245+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:01:57.175245+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"331'4700","ondisk_log_start":"331'4700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112066560,"num_objects":1710,"num_object_clones":0,"num_object_copies":5130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3707,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10436,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3774,"num_bytes_recovered":247332864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":325}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":88,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:46.255547+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:01:46.255547+0000","last_peered":"2023-09-23T23:01:46.255547+0000","last_clean":"2023-09-23T23:01:46.255547+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:01:46.255547+0000","last_undegraded":"2023-09-23T23:01:46.255547+0000","last_fullsized":"2023-09-23T23:01:46.255547+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.570730+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:01:42.570730+0000","last_peered":"2023-09-23T23:01:42.570730+0000","last_clean":"2023-09-23T23:01:42.570730+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:01:42.570730+0000","last_undegraded":"2023-09-23T23:01:42.570730+0000","last_fullsized":"2023-09-23T23:01:42.570730+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":79,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574220+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:01:42.574220+0000","last_peered":"2023-09-23T23:01:42.574220+0000","last_clean":"2023-09-23T23:01:42.574220+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:01:42.574220+0000","last_undegraded":"2023-09-23T23:01:42.574220+0000","last_fullsized":"2023-09-23T23:01:42.574220+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"394'10308","reported_seq":10836,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:57.181592+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:01:57.181592+0000","last_peered":"2023-09-23T23:01:57.181592+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:01:57.181592+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"330'4600","ondisk_log_start":"330'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115474432,"num_objects":1762,"num_object_clones":0,"num_object_copies":5286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1762,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":1762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10308,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-1789}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":350,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.563720+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:01:42.563720+0000","last_peered":"2023-09-23T23:01:42.563720+0000","last_clean":"2023-09-23T23:01:42.563720+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:01:42.563720+0000","last_undegraded":"2023-09-23T23:01:42.563720+0000","last_fullsized":"2023-09-23T23:01:42.563720+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":298,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681701+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:01:52.681701+0000","last_peered":"2023-09-23T23:01:52.681701+0000","last_clean":"2023-09-23T23:01:52.681701+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:01:52.681701+0000","last_undegraded":"2023-09-23T23:01:52.681701+0000","last_fullsized":"2023-09-23T23:01:52.681701+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":368,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567485+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:01:42.567485+0000","last_peered":"2023-09-23T23:01:42.567485+0000","last_clean":"2023-09-23T23:01:42.567485+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:01:42.567485+0000","last_undegraded":"2023-09-23T23:01:42.567485+0000","last_fullsized":"2023-09-23T23:01:42.567485+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":333,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.682575+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:01:52.682575+0000","last_peered":"2023-09-23T23:01:52.682575+0000","last_clean":"2023-09-23T23:01:52.682575+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:01:52.682575+0000","last_undegraded":"2023-09-23T23:01:52.682575+0000","last_fullsized":"2023-09-23T23:01:52.682575+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":142,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.561011+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:01:42.561011+0000","last_peered":"2023-09-23T23:01:42.561011+0000","last_clean":"2023-09-23T23:01:42.561011+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:01:42.561011+0000","last_undegraded":"2023-09-23T23:01:42.561011+0000","last_fullsized":"2023-09-23T23:01:42.561011+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":203,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567733+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:01:42.567733+0000","last_peered":"2023-09-23T23:01:42.567733+0000","last_clean":"2023-09-23T23:01:42.567733+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:01:42.567733+0000","last_undegraded":"2023-09-23T23:01:42.567733+0000","last_fullsized":"2023-09-23T23:01:42.567733+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":417,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.568405+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:01:42.568405+0000","last_peered":"2023-09-23T23:01:42.568405+0000","last_clean":"2023-09-23T23:01:42.568405+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:01:42.568405+0000","last_undegraded":"2023-09-23T23:01:42.568405+0000","last_fullsized":"2023-09-23T23:01:42.568405+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":217,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567682+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:01:42.567682+0000","last_peered":"2023-09-23T23:01:42.567682+0000","last_clean":"2023-09-23T23:01:42.567682+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:01:42.567682+0000","last_undegraded":"2023-09-23T23:01:42.567682+0000","last_fullsized":"2023-09-23T23:01:42.567682+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":304,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567979+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:01:42.567979+0000","last_peered":"2023-09-23T23:01:42.567979+0000","last_clean":"2023-09-23T23:01:42.567979+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:01:42.567979+0000","last_undegraded":"2023-09-23T23:01:42.567979+0000","last_fullsized":"2023-09-23T23:01:42.567979+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":88,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574885+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:01:42.574885+0000","last_peered":"2023-09-23T23:01:42.574885+0000","last_clean":"2023-09-23T23:01:42.574885+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:01:42.574885+0000","last_undegraded":"2023-09-23T23:01:42.574885+0000","last_fullsized":"2023-09-23T23:01:42.574885+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.574348+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:01:42.574348+0000","last_peered":"2023-09-23T23:01:42.574348+0000","last_clean":"2023-09-23T23:01:42.574348+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:01:42.574348+0000","last_undegraded":"2023-09-23T23:01:42.574348+0000","last_fullsized":"2023-09-23T23:01:42.574348+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":337,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681494+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:01:52.681494+0000","last_peered":"2023-09-23T23:01:52.681494+0000","last_clean":"2023-09-23T23:01:52.681494+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:01:52.681494+0000","last_undegraded":"2023-09-23T23:01:52.681494+0000","last_fullsized":"2023-09-23T23:01:52.681494+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":379,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T23:01:42.567179+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:01:42.567179+0000","last_peered":"2023-09-23T23:01:42.567179+0000","last_clean":"2023-09-23T23:01:42.567179+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:01:42.567179+0000","last_undegraded":"2023-09-23T23:01:42.567179+0000","last_fullsized":"2023-09-23T23:01:42.567179+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1827930136,"num_objects":27893,"num_object_clones":0,"num_object_copies":83679,"num_objects_missing_on_primary":2524,"num_objects_missing":2524,"num_objects_degraded":19619,"num_objects_misplaced":29147,"num_objects_unfound":0,"num_objects_dirty":27893,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165705,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18786,"num_bytes_recovered":1139998720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6468349952,"data_stored":6468337736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100605,"ondisk_log_size":100605,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430628,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626852,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253165,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1496660,"kb_used_data":780364,"kb_used_omap":9,"kb_used_meta":716278,"kb_avail":92248492,"statfs":{"total":95995035648,"available":94462455808,"internally_reserved":0,"allocated":799092736,"data_stored":797599591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":733469673},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8250000000000002,"5min":1.258,"15min":0.65400000000000003},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":83.664000000000001,"5min":83.664000000000001,"15min":83.664000000000001},"last":20.446000000000002},{"interface":"front","average":{"1min":5.0039999999999996,"5min":1.5980000000000001,"15min":1.103},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":83.700999999999993,"5min":83.700999999999993,"15min":83.700999999999993},"last":20.472999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.48699999999999999,"15min":0.51400000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.087,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.64200000000000002,"15min":0.69799999999999995},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.74,"5min":4.048,"15min":4.048},"last":0.22900000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.29799999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":0.36199999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.371},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":0.44700000000000001}]}]},{"osd":0,"up_from":353,"seq":1516123455536,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1378620,"kb_used_data":1199412,"kb_used_omap":37,"kb_used_meta":179162,"kb_avail":92366532,"statfs":{"total":95995035648,"available":94583328768,"internally_reserved":0,"allocated":1228197888,"data_stored":1226551108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":183462050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.59199999999999997,"15min":0.55700000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.7629999999999999,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64000000000000001,"15min":0.66100000000000003},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.8650000000000002,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.5}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":0.86399999999999999,"15min":0.80800000000000005},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":0.88},{"interface":"front","average":{"1min":1.012,"5min":0.72499999999999998,"15min":0.68500000000000005},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.623,"15min":0.51000000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.76800000000000002,"15min":0.86199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6499999999999999,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":0.748}]},{"osd":4,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":12.855,"5min":3.2290000000000001,"15min":1.649},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":183.148,"5min":183.148,"15min":183.148},"last":2.6949999999999998},{"interface":"front","average":{"1min":13.087999999999999,"5min":3.194,"15min":1.51},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":183.39500000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.86799999999999999,"15min":0.74199999999999999},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4089999999999998,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":5.984},{"interface":"front","average":{"1min":1.282,"5min":1.0009999999999999,"15min":0.999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.774,"5min":7.774,"15min":7.774},"last":0.84099999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.92900000000000005,"15min":0.86599999999999999},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.7309999999999999,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.242,"5min":0.86799999999999999,"15min":0.77200000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":6.375,"5min":6.375,"15min":6.375},"last":0.85699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.159,"15min":1.2549999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.8439999999999999,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.008,"5min":1.2150000000000001,"15min":1.2569999999999999},"min":{"1min":0.49299999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":6.343,"5min":6.343,"15min":6.343},"last":0.82899999999999996}]}]},{"osd":2,"up_from":359,"seq":1541893259308,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":719928,"kb_used_data":584368,"kb_used_omap":83,"kb_used_meta":135468,"kb_avail":93025224,"statfs":{"total":95995035648,"available":95257829376,"internally_reserved":0,"allocated":598392832,"data_stored":596810871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":138719685},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.311}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":9.8710000000000004}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":9.8170000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":1.708},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":1.165}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":9.9079999999999995},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":1.474}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":9.9359999999999999},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":1.881}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.81699999999999995}]}]},{"osd":3,"up_from":94,"seq":403726926112,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":516144,"kb_used_data":23784,"kb_used_omap":7,"kb_used_meta":492344,"kb_avail":93229008,"statfs":{"total":95995035648,"available":95466504192,"internally_reserved":0,"allocated":24354816,"data_stored":22775008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":504160872},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":1.5640000000000001,"15min":0.73399999999999999},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":122.289,"5min":122.289,"15min":122.289},"last":0.32100000000000001},{"interface":"front","average":{"1min":6.1609999999999996,"5min":1.5600000000000001,"15min":0.79100000000000004},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":0.23699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":0.74099999999999999,"15min":2.2999999999999998},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.1639999999999997,"5min":8.6280000000000001,"15min":123.866},"last":0.253},{"interface":"front","average":{"1min":1.3680000000000001,"5min":0.82499999999999996,"15min":2.1659999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5640000000000001,"5min":6.8010000000000002,"15min":122.265},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.63,"15min":0.68899999999999995},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.7,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.857,"5min":0.98899999999999999,"15min":0.82499999999999996},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.223},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.3220000000000001,"15min":2.0099999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.2300000000000004,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.2,"5min":2.496,"15min":1.927},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.4509999999999996,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.2769999999999999,"15min":1.4039999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5229999999999997,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.111,"5min":1.1919999999999999,"15min":1.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7000000000000002,"5min":10.246,"15min":11.951000000000001},"last":0.372}]},{"osd":6,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.554,"15min":1.7569999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5380000000000003,"5min":11.116,"15min":14.097},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.288,"15min":1.919},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.4219999999999997,"5min":9.2810000000000006,"15min":12.887},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.90700000000000003,"15min":1.7350000000000001},"min":{"1min":0.36299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6680000000000001,"5min":4.8449999999999998,"15min":11.694000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.0649999999999999,"15min":1.6279999999999999},"min":{"1min":0.45300000000000001,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.2690000000000001,"5min":8.5830000000000002,"15min":12.32},"last":0.46899999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019309,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2350308,"kb_used_data":1451036,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91394844,"statfs":{"total":95995035648,"available":93588320256,"internally_reserved":0,"allocated":1485860864,"data_stored":1485042745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":5.3819999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.4380000000000002,"15min":3.5270000000000001},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":3.081,"5min":87.325000000000003,"15min":132.304},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.6320000000000001,"15min":3.4849999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.253},"max":{"1min":3.6120000000000001,"5min":87.572000000000003,"15min":132.453},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 23:01:28 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.0549999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4870000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":2.8380000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.76200000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4770000000000003,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.96199999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.484,"15min":3.9009999999999998},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.32400000000000001},"max":{"1min":4.0830000000000002,"5min":10.337999999999999,"15min":164.453},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.784,"5min":1.4410000000000001,"15min":3.8929999999999998},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.0039999999999996,"5min":12.805999999999999,"15min":167.05199999999999},"last":5.3380000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.432},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.655,"15min":1.181},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":3.5680000000000001,"5min":16.943000000000001,"15min":16.943000000000001},"last":5.4489999999999998},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.4379999999999999,"15min":1.113},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.1859999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.5700000000000001,"15min":1.169},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0869999999999997,"5min":12.84,"15min":12.84},"last":5.415},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.585,"15min":1.117},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4199999999999999,"5min":10.587,"15min":10.587},"last":0.52400000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823083,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1793860,"kb_used_data":890108,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":91951292,"statfs":{"total":95995035648,"available":94158123008,"internally_reserved":0,"allocated":911470592,"data_stored":910680261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.878,"15min":0.69599999999999995},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.5709999999999997,"5min":14.42,"15min":14.42},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2430000000000001,"15min":1.135},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3159999999999998,"5min":14.496,"15min":14.496},"last":0.60399999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.317},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.2350000000000001,"15min":1.0629999999999999},"min":{"1min":0.45000000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.8370000000000002,"5min":12.23,"15min":12.23},"last":0.72099999999999997},{"interface":"front","average":{"1min":2.5830000000000002,"5min":1.7490000000000001,"15min":1.2170000000000001},"min":{"1min":0.378,"5min":0.31,"15min":0.31},"max":{"1min":7.6719999999999997,"5min":11.112,"15min":11.112},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1209864192,"data_stored":1209860120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":791609344,"data_stored":791609344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":592314368,"data_stored":592314368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":19464192,"data_stored":19464192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1470238720,"data_stored":1470234648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":905707520,"data_stored":905707520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1479151616,"data_stored":1479147544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:02:01.965 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:01.980 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:01 smithi151 ceph-mon[159210]: pgmap v1199: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 337 op/s; 19621/83751 objects degraded (23.428%); 29147/83751 objects misplaced (34.802%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:02:01.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:01 smithi052 ceph-mon[180979]: pgmap v1199: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 337 op/s; 19621/83751 objects degraded (23.428%); 29147/83751 objects misplaced (34.802%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:02:02.244 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:02.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:02.348 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:02.377 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:02.423 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:02.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:02.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:02.526 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:02.565 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:02.851 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:02.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:02.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:02.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:02 smithi052 ceph-mon[187743]: from='client.46658 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:02 smithi052 ceph-mon[180979]: from='client.46658 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:03.000 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:02 smithi151 ceph-mon[159210]: from='client.46658 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:03.010 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:03.057 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:02:03.126 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:03.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:03.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:03.231 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:03.321 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:03.380 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:03.380 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:03.417 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:03.439 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:03.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:03.449 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:03.477 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:03.511 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:03.543 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:03.549 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:03.589 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:03.596 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:03.621 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:03.652 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:03.653 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:03.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:03.658 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:03.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.667 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.668 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:02:03.696 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:02:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:02:03] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.33.4" 2023-09-23T23:02:03.756 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:03.784 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:03.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:03.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:03.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:03.962 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:03 smithi052 ceph-mon[180979]: pgmap v1200: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 393 op/s; 18891/80124 objects degraded (23.577%); 28927/80124 objects misplaced (36.103%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T23:02:03.963 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:03.963 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2389011303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:03.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:03 smithi052 ceph-mon[187743]: pgmap v1200: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 393 op/s; 18891/80124 objects degraded (23.577%); 28927/80124 objects misplaced (36.103%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T23:02:03.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:03.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2389011303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:04.037 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:04.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:03 smithi151 ceph-mon[159210]: pgmap v1200: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 393 op/s; 18891/80124 objects degraded (23.577%); 28927/80124 objects misplaced (36.103%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T23:02:04.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:02:04.174 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2389011303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:04.203 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:02:04.926 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:04.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:04.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:04.940 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:04.963 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:04.966 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:02:05.034 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:05.079 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:05.082 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:05.347 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:05.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:05.455 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:05.494 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:05.840 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:05.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:05.944 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:05.951 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:05.995 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:06.122 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:05 smithi052 ceph-mon[187743]: pgmap v1201: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 391 op/s; 19009/79770 objects degraded (23.830%); 28927/79770 objects misplaced (36.263%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:02:06.123 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 19009/79770 objects degraded (23.830%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:06.123 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:05 smithi052 ceph-mon[180979]: pgmap v1201: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 391 op/s; 19009/79770 objects degraded (23.830%); 28927/79770 objects misplaced (36.263%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:02:06.124 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 19009/79770 objects degraded (23.830%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:05 smithi151 ceph-mon[159210]: pgmap v1201: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 391 op/s; 19009/79770 objects degraded (23.830%); 28927/79770 objects misplaced (36.263%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:02:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 19009/79770 objects degraded (23.830%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:06.677 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:06.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:06.781 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:06.852 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:06.942 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:02:06.943 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:06.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:07.055 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:07.117 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:07.314 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:07.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:07.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:07.408 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:02:07.408 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:02:07.408 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.408 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.45953.0:169173 10.8 e395/360 10:1fc4a77f:::benchmark_data_smithi052_461816_object72372:head v 395'10641, mlcod=323'2535)", 2023-09-23T23:02:07.408 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:02:07.405973+0000", 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0017154780000000001, 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0017476950000000001, 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.405973+0000", 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:07.409 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.405973+0000", 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.405972+0000", 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1975e-05 2023-09-23T23:02:07.410 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.405984+0000", 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4715999999999999e-05 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.405999+0000", 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8709999999999999e-06 2023-09-23T23:02:07.411 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.406006+0000", 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1366999999999999e-05 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.406047+0000", 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4671000000000001e-05 2023-09-23T23:02:07.412 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:07.406072+0000", 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8507999999999999e-05 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T23:02:07.413 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 1 2023-09-23T23:02:07.414 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:02:07.421 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:07.481 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:07.702 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:07.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:07.778 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:02:07.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:07.845 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:07.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:07 smithi052 ceph-mon[187743]: pgmap v1202: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 18828/77571 objects degraded (24.272%); 28734/77571 objects misplaced (37.042%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:02:07.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/108815386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:07.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:07 smithi052 ceph-mon[180979]: pgmap v1202: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 18828/77571 objects degraded (24.272%); 28734/77571 objects misplaced (37.042%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:02:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/108815386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:07 smithi151 ceph-mon[159210]: pgmap v1202: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 18828/77571 objects degraded (24.272%); 28734/77571 objects misplaced (37.042%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T23:02:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/108815386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:08.236 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:08.376 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:08.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:08.481 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:08.515 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:02:08.544 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:08.739 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:08.787 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:08.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:08.903 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:08.932 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: osdmap e396: 8 total, 8 up, 6 in 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[187743]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:08.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: osdmap e396: 8 total, 8 up, 6 in 2023-09-23T23:02:08.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: failed to encode map e396 with expected crc 2023-09-23T23:02:08.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:08 smithi052 ceph-mon[180979]: failed to encode map e396 with expected crc 2023-09-23T23:02:09.142 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:09.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: failed to encode map e396 with expected crc 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: osdmap e396: 8 total, 8 up, 6 in 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: failed to encode map e396 with expected crc 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: failed to encode map e396 with expected crc 2023-09-23T23:02:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:08 smithi151 ceph-mon[159210]: failed to encode map e396 with expected crc 2023-09-23T23:02:09.222 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:09.247 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:09.293 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:09.301 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1203,"stamp":"2023-09-23T23:02:07.764849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1634342840,"num_objects":24954,"num_object_clones":0,"num_object_copies":74862,"num_objects_missing_on_primary":2519,"num_objects_missing":2519,"num_objects_degraded":18239,"num_objects_misplaced":28519,"num_objects_unfound":0,"num_objects_dirty":24954,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":168928,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19297,"num_bytes_recovered":1171209148,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100762,"ondisk_log_size":100762,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7875312,"kb_used_data":4482432,"kb_used_omap":137,"kb_used_meta":3392694,"kb_avail":554595600,"statfs":{"total":575970213888,"available":567905894400,"internally_reserved":0,"allocated":4590010368,"data_stored":4582068932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3474118840},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,1,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-233963520,"num_objects":-3570,"num_object_clones":0,"num_object_copies":-10710,"num_objects_missing_on_primary":-5,"num_objects_missing":-5,"num_objects_degraded":-1462,"num_objects_misplaced":-884,"num_objects_unfound":0,"num_objects_dirty":-3570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3570,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":31195136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.161755"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":407,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692443+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:01:52.692443+0000","last_peered":"2023-09-23T23:01:52.692443+0000","last_clean":"2023-09-23T23:01:52.692443+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:01:52.692443+0000","last_undegraded":"2023-09-23T23:01:52.692443+0000","last_fullsized":"2023-09-23T23:01:52.692443+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696469+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:01:52.696469+0000","last_peered":"2023-09-23T23:01:52.696469+0000","last_clean":"2023-09-23T23:01:52.696469+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:01:52.696469+0000","last_undegraded":"2023-09-23T23:01:52.696469+0000","last_fullsized":"2023-09-23T23:01:52.696469+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":63,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695749+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:01:52.695749+0000","last_peered":"2023-09-23T23:01:52.695749+0000","last_clean":"2023-09-23T23:01:52.695749+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:01:52.695749+0000","last_undegraded":"2023-09-23T23:01:52.695749+0000","last_fullsized":"2023-09-23T23:01:52.695749+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":392,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695984+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:01:52.695984+0000","last_peered":"2023-09-23T23:01:52.695984+0000","last_clean":"2023-09-23T23:01:52.695984+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:01:52.695984+0000","last_undegraded":"2023-09-23T23:01:52.695984+0000","last_fullsized":"2023-09-23T23:01:52.695984+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695852+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:01:52.695852+0000","last_peered":"2023-09-23T23:01:52.695852+0000","last_clean":"2023-09-23T23:01:52.695852+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:01:52.695852+0000","last_undegraded":"2023-09-23T23:01:52.695852+0000","last_fullsized":"2023-09-23T23:01:52.695852+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681132+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:01:52.681132+0000","last_peered":"2023-09-23T23:01:52.681132+0000","last_clean":"2023-09-23T23:01:52.681132+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:01:52.681132+0000","last_undegraded":"2023-09-23T23:01:52.681132+0000","last_fullsized":"2023-09-23T23:01:52.681132+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":330,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.691884+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:01:52.691884+0000","last_peered":"2023-09-23T23:01:52.691884+0000","last_clean":"2023-09-23T23:01:52.691884+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:01:52.691884+0000","last_undegraded":"2023-09-23T23:01:52.691884+0000","last_fullsized":"2023-09-23T23:01:52.691884+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695488+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:01:52.695488+0000","last_peered":"2023-09-23T23:01:52.695488+0000","last_clean":"2023-09-23T23:01:52.695488+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:01:52.695488+0000","last_undegraded":"2023-09-23T23:01:52.695488+0000","last_fullsized":"2023-09-23T23:01:52.695488+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":291,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689651+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:01:52.689651+0000","last_peered":"2023-09-23T23:01:52.689651+0000","last_clean":"2023-09-23T23:01:52.689651+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:01:52.689651+0000","last_undegraded":"2023-09-23T23:01:52.689651+0000","last_fullsized":"2023-09-23T23:01:52.689651+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695605+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:01:52.695605+0000","last_peered":"2023-09-23T23:01:52.695605+0000","last_clean":"2023-09-23T23:01:52.695605+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:01:52.695605+0000","last_undegraded":"2023-09-23T23:01:52.695605+0000","last_fullsized":"2023-09-23T23:01:52.695605+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":84,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696428+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:01:52.696428+0000","last_peered":"2023-09-23T23:01:52.696428+0000","last_clean":"2023-09-23T23:01:52.696428+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:01:52.696428+0000","last_undegraded":"2023-09-23T23:01:52.696428+0000","last_fullsized":"2023-09-23T23:01:52.696428+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":140,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.677050+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:01:52.677050+0000","last_peered":"2023-09-23T23:01:52.677050+0000","last_clean":"2023-09-23T23:01:52.677050+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:01:52.677050+0000","last_undegraded":"2023-09-23T23:01:52.677050+0000","last_fullsized":"2023-09-23T23:01:52.677050+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":466,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.693900+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:01:52.693900+0000","last_peered":"2023-09-23T23:01:52.693900+0000","last_clean":"2023-09-23T23:01:52.693900+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:01:52.693900+0000","last_undegraded":"2023-09-23T23:01:52.693900+0000","last_fullsized":"2023-09-23T23:01:52.693900+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":369,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695009+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:01:52.695009+0000","last_peered":"2023-09-23T23:01:52.695009+0000","last_clean":"2023-09-23T23:01:52.695009+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:01:52.695009+0000","last_undegraded":"2023-09-23T23:01:52.695009+0000","last_fullsized":"2023-09-23T23:01:52.695009+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":295,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695083+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:01:52.695083+0000","last_peered":"2023-09-23T23:01:52.695083+0000","last_clean":"2023-09-23T23:01:52.695083+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:01:52.695083+0000","last_undegraded":"2023-09-23T23:01:52.695083+0000","last_fullsized":"2023-09-23T23:01:52.695083+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":76,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728567+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:01:59.728567+0000","last_peered":"2023-09-23T23:01:59.728567+0000","last_clean":"2023-09-23T23:01:59.728567+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:01:59.728567+0000","last_undegraded":"2023-09-23T23:01:59.728567+0000","last_fullsized":"2023-09-23T23:01:59.728567+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":384,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695727+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:01:52.695727+0000","last_peered":"2023-09-23T23:01:52.695727+0000","last_clean":"2023-09-23T23:01:52.695727+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:01:52.695727+0000","last_undegraded":"2023-09-23T23:01:52.695727+0000","last_fullsized":"2023-09-23T23:01:52.695727+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":295,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689497+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:01:52.689497+0000","last_peered":"2023-09-23T23:01:52.689497+0000","last_clean":"2023-09-23T23:01:52.689497+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:01:52.689497+0000","last_undegraded":"2023-09-23T23:01:52.689497+0000","last_fullsized":"2023-09-23T23:01:52.689497+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":280,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692539+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:01:52.692539+0000","last_peered":"2023-09-23T23:01:52.692539+0000","last_clean":"2023-09-23T23:01:52.692539+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:01:52.692539+0000","last_undegraded":"2023-09-23T23:01:52.692539+0000","last_fullsized":"2023-09-23T23:01:52.692539+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":408,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696073+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:01:52.696073+0000","last_peered":"2023-09-23T23:01:52.696073+0000","last_clean":"2023-09-23T23:01:52.696073+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:01:52.696073+0000","last_undegraded":"2023-09-23T23:01:52.696073+0000","last_fullsized":"2023-09-23T23:01:52.696073+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":113,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680699+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:01:52.680699+0000","last_peered":"2023-09-23T23:01:52.680699+0000","last_clean":"2023-09-23T23:01:52.680699+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:01:52.680699+0000","last_undegraded":"2023-09-23T23:01:52.680699+0000","last_fullsized":"2023-09-23T23:01:52.680699+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.698411+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:01:52.698411+0000","last_peered":"2023-09-23T23:01:52.698411+0000","last_clean":"2023-09-23T23:01:52.698411+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:01:52.698411+0000","last_undegraded":"2023-09-23T23:01:52.698411+0000","last_fullsized":"2023-09-23T23:01:52.698411+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"395'10502","reported_seq":10601,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:02:04.953833+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:02:04.953833+0000","last_peered":"2023-09-23T23:02:04.953833+0000","last_clean":"2023-09-23T23:02:04.953833+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:02:04.953833+0000","last_undegraded":"2023-09-23T23:02:04.953833+0000","last_fullsized":"2023-09-23T23:02:04.953833+0000","mapping_epoch":353,"log_start":"332'5000","ondisk_log_start":"332'5000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103809024,"num_objects":1584,"num_object_clones":0,"num_object_copies":4752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10502,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":279,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696541+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:01:52.696541+0000","last_peered":"2023-09-23T23:01:52.696541+0000","last_clean":"2023-09-23T23:01:52.696541+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:01:52.696541+0000","last_undegraded":"2023-09-23T23:01:52.696541+0000","last_fullsized":"2023-09-23T23:01:52.696541+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":265,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.688901+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:01:52.688901+0000","last_peered":"2023-09-23T23:01:52.688901+0000","last_clean":"2023-09-23T23:01:52.688901+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:01:52.688901+0000","last_undegraded":"2023-09-23T23:01:52.688901+0000","last_fullsized":"2023-09-23T23:01:52.688901+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"395'10389","reported_seq":12817,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:06.126687+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:02:06.126687+0000","last_peered":"2023-09-23T23:02:06.126687+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:02:06.126687+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8589,"log_dups_size":0,"ondisk_log_size":8589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10389,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1462},{"shards":"5,6","objects":-2006}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":289,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696975+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:01:52.696975+0000","last_peered":"2023-09-23T23:01:52.696975+0000","last_clean":"2023-09-23T23:01:52.696975+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:01:52.696975+0000","last_undegraded":"2023-09-23T23:01:52.696975+0000","last_fullsized":"2023-09-23T23:01:52.696975+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":402,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692452+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:01:52.692452+0000","last_peered":"2023-09-23T23:01:52.692452+0000","last_clean":"2023-09-23T23:01:52.692452+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:01:52.692452+0000","last_undegraded":"2023-09-23T23:01:52.692452+0000","last_fullsized":"2023-09-23T23:01:52.692452+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"395'10497","reported_seq":10590,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:04.812930+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:02:04.812930+0000","last_peered":"2023-09-23T23:02:04.812930+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:02:04.812930+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:02:04.812930+0000","mapping_epoch":354,"log_start":"332'5000","ondisk_log_start":"332'5000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103219200,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":1575,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10497,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1575}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":146,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689696+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:01:52.689696+0000","last_peered":"2023-09-23T23:01:52.689696+0000","last_clean":"2023-09-23T23:01:52.689696+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:01:52.689696+0000","last_undegraded":"2023-09-23T23:01:52.689696+0000","last_fullsized":"2023-09-23T23:01:52.689696+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":264,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689168+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:01:52.689168+0000","last_peered":"2023-09-23T23:01:52.689168+0000","last_clean":"2023-09-23T23:01:52.689168+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:01:52.689168+0000","last_undegraded":"2023-09-23T23:01:52.689168+0000","last_fullsized":"2023-09-23T23:01:52.689168+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"395'10559","reported_seq":10838,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:04.932265+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:02:04.932265+0000","last_peered":"2023-09-23T23:02:04.932265+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:02:04.932265+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105840640,"num_objects":1615,"num_object_clones":0,"num_object_copies":4845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1364,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":1615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10559,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":251},{"shards":"0,5","objects":-1995}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:56.262220+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:01:56.262220+0000","last_peered":"2023-09-23T23:01:56.262220+0000","last_clean":"2023-09-23T23:01:56.262220+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:01:56.262220+0000","last_undegraded":"2023-09-23T23:01:56.262220+0000","last_fullsized":"2023-09-23T23:01:56.262220+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":294,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680862+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:01:52.680862+0000","last_peered":"2023-09-23T23:01:52.680862+0000","last_clean":"2023-09-23T23:01:52.680862+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:01:52.680862+0000","last_undegraded":"2023-09-23T23:01:52.680862+0000","last_fullsized":"2023-09-23T23:01:52.680862+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"395'10543","reported_seq":10643,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:07.177751+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:02:07.177751+0000","last_peered":"2023-09-23T23:02:07.177751+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:02:07.177751+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:02:07.177751+0000","mapping_epoch":353,"log_start":"331'4800","ondisk_log_start":"331'4800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101646336,"num_objects":1551,"num_object_clones":0,"num_object_copies":4653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10543,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1551}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":84,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:56.262198+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:01:56.262198+0000","last_peered":"2023-09-23T23:01:56.262198+0000","last_clean":"2023-09-23T23:01:56.262198+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:01:56.262198+0000","last_undegraded":"2023-09-23T23:01:56.262198+0000","last_fullsized":"2023-09-23T23:01:56.262198+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":388,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.693809+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:01:52.693809+0000","last_peered":"2023-09-23T23:01:52.693809+0000","last_clean":"2023-09-23T23:01:52.693809+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:01:52.693809+0000","last_undegraded":"2023-09-23T23:01:52.693809+0000","last_fullsized":"2023-09-23T23:01:52.693809+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"395'10669","reported_seq":10777,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:07.133377+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:02:07.133377+0000","last_peered":"2023-09-23T23:02:07.133377+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:02:07.133377+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"332'5000","ondisk_log_start":"332'5000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100466688,"num_objects":1533,"num_object_clones":0,"num_object_copies":4599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1533,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":1533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10669,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-2086}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":178,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692609+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:01:52.692609+0000","last_peered":"2023-09-23T23:01:52.692609+0000","last_clean":"2023-09-23T23:01:52.692609+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:01:52.692609+0000","last_undegraded":"2023-09-23T23:01:52.692609+0000","last_fullsized":"2023-09-23T23:01:52.692609+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":405,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.693143+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:01:52.693143+0000","last_peered":"2023-09-23T23:01:52.693143+0000","last_clean":"2023-09-23T23:01:52.693143+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:01:52.693143+0000","last_undegraded":"2023-09-23T23:01:52.693143+0000","last_fullsized":"2023-09-23T23:01:52.693143+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"395'10530","reported_seq":19172,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:07.237884+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:02:07.237884+0000","last_peered":"2023-09-23T23:02:07.237884+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:02:07.237884+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:02:07.237884+0000","mapping_epoch":354,"log_start":"331'4800","ondisk_log_start":"331'4800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":337,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10530,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4263,"num_bytes_recovered":279379968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1434},{"shards":"1,4,6","objects":96}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":418,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692224+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:01:52.692224+0000","last_peered":"2023-09-23T23:01:52.692224+0000","last_clean":"2023-09-23T23:01:52.692224+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:01:52.692224+0000","last_undegraded":"2023-09-23T23:01:52.692224+0000","last_fullsized":"2023-09-23T23:01:52.692224+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":396,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.697056+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:01:52.697056+0000","last_peered":"2023-09-23T23:01:52.697056+0000","last_clean":"2023-09-23T23:01:52.697056+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:01:52.697056+0000","last_undegraded":"2023-09-23T23:01:52.697056+0000","last_fullsized":"2023-09-23T23:01:52.697056+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"395'10357","reported_seq":10457,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:07.249115+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:02:07.249115+0000","last_peered":"2023-09-23T23:02:07.249115+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:02:07.249115+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:02:07.249115+0000","mapping_epoch":353,"log_start":"331'4600","ondisk_log_start":"331'4600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105578496,"num_objects":1611,"num_object_clones":0,"num_object_copies":4833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10357,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1611}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695713+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:01:52.695713+0000","last_peered":"2023-09-23T23:01:52.695713+0000","last_clean":"2023-09-23T23:01:52.695713+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:01:52.695713+0000","last_undegraded":"2023-09-23T23:01:52.695713+0000","last_fullsized":"2023-09-23T23:01:52.695713+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":298,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.677350+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:01:52.677350+0000","last_peered":"2023-09-23T23:01:52.677350+0000","last_clean":"2023-09-23T23:01:52.677350+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:01:52.677350+0000","last_undegraded":"2023-09-23T23:01:52.677350+0000","last_fullsized":"2023-09-23T23:01:52.677350+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"395'10505","reported_seq":19791,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:07.201413+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:02:07.201413+0000","last_peered":"2023-09-23T23:02:07.201413+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:02:07.201413+0000","last_undegraded":"2023-09-23T23:02:07.201413+0000","last_fullsized":"2023-09-23T23:02:07.201413+0000","mapping_epoch":354,"log_start":"331'4800","ondisk_log_start":"331'4800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5705,"log_dups_size":0,"ondisk_log_size":5705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102957056,"num_objects":1571,"num_object_clones":0,"num_object_copies":4713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1571,"num_objects_unfound":0,"num_objects_dirty":1571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10505,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1571}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":302,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.678011+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:01:52.678011+0000","last_peered":"2023-09-23T23:01:52.678011+0000","last_clean":"2023-09-23T23:01:52.678011+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:01:52.678011+0000","last_undegraded":"2023-09-23T23:01:52.678011+0000","last_fullsized":"2023-09-23T23:01:52.678011+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":305,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689425+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:01:52.689425+0000","last_peered":"2023-09-23T23:01:52.689425+0000","last_clean":"2023-09-23T23:01:52.689425+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:01:52.689425+0000","last_undegraded":"2023-09-23T23:01:52.689425+0000","last_fullsized":"2023-09-23T23:01:52.689425+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.688996+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:01:52.688996+0000","last_peered":"2023-09-23T23:01:52.688996+0000","last_clean":"2023-09-23T23:01:52.688996+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:01:52.688996+0000","last_undegraded":"2023-09-23T23:01:52.688996+0000","last_fullsized":"2023-09-23T23:01:52.688996+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"395'10552","reported_seq":10664,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:04.898611+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:02:04.898611+0000","last_peered":"2023-09-23T23:02:04.898611+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:02:04.898611+0000","last_undegraded":"2023-09-23T23:02:04.898611+0000","last_fullsized":"2023-09-23T23:02:04.898611+0000","mapping_epoch":359,"log_start":"332'5000","ondisk_log_start":"332'5000","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103022592,"num_objects":1572,"num_object_clones":0,"num_object_copies":4716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1572,"num_objects_unfound":0,"num_objects_dirty":1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10552,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1572}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681795+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:01:52.681795+0000","last_peered":"2023-09-23T23:01:52.681795+0000","last_clean":"2023-09-23T23:01:52.681795+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:01:52.681795+0000","last_undegraded":"2023-09-23T23:01:52.681795+0000","last_fullsized":"2023-09-23T23:01:52.681795+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":415,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695132+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:01:52.695132+0000","last_peered":"2023-09-23T23:01:52.695132+0000","last_clean":"2023-09-23T23:01:52.695132+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:01:52.695132+0000","last_undegraded":"2023-09-23T23:01:52.695132+0000","last_fullsized":"2023-09-23T23:01:52.695132+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":287,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.678092+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:01:52.678092+0000","last_peered":"2023-09-23T23:01:52.678092+0000","last_clean":"2023-09-23T23:01:52.678092+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:01:52.678092+0000","last_undegraded":"2023-09-23T23:01:52.678092+0000","last_fullsized":"2023-09-23T23:01:52.678092+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"395'10515","reported_seq":18690,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:06.062532+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:02:06.062532+0000","last_peered":"2023-09-23T23:02:06.062532+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:02:06.062532+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"322'1900","ondisk_log_start":"322'1900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8615,"log_dups_size":0,"ondisk_log_size":8615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101515264,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10515,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1549}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":362,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.676872+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:01:52.676872+0000","last_peered":"2023-09-23T23:01:52.676872+0000","last_clean":"2023-09-23T23:01:52.676872+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:01:52.676872+0000","last_undegraded":"2023-09-23T23:01:52.676872+0000","last_fullsized":"2023-09-23T23:01:52.676872+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":239,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.676752+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:01:52.676752+0000","last_peered":"2023-09-23T23:01:52.676752+0000","last_clean":"2023-09-23T23:01:52.676752+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:01:52.676752+0000","last_undegraded":"2023-09-23T23:01:52.676752+0000","last_fullsized":"2023-09-23T23:01:52.676752+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":305,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.691951+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:01:52.691951+0000","last_peered":"2023-09-23T23:01:52.691951+0000","last_clean":"2023-09-23T23:01:52.691951+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:01:52.691951+0000","last_undegraded":"2023-09-23T23:01:52.691951+0000","last_fullsized":"2023-09-23T23:01:52.691951+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"395'10588","reported_seq":12574,"reported_epoch":395,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:03.034439+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:02:03.034439+0000","last_peered":"2023-09-23T23:02:03.034439+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:02:03.034439+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'500","ondisk_log_start":"321'500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10088,"log_dups_size":0,"ondisk_log_size":10088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107216896,"num_objects":1636,"num_object_clones":0,"num_object_copies":4908,"num_objects_missing_on_primary":2517,"num_objects_missing":2517,"num_objects_degraded":3346,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":1636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10588,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":939,"num_bytes_recovered":61538304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1388},{"shards":"0,1,4,6","objects":1},{"shards":"0,1,6","objects":203},{"shards":"0,6","objects":-3302}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":378,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689208+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:01:52.689208+0000","last_peered":"2023-09-23T23:01:52.689208+0000","last_clean":"2023-09-23T23:01:52.689208+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:01:52.689208+0000","last_undegraded":"2023-09-23T23:01:52.689208+0000","last_fullsized":"2023-09-23T23:01:52.689208+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":492,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:01:59.720176+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:01:59.720176+0000","last_peered":"2023-09-23T23:01:59.720176+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:01:59.720176+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695901+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:01:52.695901+0000","last_peered":"2023-09-23T23:01:52.695901+0000","last_clean":"2023-09-23T23:01:52.695901+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:01:52.695901+0000","last_undegraded":"2023-09-23T23:01:52.695901+0000","last_fullsized":"2023-09-23T23:01:52.695901+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":305,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.693361+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:01:52.693361+0000","last_peered":"2023-09-23T23:01:52.693361+0000","last_clean":"2023-09-23T23:01:52.693361+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:01:52.693361+0000","last_undegraded":"2023-09-23T23:01:52.693361+0000","last_fullsized":"2023-09-23T23:01:52.693361+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"395'10588","reported_seq":10692,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:04.749381+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:02:04.749381+0000","last_peered":"2023-09-23T23:02:04.749381+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:02:04.749381+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98041856,"num_objects":1496,"num_object_clones":0,"num_object_copies":4488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10588,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2011}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":386,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718496+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:01:59.718496+0000","last_peered":"2023-09-23T23:01:59.718496+0000","last_clean":"2023-09-23T23:01:59.718496+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:01:59.718496+0000","last_undegraded":"2023-09-23T23:01:59.718496+0000","last_fullsized":"2023-09-23T23:01:59.718496+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":356,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.680811+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:01:52.680811+0000","last_peered":"2023-09-23T23:01:52.680811+0000","last_clean":"2023-09-23T23:01:52.680811+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:01:52.680811+0000","last_undegraded":"2023-09-23T23:01:52.680811+0000","last_fullsized":"2023-09-23T23:01:52.680811+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":316,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681775+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:01:52.681775+0000","last_peered":"2023-09-23T23:01:52.681775+0000","last_clean":"2023-09-23T23:01:52.681775+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:01:52.681775+0000","last_undegraded":"2023-09-23T23:01:52.681775+0000","last_fullsized":"2023-09-23T23:01:52.681775+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"395'10656","reported_seq":10775,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:07.238536+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:02:07.238536+0000","last_peered":"2023-09-23T23:02:07.238536+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:02:07.238536+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:02:07.238536+0000","mapping_epoch":354,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101384216,"num_objects":1548,"num_object_clones":0,"num_object_copies":4644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1548,"num_objects_unfound":0,"num_objects_dirty":1548,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10656,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1548}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":331,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681222+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:01:52.681222+0000","last_peered":"2023-09-23T23:01:52.681222+0000","last_clean":"2023-09-23T23:01:52.681222+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:01:52.681222+0000","last_undegraded":"2023-09-23T23:01:52.681222+0000","last_fullsized":"2023-09-23T23:01:52.681222+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":239,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.678160+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:01:52.678160+0000","last_peered":"2023-09-23T23:01:52.678160+0000","last_clean":"2023-09-23T23:01:52.678160+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:01:52.678160+0000","last_undegraded":"2023-09-23T23:01:52.678160+0000","last_fullsized":"2023-09-23T23:01:52.678160+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":208,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.689284+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:01:52.689284+0000","last_peered":"2023-09-23T23:01:52.689284+0000","last_clean":"2023-09-23T23:01:52.689284+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:01:52.689284+0000","last_undegraded":"2023-09-23T23:01:52.689284+0000","last_fullsized":"2023-09-23T23:01:52.689284+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"395'10666","reported_seq":20889,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:07.246272+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:02:07.246272+0000","last_peered":"2023-09-23T23:02:07.246272+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:02:07.246272+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"332'5000","ondisk_log_start":"332'5000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96993280,"num_objects":1480,"num_object_clones":0,"num_object_copies":4440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3247,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10666,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4004,"num_bytes_recovered":262406144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1385},{"shards":"4,6","objects":95}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":89,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.698651+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:01:52.698651+0000","last_peered":"2023-09-23T23:01:52.698651+0000","last_clean":"2023-09-23T23:01:52.698651+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:01:52.698651+0000","last_undegraded":"2023-09-23T23:01:52.698651+0000","last_fullsized":"2023-09-23T23:01:52.698651+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.697763+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:01:52.697763+0000","last_peered":"2023-09-23T23:01:52.697763+0000","last_clean":"2023-09-23T23:01:52.697763+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:01:52.697763+0000","last_undegraded":"2023-09-23T23:01:52.697763+0000","last_fullsized":"2023-09-23T23:01:52.697763+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695577+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:01:52.695577+0000","last_peered":"2023-09-23T23:01:52.695577+0000","last_clean":"2023-09-23T23:01:52.695577+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:01:52.695577+0000","last_undegraded":"2023-09-23T23:01:52.695577+0000","last_fullsized":"2023-09-23T23:01:52.695577+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"395'10552","reported_seq":11081,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:07.223159+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:02:07.223159+0000","last_peered":"2023-09-23T23:02:07.223159+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:02:07.223159+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"331'4800","ondisk_log_start":"331'4800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99483648,"num_objects":1518,"num_object_clones":0,"num_object_copies":4554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1518,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":1518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10552,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1439},{"shards":"4,6","objects":-2033}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":351,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.677495+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:01:52.677495+0000","last_peered":"2023-09-23T23:01:52.677495+0000","last_clean":"2023-09-23T23:01:52.677495+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:01:52.677495+0000","last_undegraded":"2023-09-23T23:01:52.677495+0000","last_fullsized":"2023-09-23T23:01:52.677495+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":298,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681701+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:01:52.681701+0000","last_peered":"2023-09-23T23:01:52.681701+0000","last_clean":"2023-09-23T23:01:52.681701+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:01:52.681701+0000","last_undegraded":"2023-09-23T23:01:52.681701+0000","last_fullsized":"2023-09-23T23:01:52.681701+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":369,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692309+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:01:52.692309+0000","last_peered":"2023-09-23T23:01:52.692309+0000","last_clean":"2023-09-23T23:01:52.692309+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:01:52.692309+0000","last_undegraded":"2023-09-23T23:01:52.692309+0000","last_fullsized":"2023-09-23T23:01:52.692309+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":333,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.682575+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:01:52.682575+0000","last_peered":"2023-09-23T23:01:52.682575+0000","last_clean":"2023-09-23T23:01:52.682575+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:01:52.682575+0000","last_undegraded":"2023-09-23T23:01:52.682575+0000","last_fullsized":"2023-09-23T23:01:52.682575+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":143,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.677814+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:01:52.677814+0000","last_peered":"2023-09-23T23:01:52.677814+0000","last_clean":"2023-09-23T23:01:52.677814+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:01:52.677814+0000","last_undegraded":"2023-09-23T23:01:52.677814+0000","last_fullsized":"2023-09-23T23:01:52.677814+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":204,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.694020+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:01:52.694020+0000","last_peered":"2023-09-23T23:01:52.694020+0000","last_clean":"2023-09-23T23:01:52.694020+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:01:52.694020+0000","last_undegraded":"2023-09-23T23:01:52.694020+0000","last_fullsized":"2023-09-23T23:01:52.694020+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":418,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695174+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:01:52.695174+0000","last_peered":"2023-09-23T23:01:52.695174+0000","last_clean":"2023-09-23T23:01:52.695174+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:01:52.695174+0000","last_undegraded":"2023-09-23T23:01:52.695174+0000","last_fullsized":"2023-09-23T23:01:52.695174+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":218,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.693219+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:01:52.693219+0000","last_peered":"2023-09-23T23:01:52.693219+0000","last_clean":"2023-09-23T23:01:52.693219+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:01:52.693219+0000","last_undegraded":"2023-09-23T23:01:52.693219+0000","last_fullsized":"2023-09-23T23:01:52.693219+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":305,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692518+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:01:52.692518+0000","last_peered":"2023-09-23T23:01:52.692518+0000","last_clean":"2023-09-23T23:01:52.692518+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:01:52.692518+0000","last_undegraded":"2023-09-23T23:01:52.692518+0000","last_fullsized":"2023-09-23T23:01:52.692518+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":89,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.696160+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:01:52.696160+0000","last_peered":"2023-09-23T23:01:52.696160+0000","last_clean":"2023-09-23T23:01:52.696160+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:01:52.696160+0000","last_undegraded":"2023-09-23T23:01:52.696160+0000","last_fullsized":"2023-09-23T23:01:52.696160+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.695829+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:01:52.695829+0000","last_peered":"2023-09-23T23:01:52.695829+0000","last_clean":"2023-09-23T23:01:52.695829+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:01:52.695829+0000","last_undegraded":"2023-09-23T23:01:52.695829+0000","last_fullsized":"2023-09-23T23:01:52.695829+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":337,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.681494+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:01:52.681494+0000","last_peered":"2023-09-23T23:01:52.681494+0000","last_clean":"2023-09-23T23:01:52.681494+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:01:52.681494+0000","last_undegraded":"2023-09-23T23:01:52.681494+0000","last_fullsized":"2023-09-23T23:01:52.681494+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":380,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T23:01:52.692834+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:01:52.692834+0000","last_peered":"2023-09-23T23:01:52.692834+0000","last_clean":"2023-09-23T23:01:52.692834+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:01:52.692834+0000","last_undegraded":"2023-09-23T23:01:52.692834+0000","last_fullsized":"2023-09-23T23:01:52.692834+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1633746968,"num_objects":24930,"num_object_clones":0,"num_object_copies":74790,"num_objects_missing_on_primary":2519,"num_objects_missing":2519,"num_objects_degraded":18237,"num_objects_misplaced":28519,"num_objects_unfound":0,"num_objects_dirty":24930,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168668,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19262,"num_bytes_recovered":1171193856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5975846912,"data_stored":5975834696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100568,"ondisk_log_size":100568,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430629,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626853,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253167,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1429600,"kb_used_data":691160,"kb_used_omap":9,"kb_used_meta":738422,"kb_avail":92315552,"statfs":{"total":95995035648,"available":94531125248,"internally_reserved":0,"allocated":707747840,"data_stored":706250725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":756145129},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8250000000000002,"5min":1.258,"15min":0.65400000000000003},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":83.664000000000001,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":5.0039999999999996,"5min":1.5980000000000001,"15min":1.103},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":83.700999999999993,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.30499999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.48699999999999999,"15min":0.51400000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.087,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.64200000000000002,"15min":0.69799999999999995},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.74,"5min":4.048,"15min":4.048},"last":0.60099999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":3.008},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.63100000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":0.378},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":3.1880000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":0.47499999999999998}]}]},{"osd":0,"up_from":353,"seq":1516123455537,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1318924,"kb_used_data":1131524,"kb_used_omap":37,"kb_used_meta":187354,"kb_avail":92426228,"statfs":{"total":95995035648,"available":94644457472,"internally_reserved":0,"allocated":1158680576,"data_stored":1157026090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":191850658},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.59199999999999997,"15min":0.55700000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.7629999999999999,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":2.7349999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64000000000000001,"15min":0.66100000000000003},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.8650000000000002,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":3.395}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":0.86399999999999999,"15min":0.80800000000000005},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":2.694},{"interface":"front","average":{"1min":1.012,"5min":0.72499999999999998,"15min":0.68500000000000005},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"last":3.8980000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.623,"15min":0.51000000000000001},"min":{"1min":0.29299999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":2.7669999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.76800000000000002,"15min":0.86199999999999999},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6499999999999999,"5min":4.0179999999999998,"15min":4.0179999999999998},"last":2.0699999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":12.855,"5min":3.2290000000000001,"15min":1.649},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":183.148,"5min":183.148,"15min":183.148},"last":0.35599999999999998},{"interface":"front","average":{"1min":13.087999999999999,"5min":3.194,"15min":1.51},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":183.39500000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":3.54}]},{"osd":5,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.86799999999999999,"15min":0.74199999999999999},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4089999999999998,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":2.823},{"interface":"front","average":{"1min":1.282,"5min":1.0009999999999999,"15min":0.999},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.774,"5min":7.774,"15min":7.774},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.92900000000000005,"15min":0.86599999999999999},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":5.7309999999999999,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":3.5070000000000001},{"interface":"front","average":{"1min":1.242,"5min":0.86799999999999999,"15min":0.77200000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":6.375,"5min":6.375,"15min":6.375},"last":2.8519999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.159,"15min":1.2549999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.8439999999999999,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":3.431},{"interface":"front","average":{"1min":1.008,"5min":1.2150000000000001,"15min":1.2569999999999999},"min":{"1min":0.49299999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":6.343,"5min":6.343,"15min":6.343},"last":2.8759999999999999}]}]},{"osd":2,"up_from":359,"seq":1541893259310,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":713928,"kb_used_data":568512,"kb_used_omap":83,"kb_used_meta":145324,"kb_avail":93031224,"statfs":{"total":95995035648,"available":95263973376,"internally_reserved":0,"allocated":582156288,"data_stored":580566621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":148812229},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":1.9850000000000001},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.64900000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.68700000000000006}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":1.853},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":1.5860000000000001},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":1.948}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":1.903}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":2.0169999999999999},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.59399999999999997}]}]},{"osd":3,"up_from":94,"seq":403726926114,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":535100,"kb_used_data":20596,"kb_used_omap":7,"kb_used_meta":514488,"kb_avail":93210052,"statfs":{"total":95995035648,"available":95447093248,"internally_reserved":0,"allocated":21090304,"data_stored":19506526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":526836328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:13 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":1.5640000000000001,"15min":0.73399999999999999},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":122.289,"5min":122.289,"15min":122.289},"last":0.61599999999999999},{"interface":"front","average":{"1min":6.1609999999999996,"5min":1.5600000000000001,"15min":0.79100000000000004},"min":{"1min":0.224,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":122.041,"5min":122.041,"15min":122.041},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":0.74099999999999999,"15min":2.2999999999999998},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.1639999999999997,"5min":8.6280000000000001,"15min":123.866},"last":4.6100000000000003},{"interface":"front","average":{"1min":1.3680000000000001,"5min":0.82499999999999996,"15min":2.1659999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5640000000000001,"5min":6.8010000000000002,"15min":122.265},"last":4.5179999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.63,"15min":0.68899999999999995},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.7,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.857,"5min":0.98899999999999999,"15min":0.82499999999999996},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.223},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":4.5490000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.3220000000000001,"15min":2.0099999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.2300000000000004,"5min":87.046999999999997,"15min":87.046999999999997},"last":4.6459999999999999},{"interface":"front","average":{"1min":1.2,"5min":2.496,"15min":1.927},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.4509999999999996,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.68999999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.2769999999999999,"15min":1.4039999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5229999999999997,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.111,"5min":1.1919999999999999,"15min":1.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7000000000000002,"5min":10.246,"15min":11.951000000000001},"last":2.4089999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.554,"15min":1.7569999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5380000000000003,"5min":11.116,"15min":14.097},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.288,"15min":1.919},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.4219999999999997,"5min":9.2810000000000006,"15min":12.887},"last":2.3599999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.90700000000000003,"15min":1.7350000000000001},"min":{"1min":0.36299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6680000000000001,"5min":4.8449999999999998,"15min":11.694000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.0649999999999999,"15min":1.6279999999999999},"min":{"1min":0.45300000000000001,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.2690000000000001,"5min":8.5830000000000002,"15min":12.32},"last":4.6799999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2194096,"kb_used_data":1282536,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":91551056,"statfs":{"total":95995035648,"available":93748281344,"internally_reserved":0,"allocated":1313316864,"data_stored":1312494775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":0.245},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":1.8660000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":1.9039999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:01:28 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.0549999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4870000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.76200000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4770000000000003,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":1.9390000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":2.3740000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":1.8240000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":1.97}]},{"osd":7,"last update":"Sat Sep 23 23:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.5700000000000001,"15min":1.169},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0869999999999997,"5min":12.84,"15min":12.84},"last":2.0009999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.585,"15min":1.117},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4199999999999999,"5min":10.587,"15min":10.587},"last":1.0780000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823085,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1683664,"kb_used_data":788104,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92061488,"statfs":{"total":95995035648,"available":94270963712,"internally_reserved":0,"allocated":807018496,"data_stored":806224195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.878,"15min":0.69599999999999995},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.5709999999999997,"5min":14.42,"15min":14.42},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2430000000000001,"15min":1.135},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.3159999999999998,"5min":14.496,"15min":14.496},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.498},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.41199999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.58299999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.55600000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 23:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.2350000000000001,"15min":1.0629999999999999},"min":{"1min":0.45000000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.8370000000000002,"5min":12.23,"15min":12.23},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.5830000000000002,"5min":1.7490000000000001,"15min":1.2170000000000001},"min":{"1min":0.378,"5min":0.31,"15min":0.31},"max":{"1min":7.6719999999999997,"5min":11.112,"15min":11.112},"last":0.47399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1137577984,"data_stored":1137573912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":700841984,"data_stored":700841984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":576978944,"data_stored":576978944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16187392,"data_stored":16187392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1301352448,"data_stored":1301348376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":800391168,"data_stored":800391168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1442516992,"data_stored":1442512920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:02:09.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:02:09.433 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:09.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:09.539 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:09.609 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:09.973 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:09.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:10.081 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:10.106 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:09 smithi052 ceph-mon[187743]: from='client.46676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:10.107 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:09 smithi052 ceph-mon[187743]: pgmap v1204: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 414 op/s; 18361/74496 objects degraded (24.647%); 28519/74496 objects misplaced (38.283%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:02:10.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:09 smithi052 ceph-mon[180979]: from='client.46676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:10.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:09 smithi052 ceph-mon[180979]: pgmap v1204: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 414 op/s; 18361/74496 objects degraded (24.647%); 28519/74496 objects misplaced (38.283%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:02:10.110 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:09 smithi151 ceph-mon[159210]: from='client.46676 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:09 smithi151 ceph-mon[159210]: pgmap v1204: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 414 op/s; 18361/74496 objects degraded (24.647%); 28519/74496 objects misplaced (38.283%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T23:02:10.426 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:10.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:10.537 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:10.593 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:10.773 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:10.782 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:02:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:10.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:10.887 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:10.895 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:10.932 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:11.053 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:11.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:11.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 18361/74496 objects degraded (24.647%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[187743]: pgmap v1205: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 368 op/s; 18263/72684 objects degraded (25.127%); 28267/72684 objects misplaced (38.890%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:11.067 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 18361/74496 objects degraded (24.647%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:11.068 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:11 smithi052 ceph-mon[180979]: pgmap v1205: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 368 op/s; 18263/72684 objects degraded (25.127%); 28267/72684 objects misplaced (38.890%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:02:11.159 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:11.218 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:11.266 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:11.298 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:11.312 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:02:11.312 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:02:11.312 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:02:11.312 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:02:11.312 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:02:11.339 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:11 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:11.339 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:11 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:11.339 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:11 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 18361/74496 objects degraded (24.647%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:11.339 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:11 smithi151 ceph-mon[159210]: pgmap v1205: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 368 op/s; 18263/72684 objects degraded (25.127%); 28267/72684 objects misplaced (38.890%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T23:02:11.340 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:11.383 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:11.385 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:11.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:11.409 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:11.446 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:11.478 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:11.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:11.509 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:11.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.518 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:02:11.535 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:11.961 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:11.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:12.064 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:12.120 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:12.121 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:02:12.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1015047328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:12.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:12.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:12.454 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1015047328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:12.454 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:12.455 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:12.456 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1015047328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:12.456 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:12.456 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:12.651 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:12.931 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:12.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:13.031 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:13.036 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:13.076 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:13 smithi151 ceph-mon[159210]: pgmap v1206: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 17633/69084 objects degraded (25.524%); 28001/69084 objects misplaced (40.532%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:02:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:13.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[180979]: pgmap v1206: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 17633/69084 objects degraded (25.524%); 28001/69084 objects misplaced (40.532%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:02:13.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:02:13.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:02:13.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:02:13.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:13 smithi052 ceph-mon[187743]: pgmap v1206: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 17633/69084 objects degraded (25.524%); 28001/69084 objects misplaced (40.532%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:02:13.288 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:13.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:13.392 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:13.424 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:13.475 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:13.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:13.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:13.580 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:13.608 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:02:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:02:13] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.33.4" 2023-09-23T23:02:13.612 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:13.700 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:13.700 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 17633/69084 objects degraded (25.524%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1666,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":396,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":23028,"data_bytes":1508120504,"bytes_used":7821578240,"bytes_avail":568148635648,"bytes_total":575970213888,"degraded_objects":17633,"degraded_total":69084,"degraded_ratio":0.25523999768397893,"misplaced_objects":28001,"misplaced_total":69084,"misplaced_ratio":0.40531816339528687,"recovering_objects_per_sec":43,"recovering_bytes_per_sec":2823687,"recovering_keys_per_sec":0,"num_objects_recovered":431,"num_bytes_recovered":28246016,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":367},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (6m)\n [======================......] (remaining: 80s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:02:13.768 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:13.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:13.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:13.905 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:14.100 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:14.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:14.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:14.236 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:14.494 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:14.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:14.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:14.598 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:14.617 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:14.617 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:14.627 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:14.643 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/87967924' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:14.772 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/87967924' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:14.773 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:02:14.773 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:02:14.774 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/87967924' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:14.786 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:14.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:14.892 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:14.921 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:14.974 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:14.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:15.018 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:02:15.079 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:15.118 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:15.354 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:15.372 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:15.381 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:02:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:15.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:15.483 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:15.537 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:15.621 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:15.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:15 smithi151 ceph-mon[159210]: pgmap v1207: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 369 op/s; 17714/68691 objects degraded (25.788%); 28001/68691 objects misplaced (40.764%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:02:15.633 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 17633/69084 objects degraded (25.524%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:15.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:15.736 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:15.776 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:15.812 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:15 smithi052 ceph-mon[180979]: pgmap v1207: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 369 op/s; 17714/68691 objects degraded (25.788%); 28001/68691 objects misplaced (40.764%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:02:15.812 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 17633/69084 objects degraded (25.524%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:15.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:15 smithi052 ceph-mon[187743]: pgmap v1207: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 369 op/s; 17714/68691 objects degraded (25.788%); 28001/68691 objects misplaced (40.764%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T23:02:15.813 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 17633/69084 objects degraded (25.524%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:15.843 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:15.887 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:15.928 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:15.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:15.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:16.035 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:16.070 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:16.474 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:16.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:16.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:16.555 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:02:16.578 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:16.607 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:16.649 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:16.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:16.739 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1864293416' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:16.739 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:16.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1864293416' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:16.740 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:16.753 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:16.807 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1864293416' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:16.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:17.378 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:17.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:17.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:17.481 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:17.512 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:17.544 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:17.544 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:17.567 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:17.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:02:17.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: osdmap e397: 8 total, 8 up, 6 in 2023-09-23T23:02:17.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: pgmap v1209: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 17544/66732 objects degraded (26.290%); 27828/66732 objects misplaced (41.701%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T23:02:17.568 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:17 smithi151 ceph-mon[159210]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.568 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:17.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:17.673 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:17.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:17.698 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: osdmap e397: 8 total, 8 up, 6 in 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: pgmap v1209: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 17544/66732 objects degraded (26.290%); 27828/66732 objects misplaced (41.701%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[187743]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:02:17.699 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: osdmap e397: 8 total, 8 up, 6 in 2023-09-23T23:02:17.700 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.700 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: pgmap v1209: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 17544/66732 objects degraded (26.290%); 27828/66732 objects misplaced (41.701%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T23:02:17.700 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:17 smithi052 ceph-mon[180979]: failed to encode map e397 with expected crc 2023-09-23T23:02:17.702 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:18.107 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:18.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:18.212 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:18.246 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:18.282 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:02:18.320 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:18.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:18.425 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:18.469 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:18.502 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:18.511 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:18.553 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:18 smithi151 ceph-mon[159210]: failed to encode map e397 with expected crc 2023-09-23T23:02:18.570 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:18.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:18.676 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:18.714 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:18.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:18 smithi052 ceph-mon[187743]: failed to encode map e397 with expected crc 2023-09-23T23:02:18.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:18 smithi052 ceph-mon[180979]: failed to encode map e397 with expected crc 2023-09-23T23:02:18.754 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:18.772 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:18.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:18.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:18.792 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:18.819 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:18.844 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:18.870 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:18.882 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:18.890 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:18.915 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:18.916 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:18.942 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:18.964 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:18.965 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.965 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.966 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.966 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.966 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.966 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.966 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.966 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:02:18.967 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:18.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:18.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:19.070 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:19.112 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:19.174 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:19.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:19.282 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:19.316 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:19.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:02:19.373 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:19.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:19.479 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:19.505 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:19.555 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:19.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:19.660 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:19.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:19 smithi151 ceph-mon[159210]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:19.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:19 smithi151 ceph-mon[159210]: pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.3 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 17382/64542 objects degraded (26.931%); 27673/64542 objects misplaced (42.876%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:02:19.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2763940739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:19.692 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:19.694 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:19.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[187743]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:19.736 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[187743]: pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.3 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 17382/64542 objects degraded (26.931%); 27673/64542 objects misplaced (42.876%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:02:19.736 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2763940739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:19.736 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:19.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[180979]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:19.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[180979]: pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.3 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 17382/64542 objects degraded (26.931%); 27673/64542 objects misplaced (42.876%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:02:19.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2763940739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:19.739 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:19.756 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1209,"stamp":"2023-09-23T23:02:16.608769+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1456740280,"num_objects":22244,"num_object_clones":0,"num_object_copies":66732,"num_objects_missing_on_primary":2506,"num_objects_missing":2506,"num_objects_degraded":17544,"num_objects_misplaced":27828,"num_objects_unfound":0,"num_objects_dirty":22244,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":171638,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19502,"num_bytes_recovered":1184644028,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100672,"ondisk_log_size":100672,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7473088,"kb_used_data":4076112,"kb_used_omap":137,"kb_used_meta":3396790,"kb_avail":554997824,"statfs":{"total":575970213888,"available":568317771776,"internally_reserved":0,"allocated":4173938688,"data_stored":4165965960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3478313144},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-177602560,"num_objects":-2710,"num_object_clones":0,"num_object_copies":-8130,"num_objects_missing_on_primary":-13,"num_objects_missing":-13,"num_objects_degraded":-695,"num_objects_misplaced":-691,"num_objects_unfound":0,"num_objects_dirty":-2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2710,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":205,"num_bytes_recovered":13434880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.843920"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":408,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718652+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:01:59.718652+0000","last_peered":"2023-09-23T23:01:59.718652+0000","last_clean":"2023-09-23T23:01:59.718652+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:01:59.718652+0000","last_undegraded":"2023-09-23T23:01:59.718652+0000","last_fullsized":"2023-09-23T23:01:59.718652+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":65,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.729328+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:01:59.729328+0000","last_peered":"2023-09-23T23:01:59.729328+0000","last_clean":"2023-09-23T23:01:59.729328+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:01:59.729328+0000","last_undegraded":"2023-09-23T23:01:59.729328+0000","last_fullsized":"2023-09-23T23:01:59.729328+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":64,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728268+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:01:59.728268+0000","last_peered":"2023-09-23T23:01:59.728268+0000","last_clean":"2023-09-23T23:01:59.728268+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:01:59.728268+0000","last_undegraded":"2023-09-23T23:01:59.728268+0000","last_fullsized":"2023-09-23T23:01:59.728268+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":393,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728434+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:01:59.728434+0000","last_peered":"2023-09-23T23:01:59.728434+0000","last_clean":"2023-09-23T23:01:59.728434+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:01:59.728434+0000","last_undegraded":"2023-09-23T23:01:59.728434+0000","last_fullsized":"2023-09-23T23:01:59.728434+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.791034+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:02:07.791034+0000","last_peered":"2023-09-23T23:02:07.791034+0000","last_clean":"2023-09-23T23:02:07.791034+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:02:07.791034+0000","last_undegraded":"2023-09-23T23:02:07.791034+0000","last_fullsized":"2023-09-23T23:02:07.791034+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":67,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.773158+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:02:07.773158+0000","last_peered":"2023-09-23T23:02:07.773158+0000","last_clean":"2023-09-23T23:02:07.773158+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:02:07.773158+0000","last_undegraded":"2023-09-23T23:02:07.773158+0000","last_fullsized":"2023-09-23T23:02:07.773158+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":331,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.731832+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:01:59.731832+0000","last_peered":"2023-09-23T23:01:59.731832+0000","last_clean":"2023-09-23T23:01:59.731832+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:01:59.731832+0000","last_undegraded":"2023-09-23T23:01:59.731832+0000","last_fullsized":"2023-09-23T23:01:59.731832+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.790908+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:02:07.790908+0000","last_peered":"2023-09-23T23:02:07.790908+0000","last_clean":"2023-09-23T23:02:07.790908+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:02:07.790908+0000","last_undegraded":"2023-09-23T23:02:07.790908+0000","last_fullsized":"2023-09-23T23:02:07.790908+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":292,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.730269+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:01:59.730269+0000","last_peered":"2023-09-23T23:01:59.730269+0000","last_clean":"2023-09-23T23:01:59.730269+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:01:59.730269+0000","last_undegraded":"2023-09-23T23:01:59.730269+0000","last_fullsized":"2023-09-23T23:01:59.730269+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":80,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.793271+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:02:07.793271+0000","last_peered":"2023-09-23T23:02:07.793271+0000","last_clean":"2023-09-23T23:02:07.793271+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:02:07.793271+0000","last_undegraded":"2023-09-23T23:02:07.793271+0000","last_fullsized":"2023-09-23T23:02:07.793271+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":85,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.729230+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:01:59.729230+0000","last_peered":"2023-09-23T23:01:59.729230+0000","last_clean":"2023-09-23T23:01:59.729230+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:01:59.729230+0000","last_undegraded":"2023-09-23T23:01:59.729230+0000","last_fullsized":"2023-09-23T23:01:59.729230+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.708396+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:01:59.708396+0000","last_peered":"2023-09-23T23:01:59.708396+0000","last_clean":"2023-09-23T23:01:59.708396+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:01:59.708396+0000","last_undegraded":"2023-09-23T23:01:59.708396+0000","last_fullsized":"2023-09-23T23:01:59.708396+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":467,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719346+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:01:59.719346+0000","last_peered":"2023-09-23T23:01:59.719346+0000","last_clean":"2023-09-23T23:01:59.719346+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:01:59.719346+0000","last_undegraded":"2023-09-23T23:01:59.719346+0000","last_fullsized":"2023-09-23T23:01:59.719346+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":370,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.720939+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:01:59.720939+0000","last_peered":"2023-09-23T23:01:59.720939+0000","last_clean":"2023-09-23T23:01:59.720939+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:01:59.720939+0000","last_undegraded":"2023-09-23T23:01:59.720939+0000","last_fullsized":"2023-09-23T23:01:59.720939+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":296,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.721036+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:01:59.721036+0000","last_peered":"2023-09-23T23:01:59.721036+0000","last_clean":"2023-09-23T23:01:59.721036+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:01:59.721036+0000","last_undegraded":"2023-09-23T23:01:59.721036+0000","last_fullsized":"2023-09-23T23:01:59.721036+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:02:06.265817+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:02:06.265817+0000","last_peered":"2023-09-23T23:02:06.265817+0000","last_clean":"2023-09-23T23:02:06.265817+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:02:06.265817+0000","last_undegraded":"2023-09-23T23:02:06.265817+0000","last_fullsized":"2023-09-23T23:02:06.265817+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":386,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.793371+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:02:07.793371+0000","last_peered":"2023-09-23T23:02:07.793371+0000","last_clean":"2023-09-23T23:02:07.793371+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:02:07.793371+0000","last_undegraded":"2023-09-23T23:02:07.793371+0000","last_fullsized":"2023-09-23T23:02:07.793371+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":296,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.730321+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:01:59.730321+0000","last_peered":"2023-09-23T23:01:59.730321+0000","last_clean":"2023-09-23T23:01:59.730321+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:01:59.730321+0000","last_undegraded":"2023-09-23T23:01:59.730321+0000","last_fullsized":"2023-09-23T23:01:59.730321+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":281,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719549+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:01:59.719549+0000","last_peered":"2023-09-23T23:01:59.719549+0000","last_clean":"2023-09-23T23:01:59.719549+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:01:59.719549+0000","last_undegraded":"2023-09-23T23:01:59.719549+0000","last_fullsized":"2023-09-23T23:01:59.719549+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":409,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728606+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:01:59.728606+0000","last_peered":"2023-09-23T23:01:59.728606+0000","last_clean":"2023-09-23T23:01:59.728606+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:01:59.728606+0000","last_undegraded":"2023-09-23T23:01:59.728606+0000","last_fullsized":"2023-09-23T23:01:59.728606+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":114,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718506+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:01:59.718506+0000","last_peered":"2023-09-23T23:01:59.718506+0000","last_clean":"2023-09-23T23:01:59.718506+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:01:59.718506+0000","last_undegraded":"2023-09-23T23:01:59.718506+0000","last_fullsized":"2023-09-23T23:01:59.718506+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.729526+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:01:59.729526+0000","last_peered":"2023-09-23T23:01:59.729526+0000","last_clean":"2023-09-23T23:01:59.729526+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:01:59.729526+0000","last_undegraded":"2023-09-23T23:01:59.729526+0000","last_fullsized":"2023-09-23T23:01:59.729526+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"396'10725","reported_seq":10825,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:14.761118+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:02:14.761118+0000","last_peered":"2023-09-23T23:02:14.761118+0000","last_clean":"2023-09-23T23:02:14.761118+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:02:14.761118+0000","last_undegraded":"2023-09-23T23:02:14.761118+0000","last_fullsized":"2023-09-23T23:02:14.761118+0000","mapping_epoch":353,"log_start":"334'5200","ondisk_log_start":"334'5200","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89194496,"num_objects":1361,"num_object_clones":0,"num_object_copies":4083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10725,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":280,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.725096+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:01:59.725096+0000","last_peered":"2023-09-23T23:01:59.725096+0000","last_clean":"2023-09-23T23:01:59.725096+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:01:59.725096+0000","last_undegraded":"2023-09-23T23:01:59.725096+0000","last_fullsized":"2023-09-23T23:01:59.725096+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.731402+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:01:59.731402+0000","last_peered":"2023-09-23T23:01:59.731402+0000","last_clean":"2023-09-23T23:01:59.731402+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:01:59.731402+0000","last_undegraded":"2023-09-23T23:01:59.731402+0000","last_fullsized":"2023-09-23T23:01:59.731402+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"396'10588","reported_seq":13017,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:16.221676+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:02:16.221676+0000","last_peered":"2023-09-23T23:02:16.221676+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:02:16.221676+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"322'2000","ondisk_log_start":"322'2000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89260032,"num_objects":1362,"num_object_clones":0,"num_object_copies":4086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1362,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":1362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10588,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1362},{"shards":"5,6","objects":-2205}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":290,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719614+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:01:59.719614+0000","last_peered":"2023-09-23T23:01:59.719614+0000","last_clean":"2023-09-23T23:01:59.719614+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:01:59.719614+0000","last_undegraded":"2023-09-23T23:01:59.719614+0000","last_fullsized":"2023-09-23T23:01:59.719614+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":403,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719477+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:01:59.719477+0000","last_peered":"2023-09-23T23:01:59.719477+0000","last_clean":"2023-09-23T23:01:59.719477+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:01:59.719477+0000","last_undegraded":"2023-09-23T23:01:59.719477+0000","last_fullsized":"2023-09-23T23:01:59.719477+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"396'10716","reported_seq":10811,"reported_epoch":396,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:14.696221+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:02:14.696221+0000","last_peered":"2023-09-23T23:02:14.696221+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:02:14.696221+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:02:14.696221+0000","mapping_epoch":354,"log_start":"334'5200","ondisk_log_start":"334'5200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88866816,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":1356,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10716,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1356}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":147,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.730450+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:01:59.730450+0000","last_peered":"2023-09-23T23:01:59.730450+0000","last_clean":"2023-09-23T23:01:59.730450+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:01:59.730450+0000","last_undegraded":"2023-09-23T23:01:59.730450+0000","last_fullsized":"2023-09-23T23:01:59.730450+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":265,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.729524+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:01:59.729524+0000","last_peered":"2023-09-23T23:01:59.729524+0000","last_clean":"2023-09-23T23:01:59.729524+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:01:59.729524+0000","last_undegraded":"2023-09-23T23:01:59.729524+0000","last_fullsized":"2023-09-23T23:01:59.729524+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"396'10767","reported_seq":11048,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:14.839605+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:02:14.839605+0000","last_peered":"2023-09-23T23:02:14.839605+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:02:14.839605+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"334'5300","ondisk_log_start":"334'5300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92209152,"num_objects":1407,"num_object_clones":0,"num_object_copies":4221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1183,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":1407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10767,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":224},{"shards":"0,5","objects":-2203}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":66,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728522+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:01:59.728522+0000","last_peered":"2023-09-23T23:01:59.728522+0000","last_clean":"2023-09-23T23:01:59.728522+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:01:59.728522+0000","last_undegraded":"2023-09-23T23:01:59.728522+0000","last_fullsized":"2023-09-23T23:01:59.728522+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":295,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718930+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:01:59.718930+0000","last_peered":"2023-09-23T23:01:59.718930+0000","last_clean":"2023-09-23T23:01:59.718930+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:01:59.718930+0000","last_undegraded":"2023-09-23T23:01:59.718930+0000","last_fullsized":"2023-09-23T23:01:59.718930+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"396'10656","reported_seq":10757,"reported_epoch":396,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:12.222999+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:02:12.222999+0000","last_peered":"2023-09-23T23:02:12.222999+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:02:12.222999+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:02:12.222999+0000","mapping_epoch":353,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94240768,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10656,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1438}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":85,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728662+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:01:59.728662+0000","last_peered":"2023-09-23T23:01:59.728662+0000","last_clean":"2023-09-23T23:01:59.728662+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:01:59.728662+0000","last_undegraded":"2023-09-23T23:01:59.728662+0000","last_fullsized":"2023-09-23T23:01:59.728662+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":389,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719267+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:01:59.719267+0000","last_peered":"2023-09-23T23:01:59.719267+0000","last_clean":"2023-09-23T23:01:59.719267+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:01:59.719267+0000","last_undegraded":"2023-09-23T23:01:59.719267+0000","last_fullsized":"2023-09-23T23:01:59.719267+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"396'10798","reported_seq":10907,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:12.224894+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:02:12.224894+0000","last_peered":"2023-09-23T23:02:12.224894+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:02:12.224894+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1404,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10798,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-2215}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":179,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718845+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:01:59.718845+0000","last_peered":"2023-09-23T23:01:59.718845+0000","last_clean":"2023-09-23T23:01:59.718845+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:01:59.718845+0000","last_undegraded":"2023-09-23T23:01:59.718845+0000","last_fullsized":"2023-09-23T23:01:59.718845+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":406,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718552+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:01:59.718552+0000","last_peered":"2023-09-23T23:01:59.718552+0000","last_clean":"2023-09-23T23:01:59.718552+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:01:59.718552+0000","last_undegraded":"2023-09-23T23:01:59.718552+0000","last_fullsized":"2023-09-23T23:01:59.718552+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"396'10650","reported_seq":19485,"reported_epoch":396,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:12.166483+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:02:12.166483+0000","last_peered":"2023-09-23T23:02:12.166483+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:02:12.166483+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:02:12.166483+0000","mapping_epoch":354,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92405760,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10650,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1410}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":419,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719269+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:01:59.719269+0000","last_peered":"2023-09-23T23:01:59.719269+0000","last_clean":"2023-09-23T23:01:59.719269+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:01:59.719269+0000","last_undegraded":"2023-09-23T23:01:59.719269+0000","last_fullsized":"2023-09-23T23:01:59.719269+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":397,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719707+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:01:59.719707+0000","last_peered":"2023-09-23T23:01:59.719707+0000","last_clean":"2023-09-23T23:01:59.719707+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:01:59.719707+0000","last_undegraded":"2023-09-23T23:01:59.719707+0000","last_fullsized":"2023-09-23T23:01:59.719707+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"396'10477","reported_seq":10578,"reported_epoch":396,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:12.167270+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:02:12.167270+0000","last_peered":"2023-09-23T23:02:12.167270+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:02:12.167270+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:02:12.167270+0000","mapping_epoch":353,"log_start":"332'4800","ondisk_log_start":"332'4800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97714176,"num_objects":1491,"num_object_clones":0,"num_object_copies":4473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10477,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1491}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":66,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728331+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:01:59.728331+0000","last_peered":"2023-09-23T23:01:59.728331+0000","last_clean":"2023-09-23T23:01:59.728331+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:01:59.728331+0000","last_undegraded":"2023-09-23T23:01:59.728331+0000","last_fullsized":"2023-09-23T23:01:59.728331+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":299,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.707707+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:01:59.707707+0000","last_peered":"2023-09-23T23:01:59.707707+0000","last_clean":"2023-09-23T23:01:59.707707+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:01:59.707707+0000","last_undegraded":"2023-09-23T23:01:59.707707+0000","last_fullsized":"2023-09-23T23:01:59.707707+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"396'10639","reported_seq":19926,"reported_epoch":396,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:12.249796+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:02:12.249796+0000","last_peered":"2023-09-23T23:02:12.249796+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:02:12.249796+0000","last_undegraded":"2023-09-23T23:02:12.249796+0000","last_fullsized":"2023-09-23T23:02:12.249796+0000","mapping_epoch":354,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94175232,"num_objects":1437,"num_object_clones":0,"num_object_copies":4311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1437,"num_objects_unfound":0,"num_objects_dirty":1437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10639,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1437}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":303,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.707867+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:01:59.707867+0000","last_peered":"2023-09-23T23:01:59.707867+0000","last_clean":"2023-09-23T23:01:59.707867+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:01:59.707867+0000","last_undegraded":"2023-09-23T23:01:59.707867+0000","last_fullsized":"2023-09-23T23:01:59.707867+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.730247+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:01:59.730247+0000","last_peered":"2023-09-23T23:01:59.730247+0000","last_clean":"2023-09-23T23:01:59.730247+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:01:59.730247+0000","last_undegraded":"2023-09-23T23:01:59.730247+0000","last_fullsized":"2023-09-23T23:01:59.730247+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":119,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.729380+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:01:59.729380+0000","last_peered":"2023-09-23T23:01:59.729380+0000","last_clean":"2023-09-23T23:01:59.729380+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:01:59.729380+0000","last_undegraded":"2023-09-23T23:01:59.729380+0000","last_fullsized":"2023-09-23T23:01:59.729380+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"396'10758","reported_seq":10871,"reported_epoch":396,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:14.690992+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:02:14.690992+0000","last_peered":"2023-09-23T23:02:14.690992+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:02:14.690992+0000","last_undegraded":"2023-09-23T23:02:14.690992+0000","last_fullsized":"2023-09-23T23:02:14.690992+0000","mapping_epoch":359,"log_start":"334'5300","ondisk_log_start":"334'5300","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5458,"log_dups_size":0,"ondisk_log_size":5458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89522176,"num_objects":1366,"num_object_clones":0,"num_object_copies":4098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1366,"num_objects_unfound":0,"num_objects_dirty":1366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10758,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1366}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":67,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.772849+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:02:07.772849+0000","last_peered":"2023-09-23T23:02:07.772849+0000","last_clean":"2023-09-23T23:02:07.772849+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:02:07.772849+0000","last_undegraded":"2023-09-23T23:02:07.772849+0000","last_fullsized":"2023-09-23T23:02:07.772849+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":416,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.721109+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:01:59.721109+0000","last_peered":"2023-09-23T23:01:59.721109+0000","last_clean":"2023-09-23T23:01:59.721109+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:01:59.721109+0000","last_undegraded":"2023-09-23T23:01:59.721109+0000","last_fullsized":"2023-09-23T23:01:59.721109+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":288,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.707937+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:01:59.707937+0000","last_peered":"2023-09-23T23:01:59.707937+0000","last_clean":"2023-09-23T23:01:59.707937+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:01:59.707937+0000","last_undegraded":"2023-09-23T23:01:59.707937+0000","last_fullsized":"2023-09-23T23:01:59.707937+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"396'10736","reported_seq":18912,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:16.246304+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:02:16.246304+0000","last_peered":"2023-09-23T23:02:16.246304+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:02:16.246304+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8636,"log_dups_size":0,"ondisk_log_size":8636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87031808,"num_objects":1328,"num_object_clones":0,"num_object_copies":3984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10736,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1328}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":363,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.707535+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:01:59.707535+0000","last_peered":"2023-09-23T23:01:59.707535+0000","last_clean":"2023-09-23T23:01:59.707535+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:01:59.707535+0000","last_undegraded":"2023-09-23T23:01:59.707535+0000","last_fullsized":"2023-09-23T23:01:59.707535+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":240,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.707526+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:01:59.707526+0000","last_peered":"2023-09-23T23:01:59.707526+0000","last_clean":"2023-09-23T23:01:59.707526+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:01:59.707526+0000","last_undegraded":"2023-09-23T23:01:59.707526+0000","last_fullsized":"2023-09-23T23:01:59.707526+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":306,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.731931+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:01:59.731931+0000","last_peered":"2023-09-23T23:01:59.731931+0000","last_clean":"2023-09-23T23:01:59.731931+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:01:59.731931+0000","last_undegraded":"2023-09-23T23:01:59.731931+0000","last_fullsized":"2023-09-23T23:01:59.731931+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"396'10841","reported_seq":12854,"reported_epoch":396,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:13.053309+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:02:13.053309+0000","last_peered":"2023-09-23T23:02:13.053309+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:02:13.053309+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'800","ondisk_log_start":"321'800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10041,"log_dups_size":0,"ondisk_log_size":10041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90636288,"num_objects":1383,"num_object_clones":0,"num_object_copies":4149,"num_objects_missing_on_primary":2504,"num_objects_missing":2504,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":1383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10841,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":953,"num_bytes_recovered":62455808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1375},{"shards":"0,6","objects":-3541}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":379,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.730678+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:01:59.730678+0000","last_peered":"2023-09-23T23:01:59.730678+0000","last_clean":"2023-09-23T23:01:59.730678+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:01:59.730678+0000","last_undegraded":"2023-09-23T23:01:59.730678+0000","last_fullsized":"2023-09-23T23:01:59.730678+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":493,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:04.971789+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:02:04.971789+0000","last_peered":"2023-09-23T23:02:04.971789+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:02:04.971789+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":66,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728464+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:01:59.728464+0000","last_peered":"2023-09-23T23:01:59.728464+0000","last_clean":"2023-09-23T23:01:59.728464+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:01:59.728464+0000","last_undegraded":"2023-09-23T23:01:59.728464+0000","last_fullsized":"2023-09-23T23:01:59.728464+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":306,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718828+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:01:59.718828+0000","last_peered":"2023-09-23T23:01:59.718828+0000","last_clean":"2023-09-23T23:01:59.718828+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:01:59.718828+0000","last_undegraded":"2023-09-23T23:01:59.718828+0000","last_fullsized":"2023-09-23T23:01:59.718828+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"396'10800","reported_seq":10905,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:14.500980+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:02:14.500980+0000","last_peered":"2023-09-23T23:02:14.500980+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:02:14.500980+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"334'5300","ondisk_log_start":"334'5300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84148224,"num_objects":1284,"num_object_clones":0,"num_object_copies":3852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10800,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2223}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":387,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:02:04.971756+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:02:04.971756+0000","last_peered":"2023-09-23T23:02:04.971756+0000","last_clean":"2023-09-23T23:02:04.971756+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:02:04.971756+0000","last_undegraded":"2023-09-23T23:02:04.971756+0000","last_fullsized":"2023-09-23T23:02:04.971756+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":358,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.772328+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:02:07.772328+0000","last_peered":"2023-09-23T23:02:07.772328+0000","last_clean":"2023-09-23T23:02:07.772328+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:02:07.772328+0000","last_undegraded":"2023-09-23T23:02:07.772328+0000","last_fullsized":"2023-09-23T23:02:07.772328+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":317,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719513+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:01:59.719513+0000","last_peered":"2023-09-23T23:01:59.719513+0000","last_clean":"2023-09-23T23:01:59.719513+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:01:59.719513+0000","last_undegraded":"2023-09-23T23:01:59.719513+0000","last_fullsized":"2023-09-23T23:01:59.719513+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"396'10788","reported_seq":10908,"reported_epoch":396,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:12.203306+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:02:12.203306+0000","last_peered":"2023-09-23T23:02:12.203306+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:02:12.203306+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:02:12.203306+0000","mapping_epoch":354,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92733464,"num_objects":1416,"num_object_clones":0,"num_object_copies":4248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1416,"num_objects_unfound":0,"num_objects_dirty":1416,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10788,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1416}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":333,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.774437+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:02:07.774437+0000","last_peered":"2023-09-23T23:02:07.774437+0000","last_clean":"2023-09-23T23:02:07.774437+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:02:07.774437+0000","last_undegraded":"2023-09-23T23:02:07.774437+0000","last_fullsized":"2023-09-23T23:02:07.774437+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":240,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.708015+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:01:59.708015+0000","last_peered":"2023-09-23T23:01:59.708015+0000","last_clean":"2023-09-23T23:01:59.708015+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:01:59.708015+0000","last_undegraded":"2023-09-23T23:01:59.708015+0000","last_fullsized":"2023-09-23T23:01:59.708015+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.730750+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:01:59.730750+0000","last_peered":"2023-09-23T23:01:59.730750+0000","last_clean":"2023-09-23T23:01:59.730750+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:01:59.730750+0000","last_undegraded":"2023-09-23T23:01:59.730750+0000","last_fullsized":"2023-09-23T23:01:59.730750+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"396'10784","reported_seq":21292,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:12.185370+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:02:12.185370+0000","last_peered":"2023-09-23T23:02:12.185370+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:02:12.185370+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89260032,"num_objects":1362,"num_object_clones":0,"num_object_copies":4086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10784,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-23},{"shards":"4,5,6","objects":1362}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":90,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.729659+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:01:59.729659+0000","last_peered":"2023-09-23T23:01:59.729659+0000","last_clean":"2023-09-23T23:01:59.729659+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:01:59.729659+0000","last_undegraded":"2023-09-23T23:01:59.729659+0000","last_fullsized":"2023-09-23T23:01:59.729659+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":104,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.722634+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:01:59.722634+0000","last_peered":"2023-09-23T23:01:59.722634+0000","last_clean":"2023-09-23T23:01:59.722634+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:01:59.722634+0000","last_undegraded":"2023-09-23T23:01:59.722634+0000","last_fullsized":"2023-09-23T23:01:59.722634+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728211+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:01:59.728211+0000","last_peered":"2023-09-23T23:01:59.728211+0000","last_clean":"2023-09-23T23:01:59.728211+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:01:59.728211+0000","last_undegraded":"2023-09-23T23:01:59.728211+0000","last_fullsized":"2023-09-23T23:01:59.728211+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"396'10655","reported_seq":11185,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:12.155045+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:02:12.155045+0000","last_peered":"2023-09-23T23:02:12.155045+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:02:12.155045+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92733440,"num_objects":1415,"num_object_clones":0,"num_object_copies":4245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1415,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":1415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10655,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1415},{"shards":"4,6","objects":-2136}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":352,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.716701+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:01:59.716701+0000","last_peered":"2023-09-23T23:01:59.716701+0000","last_clean":"2023-09-23T23:01:59.716701+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:01:59.716701+0000","last_undegraded":"2023-09-23T23:01:59.716701+0000","last_fullsized":"2023-09-23T23:01:59.716701+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":299,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719409+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:01:59.719409+0000","last_peered":"2023-09-23T23:01:59.719409+0000","last_clean":"2023-09-23T23:01:59.719409+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:01:59.719409+0000","last_undegraded":"2023-09-23T23:01:59.719409+0000","last_fullsized":"2023-09-23T23:01:59.719409+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":370,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719337+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:01:59.719337+0000","last_peered":"2023-09-23T23:01:59.719337+0000","last_clean":"2023-09-23T23:01:59.719337+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:01:59.719337+0000","last_undegraded":"2023-09-23T23:01:59.719337+0000","last_fullsized":"2023-09-23T23:01:59.719337+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":334,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719191+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:01:59.719191+0000","last_peered":"2023-09-23T23:01:59.719191+0000","last_clean":"2023-09-23T23:01:59.719191+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:01:59.719191+0000","last_undegraded":"2023-09-23T23:01:59.719191+0000","last_fullsized":"2023-09-23T23:01:59.719191+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":144,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.707657+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:01:59.707657+0000","last_peered":"2023-09-23T23:01:59.707657+0000","last_clean":"2023-09-23T23:01:59.707657+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:01:59.707657+0000","last_undegraded":"2023-09-23T23:01:59.707657+0000","last_fullsized":"2023-09-23T23:01:59.707657+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":205,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.719198+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:01:59.719198+0000","last_peered":"2023-09-23T23:01:59.719198+0000","last_clean":"2023-09-23T23:01:59.719198+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:01:59.719198+0000","last_undegraded":"2023-09-23T23:01:59.719198+0000","last_fullsized":"2023-09-23T23:01:59.719198+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":419,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.721203+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:01:59.721203+0000","last_peered":"2023-09-23T23:01:59.721203+0000","last_clean":"2023-09-23T23:01:59.721203+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:01:59.721203+0000","last_undegraded":"2023-09-23T23:01:59.721203+0000","last_fullsized":"2023-09-23T23:01:59.721203+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":219,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718658+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:01:59.718658+0000","last_peered":"2023-09-23T23:01:59.718658+0000","last_clean":"2023-09-23T23:01:59.718658+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:01:59.718658+0000","last_undegraded":"2023-09-23T23:01:59.718658+0000","last_fullsized":"2023-09-23T23:01:59.718658+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":306,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718731+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:01:59.718731+0000","last_peered":"2023-09-23T23:01:59.718731+0000","last_clean":"2023-09-23T23:01:59.718731+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:01:59.718731+0000","last_undegraded":"2023-09-23T23:01:59.718731+0000","last_fullsized":"2023-09-23T23:01:59.718731+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":90,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.728748+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:01:59.728748+0000","last_peered":"2023-09-23T23:01:59.728748+0000","last_clean":"2023-09-23T23:01:59.728748+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:01:59.728748+0000","last_undegraded":"2023-09-23T23:01:59.728748+0000","last_fullsized":"2023-09-23T23:01:59.728748+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.793560+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:02:07.793560+0000","last_peered":"2023-09-23T23:02:07.793560+0000","last_clean":"2023-09-23T23:02:07.793560+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:02:07.793560+0000","last_undegraded":"2023-09-23T23:02:07.793560+0000","last_fullsized":"2023-09-23T23:02:07.793560+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":339,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.773571+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:02:07.773571+0000","last_peered":"2023-09-23T23:02:07.773571+0000","last_clean":"2023-09-23T23:02:07.773571+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:02:07.773571+0000","last_undegraded":"2023-09-23T23:02:07.773571+0000","last_fullsized":"2023-09-23T23:02:07.773571+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":381,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T23:01:59.718750+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:01:59.718750+0000","last_peered":"2023-09-23T23:01:59.718750+0000","last_clean":"2023-09-23T23:01:59.718750+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:01:59.718750+0000","last_undegraded":"2023-09-23T23:01:59.718750+0000","last_fullsized":"2023-09-23T23:01:59.718750+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1456144408,"num_objects":22220,"num_object_clones":0,"num_object_copies":66660,"num_objects_missing_on_primary":2506,"num_objects_missing":2506,"num_objects_degraded":17542,"num_objects_misplaced":27828,"num_objects_unfound":0,"num_objects_dirty":22220,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171378,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19467,"num_bytes_recovered":1184628736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5493174272,"data_stored":5493162056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100478,"ondisk_log_size":100478,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430631,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626855,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253168,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1386796,"kb_used_data":644260,"kb_used_omap":9,"kb_used_meta":742518,"kb_avail":92358356,"statfs":{"total":95995035648,"available":94574956544,"internally_reserved":0,"allocated":659722240,"data_stored":658221155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":760339433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.696,"15min":0.80000000000000004},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":20.446000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":1.1659999999999999},{"interface":"front","average":{"1min":3.2440000000000002,"5min":2.0760000000000001,"15min":1.262},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.472999999999999,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.67300000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.48699999999999999,"15min":0.51400000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.087,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.64200000000000002,"15min":0.69799999999999995},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.74,"5min":4.048,"15min":4.048},"last":1.2010000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":3.7250000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":1.24}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":3.4529999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":3.3980000000000001},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":3.4929999999999999}]}]},{"osd":0,"up_from":353,"seq":1516123455539,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155356,"kb_used_data":976148,"kb_used_omap":37,"kb_used_meta":179162,"kb_avail":92589796,"statfs":{"total":95995035648,"available":94811951104,"internally_reserved":0,"allocated":999575552,"data_stored":997913360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":183462050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.71599999999999997,"15min":0.59799999999999998},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.82999999999999996,"15min":0.72399999999999998},"min":{"1min":0.27800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4459999999999997,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.27800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":0.86399999999999999,"15min":0.80800000000000005},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.012,"5min":0.72499999999999998,"15min":0.68500000000000005},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.88,"15min":0.59599999999999997},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.88500000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.8170000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2229999999999999,"15min":1.647},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2349999999999999,"5min":183.148,"15min":183.148},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.466,"5min":3.3530000000000002,"15min":1.5629999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1609999999999996,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.98499999999999999,"15min":0.78100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.984,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96199999999999997,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.1989999999999998,"5min":7.774,"15min":7.774},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.0509999999999999,"15min":0.90700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.359,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.91800000000000004,"15min":0.78800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3090000000000002,"5min":6.375,"15min":6.375},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.179,"15min":1.2609999999999999},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3929999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.147,"5min":1.1890000000000001,"15min":1.2490000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.669,"5min":6.343,"15min":6.343},"last":0.41399999999999998}]}]},{"osd":2,"up_from":359,"seq":1541893259312,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":703768,"kb_used_data":554256,"kb_used_omap":83,"kb_used_meta":149420,"kb_avail":93041384,"statfs":{"total":95995035648,"available":95274377216,"internally_reserved":0,"allocated":567558144,"data_stored":565960771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":153006533},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.432},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.623},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.69199999999999995}]}]},{"osd":3,"up_from":94,"seq":403726926116,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":541512,"kb_used_data":18816,"kb_used_omap":7,"kb_used_meta":522680,"kb_avail":93203640,"statfs":{"total":95995035648,"available":95440527360,"internally_reserved":0,"allocated":19267584,"data_stored":17679836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":535224936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:15 2023","interfaces":[{"interface":"back","average":{"1min":3.383,"5min":2.177,"15min":0.93799999999999994},"min":{"1min":0.191,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":37.249000000000002,"5min":122.289,"15min":122.289},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.919,"15min":0.91100000000000003},"min":{"1min":0.19800000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":35.991,"5min":122.041,"15min":122.041},"last":1.2150000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":0.74099999999999999,"15min":2.2999999999999998},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.1639999999999997,"5min":8.6280000000000001,"15min":123.866},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.3680000000000001,"5min":0.82499999999999996,"15min":2.1659999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5640000000000001,"5min":6.8010000000000002,"15min":122.265},"last":0.57099999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.63,"15min":0.68899999999999995},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.7,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":1.242},{"interface":"front","average":{"1min":1.857,"5min":0.98899999999999999,"15min":0.82499999999999996},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.223},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.3220000000000001,"15min":2.0099999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.2300000000000004,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.70399999999999996},{"interface":"front","average":{"1min":1.2,"5min":2.496,"15min":1.927},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.4509999999999996,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.2769999999999999,"15min":1.4039999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5229999999999997,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.111,"5min":1.1919999999999999,"15min":1.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7000000000000002,"5min":10.246,"15min":11.951000000000001},"last":1.282}]},{"osd":6,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.554,"15min":1.7569999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5380000000000003,"5min":11.116,"15min":14.097},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.288,"15min":1.919},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.4219999999999997,"5min":9.2810000000000006,"15min":12.887},"last":1.343}]},{"osd":7,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.90700000000000003,"15min":1.7350000000000001},"min":{"1min":0.36299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6680000000000001,"5min":4.8449999999999998,"15min":11.694000000000001},"last":1.266},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.0649999999999999,"15min":1.6279999999999999},"min":{"1min":0.45300000000000001,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.2690000000000001,"5min":8.5830000000000002,"15min":12.32},"last":0.67700000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019312,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2095612,"kb_used_data":1196340,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91649540,"statfs":{"total":95995035648,"available":93849128960,"internally_reserved":0,"allocated":1225052160,"data_stored":1224226101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.79900000000000004,"15min":0.58099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.8460000000000001,"5min":10.279,"15min":10.279},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.314}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":0.374},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":1.883}]},{"osd":2,"last update":"Sat Sep 23 23:01:28 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.0549999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4870000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":2.3340000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.76200000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4770000000000003,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.629}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.5700000000000001,"15min":1.169},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0869999999999997,"5min":12.84,"15min":12.84},"last":1.401},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.585,"15min":1.117},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4199999999999999,"5min":10.587,"15min":10.587},"last":0.748}]}]},{"osd":5,"up_from":36,"seq":154618823087,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1590044,"kb_used_data":686292,"kb_used_omap":0,"kb_used_meta":903744,"kb_avail":92155108,"statfs":{"total":95995035648,"available":94366830592,"internally_reserved":0,"allocated":702763008,"data_stored":701964737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925433856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.5229999999999999,"15min":0.91100000000000003},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":42.155000000000001,"5min":42.155000000000001,"15min":42.155000000000001},"last":23.346},{"interface":"front","average":{"1min":3.7839999999999998,"5min":1.784,"15min":1.3160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":43.378999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":23.428999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.58199999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.439},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.2350000000000001,"15min":1.0629999999999999},"min":{"1min":0.45000000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.8370000000000002,"5min":12.23,"15min":12.23},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.5830000000000002,"5min":1.7490000000000001,"15min":1.2170000000000001},"min":{"1min":0.378,"5min":0.31,"15min":0.31},"max":{"1min":7.6719999999999997,"5min":11.112,"15min":11.112},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":988024832,"data_stored":988020760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":652345344,"data_stored":652345344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":562429952,"data_stored":562429952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":14614528,"data_stored":14614528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1213272064,"data_stored":1213267992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":696975360,"data_stored":696975360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1365512192,"data_stored":1365508120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:02:19.926 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:19.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:20.030 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:20.069 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:20.100 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:20.123 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:20.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:20.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:20.229 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:20.259 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:20.364 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:20.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:20.468 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:20.481 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:02:20.481 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:02:20.482 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:02:20.482 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:02:20.482 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:02:20.506 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:20.676 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:20.683 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 17382/64542 objects degraded (26.931%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:20.683 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 17382/64542 objects degraded (26.931%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:20.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:20.785 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:20.814 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:20.861 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:20.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 17382/64542 objects degraded (26.931%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:20.964 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:21.001 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:21.277 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:02:21.280 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:21.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:21.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:21.422 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:21.495 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:21.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:21.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:21.601 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:21.637 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:21.698 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:21.702 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:21 smithi052 ceph-mon[187743]: pgmap v1211: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.3 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 309 op/s; 17319/63387 objects degraded (27.323%); 27510/63387 objects misplaced (43.400%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:02:21.702 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:21 smithi052 ceph-mon[180979]: pgmap v1211: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.3 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 309 op/s; 17319/63387 objects degraded (27.323%); 27510/63387 objects misplaced (43.400%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:02:21.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:21.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:21.732 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:21.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:21.835 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:21.909 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:21.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:21.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:21.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:21 smithi151 ceph-mon[159210]: pgmap v1211: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.3 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 309 op/s; 17319/63387 objects degraded (27.323%); 27510/63387 objects misplaced (43.400%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:02:21.983 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:02:22.017 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:22.071 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:22.146 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:22.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:22.250 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:22.311 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:22.378 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:22.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:22.477 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:22.483 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:22.517 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:22.564 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3354060200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:22.564 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:22 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:22.565 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3354060200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:22.565 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:22 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:22.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3354060200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:22.565 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:22 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:22.585 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:02:22.619 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:22.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:22.730 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:22.763 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:02:22.790 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:23.346 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:02:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:02:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:02:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:02:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.346 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115752 10.b 10:dd4f5f0e:::benchmark_data_smithi052_461816_object18951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 221.28853021800001, 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.395328154, 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.347 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115752 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.348 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052719+0000", 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9670000000000001e-06 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.349 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052727+0000", 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4453e-05 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052742+0000", 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5230000000000001e-06 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.350 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052748+0000", 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1202455760000001 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.172994+0000", 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010631813 2023-09-23T23:02:23.351 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.183626+0000", 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011626968 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.195253+0000", 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.037761408000000003 2023-09-23T23:02:23.352 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.233014+0000", 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097050000000000001 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.353 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.233984+0000", 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029044180000000002 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.236889+0000", 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.182741982 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.354 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.419631+0000", 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043122619999999999 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.423943+0000", 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024105932 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.355 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.448049+0000", 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.395328154 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115769 10.b 10:d0413e63:::benchmark_data_smithi052_461816_object18968:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.356 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.122591+0000", 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 221.21865973600001, 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.387893357, 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115769 2023-09-23T23:02:23.357 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122591+0000", 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.358 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122591+0000", 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122590+0000", 2023-09-23T23:02:23.359 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0461e-05 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122600+0000", 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9517999999999998e-05 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.360 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122619+0000", 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3550000000000005e-06 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.122628+0000", 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.146729326 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.361 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.269357+0000", 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012356867000000001 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.281714+0000", 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013998277 2023-09-23T23:02:23.362 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.295712+0000", 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017133461999999999 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.312846+0000", 2023-09-23T23:02:23.363 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0013916410000000001 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.314237+0000", 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028647410000000001 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.364 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.317102+0000", 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16527270599999999 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.482375+0000", 2023-09-23T23:02:23.365 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049430890000000003 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.487318+0000", 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023166771999999999 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.366 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.510485+0000", 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.387893357 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115897 10.b 10:da71a67f:::benchmark_data_smithi052_461816_object19096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.367 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 220.382054213, 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511318728, 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115897 2023-09-23T23:02:23.368 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.369 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959194+0000", 2023-09-23T23:02:23.370 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4834000000000001e-05 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959209+0000", 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6504999999999999e-05 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.371 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.372 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.372 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959226+0000", 2023-09-23T23:02:23.372 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.585e-06 2023-09-23T23:02:23.372 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.372 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.372 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959234+0000", 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.341365068 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.300599+0000", 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024846097000000001 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.373 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.374 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.374 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.325445+0000", 2023-09-23T23:02:23.374 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035146149000000002 2023-09-23T23:02:23.374 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.374 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.374 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.360592+0000", 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.066987158000000005 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.427579+0000", 2023-09-23T23:02:23.375 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050410359999999996 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.432620+0000", 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016353240000000001 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.376 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.434255+0000", 2023-09-23T23:02:23.377 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046578970000000003 2023-09-23T23:02:23.377 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.377 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.377 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.377 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.438913+0000", 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054771630000000002 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.444390+0000", 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.026125426 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.378 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.470516+0000", 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511318728 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.379 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115903 10.b 10:df88d60c:::benchmark_data_smithi052_461816_object19102:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 220.33892087199999, 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5939529189999999, 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.380 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115903 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:02:23.381 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.382 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.382 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.382 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.382 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:02:23.382 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.382 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002328+0000", 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5429999999999999e-06 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002338+0000", 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4139e-05 2023-09-23T23:02:23.383 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002352+0000", 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2870000000000001e-06 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002358+0000", 2023-09-23T23:02:23.384 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4692954739999999 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.471654+0000", 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013024641999999999 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.385 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.484678+0000", 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024981533 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.509660+0000", 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.059660008 2023-09-23T23:02:23.386 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.569320+0000", 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073572899999999996 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.387 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.570056+0000", 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021077549999999998 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.572163+0000", 2023-09-23T23:02:23.388 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027725240000000002 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.574936+0000", 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034689930000000001 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.389 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.578405+0000", 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017878277000000001 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.596283+0000", 2023-09-23T23:02:23.390 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5939529189999999 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115918 10.b 10:d203a882:::benchmark_data_smithi052_461816_object19117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.391 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:02:23.392 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 220.22051480799999, 2023-09-23T23:02:23.392 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511800968, 2023-09-23T23:02:23.392 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.392 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.392 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115918 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.393 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.394 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120735+0000", 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.841e-06 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120743+0000", 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3618000000000001e-05 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.395 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120756+0000", 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6620000000000002e-06 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120762+0000", 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4106586059999999 2023-09-23T23:02:23.396 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.531421+0000", 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012043214 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.397 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.543464+0000", 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012684575999999999 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.556148+0000", 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016711621999999999 2023-09-23T23:02:23.398 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.572860+0000", 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.025699692999999999 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.399 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.598560+0000", 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002245637 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.600805+0000", 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003157188 2023-09-23T23:02:23.400 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.603963+0000", 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049459949999999999 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.401 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.608909+0000", 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023628764999999999 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.632537+0000", 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511800968 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.402 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116102 10.b 10:d60ae293:::benchmark_data_smithi052_461816_object19301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 219.38737233699999, 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.473893417, 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.403 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116102 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:02:23.404 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999967 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.405 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953875+0000", 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4865000000000001e-05 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953890+0000", 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1577999999999999e-05 2023-09-23T23:02:23.406 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953912+0000", 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.075e-05 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953923+0000", 2023-09-23T23:02:23.407 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2057103410000001 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.159633+0000", 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0091687120000000007 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.168802+0000", 2023-09-23T23:02:23.408 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011821033999999999 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.180623+0000", 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018815996000000001 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.199439+0000", 2023-09-23T23:02:23.409 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022259570000000002 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.201665+0000", 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021258484000000001 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.410 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.222923+0000", 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.186530428 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.409453+0000", 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034981790000000001 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.411 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.412952+0000", 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014820543 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.427772+0000", 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.473893417 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.412 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116106 10.b 10:dc0e4ffc:::benchmark_data_smithi052_461816_object19305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 219.35652542, 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4803338619999999, 2023-09-23T23:02:23.413 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116106 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.414 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.415 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984724+0000", 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0258000000000001e-05 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.416 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984734+0000", 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8243000000000001e-05 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984752+0000", 2023-09-23T23:02:23.417 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0596e-05 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984763+0000", 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.238889275 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.418 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.223652+0000", 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012527140000000001 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.236179+0000", 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015698178 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.419 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.251877+0000", 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.043569531000000002 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.295447+0000", 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020602350000000001 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.420 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.297507+0000", 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020648390000000002 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.299572+0000", 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.134241317 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.421 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.433813+0000", 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068885409999999998 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.440702+0000", 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024357711000000001 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.422 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.465060+0000", 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4803338619999999 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.423 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116119 10.b 10:dbf3fcdb:::benchmark_data_smithi052_461816_object19318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 219.263462643, 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4178687299999999, 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.424 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116119 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.425 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077785+0000", 2023-09-23T23:02:23.426 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5787e-05 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077801+0000", 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3853e-05 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.427 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077825+0000", 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1316e-05 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077836+0000", 2023-09-23T23:02:23.428 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.185146891 2023-09-23T23:02:23.429 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.429 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.429 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.429 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.262983+0000", 2023-09-23T23:02:23.429 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095509050000000002 2023-09-23T23:02:23.429 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.272534+0000", 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011856833000000001 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.284391+0000", 2023-09-23T23:02:23.430 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017265624 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.301657+0000", 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002050713 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.431 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.303707+0000", 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12684677899999999 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.430554+0000", 2023-09-23T23:02:23.432 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035632981000000001 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.466187+0000", 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056081689999999997 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.433 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.471795+0000", 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023861869000000001 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.495657+0000", 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4178687299999999 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.434 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116267 10.b 10:d6c817ba:::benchmark_data_smithi052_461816_object19466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 218.21687562599999, 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4895586060000001, 2023-09-23T23:02:23.435 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116267 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:02:23.436 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124373+0000", 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0097e-05 2023-09-23T23:02:23.437 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124383+0000", 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6015000000000002e-05 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124399+0000", 2023-09-23T23:02:23.438 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4009999999999997e-06 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124408+0000", 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.33608481 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.439 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.460493+0000", 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019478128000000001 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.479971+0000", 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023908400999999999 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.440 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.503879+0000", 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.084884822999999998 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.588764+0000", 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029924790000000001 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.441 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.591756+0000", 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022099440000000001 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.593966+0000", 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00070443600000000002 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.442 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.594671+0000", 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035467929999999999 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.598218+0000", 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015716463 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.443 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.613934+0000", 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4895586060000001 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116332 10.b 10:dc130431:::benchmark_data_smithi052_461816_object19531:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.444 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 217.737049945, 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001, 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.445 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116332 2023-09-23T23:02:23.446 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.446 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.446 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.446 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.446 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:02:23.446 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.447 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604199+0000", 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1632e-05 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604211+0000", 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7592999999999999e-05 2023-09-23T23:02:23.448 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604228+0000", 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0749999999999998e-06 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.449 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604236+0000", 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.223173936 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.827410+0000", 2023-09-23T23:02:23.450 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015809969 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.843220+0000", 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013161054 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.451 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.856381+0000", 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035988506000000003 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.892370+0000", 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.027739362 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.452 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.920109+0000", 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020550299999999998 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922164+0000", 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.28725175800000002 2023-09-23T23:02:23.453 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.209416+0000", 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0092034360000000006 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.454 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.218619+0000", 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030490359000000002 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.249110+0000", 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001 2023-09-23T23:02:23.455 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116347 10.b 10:db51aae1:::benchmark_data_smithi052_461816_object19546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 217.68672006, 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001, 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.456 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116347 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.457 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.458 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.458 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.458 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:02:23.458 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.458 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.458 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654529+0000", 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5270000000000001e-06 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654537+0000", 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5132e-05 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.459 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654552+0000", 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5509999999999999e-06 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654557+0000", 2023-09-23T23:02:23.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2683612470000001 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922919+0000", 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0088446870000000004 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.931763+0000", 2023-09-23T23:02:23.461 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011420990000000001 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.943184+0000", 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040857233999999999 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.984042+0000", 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022559659999999999 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.986298+0000", 2023-09-23T23:02:23.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002071696 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.988369+0000", 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26508044400000003 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.253450+0000", 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055181509999999998 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.258968+0000", 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017719586999999998 2023-09-23T23:02:23.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.276687+0000", 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.466 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116354 10.b 10:dc4e8e26:::benchmark_data_smithi052_461816_object19553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 217.61854062699999, 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.590831154, 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.467 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116354 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722708+0000", 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1059e-05 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.469 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722720+0000", 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8337000000000001e-05 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722738+0000", 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0482999999999999e-05 2023-09-23T23:02:23.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722748+0000", 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2310586699999999 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.953807+0000", 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089079880000000004 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.962715+0000", 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011595375 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.974310+0000", 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016259483000000002 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.990570+0000", 2023-09-23T23:02:23.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002797819 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.993368+0000", 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034075324999999997 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.027443+0000", 2023-09-23T23:02:23.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.252294817 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.279738+0000", 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053108809999999999 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.285049+0000", 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.028492905999999998 2023-09-23T23:02:23.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.313542+0000", 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.590831154 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116358 10.b 10:d271a06f:::benchmark_data_smithi052_461816_object19557:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:02:23.476 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 217.583257291, 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.585204461, 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116358 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.477 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757992+0000", 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7521e-05 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758009+0000", 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3963e-05 2023-09-23T23:02:23.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758023+0000", 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0438999999999999e-05 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758034+0000", 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2363462219999999 2023-09-23T23:02:23.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.994380+0000", 2023-09-23T23:02:23.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089635630000000008 2023-09-23T23:02:23.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.003343+0000", 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014499898000000001 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.017843+0000", 2023-09-23T23:02:23.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.042956156000000002 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.060800+0000", 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068733800000000001 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.061487+0000", 2023-09-23T23:02:23.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024376810000000001 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.063925+0000", 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25300748499999998 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.316932+0000", 2023-09-23T23:02:23.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039558340000000001 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.320888+0000", 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022310436999999999 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.343198+0000", 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.585204461 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.485 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116372 10.b 10:ddfaedb1:::benchmark_data_smithi052_461816_object19571:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 217.499724837, 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.532253965, 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.486 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116372 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.488 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841525+0000", 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5699999999999999e-06 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841534+0000", 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6439999999999998e-05 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841551+0000", 2023-09-23T23:02:23.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9019999999999999e-06 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841558+0000", 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1866856050000001 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.028244+0000", 2023-09-23T23:02:23.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099314550000000005 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.038176+0000", 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011378353000000001 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.049554+0000", 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015845318000000001 2023-09-23T23:02:23.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.065399+0000", 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062175360000000001 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.071617+0000", 2023-09-23T23:02:23.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068547419999999996 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.078471+0000", 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26989495099999999 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.348366+0000", 2023-09-23T23:02:23.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045601740000000002 2023-09-23T23:02:23.494 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.494 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.352927+0000", 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020853618000000001 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.373780+0000", 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.532253965 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.495 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116379 10.b 10:d308aafa:::benchmark_data_smithi052_461816_object19578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 217.38244722100001, 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4501070380000001, 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.496 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116379 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958802+0000", 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0233e-05 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958812+0000", 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5444999999999999e-05 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958828+0000", 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6100000000000006e-06 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958836+0000", 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.12189563 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.080732+0000", 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021823031 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.500 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.102555+0000", 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018619785 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.121175+0000", 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22298790399999999 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.344162+0000", 2023-09-23T23:02:23.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035032671000000001 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.379195+0000", 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068400680000000004 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.386035+0000", 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003642756 2023-09-23T23:02:23.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.389678+0000", 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004166154 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.393844+0000", 2023-09-23T23:02:23.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015066784 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.408911+0000", 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4501070380000001 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.504 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123112 10.b 10:d1224097:::benchmark_data_smithi052_461816_object26311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 186.100686381, 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.389065591, 2023-09-23T23:02:23.505 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123112 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240563+0000", 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5110000000000003e-06 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240572+0000", 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5494e-05 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240597+0000", 2023-09-23T23:02:23.508 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.776e-06 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240604+0000", 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.150625987 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.391230+0000", 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014517555 2023-09-23T23:02:23.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.405747+0000", 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011751549 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.417499+0000", 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.041869322 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.459368+0000", 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001986991 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.461355+0000", 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.046885463000000002 2023-09-23T23:02:23.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.508241+0000", 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.076663249000000003 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.584904+0000", 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011116469 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.596020+0000", 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.033609914999999997 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.629630+0000", 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.389065591 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.513 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123247 10.b 10:df1d14f6:::benchmark_data_smithi052_461816_object26446:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 185.43143095400001, 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4117894469999999, 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.514 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123247 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909818+0000", 2023-09-23T23:02:23.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1479999999999995e-06 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909827+0000", 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4456e-05 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909842+0000", 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0669999999999997e-06 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909848+0000", 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2526637350000001 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.162512+0000", 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016769646999999999 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.518 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.179281+0000", 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015743411999999998 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.195025+0000", 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.080734345999999998 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.275759+0000", 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053276499999999997 2023-09-23T23:02:23.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.281087+0000", 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046729550000000003 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.285760+0000", 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012794950000000001 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.287039+0000", 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011935753 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.298975+0000", 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022634743999999998 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.321610+0000", 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4117894469999999 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123254 10.b 10:d637fe9a:::benchmark_data_smithi052_461816_object26453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 185.375375276, 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.394393993, 2023-09-23T23:02:23.522 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123254 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965874+0000", 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7100000000000002e-06 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965884+0000", 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4491999999999999e-05 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965898+0000", 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8060000000000001e-06 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965907+0000", 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2457147019999999 2023-09-23T23:02:23.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.211622+0000", 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014958720999999999 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.226581+0000", 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023973563 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.250554+0000", 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.072382668999999997 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.322937+0000", 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045649990000000001 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.327502+0000", 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005907359 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.333409+0000", 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006139238 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.339548+0000", 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055687870000000004 2023-09-23T23:02:23.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.345117+0000", 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015152598 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.360270+0000", 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.394393993 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.529 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123279 10.b 10:dfdd8540:::benchmark_data_smithi052_461816_object26478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 185.22664933499999, 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4014525230000001, 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.530 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123279 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114600+0000", 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0774999999999999e-05 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114610+0000", 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7652999999999999e-05 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114628+0000", 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0809999999999994e-06 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114636+0000", 2023-09-23T23:02:23.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2466315459999999 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.361268+0000", 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017441438 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.378709+0000", 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031131333000000001 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.409840+0000", 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.072885458 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.482726+0000", 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024013120000000001 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.485127+0000", 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026478410000000002 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.487775+0000", 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012406679999999999 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.489016+0000", 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.007715375 2023-09-23T23:02:23.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.496731+0000", 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019323248000000001 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.516054+0000", 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4014525230000001 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.537 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:173316 10.b 10:d6e586c9:::benchmark_data_smithi052_461816_object76515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e397)", 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:02:21.438591+0000", 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 1.902659638, 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.466487418, 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:02:23.538 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 173316 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438591+0000", 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438591+0000", 2023-09-23T23:02:23.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438590+0000", 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9659999999999999e-06 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438597+0000", 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1036000000000001e-05 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438608+0000", 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2539999999999998e-06 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438614+0000", 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014370368999999999 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.452984+0000", 2023-09-23T23:02:23.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019723929999999998 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.454957+0000", 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.72160893000000004 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.176566+0000", 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037524379999999999 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.180318+0000", 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.665614394 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.845932+0000", 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0025265259999999999 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:02:23.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.848459+0000", 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006650647 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.855110+0000", 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.036248412000000001 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.891358+0000", 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001098896 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.892457+0000", 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010248480000000001 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.893482+0000", 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001292353 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.894774+0000", 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018811380000000001 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.896655+0000", 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0084236209999999992 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.905079+0000", 2023-09-23T23:02:23.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.466487418 2023-09-23T23:02:23.547 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.547 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.547 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.547 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:02:23.547 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:02:23.547 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:02:23.547 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:02:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:02:23] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-09-23T23:02:23.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:23 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:02:23.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:23 smithi151 ceph-mon[159210]: failed to encode map e398 with expected crc 2023-09-23T23:02:23.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:23.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:23 smithi151 ceph-mon[159210]: osdmap e398: 8 total, 8 up, 6 in 2023-09-23T23:02:23.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:23 smithi151 ceph-mon[159210]: failed to encode map e398 with expected crc 2023-09-23T23:02:23.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:23 smithi151 ceph-mon[159210]: pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 317 op/s; 17084/61065 objects degraded (27.977%); 27349/61065 objects misplaced (44.787%); 536 KiB/s, 8 objects/s recovering 2023-09-23T23:02:23.877 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:02:23.877 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[187743]: failed to encode map e398 with expected crc 2023-09-23T23:02:23.877 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:23.877 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[187743]: osdmap e398: 8 total, 8 up, 6 in 2023-09-23T23:02:23.878 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[187743]: failed to encode map e398 with expected crc 2023-09-23T23:02:23.878 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[187743]: pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 317 op/s; 17084/61065 objects degraded (27.977%); 27349/61065 objects misplaced (44.787%); 536 KiB/s, 8 objects/s recovering 2023-09-23T23:02:23.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:02:23.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[180979]: failed to encode map e398 with expected crc 2023-09-23T23:02:23.879 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:23.879 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[180979]: osdmap e398: 8 total, 8 up, 6 in 2023-09-23T23:02:23.879 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[180979]: failed to encode map e398 with expected crc 2023-09-23T23:02:23.879 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:23 smithi052 ceph-mon[180979]: pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 317 op/s; 17084/61065 objects degraded (27.977%); 27349/61065 objects misplaced (44.787%); 536 KiB/s, 8 objects/s recovering 2023-09-23T23:02:24.135 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:24.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:24.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:24.144 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:02:24.198 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:24.241 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:24.302 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:24.386 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:24.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:24.491 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:24.523 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:24.599 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:24.624 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:24.626 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:24.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:24.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:24.652 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2820599292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:24.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2820599292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:24.676 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:24.699 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:24.726 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:24.734 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:24.749 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:24.772 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:24.778 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:24.817 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.823 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.824 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:02:24.852 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:24.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:24.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2820599292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:24.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:24.988 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:25.035 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:25.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:25.139 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:25.170 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:25.465 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:25.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:25.570 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:25.608 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:25.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:25 smithi052 ceph-mon[187743]: pgmap v1214: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 243 op/s; 17084/60834 objects degraded (28.083%); 27349/60834 objects misplaced (44.957%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T23:02:25.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 17084/60834 objects degraded (28.083%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:25 smithi052 ceph-mon[180979]: pgmap v1214: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 243 op/s; 17084/60834 objects degraded (28.083%); 27349/60834 objects misplaced (44.957%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T23:02:25.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 17084/60834 objects degraded (28.083%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:25.900 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:02:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:25 smithi151 ceph-mon[159210]: pgmap v1214: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 243 op/s; 17084/60834 objects degraded (28.083%); 27349/60834 objects misplaced (44.957%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T23:02:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 17084/60834 objects degraded (28.083%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:25.988 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:25.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:26.092 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:26.122 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:26.231 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:26.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:26.298 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:26.298 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:26.336 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:26.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:26.517 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:26.522 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:26.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:26.626 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:26.654 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:26.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:26.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:26.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:26.752 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:26.754 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:26.813 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:26.849 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:27.174 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:02:27.664 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:27.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:27 smithi151 ceph-mon[159210]: pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 16916/59061 objects degraded (28.642%); 27190/59061 objects misplaced (46.037%); 921 KiB/s, 14 objects/s recovering 2023-09-23T23:02:27.919 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:27 smithi151 ceph-mon[159210]: from='client.36642 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:27.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:27 smithi052 ceph-mon[187743]: pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 16916/59061 objects degraded (28.642%); 27190/59061 objects misplaced (46.037%); 921 KiB/s, 14 objects/s recovering 2023-09-23T23:02:27.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:27 smithi052 ceph-mon[187743]: from='client.36642 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:27.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:27 smithi052 ceph-mon[180979]: pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 16916/59061 objects degraded (28.642%); 27190/59061 objects misplaced (46.037%); 921 KiB/s, 14 objects/s recovering 2023-09-23T23:02:27.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:27 smithi052 ceph-mon[180979]: from='client.36642 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:27.933 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1215,"stamp":"2023-09-23T23:02:26.612075+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1289164728,"num_objects":19687,"num_object_clones":0,"num_object_copies":59061,"num_objects_missing_on_primary":2347,"num_objects_missing":2347,"num_objects_degraded":16916,"num_objects_misplaced":27190,"num_objects_unfound":0,"num_objects_dirty":19687,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":174195,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19646,"num_bytes_recovered":1194081212,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100829,"ondisk_log_size":100829,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7091908,"kb_used_data":3694932,"kb_used_omap":137,"kb_used_meta":3396790,"kb_avail":555379004,"statfs":{"total":575970213888,"available":568708100096,"internally_reserved":0,"allocated":3783610368,"data_stored":3775586732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3478313144},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-167575552,"num_objects":-2557,"num_object_clones":0,"num_object_copies":-7671,"num_objects_missing_on_primary":-159,"num_objects_missing":-159,"num_objects_degraded":-628,"num_objects_misplaced":-638,"num_objects_unfound":0,"num_objects_dirty":-2557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2557,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003305"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":409,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781031+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:02:07.781031+0000","last_peered":"2023-09-23T23:02:07.781031+0000","last_clean":"2023-09-23T23:02:07.781031+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:02:07.781031+0000","last_undegraded":"2023-09-23T23:02:07.781031+0000","last_fullsized":"2023-09-23T23:02:07.781031+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":68,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.592049+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:02:22.592049+0000","last_peered":"2023-09-23T23:02:22.592049+0000","last_clean":"2023-09-23T23:02:22.592049+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:02:22.592049+0000","last_undegraded":"2023-09-23T23:02:22.592049+0000","last_fullsized":"2023-09-23T23:02:22.592049+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:21.267564+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:02:21.267564+0000","last_peered":"2023-09-23T23:02:21.267564+0000","last_clean":"2023-09-23T23:02:21.267564+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:02:21.267564+0000","last_undegraded":"2023-09-23T23:02:21.267564+0000","last_fullsized":"2023-09-23T23:02:21.267564+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":395,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735515+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:02:16.735515+0000","last_peered":"2023-09-23T23:02:16.735515+0000","last_clean":"2023-09-23T23:02:16.735515+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:02:16.735515+0000","last_undegraded":"2023-09-23T23:02:16.735515+0000","last_fullsized":"2023-09-23T23:02:16.735515+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":83,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735340+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:02:16.735340+0000","last_peered":"2023-09-23T23:02:16.735340+0000","last_clean":"2023-09-23T23:02:16.735340+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:02:16.735340+0000","last_undegraded":"2023-09-23T23:02:16.735340+0000","last_fullsized":"2023-09-23T23:02:16.735340+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.549809+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:02:16.549809+0000","last_peered":"2023-09-23T23:02:16.549809+0000","last_clean":"2023-09-23T23:02:16.549809+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:02:16.549809+0000","last_undegraded":"2023-09-23T23:02:16.549809+0000","last_fullsized":"2023-09-23T23:02:16.549809+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":333,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.554193+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:02:16.554193+0000","last_peered":"2023-09-23T23:02:16.554193+0000","last_clean":"2023-09-23T23:02:16.554193+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:02:16.554193+0000","last_undegraded":"2023-09-23T23:02:16.554193+0000","last_fullsized":"2023-09-23T23:02:16.554193+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735282+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:02:16.735282+0000","last_peered":"2023-09-23T23:02:16.735282+0000","last_clean":"2023-09-23T23:02:16.735282+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:02:16.735282+0000","last_undegraded":"2023-09-23T23:02:16.735282+0000","last_fullsized":"2023-09-23T23:02:16.735282+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":294,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553962+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:02:16.553962+0000","last_peered":"2023-09-23T23:02:16.553962+0000","last_clean":"2023-09-23T23:02:16.553962+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:02:16.553962+0000","last_undegraded":"2023-09-23T23:02:16.553962+0000","last_fullsized":"2023-09-23T23:02:16.553962+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":82,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735700+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:02:16.735700+0000","last_peered":"2023-09-23T23:02:16.735700+0000","last_clean":"2023-09-23T23:02:16.735700+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:02:16.735700+0000","last_undegraded":"2023-09-23T23:02:16.735700+0000","last_fullsized":"2023-09-23T23:02:16.735700+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":88,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:21.267663+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:02:21.267663+0000","last_peered":"2023-09-23T23:02:21.267663+0000","last_clean":"2023-09-23T23:02:21.267663+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:02:21.267663+0000","last_undegraded":"2023-09-23T23:02:21.267663+0000","last_fullsized":"2023-09-23T23:02:21.267663+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":143,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.730985+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:02:16.730985+0000","last_peered":"2023-09-23T23:02:16.730985+0000","last_clean":"2023-09-23T23:02:16.730985+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:02:16.730985+0000","last_undegraded":"2023-09-23T23:02:16.730985+0000","last_fullsized":"2023-09-23T23:02:16.730985+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":468,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781088+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:02:07.781088+0000","last_peered":"2023-09-23T23:02:07.781088+0000","last_clean":"2023-09-23T23:02:07.781088+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:02:07.781088+0000","last_undegraded":"2023-09-23T23:02:07.781088+0000","last_fullsized":"2023-09-23T23:02:07.781088+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":371,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.780818+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:02:07.780818+0000","last_peered":"2023-09-23T23:02:07.780818+0000","last_clean":"2023-09-23T23:02:07.780818+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:02:07.780818+0000","last_undegraded":"2023-09-23T23:02:07.780818+0000","last_fullsized":"2023-09-23T23:02:07.780818+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":297,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.780918+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:02:07.780918+0000","last_peered":"2023-09-23T23:02:07.780918+0000","last_clean":"2023-09-23T23:02:07.780918+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:02:07.780918+0000","last_undegraded":"2023-09-23T23:02:07.780918+0000","last_fullsized":"2023-09-23T23:02:07.780918+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":79,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.736379+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:02:16.736379+0000","last_peered":"2023-09-23T23:02:16.736379+0000","last_clean":"2023-09-23T23:02:16.736379+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:02:16.736379+0000","last_undegraded":"2023-09-23T23:02:16.736379+0000","last_fullsized":"2023-09-23T23:02:16.736379+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":388,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.736708+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:02:16.736708+0000","last_peered":"2023-09-23T23:02:16.736708+0000","last_clean":"2023-09-23T23:02:16.736708+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:02:16.736708+0000","last_undegraded":"2023-09-23T23:02:16.736708+0000","last_fullsized":"2023-09-23T23:02:16.736708+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":298,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553428+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:02:16.553428+0000","last_peered":"2023-09-23T23:02:16.553428+0000","last_clean":"2023-09-23T23:02:16.553428+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:02:16.553428+0000","last_undegraded":"2023-09-23T23:02:16.553428+0000","last_fullsized":"2023-09-23T23:02:16.553428+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":282,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781478+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:02:07.781478+0000","last_peered":"2023-09-23T23:02:07.781478+0000","last_clean":"2023-09-23T23:02:07.781478+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:02:07.781478+0000","last_undegraded":"2023-09-23T23:02:07.781478+0000","last_fullsized":"2023-09-23T23:02:07.781478+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":412,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.591551+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:02:22.591551+0000","last_peered":"2023-09-23T23:02:22.591551+0000","last_clean":"2023-09-23T23:02:22.591551+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:02:22.591551+0000","last_undegraded":"2023-09-23T23:02:22.591551+0000","last_fullsized":"2023-09-23T23:02:22.591551+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":116,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.549463+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:02:16.549463+0000","last_peered":"2023-09-23T23:02:16.549463+0000","last_clean":"2023-09-23T23:02:16.549463+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:02:16.549463+0000","last_undegraded":"2023-09-23T23:02:16.549463+0000","last_fullsized":"2023-09-23T23:02:16.549463+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:21.267730+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:02:21.267730+0000","last_peered":"2023-09-23T23:02:21.267730+0000","last_clean":"2023-09-23T23:02:21.267730+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:02:21.267730+0000","last_undegraded":"2023-09-23T23:02:21.267730+0000","last_fullsized":"2023-09-23T23:02:21.267730+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"398'10894","reported_seq":10997,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:24.868389+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:02:24.868389+0000","last_peered":"2023-09-23T23:02:24.868389+0000","last_clean":"2023-09-23T23:02:24.868389+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:02:24.868389+0000","last_undegraded":"2023-09-23T23:02:24.868389+0000","last_fullsized":"2023-09-23T23:02:24.868389+0000","mapping_epoch":353,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5494,"log_dups_size":0,"ondisk_log_size":5494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78118912,"num_objects":1192,"num_object_clones":0,"num_object_copies":3576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10894,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":281,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781473+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:02:07.781473+0000","last_peered":"2023-09-23T23:02:07.781473+0000","last_clean":"2023-09-23T23:02:07.781473+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:02:07.781473+0000","last_undegraded":"2023-09-23T23:02:07.781473+0000","last_fullsized":"2023-09-23T23:02:07.781473+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553149+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:02:16.553149+0000","last_peered":"2023-09-23T23:02:16.553149+0000","last_clean":"2023-09-23T23:02:16.553149+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:02:16.553149+0000","last_undegraded":"2023-09-23T23:02:16.553149+0000","last_fullsized":"2023-09-23T23:02:16.553149+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"398'10749","reported_seq":13180,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:26.173238+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:02:26.173238+0000","last_peered":"2023-09-23T23:02:26.173238+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:02:26.173238+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8649,"log_dups_size":0,"ondisk_log_size":8649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78708736,"num_objects":1201,"num_object_clones":0,"num_object_copies":3603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1201,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":1201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10749,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1201},{"shards":"5,6","objects":-2366}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":291,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781216+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:02:07.781216+0000","last_peered":"2023-09-23T23:02:07.781216+0000","last_clean":"2023-09-23T23:02:07.781216+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:02:07.781216+0000","last_undegraded":"2023-09-23T23:02:07.781216+0000","last_fullsized":"2023-09-23T23:02:07.781216+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":404,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781409+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:02:07.781409+0000","last_peered":"2023-09-23T23:02:07.781409+0000","last_clean":"2023-09-23T23:02:07.781409+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:02:07.781409+0000","last_undegraded":"2023-09-23T23:02:07.781409+0000","last_fullsized":"2023-09-23T23:02:07.781409+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"398'10872","reported_seq":10969,"reported_epoch":398,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:24.822974+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:02:24.822974+0000","last_peered":"2023-09-23T23:02:24.822974+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:02:24.822974+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:02:24.822974+0000","mapping_epoch":354,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":1200,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10872,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1200}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":149,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553516+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:02:16.553516+0000","last_peered":"2023-09-23T23:02:16.553516+0000","last_clean":"2023-09-23T23:02:16.553516+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:02:16.553516+0000","last_undegraded":"2023-09-23T23:02:16.553516+0000","last_fullsized":"2023-09-23T23:02:16.553516+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":267,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553568+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:02:16.553568+0000","last_peered":"2023-09-23T23:02:16.553568+0000","last_clean":"2023-09-23T23:02:16.553568+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:02:16.553568+0000","last_undegraded":"2023-09-23T23:02:16.553568+0000","last_fullsized":"2023-09-23T23:02:16.553568+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"398'10937","reported_seq":11220,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:24.811138+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:02:24.811138+0000","last_peered":"2023-09-23T23:02:24.811138+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:02:24.811138+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5537,"log_dups_size":0,"ondisk_log_size":5537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81068032,"num_objects":1237,"num_object_clones":0,"num_object_copies":3711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1032,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":1237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10937,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":205},{"shards":"0,5","objects":-2373}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.736338+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:02:16.736338+0000","last_peered":"2023-09-23T23:02:16.736338+0000","last_clean":"2023-09-23T23:02:16.736338+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:02:16.736338+0000","last_undegraded":"2023-09-23T23:02:16.736338+0000","last_fullsized":"2023-09-23T23:02:16.736338+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":297,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.551060+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:02:16.551060+0000","last_peered":"2023-09-23T23:02:16.551060+0000","last_clean":"2023-09-23T23:02:16.551060+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:02:16.551060+0000","last_undegraded":"2023-09-23T23:02:16.551060+0000","last_fullsized":"2023-09-23T23:02:16.551060+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"397'10814","reported_seq":10916,"reported_epoch":397,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:22.175233+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:02:22.175233+0000","last_peered":"2023-09-23T23:02:22.175233+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:02:22.175233+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:02:22.175233+0000","mapping_epoch":353,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10814,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1280}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":87,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.737184+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:02:16.737184+0000","last_peered":"2023-09-23T23:02:16.737184+0000","last_clean":"2023-09-23T23:02:16.737184+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:02:16.737184+0000","last_undegraded":"2023-09-23T23:02:16.737184+0000","last_fullsized":"2023-09-23T23:02:16.737184+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":390,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.780885+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:02:07.780885+0000","last_peered":"2023-09-23T23:02:07.780885+0000","last_clean":"2023-09-23T23:02:07.780885+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:02:07.780885+0000","last_undegraded":"2023-09-23T23:02:07.780885+0000","last_fullsized":"2023-09-23T23:02:07.780885+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"397'10946","reported_seq":11056,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:22.233440+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:02:22.233440+0000","last_peered":"2023-09-23T23:02:22.233440+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:02:22.233440+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"334'5200","ondisk_log_start":"334'5200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82313216,"num_objects":1256,"num_object_clones":0,"num_object_copies":3768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1256,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":1256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10946,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-2363}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":180,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781172+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:02:07.781172+0000","last_peered":"2023-09-23T23:02:07.781172+0000","last_clean":"2023-09-23T23:02:07.781172+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:02:07.781172+0000","last_undegraded":"2023-09-23T23:02:07.781172+0000","last_fullsized":"2023-09-23T23:02:07.781172+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":407,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781387+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:02:07.781387+0000","last_peered":"2023-09-23T23:02:07.781387+0000","last_clean":"2023-09-23T23:02:07.781387+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:02:07.781387+0000","last_undegraded":"2023-09-23T23:02:07.781387+0000","last_fullsized":"2023-09-23T23:02:07.781387+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"397'10808","reported_seq":19644,"reported_epoch":397,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:22.239334+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:02:22.239334+0000","last_peered":"2023-09-23T23:02:22.239334+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:02:22.239334+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:02:22.239334+0000","mapping_epoch":354,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10808,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1252}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":420,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781187+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:02:07.781187+0000","last_peered":"2023-09-23T23:02:07.781187+0000","last_clean":"2023-09-23T23:02:07.781187+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:02:07.781187+0000","last_undegraded":"2023-09-23T23:02:07.781187+0000","last_fullsized":"2023-09-23T23:02:07.781187+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":398,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781293+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:02:07.781293+0000","last_peered":"2023-09-23T23:02:07.781293+0000","last_clean":"2023-09-23T23:02:07.781293+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:02:07.781293+0000","last_undegraded":"2023-09-23T23:02:07.781293+0000","last_fullsized":"2023-09-23T23:02:07.781293+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"397'10659","reported_seq":10761,"reported_epoch":397,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:22.253538+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:02:22.253538+0000","last_peered":"2023-09-23T23:02:22.253538+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:02:22.253538+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:02:22.253538+0000","mapping_epoch":353,"log_start":"332'4900","ondisk_log_start":"332'4900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85786624,"num_objects":1309,"num_object_clones":0,"num_object_copies":3927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10659,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":68,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735467+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:02:16.735467+0000","last_peered":"2023-09-23T23:02:16.735467+0000","last_clean":"2023-09-23T23:02:16.735467+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:02:16.735467+0000","last_undegraded":"2023-09-23T23:02:16.735467+0000","last_fullsized":"2023-09-23T23:02:16.735467+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":301,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.731258+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:02:16.731258+0000","last_peered":"2023-09-23T23:02:16.731258+0000","last_clean":"2023-09-23T23:02:16.731258+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:02:16.731258+0000","last_undegraded":"2023-09-23T23:02:16.731258+0000","last_fullsized":"2023-09-23T23:02:16.731258+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"397'10817","reported_seq":20105,"reported_epoch":397,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:22.170496+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:02:22.170496+0000","last_peered":"2023-09-23T23:02:22.170496+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:02:22.170496+0000","last_undegraded":"2023-09-23T23:02:22.170496+0000","last_fullsized":"2023-09-23T23:02:22.170496+0000","mapping_epoch":354,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1259,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10817,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1259}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":305,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.731646+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:02:16.731646+0000","last_peered":"2023-09-23T23:02:16.731646+0000","last_clean":"2023-09-23T23:02:16.731646+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:02:16.731646+0000","last_undegraded":"2023-09-23T23:02:16.731646+0000","last_fullsized":"2023-09-23T23:02:16.731646+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":308,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553339+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:02:16.553339+0000","last_peered":"2023-09-23T23:02:16.553339+0000","last_clean":"2023-09-23T23:02:16.553339+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:02:16.553339+0000","last_undegraded":"2023-09-23T23:02:16.553339+0000","last_fullsized":"2023-09-23T23:02:16.553339+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553197+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:02:16.553197+0000","last_peered":"2023-09-23T23:02:16.553197+0000","last_clean":"2023-09-23T23:02:16.553197+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:02:16.553197+0000","last_undegraded":"2023-09-23T23:02:16.553197+0000","last_fullsized":"2023-09-23T23:02:16.553197+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"398'10924","reported_seq":11040,"reported_epoch":398,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:24.864993+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:02:24.864993+0000","last_peered":"2023-09-23T23:02:24.864993+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:02:24.864993+0000","last_undegraded":"2023-09-23T23:02:24.864993+0000","last_fullsized":"2023-09-23T23:02:24.864993+0000","mapping_epoch":359,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1200,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10924,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1200}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":69,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.551537+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:02:16.551537+0000","last_peered":"2023-09-23T23:02:16.551537+0000","last_clean":"2023-09-23T23:02:16.551537+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:02:16.551537+0000","last_undegraded":"2023-09-23T23:02:16.551537+0000","last_fullsized":"2023-09-23T23:02:16.551537+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":417,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781016+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:02:07.781016+0000","last_peered":"2023-09-23T23:02:07.781016+0000","last_clean":"2023-09-23T23:02:07.781016+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:02:07.781016+0000","last_undegraded":"2023-09-23T23:02:07.781016+0000","last_fullsized":"2023-09-23T23:02:07.781016+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":290,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.731716+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:02:16.731716+0000","last_peered":"2023-09-23T23:02:16.731716+0000","last_clean":"2023-09-23T23:02:16.731716+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:02:16.731716+0000","last_undegraded":"2023-09-23T23:02:16.731716+0000","last_fullsized":"2023-09-23T23:02:16.731716+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"398'10873","reported_seq":19051,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:25.951011+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:02:25.951011+0000","last_peered":"2023-09-23T23:02:25.951011+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:02:25.951011+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"322'2300","ondisk_log_start":"322'2300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8573,"log_dups_size":0,"ondisk_log_size":8573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10873,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1191}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":365,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.730840+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:02:16.730840+0000","last_peered":"2023-09-23T23:02:16.730840+0000","last_clean":"2023-09-23T23:02:16.730840+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:02:16.730840+0000","last_undegraded":"2023-09-23T23:02:16.730840+0000","last_fullsized":"2023-09-23T23:02:16.730840+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:21.341969+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:02:21.341969+0000","last_peered":"2023-09-23T23:02:21.341969+0000","last_clean":"2023-09-23T23:02:21.341969+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:02:21.341969+0000","last_undegraded":"2023-09-23T23:02:21.341969+0000","last_fullsized":"2023-09-23T23:02:21.341969+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":308,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.554234+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:02:16.554234+0000","last_peered":"2023-09-23T23:02:16.554234+0000","last_clean":"2023-09-23T23:02:16.554234+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:02:16.554234+0000","last_undegraded":"2023-09-23T23:02:16.554234+0000","last_fullsized":"2023-09-23T23:02:16.554234+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"398'10993","reported_seq":13312,"reported_epoch":398,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:23.128474+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:02:23.128474+0000","last_peered":"2023-09-23T23:02:23.128474+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:02:23.128474+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'900","ondisk_log_start":"321'900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80674816,"num_objects":1231,"num_object_clones":0,"num_object_copies":3693,"num_objects_missing_on_primary":2345,"num_objects_missing":2345,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":1231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10993,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1097,"num_bytes_recovered":71892992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1216},{"shards":"0,1,4,6","objects":15},{"shards":"0,6","objects":-3549}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":381,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553562+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:02:16.553562+0000","last_peered":"2023-09-23T23:02:16.553562+0000","last_clean":"2023-09-23T23:02:16.553562+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:02:16.553562+0000","last_undegraded":"2023-09-23T23:02:16.553562+0000","last_fullsized":"2023-09-23T23:02:16.553562+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":495,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:16.549851+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:02:16.549851+0000","last_peered":"2023-09-23T23:02:16.549851+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:02:16.549851+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.736826+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:02:16.736826+0000","last_peered":"2023-09-23T23:02:16.736826+0000","last_clean":"2023-09-23T23:02:16.736826+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:02:16.736826+0000","last_undegraded":"2023-09-23T23:02:16.736826+0000","last_fullsized":"2023-09-23T23:02:16.736826+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781240+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:02:07.781240+0000","last_peered":"2023-09-23T23:02:07.781240+0000","last_clean":"2023-09-23T23:02:07.781240+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:02:07.781240+0000","last_undegraded":"2023-09-23T23:02:07.781240+0000","last_fullsized":"2023-09-23T23:02:07.781240+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"398'10959","reported_seq":11067,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:24.664578+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:02:24.664578+0000","last_peered":"2023-09-23T23:02:24.664578+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:02:24.664578+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728000,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10959,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2382}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":389,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.549449+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:02:16.549449+0000","last_peered":"2023-09-23T23:02:16.549449+0000","last_clean":"2023-09-23T23:02:16.549449+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:02:16.549449+0000","last_undegraded":"2023-09-23T23:02:16.549449+0000","last_fullsized":"2023-09-23T23:02:16.549449+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":360,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.549514+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:02:16.549514+0000","last_peered":"2023-09-23T23:02:16.549514+0000","last_clean":"2023-09-23T23:02:16.549514+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:02:16.549514+0000","last_undegraded":"2023-09-23T23:02:16.549514+0000","last_fullsized":"2023-09-23T23:02:16.549514+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":320,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.578554+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:02:22.578554+0000","last_peered":"2023-09-23T23:02:22.578554+0000","last_clean":"2023-09-23T23:02:22.578554+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:02:22.578554+0000","last_undegraded":"2023-09-23T23:02:22.578554+0000","last_fullsized":"2023-09-23T23:02:22.578554+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"397'10926","reported_seq":11047,"reported_epoch":397,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:22.185930+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:02:22.185930+0000","last_peered":"2023-09-23T23:02:22.185930+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:02:22.185930+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:02:22.185930+0000","mapping_epoch":354,"log_start":"334'5200","ondisk_log_start":"334'5200","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5726,"log_dups_size":0,"ondisk_log_size":5726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689496,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1278,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10926,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1278}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":335,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.550228+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:02:16.550228+0000","last_peered":"2023-09-23T23:02:16.550228+0000","last_clean":"2023-09-23T23:02:16.550228+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:02:16.550228+0000","last_undegraded":"2023-09-23T23:02:16.550228+0000","last_fullsized":"2023-09-23T23:02:16.550228+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":242,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.731785+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:02:16.731785+0000","last_peered":"2023-09-23T23:02:16.731785+0000","last_clean":"2023-09-23T23:02:16.731785+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:02:16.731785+0000","last_undegraded":"2023-09-23T23:02:16.731785+0000","last_fullsized":"2023-09-23T23:02:16.731785+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":211,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.553646+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:02:16.553646+0000","last_peered":"2023-09-23T23:02:16.553646+0000","last_clean":"2023-09-23T23:02:16.553646+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:02:16.553646+0000","last_undegraded":"2023-09-23T23:02:16.553646+0000","last_fullsized":"2023-09-23T23:02:16.553646+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"397'10941","reported_seq":21450,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:22.171073+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:02:22.171073+0000","last_peered":"2023-09-23T23:02:22.171073+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:02:22.171073+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"333'5200","ondisk_log_start":"333'5200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5741,"log_dups_size":0,"ondisk_log_size":5741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78970880,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10941,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-180},{"shards":"4,5,6","objects":1205}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":92,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.737640+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:02:16.737640+0000","last_peered":"2023-09-23T23:02:16.737640+0000","last_clean":"2023-09-23T23:02:16.737640+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:02:16.737640+0000","last_undegraded":"2023-09-23T23:02:16.737640+0000","last_fullsized":"2023-09-23T23:02:16.737640+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.783707+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:02:07.783707+0000","last_peered":"2023-09-23T23:02:07.783707+0000","last_clean":"2023-09-23T23:02:07.783707+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:02:07.783707+0000","last_undegraded":"2023-09-23T23:02:07.783707+0000","last_fullsized":"2023-09-23T23:02:07.783707+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735377+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:02:16.735377+0000","last_peered":"2023-09-23T23:02:16.735377+0000","last_clean":"2023-09-23T23:02:16.735377+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:02:16.735377+0000","last_undegraded":"2023-09-23T23:02:16.735377+0000","last_fullsized":"2023-09-23T23:02:16.735377+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"397'10823","reported_seq":11354,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:22.195344+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:02:22.195344+0000","last_peered":"2023-09-23T23:02:22.195344+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:02:22.195344+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"333'5100","ondisk_log_start":"333'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81723392,"num_objects":1247,"num_object_clones":0,"num_object_copies":3741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1247,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":1247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10823,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1247},{"shards":"4,6","objects":-2304}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":354,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.731546+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:02:16.731546+0000","last_peered":"2023-09-23T23:02:16.731546+0000","last_clean":"2023-09-23T23:02:16.731546+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:02:16.731546+0000","last_undegraded":"2023-09-23T23:02:16.731546+0000","last_fullsized":"2023-09-23T23:02:16.731546+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":302,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.578492+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:02:22.578492+0000","last_peered":"2023-09-23T23:02:22.578492+0000","last_clean":"2023-09-23T23:02:22.578492+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:02:22.578492+0000","last_undegraded":"2023-09-23T23:02:22.578492+0000","last_fullsized":"2023-09-23T23:02:22.578492+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":371,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781271+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:02:07.781271+0000","last_peered":"2023-09-23T23:02:07.781271+0000","last_clean":"2023-09-23T23:02:07.781271+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:02:07.781271+0000","last_undegraded":"2023-09-23T23:02:07.781271+0000","last_fullsized":"2023-09-23T23:02:07.781271+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":336,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.551285+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:02:16.551285+0000","last_peered":"2023-09-23T23:02:16.551285+0000","last_clean":"2023-09-23T23:02:16.551285+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:02:16.551285+0000","last_undegraded":"2023-09-23T23:02:16.551285+0000","last_fullsized":"2023-09-23T23:02:16.551285+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":146,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.731472+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:02:16.731472+0000","last_peered":"2023-09-23T23:02:16.731472+0000","last_clean":"2023-09-23T23:02:16.731472+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:02:16.731472+0000","last_undegraded":"2023-09-23T23:02:16.731472+0000","last_fullsized":"2023-09-23T23:02:16.731472+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":206,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.780765+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:02:07.780765+0000","last_peered":"2023-09-23T23:02:07.780765+0000","last_clean":"2023-09-23T23:02:07.780765+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:02:07.780765+0000","last_undegraded":"2023-09-23T23:02:07.780765+0000","last_fullsized":"2023-09-23T23:02:07.780765+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":420,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.786673+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:02:07.786673+0000","last_peered":"2023-09-23T23:02:07.786673+0000","last_clean":"2023-09-23T23:02:07.786673+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:02:07.786673+0000","last_undegraded":"2023-09-23T23:02:07.786673+0000","last_fullsized":"2023-09-23T23:02:07.786673+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":220,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781017+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:02:07.781017+0000","last_peered":"2023-09-23T23:02:07.781017+0000","last_clean":"2023-09-23T23:02:07.781017+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:02:07.781017+0000","last_undegraded":"2023-09-23T23:02:07.781017+0000","last_fullsized":"2023-09-23T23:02:07.781017+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":307,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.781163+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:02:07.781163+0000","last_peered":"2023-09-23T23:02:07.781163+0000","last_clean":"2023-09-23T23:02:07.781163+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:02:07.781163+0000","last_undegraded":"2023-09-23T23:02:07.781163+0000","last_fullsized":"2023-09-23T23:02:07.781163+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":92,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735694+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:02:16.735694+0000","last_peered":"2023-09-23T23:02:16.735694+0000","last_clean":"2023-09-23T23:02:16.735694+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:02:16.735694+0000","last_undegraded":"2023-09-23T23:02:16.735694+0000","last_fullsized":"2023-09-23T23:02:16.735694+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":95,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.735948+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:02:16.735948+0000","last_peered":"2023-09-23T23:02:16.735948+0000","last_clean":"2023-09-23T23:02:16.735948+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:02:16.735948+0000","last_undegraded":"2023-09-23T23:02:16.735948+0000","last_fullsized":"2023-09-23T23:02:16.735948+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":341,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T23:02:16.550168+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:02:16.550168+0000","last_peered":"2023-09-23T23:02:16.550168+0000","last_clean":"2023-09-23T23:02:16.550168+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:02:16.550168+0000","last_undegraded":"2023-09-23T23:02:16.550168+0000","last_fullsized":"2023-09-23T23:02:16.550168+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":382,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T23:02:07.780546+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:02:07.780546+0000","last_peered":"2023-09-23T23:02:07.780546+0000","last_clean":"2023-09-23T23:02:07.780546+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:02:07.780546+0000","last_undegraded":"2023-09-23T23:02:07.780546+0000","last_fullsized":"2023-09-23T23:02:07.780546+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1288568856,"num_objects":19663,"num_object_clones":0,"num_object_copies":58989,"num_objects_missing_on_primary":2347,"num_objects_missing":2347,"num_objects_degraded":16914,"num_objects_misplaced":27190,"num_objects_unfound":0,"num_objects_dirty":19663,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173935,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19611,"num_bytes_recovered":1194065920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5063520256,"data_stored":5063508040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100635,"ondisk_log_size":100635,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430633,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626857,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253170,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1324216,"kb_used_data":581680,"kb_used_omap":9,"kb_used_meta":742518,"kb_avail":92420936,"statfs":{"total":95995035648,"available":94639038464,"internally_reserved":0,"allocated":595640320,"data_stored":594135265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":760339433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.696,"15min":0.80000000000000004},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":20.446000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":3.2440000000000002,"5min":2.0760000000000001,"15min":1.262},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.472999999999999,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.48699999999999999,"15min":0.51400000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.087,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.316},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.64200000000000002,"15min":0.69799999999999995},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.74,"5min":4.048,"15min":4.048},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.38600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":0.498},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":0.504},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":1.488},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":0.59699999999999998}]}]},{"osd":0,"up_from":353,"seq":1516123455541,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054652,"kb_used_data":867252,"kb_used_omap":37,"kb_used_meta":187354,"kb_avail":92690500,"statfs":{"total":95995035648,"available":94915072000,"internally_reserved":0,"allocated":888066048,"data_stored":886388444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":191850658},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.71599999999999997,"15min":0.59799999999999998},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.82999999999999996,"15min":0.72399999999999998},"min":{"1min":0.27800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4459999999999997,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":1.595}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":0.86399999999999999,"15min":0.80800000000000005},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.012,"5min":0.72499999999999998,"15min":0.68500000000000005},"min":{"1min":0.246,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"last":1.6919999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.88,"15min":0.59599999999999997},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.88500000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.8170000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":1.762}]},{"osd":4,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2229999999999999,"15min":1.647},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2349999999999999,"5min":183.148,"15min":183.148},"last":1.73},{"interface":"front","average":{"1min":1.466,"5min":3.3530000000000002,"15min":1.5629999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1609999999999996,"5min":183.39500000000001,"15min":183.39500000000001},"last":1.677}]},{"osd":5,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.98499999999999999,"15min":0.78100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.984,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96199999999999997,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.1989999999999998,"5min":7.774,"15min":7.774},"last":1.794}]},{"osd":6,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.0509999999999999,"15min":0.90700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.359,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":1.651},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.91800000000000004,"15min":0.78800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3090000000000002,"5min":6.375,"15min":6.375},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.179,"15min":1.2609999999999999},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3929999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":1.629},{"interface":"front","average":{"1min":1.147,"5min":1.1890000000000001,"15min":1.2490000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.669,"5min":6.343,"15min":6.343},"last":0.624}]}]},{"osd":2,"up_from":359,"seq":1541893259314,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":703028,"kb_used_data":545324,"kb_used_omap":83,"kb_used_meta":157612,"kb_avail":93042124,"statfs":{"total":95995035648,"available":95275134976,"internally_reserved":0,"allocated":558411776,"data_stored":556802727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":161395141},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":1.335},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":1.629}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":1.5920000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":1.478}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":1.78},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":1.6100000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":1.405},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":1.6579999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":1.5229999999999999},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":1.4359999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":1.5600000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":1.373}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":1.468},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":1.6879999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926118,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":527776,"kb_used_data":17368,"kb_used_omap":7,"kb_used_meta":510392,"kb_avail":93217376,"statfs":{"total":95995035648,"available":95454593024,"internally_reserved":0,"allocated":17784832,"data_stored":16189144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":522642024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:15 2023","interfaces":[{"interface":"back","average":{"1min":3.383,"5min":2.177,"15min":0.93799999999999994},"min":{"1min":0.191,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":37.249000000000002,"5min":122.289,"15min":122.289},"last":0.22},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.919,"15min":0.91100000000000003},"min":{"1min":0.19800000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":35.991,"5min":122.041,"15min":122.041},"last":0.27900000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":0.74099999999999999,"15min":2.2999999999999998},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.1639999999999997,"5min":8.6280000000000001,"15min":123.866},"last":1.3999999999999999},{"interface":"front","average":{"1min":1.3680000000000001,"5min":0.82499999999999996,"15min":2.1659999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5640000000000001,"5min":6.8010000000000002,"15min":122.265},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.63,"15min":0.68899999999999995},"min":{"1min":0.22600000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.7,"5min":3.1469999999999998,"15min":3.1469999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.857,"5min":0.98899999999999999,"15min":0.82499999999999996},"min":{"1min":0.24299999999999999,"5min":0.223,"15min":0.223},"max":{"1min":8.6590000000000007,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":1.4319999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.3220000000000001,"15min":2.0099999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.2300000000000004,"5min":87.046999999999997,"15min":87.046999999999997},"last":1.4590000000000001},{"interface":"front","average":{"1min":1.2,"5min":2.496,"15min":1.927},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.4509999999999996,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.2769999999999999,"15min":1.4039999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5229999999999997,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.111,"5min":1.1919999999999999,"15min":1.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7000000000000002,"5min":10.246,"15min":11.951000000000001},"last":0.504}]},{"osd":6,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.554,"15min":1.7569999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5380000000000003,"5min":11.116,"15min":14.097},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.288,"15min":1.919},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.4219999999999997,"5min":9.2810000000000006,"15min":12.887},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.90700000000000003,"15min":1.7350000000000001},"min":{"1min":0.36299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6680000000000001,"5min":4.8449999999999998,"15min":11.694000000000001},"last":0.871},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.0649999999999999,"15min":1.6279999999999999},"min":{"1min":0.45300000000000001,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.2690000000000001,"5min":8.5830000000000002,"15min":12.32},"last":1.482}]}]},{"osd":4,"up_from":30,"seq":128849019314,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1980232,"kb_used_data":1072768,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91764920,"statfs":{"total":95995035648,"available":93967278080,"internally_reserved":0,"allocated":1098514432,"data_stored":1097684403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":10.303000000000001,"5min":2.7650000000000001,"15min":1.236},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.677000000000007,"5min":71.677000000000007,"15min":71.677000000000007},"last":0.245},{"interface":"front","average":{"1min":11.896000000000001,"5min":2.9740000000000002,"15min":1.3879999999999999},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":72.147999999999996,"5min":72.147999999999996,"15min":72.147999999999996},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 23:01:28 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.123,"15min":1.0549999999999999},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4870000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.85199999999999998,"15min":0.76200000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4770000000000003,"5min":4.9550000000000001,"15min":4.9550000000000001},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.375},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.501}]},{"osd":7,"last update":"Sat Sep 23 23:01:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.5700000000000001,"15min":1.169},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.0869999999999997,"5min":12.84,"15min":12.84},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.585,"15min":1.117},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4199999999999999,"5min":10.587,"15min":10.587},"last":1.2210000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823089,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1502004,"kb_used_data":610540,"kb_used_omap":0,"kb_used_meta":891456,"kb_avail":92243148,"statfs":{"total":95995035648,"available":94456983552,"internally_reserved":0,"allocated":625192960,"data_stored":624386749,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912850944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.5229999999999999,"15min":0.91100000000000003},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":42.155000000000001,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.7839999999999998,"5min":1.784,"15min":1.3160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":43.378999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.375},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.317},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1259999999999999,"15min":1.0900000000000001},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7830000000000004,"5min":9.5980000000000008,"15min":12.23},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.538,"15min":1.22},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.31},"max":{"1min":2.4580000000000002,"5min":10.638999999999999,"15min":11.112},"last":0.36099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":874516480,"data_stored":874512408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":589299712,"data_stored":589299712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":553517056,"data_stored":553517056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":13041664,"data_stored":13041664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1087377408,"data_stored":1087373336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":618266624,"data_stored":618266624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1327501312,"data_stored":1327497240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:02:27.939 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:02:28.053 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:28.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:28.157 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:28.173 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:02:28.195 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:28.309 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:28.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:28.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:28.457 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:28.562 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:28.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:28.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:28.603 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:28.667 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:28.724 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:28.824 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:28.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:28.928 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:28.960 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:29.023 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:29.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:29.090 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:29.135 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:29.170 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:29.225 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:29.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:29.329 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:29.368 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:29.418 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:29.418 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:29.427 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:29.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:29.532 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:29.573 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:29.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:29 smithi052 ceph-mon[187743]: pgmap v1216: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 249 op/s; 16770/57057 objects degraded (29.392%); 27050/57057 objects misplaced (47.409%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T23:02:29.676 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1809305702' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:02:29.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:29 smithi151 ceph-mon[159210]: pgmap v1216: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 249 op/s; 16770/57057 objects degraded (29.392%); 27050/57057 objects misplaced (47.409%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T23:02:29.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1809305702' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:02:29.678 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:02:29.931 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:29 smithi052 ceph-mon[180979]: pgmap v1216: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.2 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 249 op/s; 16770/57057 objects degraded (29.392%); 27050/57057 objects misplaced (47.409%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T23:02:29.931 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1809305702' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:02:29.956 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:29.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:30.061 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:30.088 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:30.279 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:02:30.513 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:30.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:30.621 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:30.663 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:30.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:30 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:30.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:30 smithi151 ceph-mon[159210]: failed to encode map e399 with expected crc 2023-09-23T23:02:30.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:30 smithi151 ceph-mon[159210]: failed to encode map e399 with expected crc 2023-09-23T23:02:30.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1809305702' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:02:30.680 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:30 smithi151 ceph-mon[159210]: osdmap e399: 8 total, 8 up, 6 in 2023-09-23T23:02:30.681 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 16770/57057 objects degraded (29.392%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:30.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[187743]: failed to encode map e399 with expected crc 2023-09-23T23:02:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[187743]: failed to encode map e399 with expected crc 2023-09-23T23:02:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1809305702' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:02:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[187743]: osdmap e399: 8 total, 8 up, 6 in 2023-09-23T23:02:30.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 16770/57057 objects degraded (29.392%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:30.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[180979]: failed to encode map e399 with expected crc 2023-09-23T23:02:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[180979]: failed to encode map e399 with expected crc 2023-09-23T23:02:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1809305702' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:02:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[180979]: osdmap e399: 8 total, 8 up, 6 in 2023-09-23T23:02:30.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 16770/57057 objects degraded (29.392%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:30.764 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:02:30.829 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:30.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:30.932 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:30.962 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:31.018 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:31.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:31.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:31.121 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:31.157 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:31.219 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:31.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:31.325 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:31.356 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:31.412 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:31.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:31.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:31.515 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:31.548 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:31.625 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:31.626 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16711/55983 objects degraded (29.850%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1684,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":399,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":18661,"data_bytes":1221924792,"bytes_used":7116980224,"bytes_avail":568853233664,"bytes_total":575970213888,"degraded_objects":16711,"degraded_total":55983,"degraded_ratio":0.2985013307611239,"misplaced_objects":26917,"misplaced_total":55983,"misplaced_ratio":0.4808066734544415,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1303512,"recovering_keys_per_sec":0,"num_objects_recovered":160,"num_bytes_recovered":10485760,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":210},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (6m)\n [======================......] (remaining: 84s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:02:31.678 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:31 smithi151 ceph-mon[159210]: pgmap v1218: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 16711/55983 objects degraded (29.850%); 26917/55983 objects misplaced (48.081%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:02:31.678 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1601647777' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:31.702 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:31.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:31.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:31.836 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:31.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:31 smithi052 ceph-mon[187743]: pgmap v1218: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 16711/55983 objects degraded (29.850%); 26917/55983 objects misplaced (48.081%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:02:31.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1601647777' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:31.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:31 smithi052 ceph-mon[180979]: pgmap v1218: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 16711/55983 objects degraded (29.850%); 26917/55983 objects misplaced (48.081%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:02:31.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1601647777' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:32.036 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:32.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:32.146 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:32.184 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:32.306 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:32.318 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:32.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:32.340 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:32.370 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:32.392 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:32.412 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:32.428 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:32.433 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:32.457 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:32.463 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:32.477 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:32.479 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.479 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.479 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.479 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.479 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.479 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.480 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.480 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:02:32.523 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:32.524 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:02:32.524 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:02:32.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:32.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:32.531 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:02:32.531 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:02:32.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:02:32.531 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [] 2023-09-23T23:02:32.532 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:02:32.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:02:32.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:32.660 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:32.738 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:32.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:32.742 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1028415347' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:32.742 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1028415347' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:32.843 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:32.881 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1028415347' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:32.890 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:32.971 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:32.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:33.079 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:33.170 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:33.308 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:33.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:33.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:33.398 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:33.412 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:33.412 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:33.461 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:02:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:02:33] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-09-23T23:02:33.466 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:33.493 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:02:33.596 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:33.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:33.704 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:33.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:33 smithi052 ceph-mon[187743]: pgmap v1219: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 16469/53457 objects degraded (30.808%); 26764/53457 objects misplaced (50.066%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T23:02:33.723 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:33 smithi052 ceph-mon[180979]: pgmap v1219: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 16469/53457 objects degraded (30.808%); 26764/53457 objects misplaced (50.066%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T23:02:33.743 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:33.897 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:33.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:33.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:33 smithi151 ceph-mon[159210]: pgmap v1219: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 16469/53457 objects degraded (30.808%); 26764/53457 objects misplaced (50.066%); 1024 KiB/s, 15 objects/s recovering 2023-09-23T23:02:34.001 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:34.036 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:34.096 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:34.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:34.205 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:34.237 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:35.057 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:35.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:35.161 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:35.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:02:35.187 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:35.189 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:35.242 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:35.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:35.328 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:35.348 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:35.451 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:35.572 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:35.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:35.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:35.681 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:35.754 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:35.961 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:35.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:35.966 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:35 smithi052 ceph-mon[180979]: pgmap v1220: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 16469/53214 objects degraded (30.949%); 26764/53214 objects misplaced (50.295%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:02:35.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 16469/53214 objects degraded (30.949%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:35.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:35 smithi052 ceph-mon[187743]: pgmap v1220: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 16469/53214 objects degraded (30.949%); 26764/53214 objects misplaced (50.295%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:02:35.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 16469/53214 objects degraded (30.949%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:36.067 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:36.092 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:35 smithi151 ceph-mon[159210]: pgmap v1220: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.1 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 16469/53214 objects degraded (30.949%); 26764/53214 objects misplaced (50.295%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:02:36.092 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 16469/53214 objects degraded (30.949%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:36.094 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:36.140 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:36.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:36.183 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:36.190 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:36.246 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:36.277 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:36.299 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:36.330 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:36.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:36.436 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:36.474 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:36.717 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:02:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2622082070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:36.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2622082070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:36.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:37.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2622082070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:37.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:37.210 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:37.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:37.317 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:37.346 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:37.430 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:37.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:37.536 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:37.575 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:37.639 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:37.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:37.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:37.746 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:37.773 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:37.777 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1220,"stamp":"2023-09-23T23:02:34.614932+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1161435064,"num_objects":17738,"num_object_clones":0,"num_object_copies":53214,"num_objects_missing_on_primary":2189,"num_objects_missing":2189,"num_objects_degraded":16469,"num_objects_misplaced":26764,"num_objects_unfound":0,"num_objects_dirty":17738,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":176144,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19810,"num_bytes_recovered":1204829116,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100678,"ondisk_log_size":100678,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6783260,"kb_used_data":3410860,"kb_used_omap":137,"kb_used_meta":3372214,"kb_avail":555687652,"statfs":{"total":575970213888,"available":569024155648,"internally_reserved":0,"allocated":3492720640,"data_stored":3484665478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3453147320},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-166461440,"num_objects":-2540,"num_object_clones":0,"num_object_copies":-7620,"num_objects_missing_on_primary":-158,"num_objects_missing":-158,"num_objects_degraded":-615,"num_objects_misplaced":-585,"num_objects_unfound":0,"num_objects_dirty":-2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2540,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":10747904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003560"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":411,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579425+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:02:22.579425+0000","last_peered":"2023-09-23T23:02:22.579425+0000","last_clean":"2023-09-23T23:02:22.579425+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:02:22.579425+0000","last_undegraded":"2023-09-23T23:02:22.579425+0000","last_fullsized":"2023-09-23T23:02:22.579425+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:26.268429+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:02:26.268429+0000","last_peered":"2023-09-23T23:02:26.268429+0000","last_clean":"2023-09-23T23:02:26.268429+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:02:26.268429+0000","last_undegraded":"2023-09-23T23:02:26.268429+0000","last_fullsized":"2023-09-23T23:02:26.268429+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593902+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:02:22.593902+0000","last_peered":"2023-09-23T23:02:22.593902+0000","last_clean":"2023-09-23T23:02:22.593902+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:02:22.593902+0000","last_undegraded":"2023-09-23T23:02:22.593902+0000","last_fullsized":"2023-09-23T23:02:22.593902+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":396,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.591418+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:02:22.591418+0000","last_peered":"2023-09-23T23:02:22.591418+0000","last_clean":"2023-09-23T23:02:22.591418+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:02:22.591418+0000","last_undegraded":"2023-09-23T23:02:22.591418+0000","last_fullsized":"2023-09-23T23:02:22.591418+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":84,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593274+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:02:22.593274+0000","last_peered":"2023-09-23T23:02:22.593274+0000","last_clean":"2023-09-23T23:02:22.593274+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:02:22.593274+0000","last_undegraded":"2023-09-23T23:02:22.593274+0000","last_fullsized":"2023-09-23T23:02:22.593274+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.578837+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:02:22.578837+0000","last_peered":"2023-09-23T23:02:22.578837+0000","last_clean":"2023-09-23T23:02:22.578837+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:02:22.578837+0000","last_undegraded":"2023-09-23T23:02:22.578837+0000","last_fullsized":"2023-09-23T23:02:22.578837+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":334,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.586274+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:02:22.586274+0000","last_peered":"2023-09-23T23:02:22.586274+0000","last_clean":"2023-09-23T23:02:22.586274+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:02:22.586274+0000","last_undegraded":"2023-09-23T23:02:22.586274+0000","last_fullsized":"2023-09-23T23:02:22.586274+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593133+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:02:22.593133+0000","last_peered":"2023-09-23T23:02:22.593133+0000","last_clean":"2023-09-23T23:02:22.593133+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:02:22.593133+0000","last_undegraded":"2023-09-23T23:02:22.593133+0000","last_fullsized":"2023-09-23T23:02:22.593133+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":295,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.582919+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:02:22.582919+0000","last_peered":"2023-09-23T23:02:22.582919+0000","last_clean":"2023-09-23T23:02:22.582919+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:02:22.582919+0000","last_undegraded":"2023-09-23T23:02:22.582919+0000","last_fullsized":"2023-09-23T23:02:22.582919+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":83,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593747+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:02:22.593747+0000","last_peered":"2023-09-23T23:02:22.593747+0000","last_clean":"2023-09-23T23:02:22.593747+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:02:22.593747+0000","last_undegraded":"2023-09-23T23:02:22.593747+0000","last_fullsized":"2023-09-23T23:02:22.593747+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":89,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.592009+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:02:22.592009+0000","last_peered":"2023-09-23T23:02:22.592009+0000","last_clean":"2023-09-23T23:02:22.592009+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:02:22.592009+0000","last_undegraded":"2023-09-23T23:02:22.592009+0000","last_fullsized":"2023-09-23T23:02:22.592009+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.584298+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:02:22.584298+0000","last_peered":"2023-09-23T23:02:22.584298+0000","last_clean":"2023-09-23T23:02:22.584298+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:02:22.584298+0000","last_undegraded":"2023-09-23T23:02:22.584298+0000","last_fullsized":"2023-09-23T23:02:22.584298+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":470,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579649+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:02:22.579649+0000","last_peered":"2023-09-23T23:02:22.579649+0000","last_clean":"2023-09-23T23:02:22.579649+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:02:22.579649+0000","last_undegraded":"2023-09-23T23:02:22.579649+0000","last_fullsized":"2023-09-23T23:02:22.579649+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":373,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593171+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:02:22.593171+0000","last_peered":"2023-09-23T23:02:22.593171+0000","last_clean":"2023-09-23T23:02:22.593171+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:02:22.593171+0000","last_undegraded":"2023-09-23T23:02:22.593171+0000","last_fullsized":"2023-09-23T23:02:22.593171+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":299,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593287+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:02:22.593287+0000","last_peered":"2023-09-23T23:02:22.593287+0000","last_clean":"2023-09-23T23:02:22.593287+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:02:22.593287+0000","last_undegraded":"2023-09-23T23:02:22.593287+0000","last_fullsized":"2023-09-23T23:02:22.593287+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":80,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.591530+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:02:22.591530+0000","last_peered":"2023-09-23T23:02:22.591530+0000","last_clean":"2023-09-23T23:02:22.591530+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:02:22.591530+0000","last_undegraded":"2023-09-23T23:02:22.591530+0000","last_fullsized":"2023-09-23T23:02:22.591530+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":389,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.594133+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:02:22.594133+0000","last_peered":"2023-09-23T23:02:22.594133+0000","last_clean":"2023-09-23T23:02:22.594133+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:02:22.594133+0000","last_undegraded":"2023-09-23T23:02:22.594133+0000","last_fullsized":"2023-09-23T23:02:22.594133+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":299,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583377+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:02:22.583377+0000","last_peered":"2023-09-23T23:02:22.583377+0000","last_clean":"2023-09-23T23:02:22.583377+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:02:22.583377+0000","last_undegraded":"2023-09-23T23:02:22.583377+0000","last_fullsized":"2023-09-23T23:02:22.583377+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":284,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579232+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:02:22.579232+0000","last_peered":"2023-09-23T23:02:22.579232+0000","last_clean":"2023-09-23T23:02:22.579232+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:02:22.579232+0000","last_undegraded":"2023-09-23T23:02:22.579232+0000","last_fullsized":"2023-09-23T23:02:22.579232+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":413,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:26.268476+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:02:26.268476+0000","last_peered":"2023-09-23T23:02:26.268476+0000","last_clean":"2023-09-23T23:02:26.268476+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:02:26.268476+0000","last_undegraded":"2023-09-23T23:02:26.268476+0000","last_fullsized":"2023-09-23T23:02:26.268476+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":117,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.577397+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:02:22.577397+0000","last_peered":"2023-09-23T23:02:22.577397+0000","last_clean":"2023-09-23T23:02:22.577397+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:02:22.577397+0000","last_undegraded":"2023-09-23T23:02:22.577397+0000","last_fullsized":"2023-09-23T23:02:22.577397+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":93,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.592189+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:02:22.592189+0000","last_peered":"2023-09-23T23:02:22.592189+0000","last_clean":"2023-09-23T23:02:22.592189+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:02:22.592189+0000","last_undegraded":"2023-09-23T23:02:22.592189+0000","last_fullsized":"2023-09-23T23:02:22.592189+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"399'10968","reported_seq":11072,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.975981+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:02:29.975981+0000","last_peered":"2023-09-23T23:02:29.975981+0000","last_clean":"2023-09-23T23:02:29.975981+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:02:29.975981+0000","last_undegraded":"2023-09-23T23:02:29.975981+0000","last_fullsized":"2023-09-23T23:02:29.975981+0000","mapping_epoch":353,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73269248,"num_objects":1118,"num_object_clones":0,"num_object_copies":3354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10968,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":283,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.580087+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:02:22.580087+0000","last_peered":"2023-09-23T23:02:22.580087+0000","last_clean":"2023-09-23T23:02:22.580087+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:02:22.580087+0000","last_undegraded":"2023-09-23T23:02:22.580087+0000","last_fullsized":"2023-09-23T23:02:22.580087+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.582463+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:02:22.582463+0000","last_peered":"2023-09-23T23:02:22.582463+0000","last_clean":"2023-09-23T23:02:22.582463+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:02:22.582463+0000","last_undegraded":"2023-09-23T23:02:22.582463+0000","last_fullsized":"2023-09-23T23:02:22.582463+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"399'10833","reported_seq":13265,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:31.119428+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:02:31.119428+0000","last_peered":"2023-09-23T23:02:31.119428+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:02:31.119428+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8633,"log_dups_size":0,"ondisk_log_size":8633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1117,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10833,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1117},{"shards":"5,6","objects":-2450}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":293,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.581383+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:02:22.581383+0000","last_peered":"2023-09-23T23:02:22.581383+0000","last_clean":"2023-09-23T23:02:22.581383+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:02:22.581383+0000","last_undegraded":"2023-09-23T23:02:22.581383+0000","last_fullsized":"2023-09-23T23:02:22.581383+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":406,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579180+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:02:22.579180+0000","last_peered":"2023-09-23T23:02:22.579180+0000","last_clean":"2023-09-23T23:02:22.579180+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:02:22.579180+0000","last_undegraded":"2023-09-23T23:02:22.579180+0000","last_fullsized":"2023-09-23T23:02:22.579180+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"399'10945","reported_seq":11043,"reported_epoch":399,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:29.924313+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:02:29.924313+0000","last_peered":"2023-09-23T23:02:29.924313+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:02:29.924313+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:02:29.924313+0000","mapping_epoch":354,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73859072,"num_objects":1127,"num_object_clones":0,"num_object_copies":3381,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":1127,"num_objects_unfound":0,"num_objects_dirty":1127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10945,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1127}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":150,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583467+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:02:22.583467+0000","last_peered":"2023-09-23T23:02:22.583467+0000","last_clean":"2023-09-23T23:02:22.583467+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:02:22.583467+0000","last_undegraded":"2023-09-23T23:02:22.583467+0000","last_fullsized":"2023-09-23T23:02:22.583467+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":268,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583288+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:02:22.583288+0000","last_peered":"2023-09-23T23:02:22.583288+0000","last_clean":"2023-09-23T23:02:22.583288+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:02:22.583288+0000","last_undegraded":"2023-09-23T23:02:22.583288+0000","last_fullsized":"2023-09-23T23:02:22.583288+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"399'11012","reported_seq":11296,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:29.969694+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:02:29.969694+0000","last_peered":"2023-09-23T23:02:29.969694+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:02:29.969694+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76152832,"num_objects":1162,"num_object_clones":0,"num_object_copies":3486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":973,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":1162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11012,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":189},{"shards":"0,5","objects":-2448}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.591455+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:02:22.591455+0000","last_peered":"2023-09-23T23:02:22.591455+0000","last_clean":"2023-09-23T23:02:22.591455+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:02:22.591455+0000","last_undegraded":"2023-09-23T23:02:22.591455+0000","last_fullsized":"2023-09-23T23:02:22.591455+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":298,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.578166+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:02:22.578166+0000","last_peered":"2023-09-23T23:02:22.578166+0000","last_clean":"2023-09-23T23:02:22.578166+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:02:22.578166+0000","last_undegraded":"2023-09-23T23:02:22.578166+0000","last_fullsized":"2023-09-23T23:02:22.578166+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"399'10988","reported_seq":11092,"reported_epoch":399,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:32.141839+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:02:32.141839+0000","last_peered":"2023-09-23T23:02:32.141839+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:02:32.141839+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:02:32.141839+0000","mapping_epoch":353,"log_start":"334'5300","ondisk_log_start":"334'5300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72482816,"num_objects":1106,"num_object_clones":0,"num_object_copies":3318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10988,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1106}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":88,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.591813+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:02:22.591813+0000","last_peered":"2023-09-23T23:02:22.591813+0000","last_clean":"2023-09-23T23:02:22.591813+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:02:22.591813+0000","last_undegraded":"2023-09-23T23:02:22.591813+0000","last_fullsized":"2023-09-23T23:02:22.591813+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":392,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579552+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:02:22.579552+0000","last_peered":"2023-09-23T23:02:22.579552+0000","last_clean":"2023-09-23T23:02:22.579552+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:02:22.579552+0000","last_undegraded":"2023-09-23T23:02:22.579552+0000","last_fullsized":"2023-09-23T23:02:22.579552+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"399'11085","reported_seq":11197,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:32.254395+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:02:32.254395+0000","last_peered":"2023-09-23T23:02:32.254395+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:02:32.254395+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1117,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11085,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-2502}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":182,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579603+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:02:22.579603+0000","last_peered":"2023-09-23T23:02:22.579603+0000","last_clean":"2023-09-23T23:02:22.579603+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:02:22.579603+0000","last_undegraded":"2023-09-23T23:02:22.579603+0000","last_fullsized":"2023-09-23T23:02:22.579603+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":409,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579232+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:02:22.579232+0000","last_peered":"2023-09-23T23:02:22.579232+0000","last_clean":"2023-09-23T23:02:22.579232+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:02:22.579232+0000","last_undegraded":"2023-09-23T23:02:22.579232+0000","last_fullsized":"2023-09-23T23:02:22.579232+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"399'10958","reported_seq":19796,"reported_epoch":399,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:32.216815+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:02:32.216815+0000","last_peered":"2023-09-23T23:02:32.216815+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:02:32.216815+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:02:32.216815+0000","mapping_epoch":354,"log_start":"334'5200","ondisk_log_start":"334'5200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72220672,"num_objects":1102,"num_object_clones":0,"num_object_copies":3306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10958,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1102}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":422,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.578970+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:02:22.578970+0000","last_peered":"2023-09-23T23:02:22.578970+0000","last_clean":"2023-09-23T23:02:22.578970+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:02:22.578970+0000","last_undegraded":"2023-09-23T23:02:22.578970+0000","last_fullsized":"2023-09-23T23:02:22.578970+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":400,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.581452+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:02:22.581452+0000","last_peered":"2023-09-23T23:02:22.581452+0000","last_clean":"2023-09-23T23:02:22.581452+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:02:22.581452+0000","last_undegraded":"2023-09-23T23:02:22.581452+0000","last_fullsized":"2023-09-23T23:02:22.581452+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"399'10824","reported_seq":10928,"reported_epoch":399,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:32.221046+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:02:32.221046+0000","last_peered":"2023-09-23T23:02:32.221046+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:02:32.221046+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:02:32.221046+0000","mapping_epoch":353,"log_start":"334'5100","ondisk_log_start":"334'5100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74973184,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10824,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1144}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":69,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593502+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:02:22.593502+0000","last_peered":"2023-09-23T23:02:22.593502+0000","last_clean":"2023-09-23T23:02:22.593502+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:02:22.593502+0000","last_undegraded":"2023-09-23T23:02:22.593502+0000","last_fullsized":"2023-09-23T23:02:22.593502+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":302,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.584823+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:02:22.584823+0000","last_peered":"2023-09-23T23:02:22.584823+0000","last_clean":"2023-09-23T23:02:22.584823+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:02:22.584823+0000","last_undegraded":"2023-09-23T23:02:22.584823+0000","last_fullsized":"2023-09-23T23:02:22.584823+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"399'10957","reported_seq":20247,"reported_epoch":399,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:32.209365+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:02:32.209365+0000","last_peered":"2023-09-23T23:02:32.209365+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:02:32.209365+0000","last_undegraded":"2023-09-23T23:02:32.209365+0000","last_fullsized":"2023-09-23T23:02:32.209365+0000","mapping_epoch":354,"log_start":"334'5200","ondisk_log_start":"334'5200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73334784,"num_objects":1119,"num_object_clones":0,"num_object_copies":3357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1119,"num_objects_unfound":0,"num_objects_dirty":1119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10957,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1119}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":306,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.584630+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:02:22.584630+0000","last_peered":"2023-09-23T23:02:22.584630+0000","last_clean":"2023-09-23T23:02:22.584630+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:02:22.584630+0000","last_undegraded":"2023-09-23T23:02:22.584630+0000","last_fullsized":"2023-09-23T23:02:22.584630+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":309,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583298+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:02:22.583298+0000","last_peered":"2023-09-23T23:02:22.583298+0000","last_clean":"2023-09-23T23:02:22.583298+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:02:22.583298+0000","last_undegraded":"2023-09-23T23:02:22.583298+0000","last_fullsized":"2023-09-23T23:02:22.583298+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583129+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:02:22.583129+0000","last_peered":"2023-09-23T23:02:22.583129+0000","last_clean":"2023-09-23T23:02:22.583129+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:02:22.583129+0000","last_undegraded":"2023-09-23T23:02:22.583129+0000","last_fullsized":"2023-09-23T23:02:22.583129+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"399'10984","reported_seq":11101,"reported_epoch":399,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:29.918296+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:02:29.918296+0000","last_peered":"2023-09-23T23:02:29.918296+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:02:29.918296+0000","last_undegraded":"2023-09-23T23:02:29.918296+0000","last_fullsized":"2023-09-23T23:02:29.918296+0000","mapping_epoch":359,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74711040,"num_objects":1140,"num_object_clones":0,"num_object_copies":3420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1140,"num_objects_unfound":0,"num_objects_dirty":1140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10984,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1140}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":70,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.577660+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:02:22.577660+0000","last_peered":"2023-09-23T23:02:22.577660+0000","last_clean":"2023-09-23T23:02:22.577660+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:02:22.577660+0000","last_undegraded":"2023-09-23T23:02:22.577660+0000","last_fullsized":"2023-09-23T23:02:22.577660+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":419,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593375+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:02:22.593375+0000","last_peered":"2023-09-23T23:02:22.593375+0000","last_clean":"2023-09-23T23:02:22.593375+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:02:22.593375+0000","last_undegraded":"2023-09-23T23:02:22.593375+0000","last_fullsized":"2023-09-23T23:02:22.593375+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":292,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803023+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:02:29.803023+0000","last_peered":"2023-09-23T23:02:29.803023+0000","last_clean":"2023-09-23T23:02:29.803023+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:02:29.803023+0000","last_undegraded":"2023-09-23T23:02:29.803023+0000","last_fullsized":"2023-09-23T23:02:29.803023+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"399'10976","reported_seq":19155,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:31.205448+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:02:31.205448+0000","last_peered":"2023-09-23T23:02:31.205448+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:02:31.205448+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8576,"log_dups_size":0,"ondisk_log_size":8576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71303168,"num_objects":1088,"num_object_clones":0,"num_object_copies":3264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10976,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1088}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":367,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.802808+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:02:29.802808+0000","last_peered":"2023-09-23T23:02:29.802808+0000","last_clean":"2023-09-23T23:02:29.802808+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:02:29.802808+0000","last_undegraded":"2023-09-23T23:02:29.802808+0000","last_fullsized":"2023-09-23T23:02:29.802808+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":244,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583992+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:02:22.583992+0000","last_peered":"2023-09-23T23:02:22.583992+0000","last_clean":"2023-09-23T23:02:22.583992+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:02:22.583992+0000","last_undegraded":"2023-09-23T23:02:22.583992+0000","last_fullsized":"2023-09-23T23:02:22.583992+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":309,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.586356+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:02:22.586356+0000","last_peered":"2023-09-23T23:02:22.586356+0000","last_clean":"2023-09-23T23:02:22.586356+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:02:22.586356+0000","last_undegraded":"2023-09-23T23:02:22.586356+0000","last_fullsized":"2023-09-23T23:02:22.586356+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"399'11156","reported_seq":13798,"reported_epoch":399,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:33.162949+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:02:33.162949+0000","last_peered":"2023-09-23T23:02:33.162949+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:02:33.162949+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'1100","ondisk_log_start":"321'1100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69992448,"num_objects":1068,"num_object_clones":0,"num_object_copies":3204,"num_objects_missing_on_primary":2187,"num_objects_missing":2187,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":1068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11156,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1261,"num_bytes_recovered":82640896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":1058},{"shards":"0,1,4,6","objects":9},{"shards":"0,6","objects":-3548}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":382,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583060+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:02:22.583060+0000","last_peered":"2023-09-23T23:02:22.583060+0000","last_clean":"2023-09-23T23:02:22.583060+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:02:22.583060+0000","last_undegraded":"2023-09-23T23:02:22.583060+0000","last_fullsized":"2023-09-23T23:02:22.583060+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":496,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:22.577808+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:02:22.577808+0000","last_peered":"2023-09-23T23:02:22.577808+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:02:22.577808+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.594318+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:02:22.594318+0000","last_peered":"2023-09-23T23:02:22.594318+0000","last_clean":"2023-09-23T23:02:22.594318+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:02:22.594318+0000","last_undegraded":"2023-09-23T23:02:22.594318+0000","last_fullsized":"2023-09-23T23:02:22.594318+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":309,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579454+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:02:22.579454+0000","last_peered":"2023-09-23T23:02:22.579454+0000","last_clean":"2023-09-23T23:02:22.579454+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:02:22.579454+0000","last_undegraded":"2023-09-23T23:02:22.579454+0000","last_fullsized":"2023-09-23T23:02:22.579454+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"399'11035","reported_seq":11144,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:29.957538+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:02:29.957538+0000","last_peered":"2023-09-23T23:02:29.957538+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:02:29.957538+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5535,"log_dups_size":0,"ondisk_log_size":5535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68747264,"num_objects":1049,"num_object_clones":0,"num_object_copies":3147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11035,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2458}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":390,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.577412+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:02:22.577412+0000","last_peered":"2023-09-23T23:02:22.577412+0000","last_clean":"2023-09-23T23:02:22.577412+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:02:22.577412+0000","last_undegraded":"2023-09-23T23:02:22.577412+0000","last_fullsized":"2023-09-23T23:02:22.577412+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":361,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.577498+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:02:22.577498+0000","last_peered":"2023-09-23T23:02:22.577498+0000","last_clean":"2023-09-23T23:02:22.577498+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:02:22.577498+0000","last_undegraded":"2023-09-23T23:02:22.577498+0000","last_fullsized":"2023-09-23T23:02:22.577498+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":321,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:24.976561+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:02:24.976561+0000","last_peered":"2023-09-23T23:02:24.976561+0000","last_clean":"2023-09-23T23:02:24.976561+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:02:24.976561+0000","last_undegraded":"2023-09-23T23:02:24.976561+0000","last_fullsized":"2023-09-23T23:02:24.976561+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"399'11079","reported_seq":11202,"reported_epoch":399,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:32.181031+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:02:32.181031+0000","last_peered":"2023-09-23T23:02:32.181031+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:02:32.181031+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:02:32.181031+0000","mapping_epoch":354,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5679,"log_dups_size":0,"ondisk_log_size":5679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73662488,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1125,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11079,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1125}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":336,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579319+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:02:22.579319+0000","last_peered":"2023-09-23T23:02:22.579319+0000","last_clean":"2023-09-23T23:02:22.579319+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:02:22.579319+0000","last_undegraded":"2023-09-23T23:02:22.579319+0000","last_fullsized":"2023-09-23T23:02:22.579319+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":244,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803072+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:02:29.803072+0000","last_peered":"2023-09-23T23:02:29.803072+0000","last_clean":"2023-09-23T23:02:29.803072+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:02:29.803072+0000","last_undegraded":"2023-09-23T23:02:29.803072+0000","last_fullsized":"2023-09-23T23:02:29.803072+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":212,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583140+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:02:22.583140+0000","last_peered":"2023-09-23T23:02:22.583140+0000","last_clean":"2023-09-23T23:02:22.583140+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:02:22.583140+0000","last_undegraded":"2023-09-23T23:02:22.583140+0000","last_fullsized":"2023-09-23T23:02:22.583140+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"399'11096","reported_seq":21607,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:32.227527+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:02:32.227527+0000","last_peered":"2023-09-23T23:02:32.227527+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:02:32.227527+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11096,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-335},{"shards":"4,5,6","objects":1050}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":93,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.592313+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:02:22.592313+0000","last_peered":"2023-09-23T23:02:22.592313+0000","last_clean":"2023-09-23T23:02:22.592313+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:02:22.592313+0000","last_undegraded":"2023-09-23T23:02:22.592313+0000","last_fullsized":"2023-09-23T23:02:22.592313+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.583977+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:02:22.583977+0000","last_peered":"2023-09-23T23:02:22.583977+0000","last_clean":"2023-09-23T23:02:22.583977+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:02:22.583977+0000","last_undegraded":"2023-09-23T23:02:22.583977+0000","last_fullsized":"2023-09-23T23:02:22.583977+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593361+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:02:22.593361+0000","last_peered":"2023-09-23T23:02:22.593361+0000","last_clean":"2023-09-23T23:02:22.593361+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:02:22.593361+0000","last_undegraded":"2023-09-23T23:02:22.593361+0000","last_fullsized":"2023-09-23T23:02:22.593361+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"399'10988","reported_seq":11521,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:32.214295+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:02:32.214295+0000","last_peered":"2023-09-23T23:02:32.214295+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:02:32.214295+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"334'5300","ondisk_log_start":"334'5300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70909952,"num_objects":1082,"num_object_clones":0,"num_object_copies":3246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1082,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":1082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10988,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1082},{"shards":"4,6","objects":-2469}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":355,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.584951+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:02:22.584951+0000","last_peered":"2023-09-23T23:02:22.584951+0000","last_clean":"2023-09-23T23:02:22.584951+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:02:22.584951+0000","last_undegraded":"2023-09-23T23:02:22.584951+0000","last_fullsized":"2023-09-23T23:02:22.584951+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":303,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:24.976551+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:02:24.976551+0000","last_peered":"2023-09-23T23:02:24.976551+0000","last_clean":"2023-09-23T23:02:24.976551+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:02:24.976551+0000","last_undegraded":"2023-09-23T23:02:24.976551+0000","last_fullsized":"2023-09-23T23:02:24.976551+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":373,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579054+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:02:22.579054+0000","last_peered":"2023-09-23T23:02:22.579054+0000","last_clean":"2023-09-23T23:02:22.579054+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:02:22.579054+0000","last_undegraded":"2023-09-23T23:02:22.579054+0000","last_fullsized":"2023-09-23T23:02:22.579054+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":337,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.578274+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:02:22.578274+0000","last_peered":"2023-09-23T23:02:22.578274+0000","last_clean":"2023-09-23T23:02:22.578274+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:02:22.578274+0000","last_undegraded":"2023-09-23T23:02:22.578274+0000","last_fullsized":"2023-09-23T23:02:22.578274+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.584414+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:02:22.584414+0000","last_peered":"2023-09-23T23:02:22.584414+0000","last_clean":"2023-09-23T23:02:22.584414+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:02:22.584414+0000","last_undegraded":"2023-09-23T23:02:22.584414+0000","last_fullsized":"2023-09-23T23:02:22.584414+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":208,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579479+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:02:22.579479+0000","last_peered":"2023-09-23T23:02:22.579479+0000","last_clean":"2023-09-23T23:02:22.579479+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:02:22.579479+0000","last_undegraded":"2023-09-23T23:02:22.579479+0000","last_fullsized":"2023-09-23T23:02:22.579479+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":422,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.593453+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:02:22.593453+0000","last_peered":"2023-09-23T23:02:22.593453+0000","last_clean":"2023-09-23T23:02:22.593453+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:02:22.593453+0000","last_undegraded":"2023-09-23T23:02:22.593453+0000","last_fullsized":"2023-09-23T23:02:22.593453+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":222,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579317+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:02:22.579317+0000","last_peered":"2023-09-23T23:02:22.579317+0000","last_clean":"2023-09-23T23:02:22.579317+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:02:22.579317+0000","last_undegraded":"2023-09-23T23:02:22.579317+0000","last_fullsized":"2023-09-23T23:02:22.579317+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":309,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579507+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:02:22.579507+0000","last_peered":"2023-09-23T23:02:22.579507+0000","last_clean":"2023-09-23T23:02:22.579507+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:02:22.579507+0000","last_undegraded":"2023-09-23T23:02:22.579507+0000","last_fullsized":"2023-09-23T23:02:22.579507+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":93,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.591695+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:02:22.591695+0000","last_peered":"2023-09-23T23:02:22.591695+0000","last_clean":"2023-09-23T23:02:22.591695+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:02:22.591695+0000","last_undegraded":"2023-09-23T23:02:22.591695+0000","last_fullsized":"2023-09-23T23:02:22.591695+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":96,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.594034+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:02:22.594034+0000","last_peered":"2023-09-23T23:02:22.594034+0000","last_clean":"2023-09-23T23:02:22.594034+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:02:22.594034+0000","last_undegraded":"2023-09-23T23:02:22.594034+0000","last_fullsized":"2023-09-23T23:02:22.594034+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":342,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579246+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:02:22.579246+0000","last_peered":"2023-09-23T23:02:22.579246+0000","last_clean":"2023-09-23T23:02:22.579246+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:02:22.579246+0000","last_undegraded":"2023-09-23T23:02:22.579246+0000","last_fullsized":"2023-09-23T23:02:22.579246+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":384,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T23:02:22.579069+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:02:22.579069+0000","last_peered":"2023-09-23T23:02:22.579069+0000","last_clean":"2023-09-23T23:02:22.579069+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:02:22.579069+0000","last_undegraded":"2023-09-23T23:02:22.579069+0000","last_fullsized":"2023-09-23T23:02:22.579069+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1160839192,"num_objects":17714,"num_object_clones":0,"num_object_copies":53142,"num_objects_missing_on_primary":2189,"num_objects_missing":2189,"num_objects_degraded":16467,"num_objects_misplaced":26764,"num_objects_unfound":0,"num_objects_dirty":17714,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175884,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19775,"num_bytes_recovered":1204813824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4728369152,"data_stored":4728356936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100484,"ondisk_log_size":100484,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430634,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626859,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253172,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251732,"kb_used_data":521484,"kb_used_omap":9,"kb_used_meta":730230,"kb_avail":92493420,"statfs":{"total":95995035648,"available":94713262080,"internally_reserved":0,"allocated":533999616,"data_stored":532482885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":747756521},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.696,"15min":0.80000000000000004},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":20.446000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.24299999999999999},{"interface":"front","average":{"1min":3.2440000000000002,"5min":2.0760000000000001,"15min":1.262},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.472999999999999,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.57499999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.55300000000000005,"15min":0.53600000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5359999999999996,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.67000000000000004,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.1310000000000002,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":0.52500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":0.441},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":67.738},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":67.670000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":68.259},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":70.057000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":67.349000000000004},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":67.366}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":0.47899999999999998}]}]},{"osd":0,"up_from":353,"seq":1516123455542,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1007932,"kb_used_data":816436,"kb_used_omap":37,"kb_used_meta":191450,"kb_avail":92737220,"statfs":{"total":95995035648,"available":94962913280,"internally_reserved":0,"allocated":836030464,"data_stored":834352860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":196044962},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.71599999999999997,"15min":0.59799999999999998},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.82999999999999996,"15min":0.72399999999999998},"min":{"1min":0.27800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4459999999999997,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.83399999999999996,"15min":0.79800000000000004},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.694,"5min":12.215,"15min":12.215},"last":0.437},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":5.9199999999999999},"last":2.464}]},{"osd":3,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.88,"15min":0.59599999999999997},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.88500000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.8170000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":2.5630000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2229999999999999,"15min":1.647},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2349999999999999,"5min":183.148,"15min":183.148},"last":2.524},{"interface":"front","average":{"1min":1.466,"5min":3.3530000000000002,"15min":1.5629999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1609999999999996,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.98499999999999999,"15min":0.78100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.984,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96199999999999997,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.1989999999999998,"5min":7.774,"15min":7.774},"last":2.5979999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.0509999999999999,"15min":0.90700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.359,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.91800000000000004,"15min":0.78800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3090000000000002,"5min":6.375,"15min":6.375},"last":0.66400000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.179,"15min":1.2609999999999999},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3929999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.502},{"interface":"front","average":{"1min":1.147,"5min":1.1890000000000001,"15min":1.2490000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.669,"5min":6.343,"15min":6.343},"last":0.68899999999999995}]}]},{"osd":2,"up_from":359,"seq":1541893259315,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":680128,"kb_used_data":538808,"kb_used_omap":83,"kb_used_meta":141228,"kb_avail":93065024,"statfs":{"total":95995035648,"available":95298584576,"internally_reserved":0,"allocated":551739392,"data_stored":550126373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":144617925},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.47799999999999998,"15min":0.39600000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.032,"5min":7.1520000000000001,"15min":7.1520000000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.66700000000000004,"15min":0.69399999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6379999999999999,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":0.55600000000000005}]},{"osd":1,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.738,"5min":3.738,"15min":3.738},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":1.022,"15min":1.1100000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.6779999999999999,"5min":6.3819999999999997,"15min":6.3819999999999997},"last":0.92600000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71599999999999997,"15min":0.71399999999999997},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.7349999999999999,"5min":5.7270000000000003,"15min":5.7270000000000003},"last":0.439},{"interface":"front","average":{"1min":0.752,"5min":0.84999999999999998,"15min":0.82799999999999996},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.4900000000000002,"5min":9.2140000000000004,"15min":9.2140000000000004},"last":0.96199999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.77800000000000002,"15min":0.63600000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.48,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.88900000000000001,"15min":0.84499999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.988,"5min":3.988,"15min":3.988},"last":0.66200000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96499999999999997,"15min":0.95799999999999996},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.5979999999999999,"5min":5.6029999999999998,"15min":5.6029999999999998},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.008,"5min":0.82999999999999996,"15min":0.72999999999999998},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3879999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.1060000000000001,"15min":1.0149999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":8.9990000000000006},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.872,"15min":0.69499999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":5.6680000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.153,"15min":1.0449999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":4.1580000000000004,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.091,"15min":1.149},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.6680000000000001,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.69899999999999995}]}]},{"osd":3,"up_from":94,"seq":403726926119,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":530860,"kb_used_data":16356,"kb_used_omap":7,"kb_used_meta":514488,"kb_avail":93214292,"statfs":{"total":95995035648,"available":95451435008,"internally_reserved":0,"allocated":16748544,"data_stored":15148886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":526836328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:15 2023","interfaces":[{"interface":"back","average":{"1min":3.383,"5min":2.177,"15min":0.93799999999999994},"min":{"1min":0.191,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":37.249000000000002,"5min":122.289,"15min":122.289},"last":0.79800000000000004},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.919,"15min":0.91100000000000003},"min":{"1min":0.19800000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":35.991,"5min":122.041,"15min":122.041},"last":0.433}]},{"osd":1,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":0.74099999999999999,"15min":2.2999999999999998},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.1639999999999997,"5min":8.6280000000000001,"15min":123.866},"last":2.871},{"interface":"front","average":{"1min":1.3680000000000001,"5min":0.82499999999999996,"15min":2.1659999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5640000000000001,"5min":6.8010000000000002,"15min":122.265},"last":1.2729999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":0.71499999999999997},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":2.8540000000000001},{"interface":"front","average":{"1min":1.28,"5min":1.097,"15min":0.86099999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.7919999999999998,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.3220000000000001,"15min":2.0099999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.2300000000000004,"5min":87.046999999999997,"15min":87.046999999999997},"last":2.8279999999999998},{"interface":"front","average":{"1min":1.2,"5min":2.496,"15min":1.927},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.4509999999999996,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.2769999999999999,"15min":1.4039999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5229999999999997,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.111,"5min":1.1919999999999999,"15min":1.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7000000000000002,"5min":10.246,"15min":11.951000000000001},"last":2.8919999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.554,"15min":1.7569999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5380000000000003,"5min":11.116,"15min":14.097},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.288,"15min":1.919},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.4219999999999997,"5min":9.2810000000000006,"15min":12.887},"last":2.802}]},{"osd":7,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.90700000000000003,"15min":1.7350000000000001},"min":{"1min":0.36299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6680000000000001,"5min":4.8449999999999998,"15min":11.694000000000001},"last":2.9260000000000002},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.0649999999999999,"15min":1.6279999999999999},"min":{"1min":0.45300000000000001,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.2690000000000001,"5min":8.5830000000000002,"15min":12.32},"last":2.9060000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019316,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849120,"kb_used_data":949848,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91896032,"statfs":{"total":95995035648,"available":94101536768,"internally_reserved":0,"allocated":972644352,"data_stored":971806383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":10.303000000000001,"5min":2.7650000000000001,"15min":1.236},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.677000000000007,"5min":71.677000000000007,"15min":71.677000000000007},"last":1.504},{"interface":"front","average":{"1min":11.896000000000001,"5min":2.9740000000000002,"15min":1.3879999999999999},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":72.147999999999996,"5min":72.147999999999996,"15min":72.147999999999996},"last":4.806}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":1.8009999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":1.321}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.117,"15min":1.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8380000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":1.367},{"interface":"front","average":{"1min":1.653,"5min":1.0389999999999999,"15min":0.82499999999999996},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":1.778}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":4.6189999999999998},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":1.8180000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:01:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.1760000000000002,"15min":1.7},"min":{"1min":0.34699999999999998,"5min":0.214,"15min":0.214},"max":{"1min":8.3439999999999994,"5min":148.61600000000001,"15min":148.61600000000001},"last":1.694},{"interface":"front","average":{"1min":0.72199999999999998,"5min":3.4180000000000001,"15min":1.78},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.248},"max":{"1min":3.1869999999999998,"5min":151.905,"15min":151.905},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":1.9019999999999999},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":1.7230000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.429,"15min":1.2549999999999999},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1050000000000004,"5min":12.84,"15min":12.84},"last":1.7490000000000001},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2450000000000001,"15min":1.1240000000000001},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2210000000000001,"5min":10.587,"15min":10.587},"last":0.98899999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823090,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1463488,"kb_used_data":567928,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92281664,"statfs":{"total":95995035648,"available":94496423936,"internally_reserved":0,"allocated":581558272,"data_stored":580748091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.5229999999999999,"15min":0.91100000000000003},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":42.155000000000001,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.7839999999999998,"5min":1.784,"15min":1.3160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":43.378999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.375},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:01:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":0.98199999999999998,"15min":0.86099999999999999},"min":{"1min":0.32300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.149,"5min":6.601,"15min":6.601},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.98099999999999998,"15min":0.98999999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.317},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1259999999999999,"15min":1.0900000000000001},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7830000000000004,"5min":9.5980000000000008,"15min":12.23},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.538,"15min":1.22},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.31},"max":{"1min":2.4580000000000002,"5min":10.638999999999999,"15min":11.112},"last":0.36099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":820580352,"data_stored":820576280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":527761408,"data_stored":527761408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":546766848,"data_stored":546766848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12058624,"data_stored":12058624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":962727936,"data_stored":962723864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":574423040,"data_stored":574423040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1284050944,"data_stored":1284046872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:02:37.817 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:37.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:37.927 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:37 smithi151 ceph-mon[159210]: from='client.56560 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:37 smithi151 ceph-mon[159210]: pgmap v1221: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 16294/51333 objects degraded (31.742%); 26586/51333 objects misplaced (51.791%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:02:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:37 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:37 smithi151 ceph-mon[159210]: failed to encode map e400 with expected crc 2023-09-23T23:02:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:37.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:37 smithi151 ceph-mon[159210]: osdmap e400: 8 total, 8 up, 6 in 2023-09-23T23:02:37.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[187743]: from='client.56560 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:37.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[187743]: pgmap v1221: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 16294/51333 objects degraded (31.742%); 26586/51333 objects misplaced (51.791%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:02:37.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:37.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[187743]: failed to encode map e400 with expected crc 2023-09-23T23:02:37.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:37.968 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[187743]: osdmap e400: 8 total, 8 up, 6 in 2023-09-23T23:02:37.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[180979]: from='client.56560 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:37.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[180979]: pgmap v1221: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 16294/51333 objects degraded (31.742%); 26586/51333 objects misplaced (51.791%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T23:02:37.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:37.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[180979]: failed to encode map e400 with expected crc 2023-09-23T23:02:37.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:37.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:37 smithi052 ceph-mon[180979]: osdmap e400: 8 total, 8 up, 6 in 2023-09-23T23:02:37.975 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:38.038 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:38.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:38.143 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:38.170 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:38.225 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:38.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:38.331 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:38.358 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:38.710 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:38.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:38.817 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:38.857 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:38.918 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:38.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:38.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:39.023 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:39.068 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:39.148 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:39.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:39.252 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:39.298 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:39.388 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:02:39.388 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:02:39.388 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T23:02:39.388 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:02:39.404 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:39.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:39.508 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:39.541 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:39.639 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:39.646 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:39.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:39.667 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:39.692 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:39.718 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:39.741 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:39.751 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:39.766 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:39.789 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:39.790 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:39.820 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.822 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:02:39.856 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:39.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:39.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:39.963 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:39 smithi052 ceph-mon[180979]: pgmap v1223: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1018 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 304 op/s; 16109/48897 objects degraded (32.945%); 26393/48897 objects misplaced (53.977%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:02:39.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2347915215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:39.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:39 smithi052 ceph-mon[187743]: pgmap v1223: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1018 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 304 op/s; 16109/48897 objects degraded (32.945%); 26393/48897 objects misplaced (53.977%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:02:39.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2347915215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:40.010 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:40.061 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:40.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:40.165 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:40.197 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:40.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:39 smithi151 ceph-mon[159210]: pgmap v1223: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 1018 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 304 op/s; 16109/48897 objects degraded (32.945%); 26393/48897 objects misplaced (53.977%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T23:02:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2347915215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:40.708 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 16109/48897 objects degraded (32.945%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:40.788 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:02:40.849 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:02:40.884 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:40.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:40.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:40.956 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:02:40.990 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:40.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 16109/48897 objects degraded (32.945%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:41.017 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 16109/48897 objects degraded (32.945%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:41.019 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:41.082 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:41.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:41.188 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:41.235 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:41.336 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:41.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:41.415 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:41.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:41.504 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:41.517 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:41.711 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:41 smithi052 ceph-mon[187743]: pgmap v1224: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 992 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 16046/47640 objects degraded (33.682%); 26224/47640 objects misplaced (55.046%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:02:41.711 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:41 smithi052 ceph-mon[180979]: pgmap v1224: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 992 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 16046/47640 objects degraded (33.682%); 26224/47640 objects misplaced (55.046%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:02:41.789 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:41.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:41.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:41.935 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:41.982 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:42.054 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:42.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:42.160 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:42.186 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:41 smithi151 ceph-mon[159210]: pgmap v1224: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 992 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 16046/47640 objects degraded (33.682%); 26224/47640 objects misplaced (55.046%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:02:42.189 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:42.234 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:42.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:42.340 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:42.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:42.902 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:02:42.934 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:42.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:43.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:43.067 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:43.123 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:43.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:43.228 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:43.256 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:43.309 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:43.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:43.416 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:43.446 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:43.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:02:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:02:43] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-09-23T23:02:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:43.609 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:43.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:43.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:43.746 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:43.837 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:43 smithi052 ceph-mon[180979]: pgmap v1225: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 934 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 285 op/s; 15750/44889 objects degraded (35.087%); 26055/44889 objects misplaced (58.043%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:02:43.837 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:43 smithi052 ceph-mon[187743]: pgmap v1225: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 934 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 285 op/s; 15750/44889 objects degraded (35.087%); 26055/44889 objects misplaced (58.043%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:02:43.853 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:43.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:43.959 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:43.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:43 smithi151 ceph-mon[159210]: pgmap v1225: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 934 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 285 op/s; 15750/44889 objects degraded (35.087%); 26055/44889 objects misplaced (58.043%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T23:02:44.262 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:44.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:44.366 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:44.392 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:44.437 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:44.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:44.544 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:44.570 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:44.622 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:44.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:44.730 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:44.765 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:44.772 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:44.894 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:44.900 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:45.012 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:45.013 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:45 smithi052 ceph-mon[187743]: pgmap v1226: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 928 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 15750/44601 objects degraded (35.313%); 26055/44601 objects misplaced (58.418%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T23:02:45.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:45 smithi052 ceph-mon[187743]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:45.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 15750/44601 objects degraded (35.313%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:45.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:45 smithi052 ceph-mon[180979]: pgmap v1226: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 928 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 15750/44601 objects degraded (35.313%); 26055/44601 objects misplaced (58.418%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T23:02:45.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:45 smithi052 ceph-mon[180979]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:45.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 15750/44601 objects degraded (35.313%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:45 smithi151 ceph-mon[159210]: pgmap v1226: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 928 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 15750/44601 objects degraded (35.313%); 26055/44601 objects misplaced (58.418%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T23:02:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:45 smithi151 ceph-mon[159210]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:02:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 15750/44601 objects degraded (35.313%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:46.204 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1226,"stamp":"2023-09-23T23:02:44.621618+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":973281208,"num_objects":14867,"num_object_clones":0,"num_object_copies":44601,"num_objects_missing_on_primary":2005,"num_objects_missing":2005,"num_objects_degraded":15750,"num_objects_misplaced":26055,"num_objects_unfound":0,"num_objects_dirty":14867,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":179015,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19994,"num_bytes_recovered":1216887740,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100649,"ondisk_log_size":100649,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6367228,"kb_used_data":2978444,"kb_used_omap":137,"kb_used_meta":3388598,"kb_avail":556103684,"statfs":{"total":575970213888,"available":569450172416,"internally_reserved":0,"allocated":3049926656,"data_stored":3041832496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3469924536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-188153856,"num_objects":-2871,"num_object_clones":0,"num_object_copies":-8613,"num_objects_missing_on_primary":-184,"num_objects_missing":-184,"num_objects_degraded":-719,"num_objects_misplaced":-709,"num_objects_unfound":0,"num_objects_dirty":-2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2871,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":184,"num_bytes_recovered":12058624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006686"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":412,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.876792+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:02:29.876792+0000","last_peered":"2023-09-23T23:02:29.876792+0000","last_clean":"2023-09-23T23:02:29.876792+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:02:29.876792+0000","last_undegraded":"2023-09-23T23:02:29.876792+0000","last_fullsized":"2023-09-23T23:02:29.876792+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":70,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804429+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:02:29.804429+0000","last_peered":"2023-09-23T23:02:29.804429+0000","last_clean":"2023-09-23T23:02:29.804429+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:02:29.804429+0000","last_undegraded":"2023-09-23T23:02:29.804429+0000","last_fullsized":"2023-09-23T23:02:29.804429+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803735+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:02:29.803735+0000","last_peered":"2023-09-23T23:02:29.803735+0000","last_clean":"2023-09-23T23:02:29.803735+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:02:29.803735+0000","last_undegraded":"2023-09-23T23:02:29.803735+0000","last_fullsized":"2023-09-23T23:02:29.803735+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":397,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804300+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:02:29.804300+0000","last_peered":"2023-09-23T23:02:29.804300+0000","last_clean":"2023-09-23T23:02:29.804300+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:02:29.804300+0000","last_undegraded":"2023-09-23T23:02:29.804300+0000","last_fullsized":"2023-09-23T23:02:29.804300+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":85,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803254+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:02:29.803254+0000","last_peered":"2023-09-23T23:02:29.803254+0000","last_clean":"2023-09-23T23:02:29.803254+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:02:29.803254+0000","last_undegraded":"2023-09-23T23:02:29.803254+0000","last_fullsized":"2023-09-23T23:02:29.803254+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":71,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.672924+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:02:29.672924+0000","last_peered":"2023-09-23T23:02:29.672924+0000","last_clean":"2023-09-23T23:02:29.672924+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:02:29.672924+0000","last_undegraded":"2023-09-23T23:02:29.672924+0000","last_fullsized":"2023-09-23T23:02:29.672924+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":335,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.870831+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:02:29.870831+0000","last_peered":"2023-09-23T23:02:29.870831+0000","last_clean":"2023-09-23T23:02:29.870831+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:02:29.870831+0000","last_undegraded":"2023-09-23T23:02:29.870831+0000","last_fullsized":"2023-09-23T23:02:29.870831+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":79,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.802793+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:02:29.802793+0000","last_peered":"2023-09-23T23:02:29.802793+0000","last_clean":"2023-09-23T23:02:29.802793+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:02:29.802793+0000","last_undegraded":"2023-09-23T23:02:29.802793+0000","last_fullsized":"2023-09-23T23:02:29.802793+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":296,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.869631+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:02:29.869631+0000","last_peered":"2023-09-23T23:02:29.869631+0000","last_clean":"2023-09-23T23:02:29.869631+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:02:29.869631+0000","last_undegraded":"2023-09-23T23:02:29.869631+0000","last_fullsized":"2023-09-23T23:02:29.869631+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803513+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:02:29.803513+0000","last_peered":"2023-09-23T23:02:29.803513+0000","last_clean":"2023-09-23T23:02:29.803513+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:02:29.803513+0000","last_undegraded":"2023-09-23T23:02:29.803513+0000","last_fullsized":"2023-09-23T23:02:29.803513+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":90,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804561+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:02:29.804561+0000","last_peered":"2023-09-23T23:02:29.804561+0000","last_clean":"2023-09-23T23:02:29.804561+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:02:29.804561+0000","last_undegraded":"2023-09-23T23:02:29.804561+0000","last_fullsized":"2023-09-23T23:02:29.804561+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803017+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:02:29.803017+0000","last_peered":"2023-09-23T23:02:29.803017+0000","last_clean":"2023-09-23T23:02:29.803017+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:02:29.803017+0000","last_undegraded":"2023-09-23T23:02:29.803017+0000","last_fullsized":"2023-09-23T23:02:29.803017+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":471,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873867+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:02:29.873867+0000","last_peered":"2023-09-23T23:02:29.873867+0000","last_clean":"2023-09-23T23:02:29.873867+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:02:29.873867+0000","last_undegraded":"2023-09-23T23:02:29.873867+0000","last_fullsized":"2023-09-23T23:02:29.873867+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":374,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.884246+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:02:29.884246+0000","last_peered":"2023-09-23T23:02:29.884246+0000","last_clean":"2023-09-23T23:02:29.884246+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:02:29.884246+0000","last_undegraded":"2023-09-23T23:02:29.884246+0000","last_fullsized":"2023-09-23T23:02:29.884246+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":300,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.884321+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:02:29.884321+0000","last_peered":"2023-09-23T23:02:29.884321+0000","last_clean":"2023-09-23T23:02:29.884321+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:02:29.884321+0000","last_undegraded":"2023-09-23T23:02:29.884321+0000","last_fullsized":"2023-09-23T23:02:29.884321+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":81,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804188+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:02:29.804188+0000","last_peered":"2023-09-23T23:02:29.804188+0000","last_clean":"2023-09-23T23:02:29.804188+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:02:29.804188+0000","last_undegraded":"2023-09-23T23:02:29.804188+0000","last_fullsized":"2023-09-23T23:02:29.804188+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":390,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803461+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:02:29.803461+0000","last_peered":"2023-09-23T23:02:29.803461+0000","last_clean":"2023-09-23T23:02:29.803461+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:02:29.803461+0000","last_undegraded":"2023-09-23T23:02:29.803461+0000","last_fullsized":"2023-09-23T23:02:29.803461+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":300,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.675182+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:02:29.675182+0000","last_peered":"2023-09-23T23:02:29.675182+0000","last_clean":"2023-09-23T23:02:29.675182+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:02:29.675182+0000","last_undegraded":"2023-09-23T23:02:29.675182+0000","last_fullsized":"2023-09-23T23:02:29.675182+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":285,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873410+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:02:29.873410+0000","last_peered":"2023-09-23T23:02:29.873410+0000","last_clean":"2023-09-23T23:02:29.873410+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:02:29.873410+0000","last_undegraded":"2023-09-23T23:02:29.873410+0000","last_fullsized":"2023-09-23T23:02:29.873410+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":414,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804402+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:02:29.804402+0000","last_peered":"2023-09-23T23:02:29.804402+0000","last_clean":"2023-09-23T23:02:29.804402+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:02:29.804402+0000","last_undegraded":"2023-09-23T23:02:29.804402+0000","last_fullsized":"2023-09-23T23:02:29.804402+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":118,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.671584+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:02:29.671584+0000","last_peered":"2023-09-23T23:02:29.671584+0000","last_clean":"2023-09-23T23:02:29.671584+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:02:29.671584+0000","last_undegraded":"2023-09-23T23:02:29.671584+0000","last_fullsized":"2023-09-23T23:02:29.671584+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":94,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804484+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:02:29.804484+0000","last_peered":"2023-09-23T23:02:29.804484+0000","last_clean":"2023-09-23T23:02:29.804484+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:02:29.804484+0000","last_undegraded":"2023-09-23T23:02:29.804484+0000","last_fullsized":"2023-09-23T23:02:29.804484+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"400'11152","reported_seq":11257,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:39.935871+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:02:39.935871+0000","last_peered":"2023-09-23T23:02:39.935871+0000","last_clean":"2023-09-23T23:02:39.935871+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:02:39.935871+0000","last_undegraded":"2023-09-23T23:02:39.935871+0000","last_fullsized":"2023-09-23T23:02:39.935871+0000","mapping_epoch":353,"log_start":"335'5600","ondisk_log_start":"335'5600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11152,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":284,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.874411+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:02:29.874411+0000","last_peered":"2023-09-23T23:02:29.874411+0000","last_clean":"2023-09-23T23:02:29.874411+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:02:29.874411+0000","last_undegraded":"2023-09-23T23:02:29.874411+0000","last_fullsized":"2023-09-23T23:02:29.874411+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":270,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.674822+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:02:29.674822+0000","last_peered":"2023-09-23T23:02:29.674822+0000","last_clean":"2023-09-23T23:02:29.674822+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:02:29.674822+0000","last_undegraded":"2023-09-23T23:02:29.674822+0000","last_fullsized":"2023-09-23T23:02:29.674822+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"400'11010","reported_seq":13443,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:41.148107+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:02:41.148107+0000","last_peered":"2023-09-23T23:02:41.148107+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:02:41.148107+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"323'2400","ondisk_log_start":"323'2400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8610,"log_dups_size":0,"ondisk_log_size":8610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61603840,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":940,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11010,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":940},{"shards":"5,6","objects":-2627}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":294,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.874684+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:02:29.874684+0000","last_peered":"2023-09-23T23:02:29.874684+0000","last_clean":"2023-09-23T23:02:29.874684+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:02:29.874684+0000","last_undegraded":"2023-09-23T23:02:29.874684+0000","last_fullsized":"2023-09-23T23:02:29.874684+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":407,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873385+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:02:29.873385+0000","last_peered":"2023-09-23T23:02:29.873385+0000","last_clean":"2023-09-23T23:02:29.873385+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:02:29.873385+0000","last_undegraded":"2023-09-23T23:02:29.873385+0000","last_fullsized":"2023-09-23T23:02:29.873385+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"400'11109","reported_seq":11208,"reported_epoch":400,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:39.928270+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:02:39.928270+0000","last_peered":"2023-09-23T23:02:39.928270+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:02:39.928270+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:02:39.928270+0000","mapping_epoch":354,"log_start":"335'5600","ondisk_log_start":"335'5600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5509,"log_dups_size":0,"ondisk_log_size":5509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63111168,"num_objects":963,"num_object_clones":0,"num_object_copies":2889,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":963,"num_objects_unfound":0,"num_objects_dirty":963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11109,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":963}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":151,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.675260+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:02:29.675260+0000","last_peered":"2023-09-23T23:02:29.675260+0000","last_clean":"2023-09-23T23:02:29.675260+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:02:29.675260+0000","last_undegraded":"2023-09-23T23:02:29.675260+0000","last_fullsized":"2023-09-23T23:02:29.675260+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":269,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.675088+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:02:29.675088+0000","last_peered":"2023-09-23T23:02:29.675088+0000","last_clean":"2023-09-23T23:02:29.675088+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:02:29.675088+0000","last_undegraded":"2023-09-23T23:02:29.675088+0000","last_fullsized":"2023-09-23T23:02:29.675088+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"400'11184","reported_seq":11469,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:39.904607+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:02:39.904607+0000","last_peered":"2023-09-23T23:02:39.904607+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:02:39.904607+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":833,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11184,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":157},{"shards":"0,5","objects":-2620}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804170+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:02:29.804170+0000","last_peered":"2023-09-23T23:02:29.804170+0000","last_clean":"2023-09-23T23:02:29.804170+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:02:29.804170+0000","last_undegraded":"2023-09-23T23:02:29.804170+0000","last_fullsized":"2023-09-23T23:02:29.804170+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":300,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:34.980267+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:02:34.980267+0000","last_peered":"2023-09-23T23:02:34.980267+0000","last_clean":"2023-09-23T23:02:34.980267+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:02:34.980267+0000","last_undegraded":"2023-09-23T23:02:34.980267+0000","last_fullsized":"2023-09-23T23:02:34.980267+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"400'11162","reported_seq":11267,"reported_epoch":400,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:42.024683+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:02:42.024683+0000","last_peered":"2023-09-23T23:02:42.024683+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:02:42.024683+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:02:42.024683+0000","mapping_epoch":353,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61079552,"num_objects":932,"num_object_clones":0,"num_object_copies":2796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11162,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":932}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803697+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:02:29.803697+0000","last_peered":"2023-09-23T23:02:29.803697+0000","last_clean":"2023-09-23T23:02:29.803697+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:02:29.803697+0000","last_undegraded":"2023-09-23T23:02:29.803697+0000","last_fullsized":"2023-09-23T23:02:29.803697+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":393,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873849+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:02:29.873849+0000","last_peered":"2023-09-23T23:02:29.873849+0000","last_clean":"2023-09-23T23:02:29.873849+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:02:29.873849+0000","last_undegraded":"2023-09-23T23:02:29.873849+0000","last_fullsized":"2023-09-23T23:02:29.873849+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"400'11280","reported_seq":11393,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:42.091751+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:02:42.091751+0000","last_peered":"2023-09-23T23:02:42.091751+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:02:42.091751+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"335'5600","ondisk_log_start":"335'5600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":922,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11280,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-2697}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":183,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.876932+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:02:29.876932+0000","last_peered":"2023-09-23T23:02:29.876932+0000","last_clean":"2023-09-23T23:02:29.876932+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:02:29.876932+0000","last_undegraded":"2023-09-23T23:02:29.876932+0000","last_fullsized":"2023-09-23T23:02:29.876932+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":410,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.874034+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:02:29.874034+0000","last_peered":"2023-09-23T23:02:29.874034+0000","last_clean":"2023-09-23T23:02:29.874034+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:02:29.874034+0000","last_undegraded":"2023-09-23T23:02:29.874034+0000","last_fullsized":"2023-09-23T23:02:29.874034+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"400'11138","reported_seq":19977,"reported_epoch":400,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:41.809098+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:02:41.809098+0000","last_peered":"2023-09-23T23:02:41.809098+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:02:41.809098+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:02:41.809098+0000","mapping_epoch":354,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11138,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":922}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":423,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873197+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:02:29.873197+0000","last_peered":"2023-09-23T23:02:29.873197+0000","last_clean":"2023-09-23T23:02:29.873197+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:02:29.873197+0000","last_undegraded":"2023-09-23T23:02:29.873197+0000","last_fullsized":"2023-09-23T23:02:29.873197+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":401,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.874762+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:02:29.874762+0000","last_peered":"2023-09-23T23:02:29.874762+0000","last_clean":"2023-09-23T23:02:29.874762+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:02:29.874762+0000","last_undegraded":"2023-09-23T23:02:29.874762+0000","last_fullsized":"2023-09-23T23:02:29.874762+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"400'11004","reported_seq":11109,"reported_epoch":400,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:41.872424+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:02:41.872424+0000","last_peered":"2023-09-23T23:02:41.872424+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:02:41.872424+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:02:41.872424+0000","mapping_epoch":353,"log_start":"334'5300","ondisk_log_start":"334'5300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63176704,"num_objects":964,"num_object_clones":0,"num_object_copies":2892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11004,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":964}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803403+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:02:29.803403+0000","last_peered":"2023-09-23T23:02:29.803403+0000","last_clean":"2023-09-23T23:02:29.803403+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:02:29.803403+0000","last_undegraded":"2023-09-23T23:02:29.803403+0000","last_fullsized":"2023-09-23T23:02:29.803403+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":303,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803312+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:02:29.803312+0000","last_peered":"2023-09-23T23:02:29.803312+0000","last_clean":"2023-09-23T23:02:29.803312+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:02:29.803312+0000","last_undegraded":"2023-09-23T23:02:29.803312+0000","last_fullsized":"2023-09-23T23:02:29.803312+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"400'11149","reported_seq":20440,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:42.112997+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:02:42.112997+0000","last_peered":"2023-09-23T23:02:42.112997+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:02:42.112997+0000","last_undegraded":"2023-09-23T23:02:42.112997+0000","last_fullsized":"2023-09-23T23:02:42.112997+0000","mapping_epoch":354,"log_start":"334'5400","ondisk_log_start":"334'5400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60751872,"num_objects":927,"num_object_clones":0,"num_object_copies":2781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":927,"num_objects_unfound":0,"num_objects_dirty":927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11149,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":927}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":308,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.343676+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:02:36.343676+0000","last_peered":"2023-09-23T23:02:36.343676+0000","last_clean":"2023-09-23T23:02:36.343676+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:02:36.343676+0000","last_undegraded":"2023-09-23T23:02:36.343676+0000","last_fullsized":"2023-09-23T23:02:36.343676+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":310,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.675172+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:02:29.675172+0000","last_peered":"2023-09-23T23:02:29.675172+0000","last_clean":"2023-09-23T23:02:29.675172+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:02:29.675172+0000","last_undegraded":"2023-09-23T23:02:29.675172+0000","last_fullsized":"2023-09-23T23:02:29.675172+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.674923+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:02:29.674923+0000","last_peered":"2023-09-23T23:02:29.674923+0000","last_clean":"2023-09-23T23:02:29.674923+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:02:29.674923+0000","last_undegraded":"2023-09-23T23:02:29.674923+0000","last_fullsized":"2023-09-23T23:02:29.674923+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"400'11167","reported_seq":11285,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:39.894941+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:02:39.894941+0000","last_peered":"2023-09-23T23:02:39.894941+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:02:39.894941+0000","last_undegraded":"2023-09-23T23:02:39.894941+0000","last_fullsized":"2023-09-23T23:02:39.894941+0000","mapping_epoch":359,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62717952,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":957,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11167,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":957}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":71,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.672837+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:02:29.672837+0000","last_peered":"2023-09-23T23:02:29.672837+0000","last_clean":"2023-09-23T23:02:29.672837+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:02:29.672837+0000","last_undegraded":"2023-09-23T23:02:29.672837+0000","last_fullsized":"2023-09-23T23:02:29.672837+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":420,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.884370+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:02:29.884370+0000","last_peered":"2023-09-23T23:02:29.884370+0000","last_clean":"2023-09-23T23:02:29.884370+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:02:29.884370+0000","last_undegraded":"2023-09-23T23:02:29.884370+0000","last_fullsized":"2023-09-23T23:02:29.884370+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":293,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:31.343076+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:02:31.343076+0000","last_peered":"2023-09-23T23:02:31.343076+0000","last_clean":"2023-09-23T23:02:31.343076+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:02:31.343076+0000","last_undegraded":"2023-09-23T23:02:31.343076+0000","last_fullsized":"2023-09-23T23:02:31.343076+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"400'11136","reported_seq":19316,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:41.244646+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:02:41.244646+0000","last_peered":"2023-09-23T23:02:41.244646+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:02:41.244646+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"325'2500","ondisk_log_start":"325'2500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8636,"log_dups_size":0,"ondisk_log_size":8636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60817408,"num_objects":928,"num_object_clones":0,"num_object_copies":2784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11136,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":928}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":368,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:31.342996+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:02:31.342996+0000","last_peered":"2023-09-23T23:02:31.342996+0000","last_clean":"2023-09-23T23:02:31.342996+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:02:31.342996+0000","last_undegraded":"2023-09-23T23:02:31.342996+0000","last_fullsized":"2023-09-23T23:02:31.342996+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":245,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.802748+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:02:29.802748+0000","last_peered":"2023-09-23T23:02:29.802748+0000","last_clean":"2023-09-23T23:02:29.802748+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:02:29.802748+0000","last_undegraded":"2023-09-23T23:02:29.802748+0000","last_fullsized":"2023-09-23T23:02:29.802748+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":310,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.870898+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:02:29.870898+0000","last_peered":"2023-09-23T23:02:29.870898+0000","last_clean":"2023-09-23T23:02:29.870898+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:02:29.870898+0000","last_undegraded":"2023-09-23T23:02:29.870898+0000","last_fullsized":"2023-09-23T23:02:29.870898+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"400'11340","reported_seq":14350,"reported_epoch":400,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:43.167527+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:02:43.167527+0000","last_peered":"2023-09-23T23:02:43.167527+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:02:43.167527+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"321'1300","ondisk_log_start":"321'1300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":2003,"num_objects_missing":2003,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11340,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":874},{"shards":"0,1,4,6","objects":9},{"shards":"0,6","objects":-3548}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":383,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.675060+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:02:29.675060+0000","last_peered":"2023-09-23T23:02:29.675060+0000","last_clean":"2023-09-23T23:02:29.675060+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:02:29.675060+0000","last_undegraded":"2023-09-23T23:02:29.675060+0000","last_fullsized":"2023-09-23T23:02:29.675060+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":497,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:29.672006+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:02:29.672006+0000","last_peered":"2023-09-23T23:02:29.672006+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:02:29.672006+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803632+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:02:29.803632+0000","last_peered":"2023-09-23T23:02:29.803632+0000","last_clean":"2023-09-23T23:02:29.803632+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:02:29.803632+0000","last_undegraded":"2023-09-23T23:02:29.803632+0000","last_fullsized":"2023-09-23T23:02:29.803632+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":310,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.874247+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:02:29.874247+0000","last_peered":"2023-09-23T23:02:29.874247+0000","last_clean":"2023-09-23T23:02:29.874247+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:02:29.874247+0000","last_undegraded":"2023-09-23T23:02:29.874247+0000","last_fullsized":"2023-09-23T23:02:29.874247+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"400'11198","reported_seq":11308,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:39.884280+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:02:39.884280+0000","last_peered":"2023-09-23T23:02:39.884280+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:02:39.884280+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11198,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2621}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":391,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.671612+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:02:29.671612+0000","last_peered":"2023-09-23T23:02:29.671612+0000","last_clean":"2023-09-23T23:02:29.671612+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:02:29.671612+0000","last_undegraded":"2023-09-23T23:02:29.671612+0000","last_fullsized":"2023-09-23T23:02:29.671612+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":362,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.671675+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:02:29.671675+0000","last_peered":"2023-09-23T23:02:29.671675+0000","last_clean":"2023-09-23T23:02:29.671675+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:02:29.671675+0000","last_undegraded":"2023-09-23T23:02:29.671675+0000","last_fullsized":"2023-09-23T23:02:29.671675+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":322,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.673236+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:02:29.673236+0000","last_peered":"2023-09-23T23:02:29.673236+0000","last_clean":"2023-09-23T23:02:29.673236+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:02:29.673236+0000","last_undegraded":"2023-09-23T23:02:29.673236+0000","last_fullsized":"2023-09-23T23:02:29.673236+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"400'11249","reported_seq":11373,"reported_epoch":400,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:42.106074+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:02:42.106074+0000","last_peered":"2023-09-23T23:02:42.106074+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:02:42.106074+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:02:42.106074+0000","mapping_epoch":354,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62521368,"num_objects":955,"num_object_clones":0,"num_object_copies":2865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":955,"num_objects_unfound":0,"num_objects_dirty":955,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11249,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":955}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":337,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.672761+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:02:29.672761+0000","last_peered":"2023-09-23T23:02:29.672761+0000","last_clean":"2023-09-23T23:02:29.672761+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:02:29.672761+0000","last_undegraded":"2023-09-23T23:02:29.672761+0000","last_fullsized":"2023-09-23T23:02:29.672761+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":245,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:31.343096+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:02:31.343096+0000","last_peered":"2023-09-23T23:02:31.343096+0000","last_clean":"2023-09-23T23:02:31.343096+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:02:31.343096+0000","last_undegraded":"2023-09-23T23:02:31.343096+0000","last_fullsized":"2023-09-23T23:02:31.343096+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.675082+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:02:29.675082+0000","last_peered":"2023-09-23T23:02:29.675082+0000","last_clean":"2023-09-23T23:02:29.675082+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:02:29.675082+0000","last_undegraded":"2023-09-23T23:02:29.675082+0000","last_fullsized":"2023-09-23T23:02:29.675082+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"400'11282","reported_seq":21794,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:41.926440+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:02:41.926440+0000","last_peered":"2023-09-23T23:02:41.926440+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:02:41.926440+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"335'5600","ondisk_log_start":"335'5600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56623104,"num_objects":864,"num_object_clones":0,"num_object_copies":2592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11282,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-521},{"shards":"4,5,6","objects":864}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":94,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804637+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:02:29.804637+0000","last_peered":"2023-09-23T23:02:29.804637+0000","last_clean":"2023-09-23T23:02:29.804637+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:02:29.804637+0000","last_undegraded":"2023-09-23T23:02:29.804637+0000","last_fullsized":"2023-09-23T23:02:29.804637+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":108,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.877171+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:02:29.877171+0000","last_peered":"2023-09-23T23:02:29.877171+0000","last_clean":"2023-09-23T23:02:29.877171+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:02:29.877171+0000","last_undegraded":"2023-09-23T23:02:29.877171+0000","last_fullsized":"2023-09-23T23:02:29.877171+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":85,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803313+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:02:29.803313+0000","last_peered":"2023-09-23T23:02:29.803313+0000","last_clean":"2023-09-23T23:02:29.803313+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:02:29.803313+0000","last_undegraded":"2023-09-23T23:02:29.803313+0000","last_fullsized":"2023-09-23T23:02:29.803313+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"400'11195","reported_seq":11729,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:42.097791+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:02:42.097791+0000","last_peered":"2023-09-23T23:02:42.097791+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:02:42.097791+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"334'5500","ondisk_log_start":"334'5500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57344000,"num_objects":875,"num_object_clones":0,"num_object_copies":2625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":875,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11195,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":875},{"shards":"4,6","objects":-2676}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":356,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803458+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:02:29.803458+0000","last_peered":"2023-09-23T23:02:29.803458+0000","last_clean":"2023-09-23T23:02:29.803458+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:02:29.803458+0000","last_undegraded":"2023-09-23T23:02:29.803458+0000","last_fullsized":"2023-09-23T23:02:29.803458+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":304,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.673162+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:02:29.673162+0000","last_peered":"2023-09-23T23:02:29.673162+0000","last_clean":"2023-09-23T23:02:29.673162+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:02:29.673162+0000","last_undegraded":"2023-09-23T23:02:29.673162+0000","last_fullsized":"2023-09-23T23:02:29.673162+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":374,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873276+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:02:29.873276+0000","last_peered":"2023-09-23T23:02:29.873276+0000","last_clean":"2023-09-23T23:02:29.873276+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:02:29.873276+0000","last_undegraded":"2023-09-23T23:02:29.873276+0000","last_fullsized":"2023-09-23T23:02:29.873276+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":338,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.672405+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:02:29.672405+0000","last_peered":"2023-09-23T23:02:29.672405+0000","last_clean":"2023-09-23T23:02:29.672405+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:02:29.672405+0000","last_undegraded":"2023-09-23T23:02:29.672405+0000","last_fullsized":"2023-09-23T23:02:29.672405+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":148,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.802854+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:02:29.802854+0000","last_peered":"2023-09-23T23:02:29.802854+0000","last_clean":"2023-09-23T23:02:29.802854+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:02:29.802854+0000","last_undegraded":"2023-09-23T23:02:29.802854+0000","last_fullsized":"2023-09-23T23:02:29.802854+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":209,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873749+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:02:29.873749+0000","last_peered":"2023-09-23T23:02:29.873749+0000","last_clean":"2023-09-23T23:02:29.873749+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:02:29.873749+0000","last_undegraded":"2023-09-23T23:02:29.873749+0000","last_fullsized":"2023-09-23T23:02:29.873749+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":423,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.884413+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:02:29.884413+0000","last_peered":"2023-09-23T23:02:29.884413+0000","last_clean":"2023-09-23T23:02:29.884413+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:02:29.884413+0000","last_undegraded":"2023-09-23T23:02:29.884413+0000","last_fullsized":"2023-09-23T23:02:29.884413+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":223,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.874116+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:02:29.874116+0000","last_peered":"2023-09-23T23:02:29.874116+0000","last_clean":"2023-09-23T23:02:29.874116+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:02:29.874116+0000","last_undegraded":"2023-09-23T23:02:29.874116+0000","last_fullsized":"2023-09-23T23:02:29.874116+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":310,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.876934+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:02:29.876934+0000","last_peered":"2023-09-23T23:02:29.876934+0000","last_clean":"2023-09-23T23:02:29.876934+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:02:29.876934+0000","last_undegraded":"2023-09-23T23:02:29.876934+0000","last_fullsized":"2023-09-23T23:02:29.876934+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":94,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.804483+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:02:29.804483+0000","last_peered":"2023-09-23T23:02:29.804483+0000","last_clean":"2023-09-23T23:02:29.804483+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:02:29.804483+0000","last_undegraded":"2023-09-23T23:02:29.804483+0000","last_fullsized":"2023-09-23T23:02:29.804483+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.803793+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:02:29.803793+0000","last_peered":"2023-09-23T23:02:29.803793+0000","last_clean":"2023-09-23T23:02:29.803793+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:02:29.803793+0000","last_undegraded":"2023-09-23T23:02:29.803793+0000","last_fullsized":"2023-09-23T23:02:29.803793+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":343,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.673249+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:02:29.673249+0000","last_peered":"2023-09-23T23:02:29.673249+0000","last_clean":"2023-09-23T23:02:29.673249+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:02:29.673249+0000","last_undegraded":"2023-09-23T23:02:29.673249+0000","last_fullsized":"2023-09-23T23:02:29.673249+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":385,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T23:02:29.873045+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:02:29.873045+0000","last_peered":"2023-09-23T23:02:29.873045+0000","last_clean":"2023-09-23T23:02:29.873045+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:02:29.873045+0000","last_undegraded":"2023-09-23T23:02:29.873045+0000","last_fullsized":"2023-09-23T23:02:29.873045+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":972685336,"num_objects":14843,"num_object_clones":0,"num_object_copies":44529,"num_objects_missing_on_primary":2005,"num_objects_missing":2005,"num_objects_degraded":15748,"num_objects_misplaced":26055,"num_objects_unfound":0,"num_objects_dirty":14843,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178755,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19959,"num_bytes_recovered":1216872448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4238684160,"data_stored":4238671944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100455,"ondisk_log_size":100455,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430636,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626861,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253174,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1187620,"kb_used_data":453276,"kb_used_omap":9,"kb_used_meta":734326,"kb_avail":92557532,"statfs":{"total":95995035648,"available":94778912768,"internally_reserved":0,"allocated":464154624,"data_stored":462630187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":751950825},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.696,"15min":0.80000000000000004},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":20.446000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.22700000000000001},{"interface":"front","average":{"1min":3.2440000000000002,"5min":2.0760000000000001,"15min":1.262},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.472999999999999,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.28000000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.55300000000000005,"15min":0.53600000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5359999999999996,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.5},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.67000000000000004,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.1310000000000002,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56399999999999995,"15min":1.9359999999999999},"min":{"1min":0.26900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.212,"5min":3.0190000000000001,"15min":87.787999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64900000000000002,"15min":2.2559999999999998},"min":{"1min":0.26800000000000002,"5min":0.22500000000000001,"15min":0.19500000000000001},"max":{"1min":2.0379999999999998,"5min":5.7590000000000003,"15min":89.861999999999995},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":2.6389999999999998,"15min":2.4199999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0570000000000004,"5min":151.036,"15min":151.036},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.802,"15min":2.5920000000000001},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.26100000000000001},"max":{"1min":3.694,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.26400000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.496,"15min":2.2919999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6829999999999998,"5min":116.098,"15min":116.098},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.4630000000000001,"15min":2.4620000000000002},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.2000000000000002,"5min":116.08499999999999,"15min":116.08499999999999},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.278,"15min":2.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":3.819,"5min":5.6150000000000002,"15min":79.311000000000007},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.133,"5min":0.97499999999999998,"15min":1.837},"min":{"1min":0.432,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":79.328999999999994},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.038,"15min":1.639},"min":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.1669999999999998,"5min":5.8410000000000002,"15min":11.481999999999999},"last":0.441},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.1819999999999999,"15min":1.9259999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.9129999999999998,"5min":10.676,"15min":20.902999999999999},"last":0.40200000000000002}]}]},{"osd":0,"up_from":353,"seq":1516123455544,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":873692,"kb_used_data":694484,"kb_used_omap":37,"kb_used_meta":179162,"kb_avail":92871460,"statfs":{"total":95995035648,"available":95100375040,"internally_reserved":0,"allocated":711151616,"data_stored":709458600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":183462050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.71599999999999997,"15min":0.59799999999999998},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.82999999999999996,"15min":0.72399999999999998},"min":{"1min":0.27800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4459999999999997,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.83399999999999996,"15min":0.79800000000000004},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.694,"5min":12.215,"15min":12.215},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":5.9199999999999999},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.88,"15min":0.59599999999999997},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.88500000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.8170000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":0.625}]},{"osd":4,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2229999999999999,"15min":1.647},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2349999999999999,"5min":183.148,"15min":183.148},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.466,"5min":3.3530000000000002,"15min":1.5629999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1609999999999996,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.98499999999999999,"15min":0.78100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.984,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96199999999999997,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.1989999999999998,"5min":7.774,"15min":7.774},"last":0.71999999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.0509999999999999,"15min":0.90700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.359,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.91800000000000004,"15min":0.78800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3090000000000002,"5min":6.375,"15min":6.375},"last":0.77400000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.179,"15min":1.2609999999999999},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3929999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.147,"5min":1.1890000000000001,"15min":1.2490000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.669,"5min":6.343,"15min":6.343},"last":0.65100000000000002}]}]},{"osd":2,"up_from":359,"seq":1541893259317,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678156,"kb_used_data":528644,"kb_used_omap":83,"kb_used_meta":149420,"kb_avail":93066996,"statfs":{"total":95995035648,"available":95300603904,"internally_reserved":0,"allocated":541331456,"data_stored":539714467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":153006533},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.2749999999999999,"15min":0.66100000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":74.241,"5min":74.241,"15min":74.241},"last":0.77100000000000002},{"interface":"front","average":{"1min":4.6699999999999999,"5min":1.4590000000000001,"15min":0.95799999999999996},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":74.430000000000007,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.68500000000000005,"15min":0.67200000000000004},"min":{"1min":0.26800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9710000000000001,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.376},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.083,"15min":1.131},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":0.81599999999999995,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.9530000000000003,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.94399999999999995,"15min":0.85899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8170000000000002,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.48299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.90000000000000002,"15min":0.67600000000000005},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.5359999999999996,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.96099999999999997,"15min":0.86899999999999999},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.8739999999999997,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":1.861}]},{"osd":5,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.0580000000000001,"15min":0.98899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":2.3090000000000002},{"interface":"front","average":{"1min":1.232,"5min":0.93999999999999995,"15min":0.76700000000000002},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.24,"15min":1.0600000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9359999999999999,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":2.968},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91700000000000004,"15min":0.70999999999999996},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.903,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1870000000000001,"15min":1.056},"min":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.5659999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.093,"15min":1.1499999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.9139999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":1.458}]}]},{"osd":3,"up_from":94,"seq":403726926121,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533048,"kb_used_data":14448,"kb_used_omap":7,"kb_used_meta":518584,"kb_avail":93212104,"statfs":{"total":95995035648,"available":95449194496,"internally_reserved":0,"allocated":14794752,"data_stored":13191124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":531030632},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:15 2023","interfaces":[{"interface":"back","average":{"1min":3.383,"5min":2.177,"15min":0.93799999999999994},"min":{"1min":0.191,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":37.249000000000002,"5min":122.289,"15min":122.289},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.919,"15min":0.91100000000000003},"min":{"1min":0.19800000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":35.991,"5min":122.041,"15min":122.041},"last":0.376}]},{"osd":1,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":0.74099999999999999,"15min":2.2999999999999998},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.1639999999999997,"5min":8.6280000000000001,"15min":123.866},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.3680000000000001,"5min":0.82499999999999996,"15min":2.1659999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5640000000000001,"5min":6.8010000000000002,"15min":122.265},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":0.71499999999999997},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.311},{"interface":"front","average":{"1min":1.28,"5min":1.097,"15min":0.86099999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.7919999999999998,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.3220000000000001,"15min":2.0099999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.2300000000000004,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.2,"5min":2.496,"15min":1.927},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.4509999999999996,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.2769999999999999,"15min":1.4039999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.5229999999999997,"5min":10.260999999999999,"15min":11.803000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.111,"5min":1.1919999999999999,"15min":1.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7000000000000002,"5min":10.246,"15min":11.951000000000001},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.554,"15min":1.7569999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5380000000000003,"5min":11.116,"15min":14.097},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.288,"15min":1.919},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.4219999999999997,"5min":9.2810000000000006,"15min":12.887},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:01:49 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.90700000000000003,"15min":1.7350000000000001},"min":{"1min":0.36299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6680000000000001,"5min":4.8449999999999998,"15min":11.694000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.0649999999999999,"15min":1.6279999999999999},"min":{"1min":0.45300000000000001,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.2690000000000001,"5min":8.5830000000000002,"15min":12.32},"last":0.53800000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019318,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1714924,"kb_used_data":807460,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92030228,"statfs":{"total":95995035648,"available":94238953472,"internally_reserved":0,"allocated":826839040,"data_stored":825997101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":10.303000000000001,"5min":2.7650000000000001,"15min":1.236},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.677000000000007,"5min":71.677000000000007,"15min":71.677000000000007},"last":6.5540000000000003},{"interface":"front","average":{"1min":11.896000000000001,"5min":2.9740000000000002,"15min":1.3879999999999999},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":72.147999999999996,"5min":72.147999999999996,"15min":72.147999999999996},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.117,"15min":1.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8380000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":6.2949999999999999},{"interface":"front","average":{"1min":1.653,"5min":1.0389999999999999,"15min":0.82499999999999996},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":4.9379999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":5.032}]},{"osd":5,"last update":"Sat Sep 23 23:02:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.169,"15min":1.7589999999999999},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.9870000000000001,"5min":12.961,"15min":148.61600000000001},"last":0.441},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.92300000000000004,"15min":1.829},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":8.7810000000000006,"5min":8.7810000000000006,"15min":151.905},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.429,"15min":1.2549999999999999},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1050000000000004,"5min":12.84,"15min":12.84},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2450000000000001,"15min":1.1240000000000001},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2210000000000001,"5min":10.587,"15min":10.587},"last":0.56599999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823092,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379788,"kb_used_data":480132,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":92365364,"statfs":{"total":95995035648,"available":94582132736,"internally_reserved":0,"allocated":491655168,"data_stored":490841017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.5229999999999999,"15min":0.91100000000000003},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":42.155000000000001,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.503},{"interface":"front","average":{"1min":3.7839999999999998,"5min":1.784,"15min":1.3160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":43.378999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.069,"15min":2.149},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":3.6680000000000001,"5min":12.159000000000001,"15min":110.36499999999999},"last":0.378},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.615,"15min":2.6850000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":7.1219999999999999,"5min":10.353,"15min":110.89},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.997,"15min":0.86599999999999999},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.601,"5min":6.601,"15min":6.601},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.997,"15min":0.995},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6139999999999999,"5min":4.899,"15min":4.899},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2090000000000001,"15min":3.75},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8719999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.403,"15min":3.8180000000000001},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.6150000000000002,"5min":12.185,"15min":108.15600000000001},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":3.016,"15min":1.569},"min":{"1min":0.29599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.0380000000000003,"5min":115.06699999999999,"15min":115.06699999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.5349999999999999,"5min":2.548,"15min":1.4830000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":9.5890000000000004,"5min":114.908,"15min":114.908},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.5660000000000001,"15min":1.2889999999999999},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.0780000000000003,"5min":11.121,"15min":12.615},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.25,"15min":1.1220000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6400000000000001,"5min":12.137,"15min":12.74},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1259999999999999,"15min":1.0900000000000001},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7830000000000004,"5min":9.5980000000000008,"15min":12.23},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.538,"15min":1.22},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.31},"max":{"1min":2.4580000000000002,"5min":10.638999999999999,"15min":11.112},"last":0.45800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":694489088,"data_stored":694485016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":456327168,"data_stored":456327168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":536215552,"data_stored":536215552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":10092544,"data_stored":10092544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":821039104,"data_stored":821035032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":483721216,"data_stored":483721216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1236799488,"data_stored":1236795416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:02:46.212 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:02:46.243 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:02:46.451 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:46.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:46.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:46.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:46.603 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:46.652 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:46.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:46.756 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:46.797 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:46.856 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:46.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:46.961 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:46.998 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:47.056 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:47.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:47.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:47.163 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:47.178 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:47.194 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:47.262 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:47.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:47.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:47.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:47.454 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:47.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:47.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:47.476 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:47.559 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:47.585 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:47.587 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:47.608 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:47.628 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:47.632 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:47.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:47.648 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:47.668 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:47.692 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:47.720 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:47.734 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:47 smithi151 ceph-mon[159210]: pgmap v1227: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 892 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 15589/42840 objects degraded (36.389%); 25890/42840 objects misplaced (60.434%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T23:02:47.734 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1918770359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:47.734 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1619476305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:47.735 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:02:47.737 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:47.753 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.771 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:02:47.774 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:47.842 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:47.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:47.948 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:47.977 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:47.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:47 smithi052 ceph-mon[180979]: pgmap v1227: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 892 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 15589/42840 objects degraded (36.389%); 25890/42840 objects misplaced (60.434%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T23:02:47.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1918770359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:47.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1619476305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:47 smithi052 ceph-mon[187743]: pgmap v1227: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 892 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 15589/42840 objects degraded (36.389%); 25890/42840 objects misplaced (60.434%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T23:02:47.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1918770359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:02:47.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1619476305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:48.027 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:48.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:48.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:48.137 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:48.171 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:48.226 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:48.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:48.335 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:48.365 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:48.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:48.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:48.522 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:48.564 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:48.863 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:48.863 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.863 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1918770359' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:02:48.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: osdmap e401: 8 total, 8 up, 6 in 2023-09-23T23:02:48.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.864 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[187743]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.864 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:48.864 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1918770359' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:02:48.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: osdmap e401: 8 total, 8 up, 6 in 2023-09-23T23:02:48.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: failed to encode map e401 with expected crc 2023-09-23T23:02:48.865 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:48 smithi052 ceph-mon[180979]: failed to encode map e401 with expected crc 2023-09-23T23:02:49.090 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:49.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:49.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:49.196 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:49.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:02:49.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: failed to encode map e401 with expected crc 2023-09-23T23:02:49.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1918770359' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:02:49.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: osdmap e401: 8 total, 8 up, 6 in 2023-09-23T23:02:49.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: failed to encode map e401 with expected crc 2023-09-23T23:02:49.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: failed to encode map e401 with expected crc 2023-09-23T23:02:49.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:48 smithi151 ceph-mon[159210]: failed to encode map e401 with expected crc 2023-09-23T23:02:49.230 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:49.280 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:49.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:49.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:49.385 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:49.420 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:49.480 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:49.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:49.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:49.592 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:49.641 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:49.711 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:49.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:49.815 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:49.860 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:49.894 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:49 smithi052 ceph-mon[187743]: pgmap v1229: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 839 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 15414/40311 objects degraded (38.238%); 25710/40311 objects misplaced (63.779%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:02:49.895 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:49.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:49 smithi052 ceph-mon[180979]: pgmap v1229: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 839 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 15414/40311 objects degraded (38.238%); 25710/40311 objects misplaced (63.779%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:02:49.895 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:49 smithi151 ceph-mon[159210]: pgmap v1229: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 839 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 15414/40311 objects degraded (38.238%); 25710/40311 objects misplaced (63.779%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:02:49.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:02:50.493 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:50.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:50.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:50.597 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:02:50.628 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:50.681 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:02:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:50.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:02:50.787 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:50.817 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:50.874 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:50.879 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 15414/40311 objects degraded (38.238%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:50.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:50.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:50.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 15414/40311 objects degraded (38.238%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:50.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 15414/40311 objects degraded (38.238%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:51.063 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:02:51.063 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114198 10.5 10:a53fd4ad:::benchmark_data_smithi052_461816_object17397:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.32372752200001, 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047289519, 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.064 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.065 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.065 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114198 2023-09-23T23:02:51.065 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.065 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.065 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.065 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.066 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:02:51.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.067 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.067 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.067 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.067 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:02:51.067 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.068 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.068 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.068 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.068 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735120+0000", 2023-09-23T23:02:51.068 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:51.069 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.069 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.069 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.069 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735128+0000", 2023-09-23T23:02:51.069 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6639999999999998e-06 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735140+0000", 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1888999999999999e-05 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.070 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.071 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735151+0000", 2023-09-23T23:02:51.071 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T23:02:51.071 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.071 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.072 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.072 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736424+0000", 2023-09-23T23:02:51.072 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001272942 2023-09-23T23:02:51.072 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.073 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.073 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.073 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736451+0000", 2023-09-23T23:02:51.073 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7029000000000002e-05 2023-09-23T23:02:51.073 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772313+0000", 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.035862128 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772461+0000", 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014821499999999999 2023-09-23T23:02:51.074 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772545+0000", 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3183000000000006e-05 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779124+0000", 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065792929999999999 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.075 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782207+0000", 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030828169999999998 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782247+0000", 2023-09-23T23:02:51.076 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0198999999999999e-05 2023-09-23T23:02:51.077 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.077 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.077 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.077 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782304+0000", 2023-09-23T23:02:51.077 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7172000000000001e-05 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782412+0000", 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107539 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.078 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114209 10.5 10:a8019dd6:::benchmark_data_smithi052_461816_object17408:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.32253306500002, 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.052215573, 2023-09-23T23:02:51.079 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114209 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.080 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.081 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:02:51.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.082 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.082 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.082 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.082 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736316+0000", 2023-09-23T23:02:51.082 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:02:51.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.083 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.083 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.083 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736321+0000", 2023-09-23T23:02:51.083 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6640000000000003e-06 2023-09-23T23:02:51.083 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736333+0000", 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1854e-05 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.084 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736343+0000", 2023-09-23T23:02:51.085 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7399999999999999e-06 2023-09-23T23:02:51.085 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.085 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.085 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.085 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736479+0000", 2023-09-23T23:02:51.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013588000000000001 2023-09-23T23:02:51.086 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.086 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.086 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.086 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736486+0000", 2023-09-23T23:02:51.086 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3150000000000003e-06 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774671+0000", 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0381846910000001 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.087 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.088 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774856+0000", 2023-09-23T23:02:51.088 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000185179 2023-09-23T23:02:51.088 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.088 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.088 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774979+0000", 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122843 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782160+0000", 2023-09-23T23:02:51.089 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071812610000000004 2023-09-23T23:02:51.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.090 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.090 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.090 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782451+0000", 2023-09-23T23:02:51.090 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029058300000000002 2023-09-23T23:02:51.090 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.091 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.091 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.091 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788381+0000", 2023-09-23T23:02:51.091 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059298370000000003 2023-09-23T23:02:51.091 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.092 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.092 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788422+0000", 2023-09-23T23:02:51.092 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1409999999999998e-05 2023-09-23T23:02:51.092 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.092 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.093 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.093 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788532+0000", 2023-09-23T23:02:51.093 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110103 2023-09-23T23:02:51.093 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.093 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.093 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114217 10.5 10:a151ffa4:::benchmark_data_smithi052_461816_object17416:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.32250058400001, 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068861074, 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.094 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114217 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.095 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736352+0000", 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9399999999999998e-06 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.096 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736358+0000", 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736363+0000", 2023-09-23T23:02:51.097 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3199999999999999e-06 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737029+0000", 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000666032 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.098 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737038+0000", 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5490000000000006e-06 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779153+0000", 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0421153320000001 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.099 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779339+0000", 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000186016 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779477+0000", 2023-09-23T23:02:51.100 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000137923 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782455+0000", 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029779519999999999 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.101 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.102 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790613+0000", 2023-09-23T23:02:51.102 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081575689999999999 2023-09-23T23:02:51.102 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.102 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.102 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805085+0000", 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014472021 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805126+0000", 2023-09-23T23:02:51.103 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1619000000000003e-05 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805210+0000", 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3728999999999997e-05 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.104 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.105 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.105 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.105 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.105 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114228 10.5 10:abf9b074:::benchmark_data_smithi052_461816_object17427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.105 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:02:51.105 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.322064544, 2023-09-23T23:02:51.106 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068634885, 2023-09-23T23:02:51.106 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.106 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.106 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114228 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:02:51.107 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736784+0000", 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:02:51.108 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736797+0000", 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2558000000000001e-05 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736807+0000", 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7669999999999998e-06 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.109 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736815+0000", 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737073+0000", 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000257502 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737079+0000", 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5180000000000001e-06 2023-09-23T23:02:51.110 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782471+0000", 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0453921399999999 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782629+0000", 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015792200000000001 2023-09-23T23:02:51.111 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782710+0000", 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0499000000000003e-05 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788710+0000", 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060006640000000002 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.112 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796703+0000", 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079926860000000006 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805261+0000", 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085578570000000003 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805317+0000", 2023-09-23T23:02:51.113 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5649000000000002e-05 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805420+0000", 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103467 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114233 10.5 10:adc2278f:::benchmark_data_smithi052_461816_object17432:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.114 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.32201928699999, 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068750884, 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114233 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:02:51.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736833+0000", 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0469999999999998e-06 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736840+0000", 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2130000000000002e-06 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736846+0000", 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1099999999999999e-06 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737097+0000", 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025043299999999999 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737103+0000", 2023-09-23T23:02:51.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8150000000000002e-06 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785090+0000", 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047987419 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785226+0000", 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013601299999999999 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785376+0000", 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000150407 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790542+0000", 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051658429999999998 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796752+0000", 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062099620000000003 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805468+0000", 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0087155189999999997 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805509+0000", 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1789000000000003e-05 2023-09-23T23:02:51.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805581+0000", 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1771000000000001e-05 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114234 10.5 10:a1bb5c32:::benchmark_data_smithi052_461816_object17433:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.32199866299999, 2023-09-23T23:02:51.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068915523, 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114234 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:02:51.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736853+0000", 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3379999999999999e-06 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736859+0000", 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0499999999999997e-06 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736870+0000", 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1021e-05 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737132+0000", 2023-09-23T23:02:51.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00026153199999999998 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737138+0000", 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9839999999999998e-06 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786800+0000", 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.049662885 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786974+0000", 2023-09-23T23:02:51.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017351000000000001 2023-09-23T23:02:51.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.787107+0000", 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013272100000000001 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790552+0000", 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034448650000000001 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797758+0000", 2023-09-23T23:02:51.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072065200000000001 2023-09-23T23:02:51.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805623+0000", 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0078654350000000005 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805667+0000", 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3665000000000003e-05 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805766+0000", 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9365999999999997e-05 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.133 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114235 10.5 10:a3d19bc6:::benchmark_data_smithi052_461816_object17434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.321975049, 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081414576, 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.134 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.135 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114235 2023-09-23T23:02:51.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.135 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736874+0000", 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736877+0000", 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8949999999999998e-06 2023-09-23T23:02:51.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736883+0000", 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3459999999999999e-06 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736891+0000", 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8050000000000007e-06 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737166+0000", 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027465399999999999 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737172+0000", 2023-09-23T23:02:51.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.207e-06 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788737+0000", 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0515644399999999 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788877+0000", 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014082 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788968+0000", 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0167999999999995e-05 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792267+0000", 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003299256 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797797+0000", 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.005530274 2023-09-23T23:02:51.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818146+0000", 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020349151999999999 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818185+0000", 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8575000000000003e-05 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818289+0000", 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010435899999999999 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114248 10.5 10:a5b9c0ca:::benchmark_data_smithi052_461816_object17447:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 285.32195388899999, 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081611235, 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.143 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114248 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736895+0000", 2023-09-23T23:02:51.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736898+0000", 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4250000000000001e-06 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736904+0000", 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736909+0000", 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2310000000000004e-06 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737193+0000", 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000283885 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:02:51.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737199+0000", 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9610000000000002e-06 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790684+0000", 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.053484358 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790817+0000", 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000133693 2023-09-23T23:02:51.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790895+0000", 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8218000000000002e-05 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792358+0000", 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001462758 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797833+0000", 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054744759999999998 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818343+0000", 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020510347000000002 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818398+0000", 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4530999999999997e-05 2023-09-23T23:02:51.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818507+0000", 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010940100000000001 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:116534 10.6 10:62c2756c:::benchmark_data_smithi052_461816_object19733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T23:02:51.152 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 244.17931884399999, 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18822530200000001, 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 116534 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879530+0000", 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:02:51.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879535+0000", 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879547+0000", 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1918e-05 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879555+0000", 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1459999999999996e-06 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879602+0000", 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6768999999999999e-05 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879810+0000", 2023-09-23T23:02:51.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020844799999999999 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879951+0000", 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014103400000000001 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.881979+0000", 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020274709999999999 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.883452+0000", 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001472985 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067651+0000", 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18419902199999999 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067709+0000", 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8180000000000002e-05 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067756+0000", 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7275999999999999e-05 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138140 10.8 10:10bb8653:::benchmark_data_smithi052_461816_object41339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 143.44611675600001, 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.272336093, 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.160 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138140 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612731+0000", 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612741+0000", 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4800000000000007e-06 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612762+0000", 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1761000000000001e-05 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612772+0000", 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6229999999999992e-06 2023-09-23T23:02:51.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612837+0000", 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4789999999999995e-05 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613071+0000", 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023441400000000001 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613293+0000", 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022140800000000001 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.616187+0000", 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028945080000000001 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.618246+0000", 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020593949999999999 2023-09-23T23:02:51.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883729+0000", 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265482459 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883802+0000", 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3008000000000003e-05 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.885069+0000", 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012670579999999999 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.167 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138142 10.8 10:10a11927:::benchmark_data_smithi052_461816_object41341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 143.432025291, 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26049065999999998, 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.168 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138142 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626822+0000", 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:02:51.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626834+0000", 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626866+0000", 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1934999999999998e-05 2023-09-23T23:02:51.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626881+0000", 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5491000000000001e-05 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626950+0000", 2023-09-23T23:02:51.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8701000000000005e-05 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627110+0000", 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000160347 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627220+0000", 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110009 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.629339+0000", 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021183280000000001 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.635341+0000", 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060026009999999998 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886154+0000", 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25081289200000001 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886210+0000", 2023-09-23T23:02:51.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5321000000000001e-05 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.887315+0000", 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001105396 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138146 10.6 10:6e738f12:::benchmark_data_smithi052_461816_object41345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:02:51.176 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.721322+0000", 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 143.337527188, 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16252518799999999, 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138146 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721322+0000", 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721322+0000", 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721321+0000", 2023-09-23T23:02:51.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721329+0000", 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9300000000000003e-06 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721345+0000", 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6147999999999999e-05 2023-09-23T23:02:51.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721360+0000", 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5622999999999999e-05 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721413+0000", 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2743e-05 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721524+0000", 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110302 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.721613+0000", 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9450999999999994e-05 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.723428+0000", 2023-09-23T23:02:51.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018152520000000001 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.724326+0000", 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00089747799999999995 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883765+0000", 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15943891499999999 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883809+0000", 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4713999999999999e-05 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883848+0000", 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8324000000000001e-05 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:140723 10.6 10:68c22ebc:::benchmark_data_smithi052_461816_object43922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 131.84382634400001, 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.195061394, 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140723 2023-09-23T23:02:51.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:02:51.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215022+0000", 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215028+0000", 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215039+0000", 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0998e-05 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215044+0000", 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6380000000000001e-06 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215089+0000", 2023-09-23T23:02:51.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4929000000000003e-05 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215188+0000", 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8741999999999998e-05 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215268+0000", 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0049000000000006e-05 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217087+0000", 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018184539999999999 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217880+0000", 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079313599999999995 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410012+0000", 2023-09-23T23:02:51.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19213219200000001 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410050+0000", 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8262000000000001e-05 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410085+0000", 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4588000000000002e-05 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.191 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:150925 10.8 10:17092047:::benchmark_data_smithi052_461816_object54124:head [delete] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:15.802287+0000", 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 95.256562174999999, 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15994923699999999, 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.192 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 150925 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802287+0000", 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802287+0000", 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802286+0000", 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802292+0000", 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.31e-06 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802300+0000", 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5900000000000008e-06 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802305+0000", 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9540000000000003e-06 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802353+0000", 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7713999999999998e-05 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802517+0000", 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000164406 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.802685+0000", 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000167617 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.804935+0000", 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022500649999999999 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.807958+0000", 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030225249999999999 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819443+0000", 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011485644 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819499+0000", 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6200999999999999e-05 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.962237+0000", 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14273727 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:150935 10.5 10:a2940dfc:::benchmark_data_smithi052_461816_object54134:head [delete] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:02:51.200 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 95.239181255000005, 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16991541099999999, 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 150935 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819671+0000", 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2229999999999999e-06 2023-09-23T23:02:51.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819678+0000", 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5080000000000002e-06 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819683+0000", 2023-09-23T23:02:51.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0799999999999996e-06 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.960363+0000", 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.140680312 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.960543+0000", 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018004800000000001 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.960725+0000", 2023-09-23T23:02:51.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018175299999999999 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.964365+0000", 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036400709999999999 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.964476+0000", 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111432 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.965502+0000", 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010257280000000001 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.988206+0000", 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022703556999999999 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.988258+0000", 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2590000000000003e-05 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.989584+0000", 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013256590000000001 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:168690 10.6 10:64304712:::benchmark_data_smithi052_461816_object71889:head [delete] snapc 0=[] ondisk+write+known_if_redirected e395)", 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:06.131735+0000", 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 44.927114170999999, 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.160641115, 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 168690 2023-09-23T23:02:51.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131735+0000", 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131735+0000", 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131733+0000", 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131741+0000", 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6790000000000005e-06 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131759+0000", 2023-09-23T23:02:51.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7872000000000001e-05 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131767+0000", 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2209999999999997e-06 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131816+0000", 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8860999999999999e-05 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.131975+0000", 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015866400000000001 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.132129+0000", 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000154821 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.134425+0000", 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022959199999999999 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.135723+0000", 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001297421 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.292311+0000", 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15658803900000001 2023-09-23T23:02:51.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.292345+0000", 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4709999999999998e-05 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:06.292377+0000", 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1109000000000003e-05 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.216 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:169443 10.8 10:12a1984a:::benchmark_data_smithi052_461816_object72642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e396)", 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:08.141652+0000", 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 42.917198044000003, 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16037544300000001, 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.217 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 169443 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141652+0000", 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141652+0000", 2023-09-23T23:02:51.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141650+0000", 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141656+0000", 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1800000000000004e-06 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141667+0000", 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0957e-05 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141675+0000", 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3729999999999993e-06 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141721+0000", 2023-09-23T23:02:51.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5711000000000003e-05 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141831+0000", 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110826 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.141928+0000", 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6978999999999994e-05 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.146100+0000", 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041717819999999997 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.147282+0000", 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011815110000000001 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.300997+0000", 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.153715454 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.301059+0000", 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2006000000000002e-05 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.302027+0000", 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00096789 2023-09-23T23:02:51.223 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:169446 10.8 10:1d59542c:::benchmark_data_smithi052_461816_object72645:head [delete] snapc 0=[] ondisk+write+known_if_redirected e396)", 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:08.143160+0000", 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 42.915689618999998, 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.159873337, 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.224 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 169446 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143160+0000", 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143160+0000", 2023-09-23T23:02:51.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143159+0000", 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143165+0000", 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3049999999999998e-06 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143176+0000", 2023-09-23T23:02:51.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0203999999999999e-05 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143181+0000", 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8209999999999998e-06 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143848+0000", 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00066674699999999998 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.143962+0000", 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113638 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:02:51.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.144114+0000", 2023-09-23T23:02:51.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000152146 2023-09-23T23:02:51.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.147566+0000", 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034525129999999999 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.150579+0000", 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030125939999999999 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.302154+0000", 2023-09-23T23:02:51.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15157480700000001 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.302203+0000", 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9598999999999997e-05 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.303033+0000", 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00082996399999999998 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:169449 10.5 10:a49aec38:::benchmark_data_smithi052_461816_object72648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e396)", 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:08.148422+0000", 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 42.910427466999998, 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15407968, 2023-09-23T23:02:51.233 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 169449 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148422+0000", 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148422+0000", 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148421+0000", 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148427+0000", 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4239999999999998e-06 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148440+0000", 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2500000000000001e-05 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148447+0000", 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5580000000000003e-06 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148493+0000", 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5516999999999998e-05 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148620+0000", 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012695 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.148729+0000", 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000109061 2023-09-23T23:02:51.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.300875+0000", 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15214629199999999 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.300944+0000", 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9197999999999997e-05 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.300991+0000", 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6893e-05 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.301032+0000", 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1393999999999997e-05 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.301086+0000", 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3062999999999997e-05 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:08.302502+0000", 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001416303 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:173638 10.6 10:638a9a21:::benchmark_data_smithi052_461816_object76837:head [delete] snapc 0=[] ondisk+write+known_if_redirected e397)", 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:22.400224+0000", 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 28.658625369999999, 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.154521616, 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:02:51.242 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 173638 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400224+0000", 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400224+0000", 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400221+0000", 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400234+0000", 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3062e-05 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400263+0000", 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8259000000000002e-05 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400276+0000", 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3419e-05 2023-09-23T23:02:51.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400333+0000", 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6888000000000001e-05 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400442+0000", 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000108593 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.400526+0000", 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4918000000000002e-05 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.403127+0000", 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002600766 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.404647+0000", 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001519704 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.554680+0000", 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.15003265599999999 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.554713+0000", 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3392999999999997e-05 2023-09-23T23:02:51.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:22.554746+0000", 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2926000000000003e-05 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:02:51.249 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:02:51.250 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:51.250 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:51.250 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15414/40311 objects degraded (38.238%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1703,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":401,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":13437,"data_bytes":879564728,"bytes_used":6311833600,"bytes_avail":569658380288,"bytes_total":575970213888,"degraded_objects":15414,"degraded_total":40311,"degraded_ratio":0.38237701867976481,"misplaced_objects":25710,"misplaced_total":40311,"misplaced_ratio":0.6377911736250651,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1257437,"recovering_keys_per_sec":0,"num_objects_recovered":192,"num_bytes_recovered":12582912,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":286},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (6m)\n [======================......] (remaining: 87s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:02:51.310 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:51.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:51.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:51.405 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:02:51.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:51.440 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:02:51.478 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:51 smithi052 ceph-mon[180979]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:02:51.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2917075660' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:51.856 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:51.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:51.864 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:02:51.964 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:51 smithi052 ceph-mon[187743]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:02:51.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2917075660' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:52.042 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:52.122 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:52.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:52.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:51 smithi151 ceph-mon[159210]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:02:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2917075660' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:02:52.232 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:52.261 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:52.308 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:52.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:52.413 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:52.429 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:02:52.429 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:02:52.442 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:52.492 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:52.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:52.526 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:52.548 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:52.571 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:52.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:52.629 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:52.685 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:52.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:52.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:52.792 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:52.820 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:52.875 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:52.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:52.987 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:02:53.023 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:53.296 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:02:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:53.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:02:53.405 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:53.439 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:53.494 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:53.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:53.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:53.598 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:53.620 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:02:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:02:53] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T23:02:53.630 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:53.689 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:53.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:53.799 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:53.829 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:53 smithi151 ceph-mon[159210]: pgmap v1231: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 276 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:02:53.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:53 smithi052 ceph-mon[180979]: pgmap v1231: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 276 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:02:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:53 smithi052 ceph-mon[187743]: pgmap v1231: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 276 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T23:02:55.002 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:55.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:55.112 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:02:55.145 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:55.200 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:02:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:55.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:02:55.204 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:55.305 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:55.347 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:55.781 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:55.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:55.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:55.890 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:55.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:55 smithi052 ceph-mon[180979]: pgmap v1232: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 896 KiB/s, 13 objects/s recovering 2023-09-23T23:02:55.929 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 15108/36594 objects degraded (41.285%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:55.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:55 smithi052 ceph-mon[187743]: pgmap v1232: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 896 KiB/s, 13 objects/s recovering 2023-09-23T23:02:55.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 15108/36594 objects degraded (41.285%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:55.932 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:56.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:55 smithi151 ceph-mon[159210]: pgmap v1232: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 761 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 15108/36594 objects degraded (41.285%); 25392/36594 objects misplaced (69.388%); 896 KiB/s, 13 objects/s recovering 2023-09-23T23:02:56.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 15108/36594 objects degraded (41.285%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:02:56.528 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:02:56.529 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:02:56.529 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:02:56.529 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:02:56.532 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:56.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:56.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:56.554 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:02:56.581 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:02:56.609 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:02:56.637 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:02:56.644 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:02:56.677 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:56.737 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:02:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:56.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:02:56.782 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:02:56.843 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:56.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3035291314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:56.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:56 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:56.931 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/803747661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:56.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3035291314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:56.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:56 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:56.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/803747661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:56.935 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:56.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3035291314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:56.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:56 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:02:56.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/803747661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:02:57.341 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:02:57.342 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:02:57.778 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:57.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:57.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:57.885 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:57.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 662 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 366 op/s; 14743/31845 objects degraded (46.296%); 24984/31845 objects misplaced (78.455%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:02:57.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:02:57.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: osdmap e402: 8 total, 8 up, 6 in 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[180979]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 662 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 366 op/s; 14743/31845 objects degraded (46.296%); 24984/31845 objects misplaced (78.455%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:57.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: osdmap e402: 8 total, 8 up, 6 in 2023-09-23T23:02:57.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:57 smithi052 ceph-mon[187743]: failed to encode map e402 with expected crc 2023-09-23T23:02:57.928 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:58.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 662 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 366 op/s; 14743/31845 objects degraded (46.296%); 24984/31845 objects misplaced (78.455%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T23:02:58.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:02:58.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: failed to encode map e402 with expected crc 2023-09-23T23:02:58.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: failed to encode map e402 with expected crc 2023-09-23T23:02:58.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:02:58.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: osdmap e402: 8 total, 8 up, 6 in 2023-09-23T23:02:58.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: failed to encode map e402 with expected crc 2023-09-23T23:02:58.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:57 smithi151 ceph-mon[159210]: failed to encode map e402 with expected crc 2023-09-23T23:02:58.486 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:58.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:58.517 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:02:58.593 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:02:58.631 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:59.017 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:58 smithi052 ceph-mon[180979]: failed to encode map e402 with expected crc 2023-09-23T23:02:59.018 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:58 smithi052 ceph-mon[187743]: failed to encode map e402 with expected crc 2023-09-23T23:02:59.099 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:02:59.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:59.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:02:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:58 smithi151 ceph-mon[159210]: failed to encode map e402 with expected crc 2023-09-23T23:02:59.203 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:02:59.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:02:59.256 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:59.339 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:02:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:59.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:02:59.448 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:02:59.502 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:59.516 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:02:59.517 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:02:59.557 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:02:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:59.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:02:59.662 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:59.691 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:59.737 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:59.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:59.846 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:02:59.864 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:02:59.877 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:02:59.877 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:02:59 smithi151 ceph-mon[159210]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 662 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 14743/31845 objects degraded (46.296%); 24984/31845 objects misplaced (78.455%); 947 KiB/s, 14 objects/s recovering 2023-09-23T23:02:59.935 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:02:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:02:59.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:02:59.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:02:59 smithi052 ceph-mon[187743]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 662 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 14743/31845 objects degraded (46.296%); 24984/31845 objects misplaced (78.455%); 947 KiB/s, 14 objects/s recovering 2023-09-23T23:02:59.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:02:59 smithi052 ceph-mon[180979]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 662 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 14743/31845 objects degraded (46.296%); 24984/31845 objects misplaced (78.455%); 947 KiB/s, 14 objects/s recovering 2023-09-23T23:03:00.043 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:00.096 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:00.516 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:00.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:00.620 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:00.650 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:00.713 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:00.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:00.737 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1235,"stamp":"2023-09-23T23:02:58.627253+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":694622136,"num_objects":10615,"num_object_clones":0,"num_object_copies":31845,"num_objects_missing_on_primary":1757,"num_objects_missing":1757,"num_objects_degraded":14743,"num_objects_misplaced":24984,"num_objects_unfound":0,"num_objects_dirty":10615,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":183267,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20238,"num_bytes_recovered":1232878524,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100701,"ondisk_log_size":100701,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5689884,"kb_used_data":2307564,"kb_used_omap":137,"kb_used_meta":3382070,"kb_avail":556781028,"statfs":{"total":575970213888,"available":570143772672,"internally_reserved":0,"allocated":2362945536,"data_stored":2354812378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3463239864},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-184942592,"num_objects":-2822,"num_object_clones":0,"num_object_copies":-8466,"num_objects_missing_on_primary":-150,"num_objects_missing":-150,"num_objects_degraded":-671,"num_objects_misplaced":-726,"num_objects_unfound":0,"num_objects_dirty":-2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2822,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004395"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":413,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711673+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:02:36.711673+0000","last_peered":"2023-09-23T23:02:36.711673+0000","last_clean":"2023-09-23T23:02:36.711673+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:02:36.711673+0000","last_undegraded":"2023-09-23T23:02:36.711673+0000","last_fullsized":"2023-09-23T23:02:36.711673+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.727230+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:02:36.727230+0000","last_peered":"2023-09-23T23:02:36.727230+0000","last_clean":"2023-09-23T23:02:36.727230+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:02:36.727230+0000","last_undegraded":"2023-09-23T23:02:36.727230+0000","last_fullsized":"2023-09-23T23:02:36.727230+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":70,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726192+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:02:36.726192+0000","last_peered":"2023-09-23T23:02:36.726192+0000","last_clean":"2023-09-23T23:02:36.726192+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:02:36.726192+0000","last_undegraded":"2023-09-23T23:02:36.726192+0000","last_fullsized":"2023-09-23T23:02:36.726192+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":398,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726263+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:02:36.726263+0000","last_peered":"2023-09-23T23:02:36.726263+0000","last_clean":"2023-09-23T23:02:36.726263+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:02:36.726263+0000","last_undegraded":"2023-09-23T23:02:36.726263+0000","last_fullsized":"2023-09-23T23:02:36.726263+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":86,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.725997+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:02:36.725997+0000","last_peered":"2023-09-23T23:02:36.725997+0000","last_clean":"2023-09-23T23:02:36.725997+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:02:36.725997+0000","last_undegraded":"2023-09-23T23:02:36.725997+0000","last_fullsized":"2023-09-23T23:02:36.725997+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.723532+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:02:36.723532+0000","last_peered":"2023-09-23T23:02:36.723532+0000","last_clean":"2023-09-23T23:02:36.723532+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:02:36.723532+0000","last_undegraded":"2023-09-23T23:02:36.723532+0000","last_fullsized":"2023-09-23T23:02:36.723532+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":336,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.719076+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:02:36.719076+0000","last_peered":"2023-09-23T23:02:36.719076+0000","last_clean":"2023-09-23T23:02:36.719076+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:02:36.719076+0000","last_undegraded":"2023-09-23T23:02:36.719076+0000","last_fullsized":"2023-09-23T23:02:36.719076+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":80,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.725934+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:02:36.725934+0000","last_peered":"2023-09-23T23:02:36.725934+0000","last_clean":"2023-09-23T23:02:36.725934+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:02:36.725934+0000","last_undegraded":"2023-09-23T23:02:36.725934+0000","last_fullsized":"2023-09-23T23:02:36.725934+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":297,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716430+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:02:36.716430+0000","last_peered":"2023-09-23T23:02:36.716430+0000","last_clean":"2023-09-23T23:02:36.716430+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:02:36.716430+0000","last_undegraded":"2023-09-23T23:02:36.716430+0000","last_fullsized":"2023-09-23T23:02:36.716430+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":85,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726101+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:02:36.726101+0000","last_peered":"2023-09-23T23:02:36.726101+0000","last_clean":"2023-09-23T23:02:36.726101+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:02:36.726101+0000","last_undegraded":"2023-09-23T23:02:36.726101+0000","last_fullsized":"2023-09-23T23:02:36.726101+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":91,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.727168+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:02:36.727168+0000","last_peered":"2023-09-23T23:02:36.727168+0000","last_clean":"2023-09-23T23:02:36.727168+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:02:36.727168+0000","last_undegraded":"2023-09-23T23:02:36.727168+0000","last_fullsized":"2023-09-23T23:02:36.727168+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":146,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.720518+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:02:36.720518+0000","last_peered":"2023-09-23T23:02:36.720518+0000","last_clean":"2023-09-23T23:02:36.720518+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:02:36.720518+0000","last_undegraded":"2023-09-23T23:02:36.720518+0000","last_fullsized":"2023-09-23T23:02:36.720518+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":472,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711913+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:02:36.711913+0000","last_peered":"2023-09-23T23:02:36.711913+0000","last_clean":"2023-09-23T23:02:36.711913+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:02:36.711913+0000","last_undegraded":"2023-09-23T23:02:36.711913+0000","last_fullsized":"2023-09-23T23:02:36.711913+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":375,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712980+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:02:36.712980+0000","last_peered":"2023-09-23T23:02:36.712980+0000","last_clean":"2023-09-23T23:02:36.712980+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:02:36.712980+0000","last_undegraded":"2023-09-23T23:02:36.712980+0000","last_fullsized":"2023-09-23T23:02:36.712980+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":301,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.713073+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:02:36.713073+0000","last_peered":"2023-09-23T23:02:36.713073+0000","last_clean":"2023-09-23T23:02:36.713073+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:02:36.713073+0000","last_undegraded":"2023-09-23T23:02:36.713073+0000","last_fullsized":"2023-09-23T23:02:36.713073+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":82,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726798+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:02:36.726798+0000","last_peered":"2023-09-23T23:02:36.726798+0000","last_clean":"2023-09-23T23:02:36.726798+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:02:36.726798+0000","last_undegraded":"2023-09-23T23:02:36.726798+0000","last_fullsized":"2023-09-23T23:02:36.726798+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":391,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726163+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:02:36.726163+0000","last_peered":"2023-09-23T23:02:36.726163+0000","last_clean":"2023-09-23T23:02:36.726163+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:02:36.726163+0000","last_undegraded":"2023-09-23T23:02:36.726163+0000","last_fullsized":"2023-09-23T23:02:36.726163+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":301,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716398+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:02:36.716398+0000","last_peered":"2023-09-23T23:02:36.716398+0000","last_clean":"2023-09-23T23:02:36.716398+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:02:36.716398+0000","last_undegraded":"2023-09-23T23:02:36.716398+0000","last_fullsized":"2023-09-23T23:02:36.716398+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":286,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711707+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:02:36.711707+0000","last_peered":"2023-09-23T23:02:36.711707+0000","last_clean":"2023-09-23T23:02:36.711707+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:02:36.711707+0000","last_undegraded":"2023-09-23T23:02:36.711707+0000","last_fullsized":"2023-09-23T23:02:36.711707+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":415,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726390+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:02:36.726390+0000","last_peered":"2023-09-23T23:02:36.726390+0000","last_clean":"2023-09-23T23:02:36.726390+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:02:36.726390+0000","last_undegraded":"2023-09-23T23:02:36.726390+0000","last_fullsized":"2023-09-23T23:02:36.726390+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":120,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:44.981973+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:02:44.981973+0000","last_peered":"2023-09-23T23:02:44.981973+0000","last_clean":"2023-09-23T23:02:44.981973+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:02:44.981973+0000","last_undegraded":"2023-09-23T23:02:44.981973+0000","last_fullsized":"2023-09-23T23:02:44.981973+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":95,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.728595+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:02:36.728595+0000","last_peered":"2023-09-23T23:02:36.728595+0000","last_clean":"2023-09-23T23:02:36.728595+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:02:36.728595+0000","last_undegraded":"2023-09-23T23:02:36.728595+0000","last_fullsized":"2023-09-23T23:02:36.728595+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"401'11446","reported_seq":11552,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T23:02:54.910695+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:02:54.910695+0000","last_peered":"2023-09-23T23:02:54.910695+0000","last_clean":"2023-09-23T23:02:54.910695+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:02:54.910695+0000","last_undegraded":"2023-09-23T23:02:54.910695+0000","last_fullsized":"2023-09-23T23:02:54.910695+0000","mapping_epoch":353,"log_start":"336'5900","ondisk_log_start":"336'5900","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5546,"log_dups_size":0,"ondisk_log_size":5546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41943040,"num_objects":640,"num_object_clones":0,"num_object_copies":1920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11446,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":285,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711829+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:02:36.711829+0000","last_peered":"2023-09-23T23:02:36.711829+0000","last_clean":"2023-09-23T23:02:36.711829+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:02:36.711829+0000","last_undegraded":"2023-09-23T23:02:36.711829+0000","last_fullsized":"2023-09-23T23:02:36.711829+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716022+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:02:36.716022+0000","last_peered":"2023-09-23T23:02:36.716022+0000","last_clean":"2023-09-23T23:02:36.716022+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:02:36.716022+0000","last_undegraded":"2023-09-23T23:02:36.716022+0000","last_fullsized":"2023-09-23T23:02:36.716022+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"401'11296","reported_seq":13730,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:55.291596+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:02:55.291596+0000","last_peered":"2023-09-23T23:02:55.291596+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:02:55.291596+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"325'2700","ondisk_log_start":"325'2700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8596,"log_dups_size":0,"ondisk_log_size":8596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42860544,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":654,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11296,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":654},{"shards":"5,6","objects":-2913}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":295,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711690+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:02:36.711690+0000","last_peered":"2023-09-23T23:02:36.711690+0000","last_clean":"2023-09-23T23:02:36.711690+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:02:36.711690+0000","last_undegraded":"2023-09-23T23:02:36.711690+0000","last_fullsized":"2023-09-23T23:02:36.711690+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":408,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711615+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:02:36.711615+0000","last_peered":"2023-09-23T23:02:36.711615+0000","last_clean":"2023-09-23T23:02:36.711615+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:02:36.711615+0000","last_undegraded":"2023-09-23T23:02:36.711615+0000","last_fullsized":"2023-09-23T23:02:36.711615+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"401'11405","reported_seq":11505,"reported_epoch":401,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:54.977297+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:02:54.977297+0000","last_peered":"2023-09-23T23:02:54.977297+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:02:54.977297+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:02:54.977297+0000","mapping_epoch":354,"log_start":"336'5900","ondisk_log_start":"336'5900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5505,"log_dups_size":0,"ondisk_log_size":5505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":667,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11405,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":667}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":152,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716449+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:02:36.716449+0000","last_peered":"2023-09-23T23:02:36.716449+0000","last_clean":"2023-09-23T23:02:36.716449+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:02:36.716449+0000","last_undegraded":"2023-09-23T23:02:36.716449+0000","last_fullsized":"2023-09-23T23:02:36.716449+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":270,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716151+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:02:36.716151+0000","last_peered":"2023-09-23T23:02:36.716151+0000","last_clean":"2023-09-23T23:02:36.716151+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:02:36.716151+0000","last_undegraded":"2023-09-23T23:02:36.716151+0000","last_fullsized":"2023-09-23T23:02:36.716151+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"401'11485","reported_seq":11771,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:54.908100+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:02:54.908100+0000","last_peered":"2023-09-23T23:02:54.908100+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:02:54.908100+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5485,"log_dups_size":0,"ondisk_log_size":5485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45154304,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":592,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11485,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":97},{"shards":"0,5","objects":-2921}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726724+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:02:36.726724+0000","last_peered":"2023-09-23T23:02:36.726724+0000","last_clean":"2023-09-23T23:02:36.726724+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:02:36.726724+0000","last_undegraded":"2023-09-23T23:02:36.726724+0000","last_fullsized":"2023-09-23T23:02:36.726724+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":301,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.722999+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:02:36.722999+0000","last_peered":"2023-09-23T23:02:36.722999+0000","last_clean":"2023-09-23T23:02:36.722999+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:02:36.722999+0000","last_undegraded":"2023-09-23T23:02:36.722999+0000","last_fullsized":"2023-09-23T23:02:36.722999+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"401'11431","reported_seq":11537,"reported_epoch":401,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:55.143353+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:02:55.143353+0000","last_peered":"2023-09-23T23:02:55.143353+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:02:55.143353+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:02:55.143353+0000","mapping_epoch":353,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5731,"log_dups_size":0,"ondisk_log_size":5731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43450368,"num_objects":663,"num_object_clones":0,"num_object_copies":1989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11431,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":663}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":90,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726747+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:02:36.726747+0000","last_peered":"2023-09-23T23:02:36.726747+0000","last_clean":"2023-09-23T23:02:36.726747+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:02:36.726747+0000","last_undegraded":"2023-09-23T23:02:36.726747+0000","last_fullsized":"2023-09-23T23:02:36.726747+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":394,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711819+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:02:36.711819+0000","last_peered":"2023-09-23T23:02:36.711819+0000","last_clean":"2023-09-23T23:02:36.711819+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:02:36.711819+0000","last_undegraded":"2023-09-23T23:02:36.711819+0000","last_fullsized":"2023-09-23T23:02:36.711819+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"401'11519","reported_seq":11633,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:55.337972+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:02:55.337972+0000","last_peered":"2023-09-23T23:02:55.337972+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:02:55.337972+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"335'5800","ondisk_log_start":"335'5800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44761088,"num_objects":683,"num_object_clones":0,"num_object_copies":2049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":683,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11519,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-2936}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":184,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712095+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:02:36.712095+0000","last_peered":"2023-09-23T23:02:36.712095+0000","last_clean":"2023-09-23T23:02:36.712095+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:02:36.712095+0000","last_undegraded":"2023-09-23T23:02:36.712095+0000","last_fullsized":"2023-09-23T23:02:36.712095+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":411,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712335+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:02:36.712335+0000","last_peered":"2023-09-23T23:02:36.712335+0000","last_clean":"2023-09-23T23:02:36.712335+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:02:36.712335+0000","last_undegraded":"2023-09-23T23:02:36.712335+0000","last_fullsized":"2023-09-23T23:02:36.712335+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"401'11389","reported_seq":20229,"reported_epoch":401,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:55.129478+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:02:55.129478+0000","last_peered":"2023-09-23T23:02:55.129478+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:02:55.129478+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:02:55.129478+0000","mapping_epoch":354,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43974656,"num_objects":671,"num_object_clones":0,"num_object_copies":2013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11389,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":671}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":424,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711456+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:02:36.711456+0000","last_peered":"2023-09-23T23:02:36.711456+0000","last_clean":"2023-09-23T23:02:36.711456+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:02:36.711456+0000","last_undegraded":"2023-09-23T23:02:36.711456+0000","last_fullsized":"2023-09-23T23:02:36.711456+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":402,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711792+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:02:36.711792+0000","last_peered":"2023-09-23T23:02:36.711792+0000","last_clean":"2023-09-23T23:02:36.711792+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:02:36.711792+0000","last_undegraded":"2023-09-23T23:02:36.711792+0000","last_fullsized":"2023-09-23T23:02:36.711792+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"401'11280","reported_seq":11386,"reported_epoch":401,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:02:55.256690+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:02:55.256690+0000","last_peered":"2023-09-23T23:02:55.256690+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:02:55.256690+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:02:55.256690+0000","mapping_epoch":353,"log_start":"335'5600","ondisk_log_start":"335'5600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45088768,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11280,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":688}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":71,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726125+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:02:36.726125+0000","last_peered":"2023-09-23T23:02:36.726125+0000","last_clean":"2023-09-23T23:02:36.726125+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:02:36.726125+0000","last_undegraded":"2023-09-23T23:02:36.726125+0000","last_fullsized":"2023-09-23T23:02:36.726125+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":305,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T23:02:47.729459+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:02:47.729459+0000","last_peered":"2023-09-23T23:02:47.729459+0000","last_clean":"2023-09-23T23:02:47.729459+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:02:47.729459+0000","last_undegraded":"2023-09-23T23:02:47.729459+0000","last_fullsized":"2023-09-23T23:02:47.729459+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"401'11392","reported_seq":20684,"reported_epoch":401,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:55.220047+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:02:55.220047+0000","last_peered":"2023-09-23T23:02:55.220047+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:02:55.220047+0000","last_undegraded":"2023-09-23T23:02:55.220047+0000","last_fullsized":"2023-09-23T23:02:55.220047+0000","mapping_epoch":354,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44826624,"num_objects":684,"num_object_clones":0,"num_object_copies":2052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":684,"num_objects_unfound":0,"num_objects_dirty":684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11392,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":684}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.720832+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:02:36.720832+0000","last_peered":"2023-09-23T23:02:36.720832+0000","last_clean":"2023-09-23T23:02:36.720832+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:02:36.720832+0000","last_undegraded":"2023-09-23T23:02:36.720832+0000","last_fullsized":"2023-09-23T23:02:36.720832+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716353+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:02:36.716353+0000","last_peered":"2023-09-23T23:02:36.716353+0000","last_clean":"2023-09-23T23:02:36.716353+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:02:36.716353+0000","last_undegraded":"2023-09-23T23:02:36.716353+0000","last_fullsized":"2023-09-23T23:02:36.716353+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716068+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:02:36.716068+0000","last_peered":"2023-09-23T23:02:36.716068+0000","last_clean":"2023-09-23T23:02:36.716068+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:02:36.716068+0000","last_undegraded":"2023-09-23T23:02:36.716068+0000","last_fullsized":"2023-09-23T23:02:36.716068+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"401'11439","reported_seq":11558,"reported_epoch":401,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:02:54.957798+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:02:54.957798+0000","last_peered":"2023-09-23T23:02:54.957798+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:02:54.957798+0000","last_undegraded":"2023-09-23T23:02:54.957798+0000","last_fullsized":"2023-09-23T23:02:54.957798+0000","mapping_epoch":359,"log_start":"336'5900","ondisk_log_start":"336'5900","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44892160,"num_objects":685,"num_object_clones":0,"num_object_copies":2055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":685,"num_objects_unfound":0,"num_objects_dirty":685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11439,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":685}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":72,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.722954+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:02:36.722954+0000","last_peered":"2023-09-23T23:02:36.722954+0000","last_clean":"2023-09-23T23:02:36.722954+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:02:36.722954+0000","last_undegraded":"2023-09-23T23:02:36.722954+0000","last_fullsized":"2023-09-23T23:02:36.722954+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":421,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.713154+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:02:36.713154+0000","last_peered":"2023-09-23T23:02:36.713154+0000","last_clean":"2023-09-23T23:02:36.713154+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:02:36.713154+0000","last_undegraded":"2023-09-23T23:02:36.713154+0000","last_fullsized":"2023-09-23T23:02:36.713154+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":294,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.720912+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:02:36.720912+0000","last_peered":"2023-09-23T23:02:36.720912+0000","last_clean":"2023-09-23T23:02:36.720912+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:02:36.720912+0000","last_undegraded":"2023-09-23T23:02:36.720912+0000","last_fullsized":"2023-09-23T23:02:36.720912+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"401'11408","reported_seq":19589,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:55.319277+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:02:55.319277+0000","last_peered":"2023-09-23T23:02:55.319277+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:02:55.319277+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"325'2800","ondisk_log_start":"325'2800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8608,"log_dups_size":0,"ondisk_log_size":8608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42991616,"num_objects":656,"num_object_clones":0,"num_object_copies":1968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11408,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":656}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":369,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.721463+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:02:36.721463+0000","last_peered":"2023-09-23T23:02:36.721463+0000","last_clean":"2023-09-23T23:02:36.721463+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:02:36.721463+0000","last_undegraded":"2023-09-23T23:02:36.721463+0000","last_fullsized":"2023-09-23T23:02:36.721463+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":246,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.720459+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:02:36.720459+0000","last_peered":"2023-09-23T23:02:36.720459+0000","last_clean":"2023-09-23T23:02:36.720459+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:02:36.720459+0000","last_undegraded":"2023-09-23T23:02:36.720459+0000","last_fullsized":"2023-09-23T23:02:36.720459+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":311,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.719135+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:02:36.719135+0000","last_peered":"2023-09-23T23:02:36.719135+0000","last_clean":"2023-09-23T23:02:36.719135+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:02:36.719135+0000","last_undegraded":"2023-09-23T23:02:36.719135+0000","last_fullsized":"2023-09-23T23:02:36.719135+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"401'11585","reported_seq":15089,"reported_epoch":401,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:55.371465+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:02:55.371465+0000","last_peered":"2023-09-23T23:02:55.371465+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:02:55.371465+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"322'1500","ondisk_log_start":"322'1500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10085,"log_dups_size":0,"ondisk_log_size":10085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41877504,"num_objects":639,"num_object_clones":0,"num_object_copies":1917,"num_objects_missing_on_primary":1755,"num_objects_missing":1755,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11585,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1689,"num_bytes_recovered":110690304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":626},{"shards":"0,1,4,6","objects":13},{"shards":"0,6","objects":-3549}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":384,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716372+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:02:36.716372+0000","last_peered":"2023-09-23T23:02:36.716372+0000","last_clean":"2023-09-23T23:02:36.716372+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:02:36.716372+0000","last_undegraded":"2023-09-23T23:02:36.716372+0000","last_fullsized":"2023-09-23T23:02:36.716372+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":498,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:36.724114+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:02:36.724114+0000","last_peered":"2023-09-23T23:02:36.724114+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:02:36.724114+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726286+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:02:36.726286+0000","last_peered":"2023-09-23T23:02:36.726286+0000","last_clean":"2023-09-23T23:02:36.726286+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:02:36.726286+0000","last_undegraded":"2023-09-23T23:02:36.726286+0000","last_fullsized":"2023-09-23T23:02:36.726286+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":311,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712564+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:02:36.712564+0000","last_peered":"2023-09-23T23:02:36.712564+0000","last_clean":"2023-09-23T23:02:36.712564+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:02:36.712564+0000","last_undegraded":"2023-09-23T23:02:36.712564+0000","last_fullsized":"2023-09-23T23:02:36.712564+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"401'11465","reported_seq":11576,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:54.974006+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:02:54.974006+0000","last_peered":"2023-09-23T23:02:54.974006+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:02:54.974006+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40566784,"num_objects":619,"num_object_clones":0,"num_object_copies":1857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11465,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2888}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":392,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.722119+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:02:36.722119+0000","last_peered":"2023-09-23T23:02:36.722119+0000","last_clean":"2023-09-23T23:02:36.722119+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:02:36.722119+0000","last_undegraded":"2023-09-23T23:02:36.722119+0000","last_fullsized":"2023-09-23T23:02:36.722119+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":363,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.722831+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:02:36.722831+0000","last_peered":"2023-09-23T23:02:36.722831+0000","last_clean":"2023-09-23T23:02:36.722831+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:02:36.722831+0000","last_undegraded":"2023-09-23T23:02:36.722831+0000","last_fullsized":"2023-09-23T23:02:36.722831+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":323,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.723751+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:02:36.723751+0000","last_peered":"2023-09-23T23:02:36.723751+0000","last_clean":"2023-09-23T23:02:36.723751+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:02:36.723751+0000","last_undegraded":"2023-09-23T23:02:36.723751+0000","last_fullsized":"2023-09-23T23:02:36.723751+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"401'11509","reported_seq":11634,"reported_epoch":401,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:02:55.257485+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:02:55.257485+0000","last_peered":"2023-09-23T23:02:55.257485+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:02:55.257485+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:02:55.257485+0000","mapping_epoch":354,"log_start":"335'5800","ondisk_log_start":"335'5800","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45482008,"num_objects":695,"num_object_clones":0,"num_object_copies":2085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":695,"num_objects_unfound":0,"num_objects_dirty":695,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11509,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":695}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":338,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.723406+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:02:36.723406+0000","last_peered":"2023-09-23T23:02:36.723406+0000","last_clean":"2023-09-23T23:02:36.723406+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:02:36.723406+0000","last_undegraded":"2023-09-23T23:02:36.723406+0000","last_fullsized":"2023-09-23T23:02:36.723406+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.720995+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:02:36.720995+0000","last_peered":"2023-09-23T23:02:36.720995+0000","last_clean":"2023-09-23T23:02:36.720995+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:02:36.720995+0000","last_undegraded":"2023-09-23T23:02:36.720995+0000","last_fullsized":"2023-09-23T23:02:36.720995+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":214,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.716445+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:02:36.716445+0000","last_peered":"2023-09-23T23:02:36.716445+0000","last_clean":"2023-09-23T23:02:36.716445+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:02:36.716445+0000","last_undegraded":"2023-09-23T23:02:36.716445+0000","last_fullsized":"2023-09-23T23:02:36.716445+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"401'11522","reported_seq":22035,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:55.235874+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:02:55.235874+0000","last_peered":"2023-09-23T23:02:55.235874+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:02:55.235874+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"335'5800","ondisk_log_start":"335'5800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40894464,"num_objects":624,"num_object_clones":0,"num_object_copies":1872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11522,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-761},{"shards":"4,5,6","objects":624}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":95,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.728693+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:02:36.728693+0000","last_peered":"2023-09-23T23:02:36.728693+0000","last_clean":"2023-09-23T23:02:36.728693+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:02:36.728693+0000","last_undegraded":"2023-09-23T23:02:36.728693+0000","last_fullsized":"2023-09-23T23:02:36.728693+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":109,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712238+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:02:36.712238+0000","last_peered":"2023-09-23T23:02:36.712238+0000","last_clean":"2023-09-23T23:02:36.712238+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:02:36.712238+0000","last_undegraded":"2023-09-23T23:02:36.712238+0000","last_fullsized":"2023-09-23T23:02:36.712238+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726013+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:02:36.726013+0000","last_peered":"2023-09-23T23:02:36.726013+0000","last_clean":"2023-09-23T23:02:36.726013+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:02:36.726013+0000","last_undegraded":"2023-09-23T23:02:36.726013+0000","last_fullsized":"2023-09-23T23:02:36.726013+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"401'11436","reported_seq":11971,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:55.151343+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:02:55.151343+0000","last_peered":"2023-09-23T23:02:55.151343+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:02:55.151343+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"335'5700","ondisk_log_start":"335'5700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41549824,"num_objects":634,"num_object_clones":0,"num_object_copies":1902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":634,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":634},{"shards":"4,6","objects":-2917}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":358,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:46.345980+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:02:46.345980+0000","last_peered":"2023-09-23T23:02:46.345980+0000","last_clean":"2023-09-23T23:02:46.345980+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:02:46.345980+0000","last_undegraded":"2023-09-23T23:02:46.345980+0000","last_fullsized":"2023-09-23T23:02:46.345980+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":305,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.723658+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:02:36.723658+0000","last_peered":"2023-09-23T23:02:36.723658+0000","last_clean":"2023-09-23T23:02:36.723658+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:02:36.723658+0000","last_undegraded":"2023-09-23T23:02:36.723658+0000","last_fullsized":"2023-09-23T23:02:36.723658+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":375,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711456+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:02:36.711456+0000","last_peered":"2023-09-23T23:02:36.711456+0000","last_clean":"2023-09-23T23:02:36.711456+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:02:36.711456+0000","last_undegraded":"2023-09-23T23:02:36.711456+0000","last_fullsized":"2023-09-23T23:02:36.711456+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":339,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.723186+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:02:36.723186+0000","last_peered":"2023-09-23T23:02:36.723186+0000","last_clean":"2023-09-23T23:02:36.723186+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:02:36.723186+0000","last_undegraded":"2023-09-23T23:02:36.723186+0000","last_fullsized":"2023-09-23T23:02:36.723186+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":149,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.720609+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:02:36.720609+0000","last_peered":"2023-09-23T23:02:36.720609+0000","last_clean":"2023-09-23T23:02:36.720609+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:02:36.720609+0000","last_undegraded":"2023-09-23T23:02:36.720609+0000","last_fullsized":"2023-09-23T23:02:36.720609+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":210,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711721+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:02:36.711721+0000","last_peered":"2023-09-23T23:02:36.711721+0000","last_clean":"2023-09-23T23:02:36.711721+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:02:36.711721+0000","last_undegraded":"2023-09-23T23:02:36.711721+0000","last_fullsized":"2023-09-23T23:02:36.711721+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":424,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.713228+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:02:36.713228+0000","last_peered":"2023-09-23T23:02:36.713228+0000","last_clean":"2023-09-23T23:02:36.713228+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:02:36.713228+0000","last_undegraded":"2023-09-23T23:02:36.713228+0000","last_fullsized":"2023-09-23T23:02:36.713228+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":224,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712420+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:02:36.712420+0000","last_peered":"2023-09-23T23:02:36.712420+0000","last_clean":"2023-09-23T23:02:36.712420+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:02:36.712420+0000","last_undegraded":"2023-09-23T23:02:36.712420+0000","last_fullsized":"2023-09-23T23:02:36.712420+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":311,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.712024+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:02:36.712024+0000","last_peered":"2023-09-23T23:02:36.712024+0000","last_clean":"2023-09-23T23:02:36.712024+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:02:36.712024+0000","last_undegraded":"2023-09-23T23:02:36.712024+0000","last_fullsized":"2023-09-23T23:02:36.712024+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":95,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726512+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:02:36.726512+0000","last_peered":"2023-09-23T23:02:36.726512+0000","last_clean":"2023-09-23T23:02:36.726512+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:02:36.726512+0000","last_undegraded":"2023-09-23T23:02:36.726512+0000","last_fullsized":"2023-09-23T23:02:36.726512+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.726331+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:02:36.726331+0000","last_peered":"2023-09-23T23:02:36.726331+0000","last_clean":"2023-09-23T23:02:36.726331+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:02:36.726331+0000","last_undegraded":"2023-09-23T23:02:36.726331+0000","last_fullsized":"2023-09-23T23:02:36.726331+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":344,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.723969+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:02:36.723969+0000","last_peered":"2023-09-23T23:02:36.723969+0000","last_clean":"2023-09-23T23:02:36.723969+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:02:36.723969+0000","last_undegraded":"2023-09-23T23:02:36.723969+0000","last_fullsized":"2023-09-23T23:02:36.723969+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":386,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T23:02:36.711509+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:02:36.711509+0000","last_peered":"2023-09-23T23:02:36.711509+0000","last_clean":"2023-09-23T23:02:36.711509+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:02:36.711509+0000","last_undegraded":"2023-09-23T23:02:36.711509+0000","last_fullsized":"2023-09-23T23:02:36.711509+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":694026264,"num_objects":10591,"num_object_clones":0,"num_object_copies":31773,"num_objects_missing_on_primary":1757,"num_objects_missing":1757,"num_objects_degraded":14741,"num_objects_misplaced":24984,"num_objects_unfound":0,"num_objects_dirty":10591,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183007,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20203,"num_bytes_recovered":1232863232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3481022464,"data_stored":3481010248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100507,"ondisk_log_size":100507,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430639,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626864,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253177,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1077500,"kb_used_data":334900,"kb_used_omap":9,"kb_used_meta":742518,"kb_avail":92667652,"statfs":{"total":95995035648,"available":94891675648,"internally_reserved":0,"allocated":342937600,"data_stored":341405223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":760339433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.696,"15min":0.80000000000000004},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":20.446000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":2.4780000000000002},{"interface":"front","average":{"1min":3.2440000000000002,"5min":2.0760000000000001,"15min":1.262},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.472999999999999,"5min":83.700999999999993,"15min":83.700999999999993},"last":2.593}]},{"osd":2,"last update":"Sat Sep 23 23:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.55300000000000005,"15min":0.53600000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5359999999999996,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.67000000000000004,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.1310000000000002,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":0.28199999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.56899999999999995,"15min":1.7969999999999999},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":87.787999999999997},"last":0.315},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.53300000000000003,"15min":2.0499999999999998},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":1.24,"5min":3.109,"15min":89.861999999999995},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3129999999999997,"5min":3.218,"15min":2.637},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.29199999999999998},"max":{"1min":67.738,"5min":151.036,"15min":151.036},"last":0.374},{"interface":"front","average":{"1min":5.7089999999999996,"5min":3.431,"15min":2.9260000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":67.670000000000002,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3300000000000001,"5min":1.6930000000000001,"15min":2.5699999999999998},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":68.259,"5min":68.259,"15min":116.098},"last":0.436},{"interface":"front","average":{"1min":5.5549999999999997,"5min":1.8759999999999999,"15min":2.6970000000000001},"min":{"1min":0.38200000000000001,"5min":0.33600000000000002,"15min":0.315},"max":{"1min":70.057000000000002,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":2.0600000000000001,"15min":2.5259999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":67.349000000000004,"5min":67.349000000000004,"15min":79.311000000000007},"last":0.45100000000000001},{"interface":"front","average":{"1min":5.5979999999999999,"5min":1.7529999999999999,"15min":2.125},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":67.366,"5min":67.366,"15min":79.328999999999994},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.78900000000000003,"15min":1.552},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4139999999999999,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.94499999999999995,"15min":1.925},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.4929999999999999,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.52200000000000002}]}]},{"osd":0,"up_from":353,"seq":1516123455547,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":686508,"kb_used_data":493348,"kb_used_omap":37,"kb_used_meta":193114,"kb_avail":93058644,"statfs":{"total":95995035648,"available":95292051456,"internally_reserved":0,"allocated":505188352,"data_stored":503487630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":197748898},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.71599999999999997,"15min":0.59799999999999998},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":0.501},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.82999999999999996,"15min":0.72399999999999998},"min":{"1min":0.27800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4459999999999997,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.96899999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.83399999999999996,"15min":0.79800000000000004},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.694,"5min":12.215,"15min":12.215},"last":2.4199999999999999},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":5.9199999999999999},"last":1.9159999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.88,"15min":0.59599999999999997},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.88500000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.8170000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2229999999999999,"15min":1.647},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2349999999999999,"5min":183.148,"15min":183.148},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.466,"5min":3.3530000000000002,"15min":1.5629999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1609999999999996,"5min":183.39500000000001,"15min":183.39500000000001},"last":1.032}]},{"osd":5,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.98499999999999999,"15min":0.78100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.984,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.433},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96199999999999997,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.1989999999999998,"5min":7.774,"15min":7.774},"last":0.86899999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.0509999999999999,"15min":0.90700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.359,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.91800000000000004,"15min":0.78800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3090000000000002,"5min":6.375,"15min":6.375},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.179,"15min":1.2609999999999999},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3929999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.147,"5min":1.1890000000000001,"15min":1.2490000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.669,"5min":6.343,"15min":6.343},"last":0.54200000000000004}]}]},{"osd":2,"up_from":359,"seq":1541893259320,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":651484,"kb_used_data":510164,"kb_used_omap":83,"kb_used_meta":141228,"kb_avail":93093668,"statfs":{"total":95995035648,"available":95327916032,"internally_reserved":0,"allocated":522407936,"data_stored":520783241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":144617925},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.2749999999999999,"15min":0.66100000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":74.241,"5min":74.241,"15min":74.241},"last":0.622},{"interface":"front","average":{"1min":4.6699999999999999,"5min":1.4590000000000001,"15min":0.95799999999999996},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":74.430000000000007,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.33000000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.68500000000000005,"15min":0.67200000000000004},"min":{"1min":0.26800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9710000000000001,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.438},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.083,"15min":1.131},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.70299999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":0.81599999999999995,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.9530000000000003,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.372},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.94399999999999995,"15min":0.85899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8170000000000002,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.90000000000000002,"15min":0.67600000000000005},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.5359999999999996,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.96099999999999997,"15min":0.86899999999999999},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.8739999999999997,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.0580000000000001,"15min":0.98899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.232,"5min":0.93999999999999995,"15min":0.76700000000000002},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.24,"15min":1.0600000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9359999999999999,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.496},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91700000000000004,"15min":0.70999999999999996},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.903,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1870000000000001,"15min":1.056},"min":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.5659999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.093,"15min":1.1499999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.9139999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.58999999999999997}]}]},{"osd":3,"up_from":94,"seq":403726926124,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":521096,"kb_used_data":10688,"kb_used_omap":7,"kb_used_meta":510392,"kb_avail":93224056,"statfs":{"total":95995035648,"available":95461433344,"internally_reserved":0,"allocated":10944512,"data_stored":9333178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":522642024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:15 2023","interfaces":[{"interface":"back","average":{"1min":3.383,"5min":2.177,"15min":0.93799999999999994},"min":{"1min":0.191,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":37.249000000000002,"5min":122.289,"15min":122.289},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.919,"15min":0.91100000000000003},"min":{"1min":0.19800000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":35.991,"5min":122.041,"15min":122.041},"last":0.31}]},{"osd":1,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.87,"15min":2.2480000000000002},"min":{"1min":0.22,"5min":0.187,"15min":0.187},"max":{"1min":4.6100000000000003,"5min":8.6280000000000001,"15min":123.866},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.2929999999999999,"5min":0.90200000000000002,"15min":2.1539999999999999},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1710000000000003,"5min":6.8010000000000002,"15min":122.265},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":0.71499999999999997},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.28,"5min":1.097,"15min":0.86099999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.7919999999999998,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.29799999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.1970000000000001,"15min":1.966},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.6459999999999999,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.222,"5min":2.3119999999999998,"15min":1.9199999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2149999999999999,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.36899999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.98699999999999999,"15min":1.403},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2530000000000001,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.163,"5min":1.0640000000000001,"15min":1.744},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":5.149,"5min":10.246,"15min":11.951000000000001},"last":0.41499999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.264,"15min":1.748},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":6.2830000000000004,"5min":9.2520000000000007,"15min":14.097},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.091,"5min":1.181,"15min":1.869},"min":{"1min":0.378,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":5.1970000000000001,"5min":9.2810000000000006,"15min":12.887},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95499999999999996,"15min":1.6850000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.119,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.6799999999999997,"5min":8.5830000000000002,"15min":12.32},"last":0.40300000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019321,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514296,"kb_used_data":610928,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92230856,"statfs":{"total":95995035648,"available":94444396544,"internally_reserved":0,"allocated":625590272,"data_stored":624744363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":10.303000000000001,"5min":2.7650000000000001,"15min":1.236},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.677000000000007,"5min":71.677000000000007,"15min":71.677000000000007},"last":0.36599999999999999},{"interface":"front","average":{"1min":11.896000000000001,"5min":2.9740000000000002,"15min":1.3879999999999999},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":72.147999999999996,"5min":72.147999999999996,"15min":72.147999999999996},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.117,"15min":1.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8380000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.433},{"interface":"front","average":{"1min":1.653,"5min":1.0389999999999999,"15min":0.82499999999999996},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":1.871}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.90400000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:02:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.169,"15min":1.7589999999999999},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.9870000000000001,"5min":12.961,"15min":148.61600000000001},"last":0.316},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.92300000000000004,"15min":1.829},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":8.7810000000000006,"5min":8.7810000000000006,"15min":151.905},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.39800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.429,"15min":1.2549999999999999},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1050000000000004,"5min":12.84,"15min":12.84},"last":0.44},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2450000000000001,"15min":1.1240000000000001},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2210000000000001,"5min":10.587,"15min":10.587},"last":0.50800000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823095,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1239000,"kb_used_data":347536,"kb_used_omap":0,"kb_used_meta":891456,"kb_avail":92506152,"statfs":{"total":95995035648,"available":94726299648,"internally_reserved":0,"allocated":355876864,"data_stored":355058743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912850944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.5229999999999999,"15min":0.91100000000000003},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":42.155000000000001,"5min":42.155000000000001,"15min":42.155000000000001},"last":2.5},{"interface":"front","average":{"1min":3.7839999999999998,"5min":1.784,"15min":1.3160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":43.378999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.76500000000000001,"15min":2.1429999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.9649999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.202,"15min":2.6389999999999998},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":2.3079999999999998,"5min":10.353,"15min":110.89},"last":0.42899999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.997,"15min":0.86599999999999999},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.601,"5min":6.601,"15min":6.601},"last":2.5939999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.997,"15min":0.995},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6139999999999999,"5min":4.899,"15min":4.899},"last":1.3500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0600000000000001,"15min":3.6230000000000002},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0899999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":2.5640000000000001},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.2090000000000001,"15min":3.6680000000000001},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3420000000000001,"5min":10.965,"15min":108.15600000000001},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":1.298,"15min":1.552},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":1.321,"5min":10.603,"15min":115.06699999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92300000000000004,"15min":1.482},"min":{"1min":0.34599999999999997,"5min":0.253,"15min":0.253},"max":{"1min":1.764,"5min":9.5890000000000004,"15min":114.908},"last":2.5419999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.169,"15min":1.256},"min":{"1min":0.30499999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.77300000000000002,"5min":7.4630000000000001,"15min":12.615},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.90400000000000003,"15min":1.1100000000000001},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.0569999999999999,"5min":5.4909999999999997,"15min":12.74},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1259999999999999,"15min":1.0900000000000001},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7830000000000004,"5min":9.5980000000000008,"15min":12.23},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.538,"15min":1.22},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.31},"max":{"1min":2.4580000000000002,"5min":10.638999999999999,"15min":11.112},"last":0.51000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":485363712,"data_stored":485359640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":334823424,"data_stored":334823424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":517079040,"data_stored":517079040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":6160384,"data_stored":6160384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":619515904,"data_stored":619511832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":342228992,"data_stored":342228992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1175851008,"data_stored":1175846936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:03:00.820 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:00.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:00 smithi151 ceph-mon[159210]: from='client.56608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 14743/31845 objects degraded (46.296%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:00.959 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:00.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:00.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:00 smithi052 ceph-mon[187743]: from='client.56608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:00.963 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 14743/31845 objects degraded (46.296%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:00.964 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:00 smithi052 ceph-mon[180979]: from='client.56608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:00.964 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 14743/31845 objects degraded (46.296%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:01.036 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:03:01.049 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:01.064 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:01.107 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:01.268 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:01.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:01.374 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:01.430 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:01.520 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:01.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:01.609 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:03:01.609 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:03:01.635 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:01.678 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:01.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:01 smithi052 ceph-mon[180979]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:01.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:01 smithi052 ceph-mon[187743]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:02.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:01 smithi151 ceph-mon[159210]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:02.916 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:02.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:03.021 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:03.034 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:03:03.054 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:03.109 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:03.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:03.217 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:03.257 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:03.338 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:03.400 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:03.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:03.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:03:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:03:03] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T23:03:03.515 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:03.548 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:03.604 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:03.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:03.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:03.712 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:03.744 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:03:03.792 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:03.944 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:03.952 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:03 smithi151 ceph-mon[159210]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:03.978 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:03.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:03.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:03 smithi052 ceph-mon[180979]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:03.986 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:03 smithi052 ceph-mon[187743]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:04.086 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:04.153 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:04.673 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:04.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:04.780 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:04.824 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:03:04.849 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:04.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/681633269' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/681633269' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:04.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:05.071 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/681633269' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:04 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: pgmap v1238: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: failed to encode map e403 with expected crc 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: osdmap e403: 8 total, 8 up, 6 in 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: failed to encode map e403 with expected crc 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: failed to encode map e403 with expected crc 2023-09-23T23:03:05.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 14310/26340 objects degraded (54.328%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: pgmap v1238: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: failed to encode map e403 with expected crc 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: osdmap e403: 8 total, 8 up, 6 in 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: failed to encode map e403 with expected crc 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: failed to encode map e403 with expected crc 2023-09-23T23:03:05.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 14310/26340 objects degraded (54.328%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:06.067 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:06.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:06.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:06.172 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: pgmap v1238: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 548 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 341 op/s; 14310/26340 objects degraded (54.328%); 24534/26340 objects misplaced (93.144%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: failed to encode map e403 with expected crc 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: osdmap e403: 8 total, 8 up, 6 in 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: failed to encode map e403 with expected crc 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: failed to encode map e403 with expected crc 2023-09-23T23:03:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 14310/26340 objects degraded (54.328%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:06.208 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:06.275 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:06.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:06.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:06.424 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:06.492 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:06.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:06.601 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:06.639 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:03:06.650 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:06.668 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:03:06.704 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:03:06.734 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:03:06.761 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:03:06.786 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:03:06.810 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2344009934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:06.814 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:03:06.847 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:03:06.856 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.857 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:03:06.879 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2344009934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:06.879 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2344009934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:07.404 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:07.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:07.508 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:07.570 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:07.996 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:08.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:08.080 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:03:08.105 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:08.132 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:07 smithi151 ceph-mon[159210]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 429 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 378 op/s; 13849/20649 objects degraded (67.069%); 24044/20649 objects misplaced (116.441%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T23:03:08.134 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:07 smithi052 ceph-mon[180979]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 429 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 378 op/s; 13849/20649 objects degraded (67.069%); 24044/20649 objects misplaced (116.441%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T23:03:08.135 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:07 smithi052 ceph-mon[187743]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 429 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 378 op/s; 13849/20649 objects degraded (67.069%); 24044/20649 objects misplaced (116.441%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T23:03:08.136 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:08.203 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:08.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:08.217 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:03:08.217 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:03:08.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:03:08.217 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119492 10.9 e372/360 10:9e63863c:::benchmark_data_smithi052_461816_object22691:head v 372'7491, mlcod=326'3298)", 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:00.614037+0000", 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 247.59742922300001, 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036220987000000003, 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614037+0000", 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614037+0000", 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614036+0000", 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:08.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614041+0000", 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0259999999999998e-06 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614048+0000", 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6710000000000002e-06 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.614058+0000", 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9340000000000001e-06 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.615825+0000", 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017668009999999999 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.615843+0000", 2023-09-23T23:03:08.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.872e-05 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650230+0000", 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.034386247000000002 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650258+0000", 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8113999999999999e-05 2023-09-23T23:03:08.222 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119487 10.e e372/360 10:76db422f:::benchmark_data_smithi052_461816_object22686:head v 372'7460, mlcod=0'0)", 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:00.618065+0000", 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 247.593400954, 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.037691571, 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618065+0000", 2023-09-23T23:03:08.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618065+0000", 2023-09-23T23:03:08.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618050+0000", 2023-09-23T23:03:08.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999852 2023-09-23T23:03:08.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618077+0000", 2023-09-23T23:03:08.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6390999999999999e-05 2023-09-23T23:03:08.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618091+0000", 2023-09-23T23:03:08.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4443e-05 2023-09-23T23:03:08.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618099+0000", 2023-09-23T23:03:08.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9920000000000007e-06 2023-09-23T23:03:08.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.618149+0000", 2023-09-23T23:03:08.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9818000000000003e-05 2023-09-23T23:03:08.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650207+0000", 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.032058193999999998 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.655710+0000", 2023-09-23T23:03:08.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055033679999999998 2023-09-23T23:03:08.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.655757+0000", 2023-09-23T23:03:08.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6239000000000002e-05 2023-09-23T23:03:08.232 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119496 10.f e372/360 10:f4ed15ba:::benchmark_data_smithi052_461816_object22695:head v 372'7463, mlcod=323'2484)", 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:00.625660+0000", 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 247.585805795, 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.035401146000000001, 2023-09-23T23:03:08.233 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625660+0000", 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625660+0000", 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625659+0000", 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.625672+0000", 2023-09-23T23:03:08.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.346e-05 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650169+0000", 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.024496401000000001 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650184+0000", 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5075000000000001e-05 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650234+0000", 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0618000000000002e-05 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.650259+0000", 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4414e-05 2023-09-23T23:03:08.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.661021+0000", 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010762177 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:00.661061+0000", 2023-09-23T23:03:08.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0342000000000001e-05 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119633 10.e e373/360 10:72cd2dfa:::benchmark_data_smithi052_461816_object22832:head v 373'7472, mlcod=0'0)", 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.278330+0000", 2023-09-23T23:03:08.239 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 246.93313619599999, 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10197167, 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278330+0000", 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278330+0000", 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278328+0000", 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278343+0000", 2023-09-23T23:03:08.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.449e-05 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.278364+0000", 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1593000000000001e-05 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.374216+0000", 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095851378000000001 2023-09-23T23:03:08.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.374273+0000", 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7846000000000001e-05 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.374310+0000", 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7002000000000002e-05 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.380258+0000", 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059473010000000003 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.380302+0000", 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3859999999999997e-05 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(10.9 373/360 [PushOp(10:9072a8b6:::benchmark_data_smithi052_461816_object53040:head, version: 326'3315, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:9072a8b6:::benchmark_data_smithi052_461816_object53040:head@326'3315, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 246.90639781499999, 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070204256000000007, 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:03:08.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305068+0000", 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:08.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.305148+0000", 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0175999999999999e-05 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371739+0000", 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.066591289999999997 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371773+0000", 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3893000000000002e-05 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371871+0000", 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8084000000000005e-05 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371884+0000", 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3312999999999999e-05 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.375273+0000", 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033882019999999999 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141425 10.e e385/360 10:7f34fb6d:::benchmark_data_smithi052_461816_object44624:head v 385'8905, mlcod=0'0)", 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.704109+0000", 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.50735667199999, 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071707641000000003, 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704109+0000", 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704109+0000", 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704108+0000", 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704116+0000", 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8259999999999993e-06 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704129+0000", 2023-09-23T23:03:08.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2669e-05 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704138+0000", 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6289999999999999e-06 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704193+0000", 2023-09-23T23:03:08.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5735000000000001e-05 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.704225+0000", 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1414000000000001e-05 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775742+0000", 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071517739999999996 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775817+0000", 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4647999999999994e-05 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141427 10.3 e385/359 10:c1b733eb:::benchmark_data_smithi052_461816_object44626:head v 385'8926, mlcod=323'2482)", 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.710101+0000", 2023-09-23T23:03:08.255 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.5013649, 2023-09-23T23:03:08.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.071061101000000002, 2023-09-23T23:03:08.256 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.256 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.256 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710101+0000", 2023-09-23T23:03:08.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.258 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710101+0000", 2023-09-23T23:03:08.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.258 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.258 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.258 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710100+0000", 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.710108+0000", 2023-09-23T23:03:08.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2479999999999996e-06 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775731+0000", 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065622335000000004 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775744+0000", 2023-09-23T23:03:08.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3657e-05 2023-09-23T23:03:08.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775807+0000", 2023-09-23T23:03:08.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2572999999999999e-05 2023-09-23T23:03:08.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775836+0000", 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8844999999999999e-05 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.781134+0000", 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052976380000000003 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.781162+0000", 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8767000000000001e-05 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(10.9 385/360 [PushOp(10:94cabeba:::benchmark_data_smithi052_461816_object90470:head, version: 335'5644, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:94cabeba:::benchmark_data_smithi052_461816_object90470:head@335'5644, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.736654+0000", 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 146.47481224399999, 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.042611151, 2023-09-23T23:03:08.264 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736654+0000", 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736654+0000", 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.265 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736652+0000", 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.736725+0000", 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3126000000000004e-05 2023-09-23T23:03:08.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775722+0000", 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.038996958999999998 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775747+0000", 2023-09-23T23:03:08.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4967e-05 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775836+0000", 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8986000000000005e-05 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.775850+0000", 2023-09-23T23:03:08.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4197e-05 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.779265+0000", 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003415169 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:145924 10.7 e388/360 10:eaa10b09:::benchmark_data_smithi052_461816_object49123:head v 388'8980, mlcod=323'2408)", 2023-09-23T23:03:08.269 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.178877+0000", 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 129.03258929399999, 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.044936373000000002, 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178877+0000", 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178877+0000", 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178875+0000", 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178884+0000", 2023-09-23T23:03:08.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0469999999999999e-06 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178896+0000", 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1557e-05 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178905+0000", 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5900000000000008e-06 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178950+0000", 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5021999999999997e-05 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.178970+0000", 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0276e-05 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223766+0000", 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.044796509999999998 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223813+0000", 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6841000000000001e-05 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.274 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGRecoveryDelete(10.9 e388,360 10:96ac8436:::benchmark_data_smithi052_461816_object202:head,336'6050)", 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.182567+0000", 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 129.028898672, 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.037664665, 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182567+0000", 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182567+0000", 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182566+0000", 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.182573+0000", 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0400000000000004e-06 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220077+0000", 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.037503309999999998 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220094+0000", 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6864999999999999e-05 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220154+0000", 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0449999999999999e-05 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220167+0000", 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3030000000000001e-05 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.220232+0000", 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4976999999999998e-05 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.279 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:145930 10.f e388/360 10:fcfdf602:::benchmark_data_smithi052_461816_object49129:head v 388'9116, mlcod=323'2484)", 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.184108+0000", 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 129.02735803499999, 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.039735528999999999, 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.280 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184108+0000", 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184108+0000", 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184107+0000", 2023-09-23T23:03:08.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.184120+0000", 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2561000000000001e-05 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217204+0000", 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.033084163 2023-09-23T23:03:08.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217214+0000", 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0376e-05 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217253+0000", 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9258999999999998e-05 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.217283+0000", 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9300000000000001e-05 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223816+0000", 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065333190000000001 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.223844+0000", 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7546e-05 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:145931 10.a e388/359 10:5f2e1c5d:::benchmark_data_smithi052_461816_object49130:head v 388'9151, mlcod=388'9149)", 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:59.186834+0000", 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 129.02463253499999, 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.031626363999999997, 2023-09-23T23:03:08.285 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186834+0000", 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186834+0000", 2023-09-23T23:03:08.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186833+0000", 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.186844+0000", 2023-09-23T23:03:08.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1157e-05 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216039+0000", 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.029194931 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216064+0000", 2023-09-23T23:03:08.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5401999999999998e-05 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216197+0000", 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013290700000000001 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.216219+0000", 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2169000000000001e-05 2023-09-23T23:03:08.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.218413+0000", 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021937879999999999 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:59.218460+0000", 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7030999999999997e-05 2023-09-23T23:03:08.290 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151030 10.f e389/360 10:fe8e1dd7:::benchmark_data_smithi052_461816_object54229:head v 389'9440, mlcod=323'2484)", 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 112.073676717, 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.034341280000000002, 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.291 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.292 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.292 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.292 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:03:08.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137789+0000", 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137794+0000", 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9699999999999998e-06 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137800+0000", 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5379999999999999e-06 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137806+0000", 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7939999999999999e-06 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137854+0000", 2023-09-23T23:03:08.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8021e-05 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.137879+0000", 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4388000000000001e-05 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.172083+0000", 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.034204076999999999 2023-09-23T23:03:08.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.172131+0000", 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7913e-05 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151611 10.8 e389/360 10:1dcea51c:::benchmark_data_smithi052_461816_object54810:head v 389'9542, mlcod=323'2535)", 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.700805+0000", 2023-09-23T23:03:08.297 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.510661343, 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.114821174, 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700805+0000", 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700805+0000", 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700804+0000", 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700812+0000", 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0770000000000008e-06 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700824+0000", 2023-09-23T23:03:08.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2359e-05 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700832+0000", 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0679999999999997e-06 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700898+0000", 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5370999999999995e-05 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.700922+0000", 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4831e-05 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815559+0000", 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.114636196 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815626+0000", 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7288999999999999e-05 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(10.e 389/360 [PushOp(10:72b871c2:::benchmark_data_smithi052_461816_object54811:head, version: 326'3497, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:72b871c2:::benchmark_data_smithi052_461816_object54811:head@326'3497, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.705015+0000", 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.50645107699999, 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11028885500000001, 2023-09-23T23:03:08.303 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705015+0000", 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705015+0000", 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705014+0000", 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705086+0000", 2023-09-23T23:03:08.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1656000000000006e-05 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705104+0000", 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8647999999999998e-05 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705122+0000", 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8003999999999999e-05 2023-09-23T23:03:08.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705204+0000", 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1448999999999999e-05 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.705219+0000", 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4899000000000001e-05 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.307 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815304+0000", 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.110085129 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151615 10.8 e389/360 10:1b606561:::benchmark_data_smithi052_461816_object54814:head v 389'9543, mlcod=323'2535)", 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.710146+0000", 2023-09-23T23:03:08.308 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.501320229, 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11269485899999999, 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.710146+0000", 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.309 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.710146+0000", 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.710146+0000", 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.310 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.710153+0000", 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7589999999999997e-06 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815382+0000", 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.105228537 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815393+0000", 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.152e-05 2023-09-23T23:03:08.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815770+0000", 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00037691200000000003 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815792+0000", 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.198e-05 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.822792+0000", 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0069996820000000001 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.822841+0000", 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8745999999999997e-05 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.313 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGRecoveryDelete(10.9 e390,360 10:98811a53:::benchmark_data_smithi052_461816_object7124:head,340'6481)", 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.712269+0000", 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.499197573, 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.105547619, 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.712269+0000", 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.314 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.712269+0000", 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.712268+0000", 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.712276+0000", 2023-09-23T23:03:08.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8059999999999995e-06 2023-09-23T23:03:08.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.317 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815393+0000", 2023-09-23T23:03:08.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10311759299999999 2023-09-23T23:03:08.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.317 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.317 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815406+0000", 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2601e-05 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.817732+0000", 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023265489999999998 2023-09-23T23:03:08.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.817742+0000", 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8290000000000003e-06 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.319 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.817816+0000", 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4017999999999995e-05 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(10.e 390/360 [PushOp(10:77cd3cdd:::benchmark_data_smithi052_461816_object54815:head, version: 326'3498, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:77cd3cdd:::benchmark_data_smithi052_461816_object54815:head@326'3498, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:03:08.320 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.728190+0000", 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.48327606300001, 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.090222947999999997, 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.728190+0000", 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.728190+0000", 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.728188+0000", 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.728252+0000", 2023-09-23T23:03:08.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4295999999999996e-05 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815421+0000", 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.087168994999999999 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815442+0000", 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0384e-05 2023-09-23T23:03:08.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815776+0000", 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00033379299999999998 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815789+0000", 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3152999999999999e-05 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.818413+0000", 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026242010000000001 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:182712 10.f e401/360 10:f7328e40:::benchmark_data_smithi052_461816_object85911:head v 401'11424, mlcod=323'2484)", 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:54.607238+0000", 2023-09-23T23:03:08.326 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 13.604228154999999, 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.031191648999999998, 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.607238+0000", 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.607238+0000", 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.607237+0000", 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.328 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.607245+0000", 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0530000000000007e-06 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.607257+0000", 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2130000000000001e-05 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.607265+0000", 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5349999999999999e-06 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.611584+0000", 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043188640000000004 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.611610+0000", 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5789e-05 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.638391+0000", 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.026781066999999999 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:54.638430+0000", 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8951e-05 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:187825 10.7 e403/360 10:eda2c2df:::benchmark_data_smithi052_461816_object91024:head v 403'11570, mlcod=323'2408)", 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:07.837643+0000", 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.37382285599999998, 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.028976321999999999, 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837643+0000", 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.332 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837643+0000", 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837642+0000", 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837655+0000", 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.29e-05 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837674+0000", 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8848999999999999e-05 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837684+0000", 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0239e-05 2023-09-23T23:03:08.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837798+0000", 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011451900000000001 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.837824+0000", 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5344999999999999e-05 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:08.336 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.866570+0000", 2023-09-23T23:03:08.336 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.028746412999999998 2023-09-23T23:03:08.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:08.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:08.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:08.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.866620+0000", 2023-09-23T23:03:08.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9370000000000003e-05 2023-09-23T23:03:08.337 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.338 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.338 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.338 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:08.338 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:08.338 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:03:08.338 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:08.368 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:08.427 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:08.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:08.536 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:08.573 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:08.625 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:08.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:08.732 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:08.770 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:08.808 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:08.818 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:03:09.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:03:09.661 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:09.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:09.737 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1241,"stamp":"2023-09-23T23:03:08.630800+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":450041784,"num_objects":6883,"num_object_clones":0,"num_object_copies":20649,"num_objects_missing_on_primary":1532,"num_objects_missing":1532,"num_objects_degraded":13849,"num_objects_misplaced":24044,"num_objects_unfound":0,"num_objects_dirty":6883,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":186999,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20466,"num_bytes_recovered":1247820732,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100733,"ondisk_log_size":100733,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5155668,"kb_used_data":1734884,"kb_used_omap":137,"kb_used_meta":3420598,"kb_avail":557315244,"statfs":{"total":575970213888,"available":570690809856,"internally_reserved":0,"allocated":1776521216,"data_stored":1768341120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3502692536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-244580352,"num_objects":-3732,"num_object_clones":0,"num_object_copies":-11196,"num_objects_missing_on_primary":-225,"num_objects_missing":-225,"num_objects_degraded":-894,"num_objects_misplaced":-940,"num_objects_unfound":0,"num_objects_dirty":-3732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3732,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":228,"num_bytes_recovered":14942208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003547"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":415,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781290+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:02:56.781290+0000","last_peered":"2023-09-23T23:02:56.781290+0000","last_clean":"2023-09-23T23:02:56.781290+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:02:56.781290+0000","last_undegraded":"2023-09-23T23:02:56.781290+0000","last_fullsized":"2023-09-23T23:02:56.781290+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.776690+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:02:56.776690+0000","last_peered":"2023-09-23T23:02:56.776690+0000","last_clean":"2023-09-23T23:02:56.776690+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:02:56.776690+0000","last_undegraded":"2023-09-23T23:02:56.776690+0000","last_fullsized":"2023-09-23T23:02:56.776690+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":72,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774609+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:02:56.774609+0000","last_peered":"2023-09-23T23:02:56.774609+0000","last_clean":"2023-09-23T23:02:56.774609+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:02:56.774609+0000","last_undegraded":"2023-09-23T23:02:56.774609+0000","last_fullsized":"2023-09-23T23:02:56.774609+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":400,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774935+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:02:56.774935+0000","last_peered":"2023-09-23T23:02:56.774935+0000","last_clean":"2023-09-23T23:02:56.774935+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:02:56.774935+0000","last_undegraded":"2023-09-23T23:02:56.774935+0000","last_fullsized":"2023-09-23T23:02:56.774935+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774443+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:02:56.774443+0000","last_peered":"2023-09-23T23:02:56.774443+0000","last_clean":"2023-09-23T23:02:56.774443+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:02:56.774443+0000","last_undegraded":"2023-09-23T23:02:56.774443+0000","last_fullsized":"2023-09-23T23:02:56.774443+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":74,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768595+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:02:56.768595+0000","last_peered":"2023-09-23T23:02:56.768595+0000","last_clean":"2023-09-23T23:02:56.768595+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:02:56.768595+0000","last_undegraded":"2023-09-23T23:02:56.768595+0000","last_fullsized":"2023-09-23T23:02:56.768595+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":338,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.769874+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:02:56.769874+0000","last_peered":"2023-09-23T23:02:56.769874+0000","last_clean":"2023-09-23T23:02:56.769874+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:02:56.769874+0000","last_undegraded":"2023-09-23T23:02:56.769874+0000","last_fullsized":"2023-09-23T23:02:56.769874+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":82,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774305+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:02:56.774305+0000","last_peered":"2023-09-23T23:02:56.774305+0000","last_clean":"2023-09-23T23:02:56.774305+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:02:56.774305+0000","last_undegraded":"2023-09-23T23:02:56.774305+0000","last_fullsized":"2023-09-23T23:02:56.774305+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":299,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768833+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:02:56.768833+0000","last_peered":"2023-09-23T23:02:56.768833+0000","last_clean":"2023-09-23T23:02:56.768833+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:02:56.768833+0000","last_undegraded":"2023-09-23T23:02:56.768833+0000","last_fullsized":"2023-09-23T23:02:56.768833+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":87,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774476+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:02:56.774476+0000","last_peered":"2023-09-23T23:02:56.774476+0000","last_clean":"2023-09-23T23:02:56.774476+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:02:56.774476+0000","last_undegraded":"2023-09-23T23:02:56.774476+0000","last_fullsized":"2023-09-23T23:02:56.774476+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":93,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.776604+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:02:56.776604+0000","last_peered":"2023-09-23T23:02:56.776604+0000","last_clean":"2023-09-23T23:02:56.776604+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:02:56.776604+0000","last_undegraded":"2023-09-23T23:02:56.776604+0000","last_fullsized":"2023-09-23T23:02:56.776604+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":149,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808020+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:03:04.808020+0000","last_peered":"2023-09-23T23:03:04.808020+0000","last_clean":"2023-09-23T23:03:04.808020+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:03:04.808020+0000","last_undegraded":"2023-09-23T23:03:04.808020+0000","last_fullsized":"2023-09-23T23:03:04.808020+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":474,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781128+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:02:56.781128+0000","last_peered":"2023-09-23T23:02:56.781128+0000","last_clean":"2023-09-23T23:02:56.781128+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:02:56.781128+0000","last_undegraded":"2023-09-23T23:02:56.781128+0000","last_fullsized":"2023-09-23T23:02:56.781128+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":377,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.783996+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:02:56.783996+0000","last_peered":"2023-09-23T23:02:56.783996+0000","last_clean":"2023-09-23T23:02:56.783996+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:02:56.783996+0000","last_undegraded":"2023-09-23T23:02:56.783996+0000","last_fullsized":"2023-09-23T23:02:56.783996+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":303,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.784061+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:02:56.784061+0000","last_peered":"2023-09-23T23:02:56.784061+0000","last_clean":"2023-09-23T23:02:56.784061+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:02:56.784061+0000","last_undegraded":"2023-09-23T23:02:56.784061+0000","last_fullsized":"2023-09-23T23:02:56.784061+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":84,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.776399+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:02:56.776399+0000","last_peered":"2023-09-23T23:02:56.776399+0000","last_clean":"2023-09-23T23:02:56.776399+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:02:56.776399+0000","last_undegraded":"2023-09-23T23:02:56.776399+0000","last_fullsized":"2023-09-23T23:02:56.776399+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":393,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774627+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:02:56.774627+0000","last_peered":"2023-09-23T23:02:56.774627+0000","last_clean":"2023-09-23T23:02:56.774627+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:02:56.774627+0000","last_undegraded":"2023-09-23T23:02:56.774627+0000","last_fullsized":"2023-09-23T23:02:56.774627+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":303,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768206+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:02:56.768206+0000","last_peered":"2023-09-23T23:02:56.768206+0000","last_clean":"2023-09-23T23:02:56.768206+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:02:56.768206+0000","last_undegraded":"2023-09-23T23:02:56.768206+0000","last_fullsized":"2023-09-23T23:02:56.768206+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":288,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781209+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:02:56.781209+0000","last_peered":"2023-09-23T23:02:56.781209+0000","last_clean":"2023-09-23T23:02:56.781209+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:02:56.781209+0000","last_undegraded":"2023-09-23T23:02:56.781209+0000","last_fullsized":"2023-09-23T23:02:56.781209+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":417,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.775038+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:02:56.775038+0000","last_peered":"2023-09-23T23:02:56.775038+0000","last_clean":"2023-09-23T23:02:56.775038+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:02:56.775038+0000","last_undegraded":"2023-09-23T23:02:56.775038+0000","last_fullsized":"2023-09-23T23:02:56.775038+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":122,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768000+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:02:56.768000+0000","last_peered":"2023-09-23T23:02:56.768000+0000","last_clean":"2023-09-23T23:02:56.768000+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:02:56.768000+0000","last_undegraded":"2023-09-23T23:02:56.768000+0000","last_fullsized":"2023-09-23T23:02:56.768000+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":97,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.777136+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:02:56.777136+0000","last_peered":"2023-09-23T23:02:56.777136+0000","last_clean":"2023-09-23T23:02:56.777136+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:02:56.777136+0000","last_undegraded":"2023-09-23T23:02:56.777136+0000","last_fullsized":"2023-09-23T23:02:56.777136+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"403'11677","reported_seq":11785,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.857806+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:03:04.857806+0000","last_peered":"2023-09-23T23:03:04.857806+0000","last_clean":"2023-09-23T23:03:04.857806+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:03:04.857806+0000","last_undegraded":"2023-09-23T23:03:04.857806+0000","last_fullsized":"2023-09-23T23:03:04.857806+0000","mapping_epoch":353,"log_start":"339'6200","ondisk_log_start":"339'6200","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5477,"log_dups_size":0,"ondisk_log_size":5477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26804224,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11677,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":287,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.782661+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:02:56.782661+0000","last_peered":"2023-09-23T23:02:56.782661+0000","last_clean":"2023-09-23T23:02:56.782661+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:02:56.782661+0000","last_undegraded":"2023-09-23T23:02:56.782661+0000","last_fullsized":"2023-09-23T23:02:56.782661+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":273,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.767771+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:02:56.767771+0000","last_peered":"2023-09-23T23:02:56.767771+0000","last_clean":"2023-09-23T23:02:56.767771+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:02:56.767771+0000","last_undegraded":"2023-09-23T23:02:56.767771+0000","last_fullsized":"2023-09-23T23:02:56.767771+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"403'11505","reported_seq":13941,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:05.277639+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:03:05.277639+0000","last_peered":"2023-09-23T23:03:05.277639+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:03:05.277639+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"325'2900","ondisk_log_start":"325'2900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29163520,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":445,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11505,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":445},{"shards":"5,6","objects":-3122}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":297,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781245+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:02:56.781245+0000","last_peered":"2023-09-23T23:02:56.781245+0000","last_clean":"2023-09-23T23:02:56.781245+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:02:56.781245+0000","last_undegraded":"2023-09-23T23:02:56.781245+0000","last_fullsized":"2023-09-23T23:02:56.781245+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":410,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781111+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:02:56.781111+0000","last_peered":"2023-09-23T23:02:56.781111+0000","last_clean":"2023-09-23T23:02:56.781111+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:02:56.781111+0000","last_undegraded":"2023-09-23T23:02:56.781111+0000","last_fullsized":"2023-09-23T23:02:56.781111+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"403'11628","reported_seq":11730,"reported_epoch":403,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:03:04.938739+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:03:04.938739+0000","last_peered":"2023-09-23T23:03:04.938739+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:03:04.938739+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:03:04.938739+0000","mapping_epoch":354,"log_start":"338'6100","ondisk_log_start":"338'6100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5528,"log_dups_size":0,"ondisk_log_size":5528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":444,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11628,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":444}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":154,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768299+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:02:56.768299+0000","last_peered":"2023-09-23T23:02:56.768299+0000","last_clean":"2023-09-23T23:02:56.768299+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:02:56.768299+0000","last_undegraded":"2023-09-23T23:02:56.768299+0000","last_fullsized":"2023-09-23T23:02:56.768299+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":272,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.769571+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:02:56.769571+0000","last_peered":"2023-09-23T23:02:56.769571+0000","last_clean":"2023-09-23T23:02:56.769571+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:02:56.769571+0000","last_undegraded":"2023-09-23T23:02:56.769571+0000","last_fullsized":"2023-09-23T23:02:56.769571+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"403'11715","reported_seq":12003,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:04.953357+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:03:04.953357+0000","last_peered":"2023-09-23T23:03:04.953357+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:03:04.953357+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"339'6200","ondisk_log_start":"339'6200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30081024,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":395,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11715,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":64},{"shards":"0,5","objects":-3151}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.776316+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:02:56.776316+0000","last_peered":"2023-09-23T23:02:56.776316+0000","last_clean":"2023-09-23T23:02:56.776316+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:02:56.776316+0000","last_undegraded":"2023-09-23T23:02:56.776316+0000","last_fullsized":"2023-09-23T23:02:56.776316+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":303,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768879+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:02:56.768879+0000","last_peered":"2023-09-23T23:02:56.768879+0000","last_clean":"2023-09-23T23:02:56.768879+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:02:56.768879+0000","last_undegraded":"2023-09-23T23:02:56.768879+0000","last_fullsized":"2023-09-23T23:02:56.768879+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"403'11674","reported_seq":11782,"reported_epoch":403,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:05.243521+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:03:05.243521+0000","last_peered":"2023-09-23T23:03:05.243521+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:03:05.243521+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:03:05.243521+0000","mapping_epoch":353,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27525120,"num_objects":420,"num_object_clones":0,"num_object_copies":1260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11674,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":420}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":92,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.775648+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:02:56.775648+0000","last_peered":"2023-09-23T23:02:56.775648+0000","last_clean":"2023-09-23T23:02:56.775648+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:02:56.775648+0000","last_undegraded":"2023-09-23T23:02:56.775648+0000","last_fullsized":"2023-09-23T23:02:56.775648+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":396,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781074+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:02:56.781074+0000","last_peered":"2023-09-23T23:02:56.781074+0000","last_clean":"2023-09-23T23:02:56.781074+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:02:56.781074+0000","last_undegraded":"2023-09-23T23:02:56.781074+0000","last_fullsized":"2023-09-23T23:02:56.781074+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"403'11759","reported_seq":11875,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:05.249535+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:03:05.249535+0000","last_peered":"2023-09-23T23:03:05.249535+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:03:05.249535+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29032448,"num_objects":443,"num_object_clones":0,"num_object_copies":1329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":443,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11759,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-3176}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":186,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781414+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:02:56.781414+0000","last_peered":"2023-09-23T23:02:56.781414+0000","last_clean":"2023-09-23T23:02:56.781414+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:02:56.781414+0000","last_undegraded":"2023-09-23T23:02:56.781414+0000","last_fullsized":"2023-09-23T23:02:56.781414+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":413,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781215+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:02:56.781215+0000","last_peered":"2023-09-23T23:02:56.781215+0000","last_clean":"2023-09-23T23:02:56.781215+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:02:56.781215+0000","last_undegraded":"2023-09-23T23:02:56.781215+0000","last_fullsized":"2023-09-23T23:02:56.781215+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"403'11618","reported_seq":20460,"reported_epoch":403,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:05.296617+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:03:05.296617+0000","last_peered":"2023-09-23T23:03:05.296617+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:03:05.296617+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:03:05.296617+0000","mapping_epoch":354,"log_start":"336'5900","ondisk_log_start":"336'5900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28966912,"num_objects":442,"num_object_clones":0,"num_object_copies":1326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11618,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":442}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":426,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.780897+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:02:56.780897+0000","last_peered":"2023-09-23T23:02:56.780897+0000","last_clean":"2023-09-23T23:02:56.780897+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:02:56.780897+0000","last_undegraded":"2023-09-23T23:02:56.780897+0000","last_fullsized":"2023-09-23T23:02:56.780897+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":404,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781318+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:02:56.781318+0000","last_peered":"2023-09-23T23:02:56.781318+0000","last_clean":"2023-09-23T23:02:56.781318+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:02:56.781318+0000","last_undegraded":"2023-09-23T23:02:56.781318+0000","last_fullsized":"2023-09-23T23:02:56.781318+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"403'11524","reported_seq":11632,"reported_epoch":403,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:05.140923+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:03:05.140923+0000","last_peered":"2023-09-23T23:03:05.140923+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:03:05.140923+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:03:05.140923+0000","mapping_epoch":353,"log_start":"336'5800","ondisk_log_start":"336'5800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11524,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":444}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774546+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:02:56.774546+0000","last_peered":"2023-09-23T23:02:56.774546+0000","last_clean":"2023-09-23T23:02:56.774546+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:02:56.774546+0000","last_undegraded":"2023-09-23T23:02:56.774546+0000","last_fullsized":"2023-09-23T23:02:56.774546+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":307,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.772390+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:02:56.772390+0000","last_peered":"2023-09-23T23:02:56.772390+0000","last_clean":"2023-09-23T23:02:56.772390+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:02:56.772390+0000","last_undegraded":"2023-09-23T23:02:56.772390+0000","last_fullsized":"2023-09-23T23:02:56.772390+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"403'11628","reported_seq":20922,"reported_epoch":403,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:03:05.280966+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:03:05.280966+0000","last_peered":"2023-09-23T23:03:05.280966+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:03:05.280966+0000","last_undegraded":"2023-09-23T23:03:05.280966+0000","last_fullsized":"2023-09-23T23:03:05.280966+0000","mapping_epoch":354,"log_start":"336'5900","ondisk_log_start":"336'5900","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5728,"log_dups_size":0,"ondisk_log_size":5728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":448,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11628,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":448}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":311,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.773465+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:02:56.773465+0000","last_peered":"2023-09-23T23:02:56.773465+0000","last_clean":"2023-09-23T23:02:56.773465+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:02:56.773465+0000","last_undegraded":"2023-09-23T23:02:56.773465+0000","last_fullsized":"2023-09-23T23:02:56.773465+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":313,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768083+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:02:56.768083+0000","last_peered":"2023-09-23T23:02:56.768083+0000","last_clean":"2023-09-23T23:02:56.768083+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:02:56.768083+0000","last_undegraded":"2023-09-23T23:02:56.768083+0000","last_fullsized":"2023-09-23T23:02:56.768083+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.767860+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:02:56.767860+0000","last_peered":"2023-09-23T23:02:56.767860+0000","last_clean":"2023-09-23T23:02:56.767860+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:02:56.767860+0000","last_undegraded":"2023-09-23T23:02:56.767860+0000","last_fullsized":"2023-09-23T23:02:56.767860+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"403'11669","reported_seq":11790,"reported_epoch":403,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:03:04.962904+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:03:04.962904+0000","last_peered":"2023-09-23T23:03:04.962904+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:03:04.962904+0000","last_undegraded":"2023-09-23T23:03:04.962904+0000","last_fullsized":"2023-09-23T23:03:04.962904+0000","mapping_epoch":359,"log_start":"339'6200","ondisk_log_start":"339'6200","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":455,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11669,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":455}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":74,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.770822+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:02:56.770822+0000","last_peered":"2023-09-23T23:02:56.770822+0000","last_clean":"2023-09-23T23:02:56.770822+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:02:56.770822+0000","last_undegraded":"2023-09-23T23:02:56.770822+0000","last_fullsized":"2023-09-23T23:02:56.770822+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":423,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.784109+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:02:56.784109+0000","last_peered":"2023-09-23T23:02:56.784109+0000","last_clean":"2023-09-23T23:02:56.784109+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:02:56.784109+0000","last_undegraded":"2023-09-23T23:02:56.784109+0000","last_fullsized":"2023-09-23T23:02:56.784109+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":296,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.773534+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:02:56.773534+0000","last_peered":"2023-09-23T23:02:56.773534+0000","last_clean":"2023-09-23T23:02:56.773534+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:02:56.773534+0000","last_undegraded":"2023-09-23T23:02:56.773534+0000","last_fullsized":"2023-09-23T23:02:56.773534+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"403'11647","reported_seq":19830,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:05.266957+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:03:05.266957+0000","last_peered":"2023-09-23T23:03:05.266957+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:03:05.266957+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"325'3000","ondisk_log_start":"325'3000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8647,"log_dups_size":0,"ondisk_log_size":8647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27328512,"num_objects":417,"num_object_clones":0,"num_object_copies":1251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11647,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":417}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":371,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.771945+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:02:56.771945+0000","last_peered":"2023-09-23T23:02:56.771945+0000","last_clean":"2023-09-23T23:02:56.771945+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:02:56.771945+0000","last_undegraded":"2023-09-23T23:02:56.771945+0000","last_fullsized":"2023-09-23T23:02:56.771945+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":248,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.772558+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:02:56.772558+0000","last_peered":"2023-09-23T23:02:56.772558+0000","last_clean":"2023-09-23T23:02:56.772558+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:02:56.772558+0000","last_undegraded":"2023-09-23T23:02:56.772558+0000","last_fullsized":"2023-09-23T23:02:56.772558+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":313,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.769917+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:02:56.769917+0000","last_peered":"2023-09-23T23:02:56.769917+0000","last_clean":"2023-09-23T23:02:56.769917+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:02:56.769917+0000","last_undegraded":"2023-09-23T23:02:56.769917+0000","last_fullsized":"2023-09-23T23:02:56.769917+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"403'11813","reported_seq":15771,"reported_epoch":403,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:05.414119+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:03:05.414119+0000","last_peered":"2023-09-23T23:03:05.414119+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:03:05.414119+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"322'1800","ondisk_log_start":"322'1800","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10013,"log_dups_size":0,"ondisk_log_size":10013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26935296,"num_objects":411,"num_object_clones":0,"num_object_copies":1233,"num_objects_missing_on_primary":1530,"num_objects_missing":1530,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11813,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1917,"num_bytes_recovered":125632512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":401},{"shards":"0,1,4,6","objects":10},{"shards":"0,6","objects":-3549}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":386,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768529+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:02:56.768529+0000","last_peered":"2023-09-23T23:02:56.768529+0000","last_clean":"2023-09-23T23:02:56.768529+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:02:56.768529+0000","last_undegraded":"2023-09-23T23:02:56.768529+0000","last_fullsized":"2023-09-23T23:02:56.768529+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":500,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:02:56.768404+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:02:56.768404+0000","last_peered":"2023-09-23T23:02:56.768404+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:02:56.768404+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":73,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774798+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:02:56.774798+0000","last_peered":"2023-09-23T23:02:56.774798+0000","last_clean":"2023-09-23T23:02:56.774798+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:02:56.774798+0000","last_undegraded":"2023-09-23T23:02:56.774798+0000","last_fullsized":"2023-09-23T23:02:56.774798+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":313,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781484+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:02:56.781484+0000","last_peered":"2023-09-23T23:02:56.781484+0000","last_clean":"2023-09-23T23:02:56.781484+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:02:56.781484+0000","last_undegraded":"2023-09-23T23:02:56.781484+0000","last_fullsized":"2023-09-23T23:02:56.781484+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"403'11690","reported_seq":11803,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:04.887042+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:03:04.887042+0000","last_peered":"2023-09-23T23:03:04.887042+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:03:04.887042+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"339'6200","ondisk_log_start":"339'6200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25821184,"num_objects":394,"num_object_clones":0,"num_object_copies":1182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11690,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-3113}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":394,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768011+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:02:56.768011+0000","last_peered":"2023-09-23T23:02:56.768011+0000","last_clean":"2023-09-23T23:02:56.768011+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:02:56.768011+0000","last_undegraded":"2023-09-23T23:02:56.768011+0000","last_fullsized":"2023-09-23T23:02:56.768011+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":365,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768313+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:02:56.768313+0000","last_peered":"2023-09-23T23:02:56.768313+0000","last_clean":"2023-09-23T23:02:56.768313+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:02:56.768313+0000","last_undegraded":"2023-09-23T23:02:56.768313+0000","last_fullsized":"2023-09-23T23:02:56.768313+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":325,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.769248+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:02:56.769248+0000","last_peered":"2023-09-23T23:02:56.769248+0000","last_clean":"2023-09-23T23:02:56.769248+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:02:56.769248+0000","last_undegraded":"2023-09-23T23:02:56.769248+0000","last_fullsized":"2023-09-23T23:02:56.769248+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"403'11760","reported_seq":11887,"reported_epoch":403,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:03:05.292141+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:03:05.292141+0000","last_peered":"2023-09-23T23:03:05.292141+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:03:05.292141+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:03:05.292141+0000","mapping_epoch":354,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29032472,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":444,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11760,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":444}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":340,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.769492+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:02:56.769492+0000","last_peered":"2023-09-23T23:02:56.769492+0000","last_clean":"2023-09-23T23:02:56.769492+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:02:56.769492+0000","last_undegraded":"2023-09-23T23:02:56.769492+0000","last_fullsized":"2023-09-23T23:02:56.769492+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":248,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.773633+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:02:56.773633+0000","last_peered":"2023-09-23T23:02:56.773633+0000","last_clean":"2023-09-23T23:02:56.773633+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:02:56.773633+0000","last_undegraded":"2023-09-23T23:02:56.773633+0000","last_fullsized":"2023-09-23T23:02:56.773633+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768638+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:02:56.768638+0000","last_peered":"2023-09-23T23:02:56.768638+0000","last_clean":"2023-09-23T23:02:56.768638+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:02:56.768638+0000","last_undegraded":"2023-09-23T23:02:56.768638+0000","last_fullsized":"2023-09-23T23:02:56.768638+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"403'11748","reported_seq":22263,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:05.290709+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:03:05.290709+0000","last_peered":"2023-09-23T23:03:05.290709+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:03:05.290709+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11748,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-987},{"shards":"4,5,6","objects":398}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":97,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.777443+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:02:56.777443+0000","last_peered":"2023-09-23T23:02:56.777443+0000","last_clean":"2023-09-23T23:02:56.777443+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:02:56.777443+0000","last_undegraded":"2023-09-23T23:02:56.777443+0000","last_fullsized":"2023-09-23T23:02:56.777443+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781409+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:02:56.781409+0000","last_peered":"2023-09-23T23:02:56.781409+0000","last_clean":"2023-09-23T23:02:56.781409+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:02:56.781409+0000","last_undegraded":"2023-09-23T23:02:56.781409+0000","last_fullsized":"2023-09-23T23:02:56.781409+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":88,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774443+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:02:56.774443+0000","last_peered":"2023-09-23T23:02:56.774443+0000","last_clean":"2023-09-23T23:02:56.774443+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:02:56.774443+0000","last_undegraded":"2023-09-23T23:02:56.774443+0000","last_fullsized":"2023-09-23T23:02:56.774443+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"403'11684","reported_seq":12221,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:05.288210+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:03:05.288210+0000","last_peered":"2023-09-23T23:03:05.288210+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:03:05.288210+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"336'6000","ondisk_log_start":"336'6000","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25296896,"num_objects":386,"num_object_clones":0,"num_object_copies":1158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":386,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11684,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":386},{"shards":"4,6","objects":-3165}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":360,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.772716+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:02:56.772716+0000","last_peered":"2023-09-23T23:02:56.772716+0000","last_clean":"2023-09-23T23:02:56.772716+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:02:56.772716+0000","last_undegraded":"2023-09-23T23:02:56.772716+0000","last_fullsized":"2023-09-23T23:02:56.772716+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":307,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.769177+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:02:56.769177+0000","last_peered":"2023-09-23T23:02:56.769177+0000","last_clean":"2023-09-23T23:02:56.769177+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:02:56.769177+0000","last_undegraded":"2023-09-23T23:02:56.769177+0000","last_fullsized":"2023-09-23T23:02:56.769177+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":377,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.780980+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:02:56.780980+0000","last_peered":"2023-09-23T23:02:56.780980+0000","last_clean":"2023-09-23T23:02:56.780980+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:02:56.780980+0000","last_undegraded":"2023-09-23T23:02:56.780980+0000","last_fullsized":"2023-09-23T23:02:56.780980+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":342,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797334+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:03:04.797334+0000","last_peered":"2023-09-23T23:03:04.797334+0000","last_clean":"2023-09-23T23:03:04.797334+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:03:04.797334+0000","last_undegraded":"2023-09-23T23:03:04.797334+0000","last_fullsized":"2023-09-23T23:03:04.797334+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":151,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.773185+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:02:56.773185+0000","last_peered":"2023-09-23T23:02:56.773185+0000","last_clean":"2023-09-23T23:02:56.773185+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:02:56.773185+0000","last_undegraded":"2023-09-23T23:02:56.773185+0000","last_fullsized":"2023-09-23T23:02:56.773185+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":212,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781033+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:02:56.781033+0000","last_peered":"2023-09-23T23:02:56.781033+0000","last_clean":"2023-09-23T23:02:56.781033+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:02:56.781033+0000","last_undegraded":"2023-09-23T23:02:56.781033+0000","last_fullsized":"2023-09-23T23:02:56.781033+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":426,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.784151+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:02:56.784151+0000","last_peered":"2023-09-23T23:02:56.784151+0000","last_clean":"2023-09-23T23:02:56.784151+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:02:56.784151+0000","last_undegraded":"2023-09-23T23:02:56.784151+0000","last_fullsized":"2023-09-23T23:02:56.784151+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":226,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781314+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:02:56.781314+0000","last_peered":"2023-09-23T23:02:56.781314+0000","last_clean":"2023-09-23T23:02:56.781314+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:02:56.781314+0000","last_undegraded":"2023-09-23T23:02:56.781314+0000","last_fullsized":"2023-09-23T23:02:56.781314+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":313,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.781362+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:02:56.781362+0000","last_peered":"2023-09-23T23:02:56.781362+0000","last_clean":"2023-09-23T23:02:56.781362+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:02:56.781362+0000","last_undegraded":"2023-09-23T23:02:56.781362+0000","last_fullsized":"2023-09-23T23:02:56.781362+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":97,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.775204+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:02:56.775204+0000","last_peered":"2023-09-23T23:02:56.775204+0000","last_clean":"2023-09-23T23:02:56.775204+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:02:56.775204+0000","last_undegraded":"2023-09-23T23:02:56.775204+0000","last_fullsized":"2023-09-23T23:02:56.775204+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":100,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.774778+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:02:56.774778+0000","last_peered":"2023-09-23T23:02:56.774778+0000","last_clean":"2023-09-23T23:02:56.774778+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:02:56.774778+0000","last_undegraded":"2023-09-23T23:02:56.774778+0000","last_fullsized":"2023-09-23T23:02:56.774778+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":346,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.768968+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:02:56.768968+0000","last_peered":"2023-09-23T23:02:56.768968+0000","last_clean":"2023-09-23T23:02:56.768968+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:02:56.768968+0000","last_undegraded":"2023-09-23T23:02:56.768968+0000","last_fullsized":"2023-09-23T23:02:56.768968+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":388,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T23:02:56.780762+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:02:56.780762+0000","last_peered":"2023-09-23T23:02:56.780762+0000","last_clean":"2023-09-23T23:02:56.780762+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:02:56.780762+0000","last_undegraded":"2023-09-23T23:02:56.780762+0000","last_fullsized":"2023-09-23T23:02:56.780762+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":449445912,"num_objects":6859,"num_object_clones":0,"num_object_copies":20577,"num_objects_missing_on_primary":1532,"num_objects_missing":1532,"num_objects_degraded":13847,"num_objects_misplaced":24044,"num_objects_unfound":0,"num_objects_dirty":6859,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186739,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20431,"num_bytes_recovered":1247805440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2832740352,"data_stored":2832728136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100539,"ondisk_log_size":100539,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430641,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626866,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253179,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":977736,"kb_used_data":243392,"kb_used_omap":9,"kb_used_meta":734326,"kb_avail":92767416,"statfs":{"total":95995035648,"available":94993833984,"internally_reserved":0,"allocated":249233408,"data_stored":247697061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":751950825},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.696,"15min":0.80000000000000004},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":20.446000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":1.6200000000000001},{"interface":"front","average":{"1min":3.2440000000000002,"5min":2.0760000000000001,"15min":1.262},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":20.472999999999999,"5min":83.700999999999993,"15min":83.700999999999993},"last":5.266}]},{"osd":2,"last update":"Sat Sep 23 23:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.55300000000000005,"15min":0.53600000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5359999999999996,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.67000000000000004,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.1310000000000002,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":2.9630000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.56899999999999995,"15min":1.7969999999999999},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":87.787999999999997},"last":2.363},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.53300000000000003,"15min":2.0499999999999998},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":1.24,"5min":3.109,"15min":89.861999999999995},"last":3.1000000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3129999999999997,"5min":3.218,"15min":2.637},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.29199999999999998},"max":{"1min":67.738,"5min":151.036,"15min":151.036},"last":3.004},{"interface":"front","average":{"1min":5.7089999999999996,"5min":3.431,"15min":2.9260000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":67.670000000000002,"5min":151.06899999999999,"15min":151.06899999999999},"last":1.292}]},{"osd":5,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3300000000000001,"5min":1.6930000000000001,"15min":2.5699999999999998},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":68.259,"5min":68.259,"15min":116.098},"last":0.63700000000000001},{"interface":"front","average":{"1min":5.5549999999999997,"5min":1.8759999999999999,"15min":2.6970000000000001},"min":{"1min":0.38200000000000001,"5min":0.33600000000000002,"15min":0.315},"max":{"1min":70.057000000000002,"5min":70.057000000000002,"15min":116.08499999999999},"last":3.04}]},{"osd":6,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":2.0600000000000001,"15min":2.5259999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":67.349000000000004,"5min":67.349000000000004,"15min":79.311000000000007},"last":1.357},{"interface":"front","average":{"1min":5.5979999999999999,"5min":1.7529999999999999,"15min":2.125},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":67.366,"5min":67.366,"15min":79.328999999999994},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.78900000000000003,"15min":1.552},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4139999999999999,"5min":4.1669999999999998,"15min":11.481999999999999},"last":3.069},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.94499999999999995,"15min":1.925},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.4929999999999999,"5min":3.9860000000000002,"15min":20.902999999999999},"last":1.327}]}]},{"osd":0,"up_from":353,"seq":1516123455549,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533308,"kb_used_data":327860,"kb_used_omap":37,"kb_used_meta":205402,"kb_avail":93211844,"statfs":{"total":95995035648,"available":95448928256,"internally_reserved":0,"allocated":335728640,"data_stored":334020212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":210331810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.71599999999999997,"15min":0.59799999999999998},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":8.234,"5min":8.234,"15min":8.234},"last":1.9490000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.82999999999999996,"15min":0.72399999999999998},"min":{"1min":0.27800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4459999999999997,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":1.802}]},{"osd":2,"last update":"Sat Sep 23 23:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.83399999999999996,"15min":0.79800000000000004},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.694,"5min":12.215,"15min":12.215},"last":1.9159999999999999},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":5.9199999999999999},"last":2.2050000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.88,"15min":0.59599999999999997},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"last":2.4300000000000002},{"interface":"front","average":{"1min":1.254,"5min":0.83699999999999997,"15min":0.88500000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.8170000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":1.927}]},{"osd":4,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2229999999999999,"15min":1.647},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2349999999999999,"5min":183.148,"15min":183.148},"last":1.877},{"interface":"front","average":{"1min":1.466,"5min":3.3530000000000002,"15min":1.5629999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1609999999999996,"5min":183.39500000000001,"15min":183.39500000000001},"last":1.7330000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.98499999999999999,"15min":0.78100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.984,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":1.994},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.96199999999999997,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.1989999999999998,"5min":7.774,"15min":7.774},"last":1.974}]},{"osd":6,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.0509999999999999,"15min":0.90700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.359,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":1.7709999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.91800000000000004,"15min":0.78800000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3090000000000002,"5min":6.375,"15min":6.375},"last":2.0539999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.179,"15min":1.2609999999999999},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3929999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":1.829},{"interface":"front","average":{"1min":1.147,"5min":1.1890000000000001,"15min":1.2490000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.669,"5min":6.343,"15min":6.343},"last":2.0249999999999999}]}]},{"osd":2,"up_from":359,"seq":1541893259322,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":662332,"kb_used_data":494772,"kb_used_omap":83,"kb_used_meta":167468,"kb_avail":93082820,"statfs":{"total":95995035648,"available":95316807680,"internally_reserved":0,"allocated":506646528,"data_stored":505006421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":171487685},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.2749999999999999,"15min":0.66100000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":74.241,"5min":74.241,"15min":74.241},"last":25.361000000000001},{"interface":"front","average":{"1min":4.6699999999999999,"5min":1.4590000000000001,"15min":0.95799999999999996},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":74.430000000000007,"5min":74.430000000000007,"15min":74.430000000000007},"last":25.512}]},{"osd":1,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.68500000000000005,"15min":0.67200000000000004},"min":{"1min":0.26800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9710000000000001,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.20599999999999999},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.083,"15min":1.131},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.255}]},{"osd":3,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":0.81599999999999995,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.9530000000000003,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.94399999999999995,"15min":0.85899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8170000000000002,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.28799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.90000000000000002,"15min":0.67600000000000005},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.5359999999999996,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.96099999999999997,"15min":0.86899999999999999},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.8739999999999997,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.30599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.0580000000000001,"15min":0.98899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.232,"5min":0.93999999999999995,"15min":0.76700000000000002},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.24,"15min":1.0600000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9359999999999999,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.375},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91700000000000004,"15min":0.70999999999999996},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.903,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.32200000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1870000000000001,"15min":1.056},"min":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.5659999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.093,"15min":1.1499999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.9139999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.34399999999999997}]}]},{"osd":3,"up_from":94,"seq":403726926126,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":523104,"kb_used_data":8600,"kb_used_omap":7,"kb_used_meta":514488,"kb_avail":93222048,"statfs":{"total":95995035648,"available":95459377152,"internally_reserved":0,"allocated":8806400,"data_stored":7187126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":526836328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:15 2023","interfaces":[{"interface":"back","average":{"1min":3.383,"5min":2.177,"15min":0.93799999999999994},"min":{"1min":0.191,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":37.249000000000002,"5min":122.289,"15min":122.289},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.919,"15min":0.91100000000000003},"min":{"1min":0.19800000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":35.991,"5min":122.041,"15min":122.041},"last":0.307}]},{"osd":1,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.87,"15min":2.2480000000000002},"min":{"1min":0.22,"5min":0.187,"15min":0.187},"max":{"1min":4.6100000000000003,"5min":8.6280000000000001,"15min":123.866},"last":2.758},{"interface":"front","average":{"1min":1.2929999999999999,"5min":0.90200000000000002,"15min":2.1539999999999999},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1710000000000003,"5min":6.8010000000000002,"15min":122.265},"last":0.76500000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":0.71499999999999997},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.28,"5min":1.097,"15min":0.86099999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.7919999999999998,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":2.7109999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.1970000000000001,"15min":1.966},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.6459999999999999,"5min":87.046999999999997,"15min":87.046999999999997},"last":2.8239999999999998},{"interface":"front","average":{"1min":1.222,"5min":2.3119999999999998,"15min":1.9199999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2149999999999999,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.34699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.98699999999999999,"15min":1.403},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2530000000000001,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.373},{"interface":"front","average":{"1min":1.163,"5min":1.0640000000000001,"15min":1.744},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":5.149,"5min":10.246,"15min":11.951000000000001},"last":0.92000000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.264,"15min":1.748},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":6.2830000000000004,"5min":9.2520000000000007,"15min":14.097},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.181,"15min":1.869},"min":{"1min":0.378,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":5.1970000000000001,"5min":9.2810000000000006,"15min":12.887},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95499999999999996,"15min":1.6850000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":11.694000000000001},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.119,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.6799999999999997,"5min":8.5830000000000002,"15min":12.32},"last":2.79}]}]},{"osd":4,"up_from":30,"seq":128849019323,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1328400,"kb_used_data":429128,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92416752,"statfs":{"total":95995035648,"available":94634754048,"internally_reserved":0,"allocated":439427072,"data_stored":438573223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:13 2023","interfaces":[{"interface":"back","average":{"1min":10.303000000000001,"5min":2.7650000000000001,"15min":1.236},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.677000000000007,"5min":71.677000000000007,"15min":71.677000000000007},"last":0.34000000000000002},{"interface":"front","average":{"1min":11.896000000000001,"5min":2.9740000000000002,"15min":1.3879999999999999},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":72.147999999999996,"5min":72.147999999999996,"15min":72.147999999999996},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.5660000000000001,"15min":3.3769999999999998},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":7.0640000000000001,"5min":40.860999999999997,"15min":132.304},"last":0.312},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.603,"15min":3.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8159999999999998,"5min":40.756999999999998,"15min":132.453},"last":0.51000000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.117,"15min":1.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8380000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.653,"5min":1.0389999999999999,"15min":0.82499999999999996},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.2949999999999999,"15min":3.6379999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32400000000000001},"max":{"1min":3.2909999999999999,"5min":10.337999999999999,"15min":164.453},"last":4.4800000000000004},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.5149999999999999,"15min":3.6859999999999999},"min":{"1min":0.36699999999999999,"5min":0.35599999999999998,"15min":0.32500000000000001},"max":{"1min":7.0140000000000002,"5min":12.805999999999999,"15min":167.05199999999999},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:02:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.169,"15min":1.7589999999999999},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.9870000000000001,"5min":12.961,"15min":148.61600000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.92300000000000004,"15min":1.829},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":8.7810000000000006,"5min":8.7810000000000006,"15min":151.905},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.6499999999999999,"15min":1.268},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":8.9610000000000003,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.274,"5min":1.4019999999999999,"15min":1.169},"min":{"1min":0.438,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.0430000000000001,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.429,"15min":1.2549999999999999},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1050000000000004,"5min":12.84,"15min":12.84},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2450000000000001,"15min":1.1240000000000001},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2210000000000001,"5min":10.587,"15min":10.587},"last":0.628}]}]},{"osd":5,"up_from":36,"seq":154618823097,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1130788,"kb_used_data":231132,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":92614364,"statfs":{"total":95995035648,"available":94837108736,"internally_reserved":0,"allocated":236679168,"data_stored":235857077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.5229999999999999,"15min":0.91100000000000003},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":42.155000000000001,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.245},{"interface":"front","average":{"1min":3.7839999999999998,"5min":1.784,"15min":1.3160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":43.378999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.76500000000000001,"15min":2.1429999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.9649999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.505},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.202,"15min":2.6389999999999998},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":2.3079999999999998,"5min":10.353,"15min":110.89},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.997,"15min":0.86599999999999999},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.601,"5min":6.601,"15min":6.601},"last":0.317},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.997,"15min":0.995},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6139999999999999,"5min":4.899,"15min":4.899},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0600000000000001,"15min":3.6230000000000002},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0899999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":2.6629999999999998},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.2090000000000001,"15min":3.6680000000000001},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3420000000000001,"5min":10.965,"15min":108.15600000000001},"last":2.3420000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":1.298,"15min":1.552},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":1.321,"5min":10.603,"15min":115.06699999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92300000000000004,"15min":1.482},"min":{"1min":0.34599999999999997,"5min":0.253,"15min":0.253},"max":{"1min":1.764,"5min":9.5890000000000004,"15min":114.908},"last":0.77000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.169,"15min":1.256},"min":{"1min":0.30499999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.77300000000000002,"5min":7.4630000000000001,"15min":12.615},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.90400000000000003,"15min":1.1100000000000001},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.0569999999999999,"5min":5.4909999999999997,"15min":12.74},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1259999999999999,"15min":1.0900000000000001},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7830000000000004,"5min":9.5980000000000008,"15min":12.23},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.538,"15min":1.22},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.31},"max":{"1min":2.4580000000000002,"5min":10.638999999999999,"15min":11.112},"last":0.39400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":316477440,"data_stored":316473368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":241434624,"data_stored":241434624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":501350400,"data_stored":501350400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3932160,"data_stored":3932160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":433786880,"data_stored":433782808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":221839360,"data_stored":221839360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1113919488,"data_stored":1113915416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:03:09.743 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:03:09.769 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:09.825 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:09.881 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:09.909 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:09.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:09.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:09 smithi052 ceph-mon[180979]: pgmap v1241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 429 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 13849/20649 objects degraded (67.069%); 24044/20649 objects misplaced (116.441%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:09 smithi052 ceph-mon[180979]: from='client.46790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:09 smithi052 ceph-mon[187743]: pgmap v1241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 429 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 13849/20649 objects degraded (67.069%); 24044/20649 objects misplaced (116.441%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:09 smithi052 ceph-mon[187743]: from='client.46790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:10.006 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:03:10.017 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:10.087 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:09 smithi151 ceph-mon[159210]: pgmap v1241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 429 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 13849/20649 objects degraded (67.069%); 24044/20649 objects misplaced (116.441%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:09 smithi151 ceph-mon[159210]: from='client.46790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:10.253 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:10.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:10.365 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:10.400 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:10.438 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:10.472 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:10.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:10.577 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:10.608 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:10.659 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:10.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:10.764 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:10.793 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 13849/20649 objects degraded (67.069%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:11.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 13849/20649 objects degraded (67.069%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:11.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 13849/20649 objects degraded (67.069%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:11.248 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:11.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:11.338 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:11.352 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:11.421 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:11.493 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:11.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:11.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:11.643 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:11.704 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:11.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:11.811 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:11.843 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:11.890 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:11.894 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:11 smithi151 ceph-mon[159210]: pgmap v1242: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:11.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:11.995 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:12.011 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:12.039 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:11 smithi052 ceph-mon[187743]: pgmap v1242: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:12.039 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:11 smithi052 ceph-mon[180979]: pgmap v1242: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:12.041 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:12.713 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:12.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:12.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:12.820 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:12.852 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:12.997 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:12 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:12.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:12 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:12 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:03:13.399 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:13.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:13.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:03:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:03:13] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.33.4" 2023-09-23T23:03:13.505 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:13.543 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:13.622 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:13.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:13.726 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:13.777 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:13.844 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:13.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:14.026 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:14.082 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:14.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:13 smithi151 ceph-mon[159210]: pgmap v1243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:13 smithi052 ceph-mon[187743]: pgmap v1243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:14.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:13 smithi052 ceph-mon[180979]: pgmap v1243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:14.273 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:14.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:14.384 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:14.430 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:14.555 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:14.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:14.664 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:14.707 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:14.778 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:14.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:14.816 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:03:14.816 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:03:14.816 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T23:03:14.816 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:03:14.869 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:03:14.883 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:14.961 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1826818819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:15.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1826818819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:15.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1826818819' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:15.339 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:15.340 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 13363/14547 objects degraded (91.861%), 14 pgs degraded, 9 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1728,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":404,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":4849,"data_bytes":316741560,"bytes_used":4961976320,"bytes_avail":571008237568,"bytes_total":575970213888,"degraded_objects":13363,"degraded_total":14547,"degraded_ratio":0.91860864783116791,"misplaced_objects":23504,"misplaced_total":14547,"misplaced_ratio":1.6157283288650581,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1487153,"recovering_keys_per_sec":0,"num_objects_recovered":227,"num_bytes_recovered":14876672,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":392},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (7m)\n [======================......] (remaining: 93s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:03:15.788 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:15.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:15.898 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:15.936 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:16.138 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:16.138 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:16.138 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: failed to encode map e404 with expected crc 2023-09-23T23:03:16.138 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1826818819' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: osdmap e404: 8 total, 8 up, 6 in 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 13363/14547 objects degraded (91.861%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1222287956' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: failed to encode map e404 with expected crc 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1826818819' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: osdmap e404: 8 total, 8 up, 6 in 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 13363/14547 objects degraded (91.861%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:16.139 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1222287956' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 302 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 13363/14547 objects degraded (91.861%); 23504/14547 objects misplaced (161.573%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: failed to encode map e404 with expected crc 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1826818819' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: osdmap e404: 8 total, 8 up, 6 in 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 13363/14547 objects degraded (91.861%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1222287956' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:03:16.249 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:16.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:16.307 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:03:16.346 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:03:16.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:16.379 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:03:16.411 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:03:16.437 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:16.442 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:03:16.473 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:03:16.501 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:03:16.534 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:03:16.536 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.536 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.536 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.536 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.537 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.537 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.537 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.537 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:03:16.747 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:03:16.747 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:03:16.780 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:16.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:16.800 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:03:16.888 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:16.924 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:16.982 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:16.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:16.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/422467043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:16.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:16 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:16.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:16 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:16.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/422467043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:16.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:16 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:16.998 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:16 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:17.089 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:17.123 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/422467043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:16 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:17.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:16 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:17.595 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:17.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:17.700 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:17.733 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:17.900 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:17.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:17.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:17 smithi052 ceph-mon[187743]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 187 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 12909/9018 objects degraded (143.147%); 23029/9018 objects misplaced (255.367%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:17.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:17 smithi052 ceph-mon[180979]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 187 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 12909/9018 objects degraded (143.147%); 23029/9018 objects misplaced (255.367%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:18.000 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:18.005 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:18.060 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:18.082 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:18.139 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:03:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:17 smithi151 ceph-mon[159210]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 187 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 12909/9018 objects degraded (143.147%); 23029/9018 objects misplaced (255.367%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:18.326 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:18.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:18.431 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:18.466 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:18.523 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:18.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:18.634 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:18.664 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:18.717 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:18.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:18.828 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:18.858 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:19.523 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:19.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:19.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:19.631 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:19.687 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:19.845 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:03:19.853 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:19 smithi052 ceph-mon[187743]: pgmap v1247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 187 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 12909/9018 objects degraded (143.147%); 23029/9018 objects misplaced (255.367%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:19.854 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:19.854 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:19.855 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:19 smithi052 ceph-mon[180979]: pgmap v1247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 187 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 12909/9018 objects degraded (143.147%); 23029/9018 objects misplaced (255.367%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:19.855 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:19.855 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:20.035 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:20.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:20.139 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:20.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:19 smithi151 ceph-mon[159210]: pgmap v1247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 187 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 12909/9018 objects degraded (143.147%); 23029/9018 objects misplaced (255.367%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:20.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:20.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:20.167 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:20.224 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:20.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:20.331 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:20.367 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:20.563 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:03:20.563 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:03:20.604 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:20.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:20.709 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:20.762 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:21.115 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:21.116 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:03:21.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 12909/9018 objects degraded (143.147%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:21.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 12909/9018 objects degraded (143.147%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 12909/9018 objects degraded (143.147%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:21.310 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:21.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:21.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:21.415 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:21.451 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:22.119 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:21 smithi052 ceph-mon[187743]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:22.120 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:21 smithi052 ceph-mon[180979]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:22.121 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:03:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:21 smithi151 ceph-mon[159210]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:22.216 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:22.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:22.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:22.219 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:22.286 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:22.317 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1248,"stamp":"2023-09-23T23:03:20.635949+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":69146552,"num_objects":1071,"num_object_clones":0,"num_object_copies":3213,"num_objects_missing_on_primary":1194,"num_objects_missing":1194,"num_objects_degraded":12414,"num_objects_misplaced":22526,"num_objects_unfound":0,"num_objects_dirty":1071,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":192811,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20800,"num_bytes_recovered":1269709756,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100745,"ondisk_log_size":100745,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4327296,"kb_used_data":894224,"kb_used_omap":137,"kb_used_meta":3432886,"kb_avail":558143616,"statfs":{"total":575970213888,"available":571539062784,"internally_reserved":0,"allocated":915685376,"data_stored":907462312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3515275448},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-247595008,"num_objects":-3778,"num_object_clones":0,"num_object_copies":-11334,"num_objects_missing_on_primary":-222,"num_objects_missing":-222,"num_objects_degraded":-949,"num_objects_misplaced":-978,"num_objects_unfound":0,"num_objects_dirty":-3778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3778,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":222,"num_bytes_recovered":14548992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004671"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":416,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807927+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:03:04.807927+0000","last_peered":"2023-09-23T23:03:04.807927+0000","last_clean":"2023-09-23T23:03:04.807927+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:03:04.807927+0000","last_undegraded":"2023-09-23T23:03:04.807927+0000","last_fullsized":"2023-09-23T23:03:04.807927+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820872+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:03:04.820872+0000","last_peered":"2023-09-23T23:03:04.820872+0000","last_clean":"2023-09-23T23:03:04.820872+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:03:04.820872+0000","last_undegraded":"2023-09-23T23:03:04.820872+0000","last_fullsized":"2023-09-23T23:03:04.820872+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814884+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:03:04.814884+0000","last_peered":"2023-09-23T23:03:04.814884+0000","last_clean":"2023-09-23T23:03:04.814884+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:03:04.814884+0000","last_undegraded":"2023-09-23T23:03:04.814884+0000","last_fullsized":"2023-09-23T23:03:04.814884+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":401,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.816869+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:03:04.816869+0000","last_peered":"2023-09-23T23:03:04.816869+0000","last_clean":"2023-09-23T23:03:04.816869+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:03:04.816869+0000","last_undegraded":"2023-09-23T23:03:04.816869+0000","last_fullsized":"2023-09-23T23:03:04.816869+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814531+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:03:04.814531+0000","last_peered":"2023-09-23T23:03:04.814531+0000","last_clean":"2023-09-23T23:03:04.814531+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:03:04.814531+0000","last_undegraded":"2023-09-23T23:03:04.814531+0000","last_fullsized":"2023-09-23T23:03:04.814531+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797679+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:03:04.797679+0000","last_peered":"2023-09-23T23:03:04.797679+0000","last_clean":"2023-09-23T23:03:04.797679+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:03:04.797679+0000","last_undegraded":"2023-09-23T23:03:04.797679+0000","last_fullsized":"2023-09-23T23:03:04.797679+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":339,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.821053+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:03:04.821053+0000","last_peered":"2023-09-23T23:03:04.821053+0000","last_clean":"2023-09-23T23:03:04.821053+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:03:04.821053+0000","last_undegraded":"2023-09-23T23:03:04.821053+0000","last_fullsized":"2023-09-23T23:03:04.821053+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814390+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:03:04.814390+0000","last_peered":"2023-09-23T23:03:04.814390+0000","last_clean":"2023-09-23T23:03:04.814390+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:03:04.814390+0000","last_undegraded":"2023-09-23T23:03:04.814390+0000","last_fullsized":"2023-09-23T23:03:04.814390+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":300,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820805+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:03:04.820805+0000","last_peered":"2023-09-23T23:03:04.820805+0000","last_clean":"2023-09-23T23:03:04.820805+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:03:04.820805+0000","last_undegraded":"2023-09-23T23:03:04.820805+0000","last_fullsized":"2023-09-23T23:03:04.820805+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":88,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814770+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:03:04.814770+0000","last_peered":"2023-09-23T23:03:04.814770+0000","last_clean":"2023-09-23T23:03:04.814770+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:03:04.814770+0000","last_undegraded":"2023-09-23T23:03:04.814770+0000","last_fullsized":"2023-09-23T23:03:04.814770+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":94,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820870+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:03:04.820870+0000","last_peered":"2023-09-23T23:03:04.820870+0000","last_clean":"2023-09-23T23:03:04.820870+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:03:04.820870+0000","last_undegraded":"2023-09-23T23:03:04.820870+0000","last_fullsized":"2023-09-23T23:03:04.820870+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":150,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:06.363106+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:03:06.363106+0000","last_peered":"2023-09-23T23:03:06.363106+0000","last_clean":"2023-09-23T23:03:06.363106+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:03:06.363106+0000","last_undegraded":"2023-09-23T23:03:06.363106+0000","last_fullsized":"2023-09-23T23:03:06.363106+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":475,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807606+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:03:04.807606+0000","last_peered":"2023-09-23T23:03:04.807606+0000","last_clean":"2023-09-23T23:03:04.807606+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:03:04.807606+0000","last_undegraded":"2023-09-23T23:03:04.807606+0000","last_fullsized":"2023-09-23T23:03:04.807606+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":378,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813686+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:03:04.813686+0000","last_peered":"2023-09-23T23:03:04.813686+0000","last_clean":"2023-09-23T23:03:04.813686+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:03:04.813686+0000","last_undegraded":"2023-09-23T23:03:04.813686+0000","last_fullsized":"2023-09-23T23:03:04.813686+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":304,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813796+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:03:04.813796+0000","last_peered":"2023-09-23T23:03:04.813796+0000","last_clean":"2023-09-23T23:03:04.813796+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:03:04.813796+0000","last_undegraded":"2023-09-23T23:03:04.813796+0000","last_fullsized":"2023-09-23T23:03:04.813796+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820658+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:03:04.820658+0000","last_peered":"2023-09-23T23:03:04.820658+0000","last_clean":"2023-09-23T23:03:04.820658+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:03:04.820658+0000","last_undegraded":"2023-09-23T23:03:04.820658+0000","last_fullsized":"2023-09-23T23:03:04.820658+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":394,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.815081+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:03:04.815081+0000","last_peered":"2023-09-23T23:03:04.815081+0000","last_clean":"2023-09-23T23:03:04.815081+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:03:04.815081+0000","last_undegraded":"2023-09-23T23:03:04.815081+0000","last_fullsized":"2023-09-23T23:03:04.815081+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":304,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820660+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:03:04.820660+0000","last_peered":"2023-09-23T23:03:04.820660+0000","last_clean":"2023-09-23T23:03:04.820660+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:03:04.820660+0000","last_undegraded":"2023-09-23T23:03:04.820660+0000","last_fullsized":"2023-09-23T23:03:04.820660+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":289,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809860+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:03:04.809860+0000","last_peered":"2023-09-23T23:03:04.809860+0000","last_clean":"2023-09-23T23:03:04.809860+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:03:04.809860+0000","last_undegraded":"2023-09-23T23:03:04.809860+0000","last_fullsized":"2023-09-23T23:03:04.809860+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":418,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.817027+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:03:04.817027+0000","last_peered":"2023-09-23T23:03:04.817027+0000","last_clean":"2023-09-23T23:03:04.817027+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:03:04.817027+0000","last_undegraded":"2023-09-23T23:03:04.817027+0000","last_fullsized":"2023-09-23T23:03:04.817027+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":123,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795260+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:03:04.795260+0000","last_peered":"2023-09-23T23:03:04.795260+0000","last_clean":"2023-09-23T23:03:04.795260+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:03:04.795260+0000","last_undegraded":"2023-09-23T23:03:04.795260+0000","last_fullsized":"2023-09-23T23:03:04.795260+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.816975+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:03:04.816975+0000","last_peered":"2023-09-23T23:03:04.816975+0000","last_clean":"2023-09-23T23:03:04.816975+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:03:04.816975+0000","last_undegraded":"2023-09-23T23:03:04.816975+0000","last_fullsized":"2023-09-23T23:03:04.816975+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"404'12018","reported_seq":12127,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:19.989943+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:03:19.989943+0000","last_peered":"2023-09-23T23:03:19.989943+0000","last_clean":"2023-09-23T23:03:19.989943+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:03:19.989943+0000","last_undegraded":"2023-09-23T23:03:19.989943+0000","last_fullsized":"2023-09-23T23:03:19.989943+0000","mapping_epoch":353,"log_start":"340'6500","ondisk_log_start":"340'6500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5518,"log_dups_size":0,"ondisk_log_size":5518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4456448,"num_objects":68,"num_object_clones":0,"num_object_copies":204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12018,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":288,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809675+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:03:04.809675+0000","last_peered":"2023-09-23T23:03:04.809675+0000","last_clean":"2023-09-23T23:03:04.809675+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:03:04.809675+0000","last_undegraded":"2023-09-23T23:03:04.809675+0000","last_fullsized":"2023-09-23T23:03:04.809675+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":274,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820249+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:03:04.820249+0000","last_peered":"2023-09-23T23:03:04.820249+0000","last_clean":"2023-09-23T23:03:04.820249+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:03:04.820249+0000","last_undegraded":"2023-09-23T23:03:04.820249+0000","last_fullsized":"2023-09-23T23:03:04.820249+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"404'11901","reported_seq":14338,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:20.181361+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:03:20.181361+0000","last_peered":"2023-09-23T23:03:20.181361+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:03:20.181361+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8601,"log_dups_size":0,"ondisk_log_size":8601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3211264,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":49,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11901,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"4,5,6","objects":49},{"shards":"5,6","objects":-3518}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":298,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808078+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:03:04.808078+0000","last_peered":"2023-09-23T23:03:04.808078+0000","last_clean":"2023-09-23T23:03:04.808078+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:03:04.808078+0000","last_undegraded":"2023-09-23T23:03:04.808078+0000","last_fullsized":"2023-09-23T23:03:04.808078+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":411,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809779+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:03:04.809779+0000","last_peered":"2023-09-23T23:03:04.809779+0000","last_clean":"2023-09-23T23:03:04.809779+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:03:04.809779+0000","last_undegraded":"2023-09-23T23:03:04.809779+0000","last_fullsized":"2023-09-23T23:03:04.809779+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"404'11991","reported_seq":12094,"reported_epoch":404,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:03:19.989278+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:03:19.989278+0000","last_peered":"2023-09-23T23:03:19.989278+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:03:19.989278+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:03:19.989278+0000","mapping_epoch":354,"log_start":"340'6500","ondisk_log_start":"340'6500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5491,"log_dups_size":0,"ondisk_log_size":5491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5308416,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":81,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11991,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":81}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":155,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820761+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:03:04.820761+0000","last_peered":"2023-09-23T23:03:04.820761+0000","last_clean":"2023-09-23T23:03:04.820761+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:03:04.820761+0000","last_undegraded":"2023-09-23T23:03:04.820761+0000","last_fullsized":"2023-09-23T23:03:04.820761+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":273,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820348+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:03:04.820348+0000","last_peered":"2023-09-23T23:03:04.820348+0000","last_clean":"2023-09-23T23:03:04.820348+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:03:04.820348+0000","last_undegraded":"2023-09-23T23:03:04.820348+0000","last_fullsized":"2023-09-23T23:03:04.820348+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"404'12102","reported_seq":12391,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:19.906674+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:03:19.906674+0000","last_peered":"2023-09-23T23:03:19.906674+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:03:19.906674+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"340'6600","ondisk_log_start":"340'6600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4718592,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":64,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12102,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":8},{"shards":"0,5","objects":-3538}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":74,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820537+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:03:04.820537+0000","last_peered":"2023-09-23T23:03:04.820537+0000","last_clean":"2023-09-23T23:03:04.820537+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:03:04.820537+0000","last_undegraded":"2023-09-23T23:03:04.820537+0000","last_fullsized":"2023-09-23T23:03:04.820537+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797096+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:03:04.797096+0000","last_peered":"2023-09-23T23:03:04.797096+0000","last_clean":"2023-09-23T23:03:04.797096+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:03:04.797096+0000","last_undegraded":"2023-09-23T23:03:04.797096+0000","last_fullsized":"2023-09-23T23:03:04.797096+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"404'12038","reported_seq":12147,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:20.356587+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:03:20.356587+0000","last_peered":"2023-09-23T23:03:20.356587+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:03:20.356587+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:03:20.356587+0000","mapping_epoch":353,"log_start":"339'6300","ondisk_log_start":"339'6300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3670016,"num_objects":56,"num_object_clones":0,"num_object_copies":168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12038,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":56}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":93,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.815417+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:03:04.815417+0000","last_peered":"2023-09-23T23:03:04.815417+0000","last_clean":"2023-09-23T23:03:04.815417+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:03:04.815417+0000","last_undegraded":"2023-09-23T23:03:04.815417+0000","last_fullsized":"2023-09-23T23:03:04.815417+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":397,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807474+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:03:04.807474+0000","last_peered":"2023-09-23T23:03:04.807474+0000","last_clean":"2023-09-23T23:03:04.807474+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:03:04.807474+0000","last_undegraded":"2023-09-23T23:03:04.807474+0000","last_fullsized":"2023-09-23T23:03:04.807474+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"404'12141","reported_seq":12258,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:20.209830+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:03:20.209830+0000","last_peered":"2023-09-23T23:03:20.209830+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:03:20.209830+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"340'6400","ondisk_log_start":"340'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5741,"log_dups_size":0,"ondisk_log_size":5741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3997696,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":61,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12141,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-3558}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":187,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808133+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:03:04.808133+0000","last_peered":"2023-09-23T23:03:04.808133+0000","last_clean":"2023-09-23T23:03:04.808133+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:03:04.808133+0000","last_undegraded":"2023-09-23T23:03:04.808133+0000","last_fullsized":"2023-09-23T23:03:04.808133+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":414,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807541+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:03:04.807541+0000","last_peered":"2023-09-23T23:03:04.807541+0000","last_clean":"2023-09-23T23:03:04.807541+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:03:04.807541+0000","last_undegraded":"2023-09-23T23:03:04.807541+0000","last_fullsized":"2023-09-23T23:03:04.807541+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"404'11989","reported_seq":20832,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:20.203051+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:03:20.203051+0000","last_peered":"2023-09-23T23:03:20.203051+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:03:20.203051+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:03:20.203051+0000","mapping_epoch":354,"log_start":"339'6300","ondisk_log_start":"339'6300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4653056,"num_objects":71,"num_object_clones":0,"num_object_copies":213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11989,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":71}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":427,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809515+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:03:04.809515+0000","last_peered":"2023-09-23T23:03:04.809515+0000","last_clean":"2023-09-23T23:03:04.809515+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:03:04.809515+0000","last_undegraded":"2023-09-23T23:03:04.809515+0000","last_fullsized":"2023-09-23T23:03:04.809515+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":405,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808148+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:03:04.808148+0000","last_peered":"2023-09-23T23:03:04.808148+0000","last_clean":"2023-09-23T23:03:04.808148+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:03:04.808148+0000","last_undegraded":"2023-09-23T23:03:04.808148+0000","last_fullsized":"2023-09-23T23:03:04.808148+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"404'11901","reported_seq":12010,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:20.302892+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:03:20.302892+0000","last_peered":"2023-09-23T23:03:20.302892+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:03:20.302892+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:03:20.302892+0000","mapping_epoch":353,"log_start":"339'6200","ondisk_log_start":"339'6200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4390912,"num_objects":67,"num_object_clones":0,"num_object_copies":201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11901,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":67}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814647+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:03:04.814647+0000","last_peered":"2023-09-23T23:03:04.814647+0000","last_clean":"2023-09-23T23:03:04.814647+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:03:04.814647+0000","last_undegraded":"2023-09-23T23:03:04.814647+0000","last_fullsized":"2023-09-23T23:03:04.814647+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":308,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808859+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:03:04.808859+0000","last_peered":"2023-09-23T23:03:04.808859+0000","last_clean":"2023-09-23T23:03:04.808859+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:03:04.808859+0000","last_undegraded":"2023-09-23T23:03:04.808859+0000","last_fullsized":"2023-09-23T23:03:04.808859+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"404'12014","reported_seq":21309,"reported_epoch":404,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:03:20.222172+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:03:20.222172+0000","last_peered":"2023-09-23T23:03:20.222172+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:03:20.222172+0000","last_undegraded":"2023-09-23T23:03:20.222172+0000","last_fullsized":"2023-09-23T23:03:20.222172+0000","mapping_epoch":354,"log_start":"339'6300","ondisk_log_start":"339'6300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4063232,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":62,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12014,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":62}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":312,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808354+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:03:04.808354+0000","last_peered":"2023-09-23T23:03:04.808354+0000","last_clean":"2023-09-23T23:03:04.808354+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:03:04.808354+0000","last_undegraded":"2023-09-23T23:03:04.808354+0000","last_fullsized":"2023-09-23T23:03:04.808354+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820567+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:03:04.820567+0000","last_peered":"2023-09-23T23:03:04.820567+0000","last_clean":"2023-09-23T23:03:04.820567+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:03:04.820567+0000","last_undegraded":"2023-09-23T23:03:04.820567+0000","last_fullsized":"2023-09-23T23:03:04.820567+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820196+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:03:04.820196+0000","last_peered":"2023-09-23T23:03:04.820196+0000","last_clean":"2023-09-23T23:03:04.820196+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:03:04.820196+0000","last_undegraded":"2023-09-23T23:03:04.820196+0000","last_fullsized":"2023-09-23T23:03:04.820196+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"404'12052","reported_seq":12174,"reported_epoch":404,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:03:19.925968+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:03:19.925968+0000","last_peered":"2023-09-23T23:03:19.925968+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:03:19.925968+0000","last_undegraded":"2023-09-23T23:03:19.925968+0000","last_fullsized":"2023-09-23T23:03:19.925968+0000","mapping_epoch":359,"log_start":"340'6500","ondisk_log_start":"340'6500","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4718592,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":72,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12052,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":72}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":75,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795883+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:03:04.795883+0000","last_peered":"2023-09-23T23:03:04.795883+0000","last_clean":"2023-09-23T23:03:04.795883+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:03:04.795883+0000","last_undegraded":"2023-09-23T23:03:04.795883+0000","last_fullsized":"2023-09-23T23:03:04.795883+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":424,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813886+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:03:04.813886+0000","last_peered":"2023-09-23T23:03:04.813886+0000","last_clean":"2023-09-23T23:03:04.813886+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:03:04.813886+0000","last_undegraded":"2023-09-23T23:03:04.813886+0000","last_fullsized":"2023-09-23T23:03:04.813886+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":297,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808426+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:03:04.808426+0000","last_peered":"2023-09-23T23:03:04.808426+0000","last_clean":"2023-09-23T23:03:04.808426+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:03:04.808426+0000","last_undegraded":"2023-09-23T23:03:04.808426+0000","last_fullsized":"2023-09-23T23:03:04.808426+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"404'12000","reported_seq":20184,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:20.251706+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:03:20.251706+0000","last_peered":"2023-09-23T23:03:20.251706+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:03:20.251706+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12000,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":64}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":372,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809858+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:03:04.809858+0000","last_peered":"2023-09-23T23:03:04.809858+0000","last_clean":"2023-09-23T23:03:04.809858+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:03:04.809858+0000","last_undegraded":"2023-09-23T23:03:04.809858+0000","last_fullsized":"2023-09-23T23:03:04.809858+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":249,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809617+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:03:04.809617+0000","last_peered":"2023-09-23T23:03:04.809617+0000","last_clean":"2023-09-23T23:03:04.809617+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:03:04.809617+0000","last_undegraded":"2023-09-23T23:03:04.809617+0000","last_fullsized":"2023-09-23T23:03:04.809617+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.821095+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:03:04.821095+0000","last_peered":"2023-09-23T23:03:04.821095+0000","last_clean":"2023-09-23T23:03:04.821095+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:03:04.821095+0000","last_undegraded":"2023-09-23T23:03:04.821095+0000","last_fullsized":"2023-09-23T23:03:04.821095+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"404'12147","reported_seq":16779,"reported_epoch":404,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:20.394986+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:03:20.394986+0000","last_peered":"2023-09-23T23:03:20.394986+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:03:20.394986+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"322'2100","ondisk_log_start":"322'2100","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10047,"log_dups_size":0,"ondisk_log_size":10047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5046272,"num_objects":77,"num_object_clones":0,"num_object_copies":231,"num_objects_missing_on_primary":1192,"num_objects_missing":1192,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":77,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12147,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2251,"num_bytes_recovered":147521536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1,4","objects":63},{"shards":"0,1,4,6","objects":14},{"shards":"0,6","objects":-3549}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":387,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820675+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:03:04.820675+0000","last_peered":"2023-09-23T23:03:04.820675+0000","last_clean":"2023-09-23T23:03:04.820675+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:03:04.820675+0000","last_undegraded":"2023-09-23T23:03:04.820675+0000","last_fullsized":"2023-09-23T23:03:04.820675+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":501,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:04.796492+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:03:04.796492+0000","last_peered":"2023-09-23T23:03:04.796492+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:03:04.796492+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":74,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.815190+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:03:04.815190+0000","last_peered":"2023-09-23T23:03:04.815190+0000","last_clean":"2023-09-23T23:03:04.815190+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:03:04.815190+0000","last_undegraded":"2023-09-23T23:03:04.815190+0000","last_fullsized":"2023-09-23T23:03:04.815190+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807801+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:03:04.807801+0000","last_peered":"2023-09-23T23:03:04.807801+0000","last_clean":"2023-09-23T23:03:04.807801+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:03:04.807801+0000","last_undegraded":"2023-09-23T23:03:04.807801+0000","last_fullsized":"2023-09-23T23:03:04.807801+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"404'12024","reported_seq":12138,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:19.987224+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:03:19.987224+0000","last_peered":"2023-09-23T23:03:19.987224+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:03:19.987224+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"340'6500","ondisk_log_start":"340'6500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3932160,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12024,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-3447}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":395,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795343+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:03:04.795343+0000","last_peered":"2023-09-23T23:03:04.795343+0000","last_clean":"2023-09-23T23:03:04.795343+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:03:04.795343+0000","last_undegraded":"2023-09-23T23:03:04.795343+0000","last_fullsized":"2023-09-23T23:03:04.795343+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":366,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795757+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:03:04.795757+0000","last_peered":"2023-09-23T23:03:04.795757+0000","last_clean":"2023-09-23T23:03:04.795757+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:03:04.795757+0000","last_undegraded":"2023-09-23T23:03:04.795757+0000","last_fullsized":"2023-09-23T23:03:04.795757+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":326,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.798533+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:03:04.798533+0000","last_peered":"2023-09-23T23:03:04.798533+0000","last_clean":"2023-09-23T23:03:04.798533+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:03:04.798533+0000","last_undegraded":"2023-09-23T23:03:04.798533+0000","last_fullsized":"2023-09-23T23:03:04.798533+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"404'12146","reported_seq":12274,"reported_epoch":404,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:03:20.341830+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:03:20.341830+0000","last_peered":"2023-09-23T23:03:20.341830+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:03:20.341830+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:03:20.341830+0000","mapping_epoch":354,"log_start":"339'6400","ondisk_log_start":"339'6400","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3735576,"num_objects":58,"num_object_clones":0,"num_object_copies":174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":58,"num_objects_unfound":0,"num_objects_dirty":58,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12146,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":58}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":341,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797161+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:03:04.797161+0000","last_peered":"2023-09-23T23:03:04.797161+0000","last_clean":"2023-09-23T23:03:04.797161+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:03:04.797161+0000","last_undegraded":"2023-09-23T23:03:04.797161+0000","last_fullsized":"2023-09-23T23:03:04.797161+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":249,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808499+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:03:04.808499+0000","last_peered":"2023-09-23T23:03:04.808499+0000","last_clean":"2023-09-23T23:03:04.808499+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:03:04.808499+0000","last_undegraded":"2023-09-23T23:03:04.808499+0000","last_fullsized":"2023-09-23T23:03:04.808499+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":217,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820765+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:03:04.820765+0000","last_peered":"2023-09-23T23:03:04.820765+0000","last_clean":"2023-09-23T23:03:04.820765+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:03:04.820765+0000","last_undegraded":"2023-09-23T23:03:04.820765+0000","last_fullsized":"2023-09-23T23:03:04.820765+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"404'12077","reported_seq":22593,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:20.268752+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:03:20.268752+0000","last_peered":"2023-09-23T23:03:20.268752+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:03:20.268752+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"340'6400","ondisk_log_start":"340'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4521984,"num_objects":69,"num_object_clones":0,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12077,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-1316},{"shards":"4,5,6","objects":69}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":98,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820388+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:03:04.820388+0000","last_peered":"2023-09-23T23:03:04.820388+0000","last_clean":"2023-09-23T23:03:04.820388+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:03:04.820388+0000","last_undegraded":"2023-09-23T23:03:04.820388+0000","last_fullsized":"2023-09-23T23:03:04.820388+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809691+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:03:04.809691+0000","last_peered":"2023-09-23T23:03:04.809691+0000","last_clean":"2023-09-23T23:03:04.809691+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:03:04.809691+0000","last_undegraded":"2023-09-23T23:03:04.809691+0000","last_fullsized":"2023-09-23T23:03:04.809691+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814575+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:03:04.814575+0000","last_peered":"2023-09-23T23:03:04.814575+0000","last_clean":"2023-09-23T23:03:04.814575+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:03:04.814575+0000","last_undegraded":"2023-09-23T23:03:04.814575+0000","last_fullsized":"2023-09-23T23:03:04.814575+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"404'12010","reported_seq":12548,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:20.289912+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:03:20.289912+0000","last_peered":"2023-09-23T23:03:20.289912+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:03:20.289912+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"339'6300","ondisk_log_start":"339'6300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5710,"log_dups_size":0,"ondisk_log_size":5710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3932160,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":60,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12010,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"1,4,6","objects":60},{"shards":"4,6","objects":-3491}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":361,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809117+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:03:04.809117+0000","last_peered":"2023-09-23T23:03:04.809117+0000","last_clean":"2023-09-23T23:03:04.809117+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:03:04.809117+0000","last_undegraded":"2023-09-23T23:03:04.809117+0000","last_fullsized":"2023-09-23T23:03:04.809117+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":308,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.798488+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:03:04.798488+0000","last_peered":"2023-09-23T23:03:04.798488+0000","last_clean":"2023-09-23T23:03:04.798488+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:03:04.798488+0000","last_undegraded":"2023-09-23T23:03:04.798488+0000","last_fullsized":"2023-09-23T23:03:04.798488+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":378,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809630+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:03:04.809630+0000","last_peered":"2023-09-23T23:03:04.809630+0000","last_clean":"2023-09-23T23:03:04.809630+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:03:04.809630+0000","last_undegraded":"2023-09-23T23:03:04.809630+0000","last_fullsized":"2023-09-23T23:03:04.809630+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":343,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.999363+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:03:04.999363+0000","last_peered":"2023-09-23T23:03:04.999363+0000","last_clean":"2023-09-23T23:03:04.999363+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:03:04.999363+0000","last_undegraded":"2023-09-23T23:03:04.999363+0000","last_fullsized":"2023-09-23T23:03:04.999363+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":152,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808176+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:03:04.808176+0000","last_peered":"2023-09-23T23:03:04.808176+0000","last_clean":"2023-09-23T23:03:04.808176+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:03:04.808176+0000","last_undegraded":"2023-09-23T23:03:04.808176+0000","last_fullsized":"2023-09-23T23:03:04.808176+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":213,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807401+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:03:04.807401+0000","last_peered":"2023-09-23T23:03:04.807401+0000","last_clean":"2023-09-23T23:03:04.807401+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:03:04.807401+0000","last_undegraded":"2023-09-23T23:03:04.807401+0000","last_fullsized":"2023-09-23T23:03:04.807401+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":427,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813965+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:03:04.813965+0000","last_peered":"2023-09-23T23:03:04.813965+0000","last_clean":"2023-09-23T23:03:04.813965+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:03:04.813965+0000","last_undegraded":"2023-09-23T23:03:04.813965+0000","last_fullsized":"2023-09-23T23:03:04.813965+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":227,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807663+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:03:04.807663+0000","last_peered":"2023-09-23T23:03:04.807663+0000","last_clean":"2023-09-23T23:03:04.807663+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:03:04.807663+0000","last_undegraded":"2023-09-23T23:03:04.807663+0000","last_fullsized":"2023-09-23T23:03:04.807663+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808047+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:03:04.808047+0000","last_peered":"2023-09-23T23:03:04.808047+0000","last_clean":"2023-09-23T23:03:04.808047+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:03:04.808047+0000","last_undegraded":"2023-09-23T23:03:04.808047+0000","last_fullsized":"2023-09-23T23:03:04.808047+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":98,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.818637+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:03:04.818637+0000","last_peered":"2023-09-23T23:03:04.818637+0000","last_clean":"2023-09-23T23:03:04.818637+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:03:04.818637+0000","last_undegraded":"2023-09-23T23:03:04.818637+0000","last_fullsized":"2023-09-23T23:03:04.818637+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.814963+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:03:04.814963+0000","last_peered":"2023-09-23T23:03:04.814963+0000","last_clean":"2023-09-23T23:03:04.814963+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:03:04.814963+0000","last_undegraded":"2023-09-23T23:03:04.814963+0000","last_fullsized":"2023-09-23T23:03:04.814963+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":347,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.798314+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:03:04.798314+0000","last_peered":"2023-09-23T23:03:04.798314+0000","last_clean":"2023-09-23T23:03:04.798314+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:03:04.798314+0000","last_undegraded":"2023-09-23T23:03:04.798314+0000","last_fullsized":"2023-09-23T23:03:04.798314+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":389,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807307+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:03:04.807307+0000","last_peered":"2023-09-23T23:03:04.807307+0000","last_clean":"2023-09-23T23:03:04.807307+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:03:04.807307+0000","last_undegraded":"2023-09-23T23:03:04.807307+0000","last_fullsized":"2023-09-23T23:03:04.807307+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":68550680,"num_objects":1047,"num_object_clones":0,"num_object_copies":3141,"num_objects_missing_on_primary":1194,"num_objects_missing":1194,"num_objects_degraded":12412,"num_objects_misplaced":22526,"num_objects_unfound":0,"num_objects_dirty":1047,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":192551,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20765,"num_bytes_recovered":1269694464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1882533888,"data_stored":1882521672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100551,"ondisk_log_size":100551,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430644,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626869,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253181,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":885720,"kb_used_data":147280,"kb_used_omap":9,"kb_used_meta":738422,"kb_avail":92859432,"statfs":{"total":95995035648,"available":95088058368,"internally_reserved":0,"allocated":150814720,"data_stored":149270667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":756145129},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7769999999999999,"15min":0.82699999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4780000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.995,"5min":2.1040000000000001,"15min":1.272},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.266,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 23:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.55300000000000005,"15min":0.53600000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5359999999999996,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.67000000000000004,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.1310000000000002,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":1.6779999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.56899999999999995,"15min":1.7969999999999999},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":87.787999999999997},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.53300000000000003,"15min":2.0499999999999998},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":1.24,"5min":3.109,"15min":89.861999999999995},"last":1.4890000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3129999999999997,"5min":3.218,"15min":2.637},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.29199999999999998},"max":{"1min":67.738,"5min":151.036,"15min":151.036},"last":0.36299999999999999},{"interface":"front","average":{"1min":5.7089999999999996,"5min":3.431,"15min":2.9260000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":67.670000000000002,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3300000000000001,"5min":1.6930000000000001,"15min":2.5699999999999998},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":68.259,"5min":68.259,"15min":116.098},"last":0.55300000000000005},{"interface":"front","average":{"1min":5.5549999999999997,"5min":1.8759999999999999,"15min":2.6970000000000001},"min":{"1min":0.38200000000000001,"5min":0.33600000000000002,"15min":0.315},"max":{"1min":70.057000000000002,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":2.0600000000000001,"15min":2.5259999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":67.349000000000004,"5min":67.349000000000004,"15min":79.311000000000007},"last":0.42799999999999999},{"interface":"front","average":{"1min":5.5979999999999999,"5min":1.7529999999999999,"15min":2.125},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":67.366,"5min":67.366,"15min":79.328999999999994},"last":0.73299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.78900000000000003,"15min":1.552},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4139999999999999,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.94499999999999995,"15min":1.925},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.4929999999999999,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.59999999999999998}]}]},{"osd":0,"up_from":353,"seq":1516123455552,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270876,"kb_used_data":65428,"kb_used_omap":37,"kb_used_meta":205402,"kb_avail":93474276,"statfs":{"total":95995035648,"available":95717658624,"internally_reserved":0,"allocated":66998272,"data_stored":65274432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":210331810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.83299999999999996,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.6970000000000001,"5min":8.234,"15min":8.234},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.01,"5min":0.89700000000000002,"15min":0.747},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7169999999999996,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.83399999999999996,"15min":0.79800000000000004},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.694,"5min":12.215,"15min":12.215},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":5.9199999999999999},"last":74.698999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.997,"15min":0.63500000000000001},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.1109999999999998,"5min":12.178000000000001,"15min":12.178000000000001},"last":74.744},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.93799999999999994,"15min":0.91800000000000004},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.8050000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":74.777000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":3.298,"15min":1.6719999999999999},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2690000000000001,"5min":183.148,"15min":183.148},"last":74.805000000000007},{"interface":"front","average":{"1min":0.98899999999999999,"5min":3.4169999999999998,"15min":1.585},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.9910000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.38}]},{"osd":5,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.069,"15min":0.80900000000000005},"min":{"1min":0.433,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.3090000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.026,"15min":1.0069999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.7080000000000002,"5min":7.774,"15min":7.774},"last":74.858000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.0860000000000001,"15min":0.91800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.0309999999999997,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1,"15min":0.81599999999999995},"min":{"1min":0.44900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.3410000000000002,"5min":6.375,"15min":6.375},"last":0.65300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.139,"15min":1.248},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.0579999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.432},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.1699999999999999,"15min":1.242},"min":{"1min":0.496,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.3719999999999999,"5min":6.343,"15min":6.343},"last":0.68400000000000005}]}]},{"osd":2,"up_from":359,"seq":1541893259324,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650696,"kb_used_data":479040,"kb_used_omap":83,"kb_used_meta":171564,"kb_avail":93094456,"statfs":{"total":95995035648,"available":95328722944,"internally_reserved":0,"allocated":490536960,"data_stored":488892883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":175681989},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.2749999999999999,"15min":0.66100000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":74.241,"5min":74.241,"15min":74.241},"last":0.26100000000000001},{"interface":"front","average":{"1min":4.6699999999999999,"5min":1.4590000000000001,"15min":0.95799999999999996},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":74.430000000000007,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.68500000000000005,"15min":0.67200000000000004},"min":{"1min":0.26800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9710000000000001,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.083,"15min":1.131},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":0.81599999999999995,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.9530000000000003,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.94399999999999995,"15min":0.85899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8170000000000002,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.90000000000000002,"15min":0.67600000000000005},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.5359999999999996,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.96099999999999997,"15min":0.86899999999999999},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.8739999999999997,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.77000000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.0580000000000001,"15min":0.98899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.232,"5min":0.93999999999999995,"15min":0.76700000000000002},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.24,"15min":1.0600000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9359999999999999,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91700000000000004,"15min":0.70999999999999996},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.903,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1870000000000001,"15min":1.056},"min":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.5659999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":1.1930000000000001},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.093,"15min":1.1499999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.9139999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.81599999999999995}]}]},{"osd":3,"up_from":94,"seq":403726926128,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528556,"kb_used_data":5860,"kb_used_omap":7,"kb_used_meta":522680,"kb_avail":93216596,"statfs":{"total":95995035648,"available":95453794304,"internally_reserved":0,"allocated":6000640,"data_stored":4377396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":535224936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3780000000000001,"15min":1.0049999999999999},"min":{"1min":0.22,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.3019999999999996,"5min":122.289,"15min":122.289},"last":0.75},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.0150000000000001,"15min":0.94299999999999995},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":8.9849999999999994,"5min":122.041,"15min":122.041},"last":0.88700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.87,"15min":2.2480000000000002},"min":{"1min":0.22,"5min":0.187,"15min":0.187},"max":{"1min":4.6100000000000003,"5min":8.6280000000000001,"15min":123.866},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.2929999999999999,"5min":0.90200000000000002,"15min":2.1539999999999999},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1710000000000003,"5min":6.8010000000000002,"15min":122.265},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":0.71499999999999997},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.28,"5min":1.097,"15min":0.86099999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.7919999999999998,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.623}]},{"osd":4,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.1970000000000001,"15min":1.966},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.6459999999999999,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.222,"5min":2.3119999999999998,"15min":1.9199999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2149999999999999,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.81399999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.98699999999999999,"15min":1.403},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2530000000000001,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.163,"5min":1.0640000000000001,"15min":1.744},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":5.149,"5min":10.246,"15min":11.951000000000001},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.264,"15min":1.748},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":6.2830000000000004,"5min":9.2520000000000007,"15min":14.097},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.091,"5min":1.181,"15min":1.869},"min":{"1min":0.378,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":5.1970000000000001,"5min":9.2810000000000006,"15min":12.887},"last":0.60099999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95499999999999996,"15min":1.6850000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.119,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.6799999999999997,"5min":8.5830000000000002,"15min":12.32},"last":0.65600000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019326,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1052188,"kb_used_data":152916,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92692964,"statfs":{"total":95995035648,"available":94917595136,"internally_reserved":0,"allocated":156585984,"data_stored":155728165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":7.6189999999999998,"5min":4.1950000000000003,"15min":1.7130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":156.375,"5min":156.375,"15min":156.375},"last":0.28499999999999998},{"interface":"front","average":{"1min":7.4009999999999998,"5min":4.335,"15min":1.8420000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":157.02199999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":0.97399999999999998,"15min":3.29},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8739999999999997,"5min":7.0640000000000001,"15min":132.304},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.90400000000000003,"15min":3.2410000000000001},"min":{"1min":0.308,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.181,"5min":10.361000000000001,"15min":132.453},"last":0.52600000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.117,"15min":1.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8380000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.653,"5min":1.0389999999999999,"15min":0.82499999999999996},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.071,"15min":2.79},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6189999999999998,"5min":10.337999999999999,"15min":164.453},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.2330000000000001,"15min":2.8170000000000002},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.32500000000000001},"max":{"1min":7.2519999999999998,"5min":7.2519999999999998,"15min":167.05199999999999},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:02:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.169,"15min":1.7589999999999999},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.9870000000000001,"5min":12.961,"15min":148.61600000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.92300000000000004,"15min":1.829},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":8.7810000000000006,"5min":8.7810000000000006,"15min":151.905},"last":0.373}]},{"osd":6,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.1870000000000001,"15min":1.2889999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":6.7050000000000001,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.014,"15min":1.1779999999999999},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.7409999999999997,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.429,"15min":1.2549999999999999},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1050000000000004,"5min":12.84,"15min":12.84},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2450000000000001,"15min":1.1240000000000001},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2210000000000001,"5min":10.587,"15min":10.587},"last":0.61199999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823100,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":939260,"kb_used_data":43700,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92805892,"statfs":{"total":95995035648,"available":95033233408,"internally_reserved":0,"allocated":44748800,"data_stored":43918769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.786,"15min":0.999},"min":{"1min":0.23799999999999999,"5min":0.221,"15min":0.221},"max":{"1min":23.346,"5min":42.155000000000001,"15min":42.155000000000001},"last":3.8119999999999998},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.9690000000000001,"15min":1.377},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":23.428999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.33600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.76500000000000001,"15min":2.1429999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.9649999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.202,"15min":2.6389999999999998},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":2.3079999999999998,"5min":10.353,"15min":110.89},"last":1.1739999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.997,"15min":0.86599999999999999},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.601,"5min":6.601,"15min":6.601},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.997,"15min":0.995},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6139999999999999,"5min":4.899,"15min":4.899},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0600000000000001,"15min":3.6230000000000002},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0899999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.624},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.2090000000000001,"15min":3.6680000000000001},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3420000000000001,"5min":10.965,"15min":108.15600000000001},"last":3.6789999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":1.298,"15min":1.552},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":1.321,"5min":10.603,"15min":115.06699999999999},"last":1.107},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92300000000000004,"15min":1.482},"min":{"1min":0.34599999999999997,"5min":0.253,"15min":0.253},"max":{"1min":1.764,"5min":9.5890000000000004,"15min":114.908},"last":0.32500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.169,"15min":1.256},"min":{"1min":0.30499999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.77300000000000002,"5min":7.4630000000000001,"15min":12.615},"last":1.145},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.90400000000000003,"15min":1.1100000000000001},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.0569999999999999,"5min":5.4909999999999997,"15min":12.74},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.1259999999999999,"15min":1.0900000000000001},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7830000000000004,"5min":9.5980000000000008,"15min":12.23},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.538,"15min":1.22},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.31},"max":{"1min":2.4580000000000002,"5min":10.638999999999999,"15min":11.112},"last":1.8500000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":51908608,"data_stored":51904536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":144179200,"data_stored":144179200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":484966400,"data_stored":484966400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1114112,"data_stored":1114112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":151195648,"data_stored":151191576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":32374784,"data_stored":32374784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1016795136,"data_stored":1016791064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:03:22.324 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:22.353 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:22.402 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:22.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:22.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:22.508 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:22.536 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:22.595 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:22.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:22.704 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:22.734 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:22.790 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:22.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:22.869 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:22 smithi151 ceph-mon[159210]: from='client.36732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:22.897 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:22.923 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:22 smithi052 ceph-mon[187743]: from='client.36732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:22.923 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:22 smithi052 ceph-mon[180979]: from='client.36732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:22.929 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:22.997 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:23.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:23.106 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:23.143 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:23.208 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:23.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:23.312 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:23.343 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:23.715 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Removed 96798 objects 2023-09-23T23:03:23.715 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Clean up completed and total clean up time :380.752 2023-09-23T23:03:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:03:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:03:23] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.33.4" 2023-09-23T23:03:23.899 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:23.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:23.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:24.003 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:24.032 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:24.088 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:24.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:24.192 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:23 smithi151 ceph-mon[159210]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:24.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:23 smithi052 ceph-mon[180979]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:24.218 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:23 smithi052 ceph-mon[187743]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:24.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:24.528 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:24.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:24.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:24.545 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:24.631 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:24.662 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:24.705 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:24.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:24.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:24.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:24.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:24.885 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:24.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:24.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:24.990 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:25.016 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:25.058 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:25.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:25.167 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:25.194 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:25.240 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:25.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:25.325 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:03:25.346 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:25.402 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:25.460 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:25.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:25.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:25.535 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:03:25.554 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:03:25.564 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:25.587 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:03:25.605 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:25.610 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:03:25.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:03:25.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:03:25.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:03:25.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.757 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:03:25.876 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:03:25.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[187743]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:25.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4015115786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:03:25.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:25.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:25.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 12414/3213 objects degraded (386.368%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:25.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1817310722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:26.124 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:26.126 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[180979]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:26.126 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4015115786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:03:26.126 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:26.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:26.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 12414/3213 objects degraded (386.368%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:26.127 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1817310722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:26.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:25 smithi151 ceph-mon[159210]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 66 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 12414/3213 objects degraded (386.368%); 22526/3213 objects misplaced (701.089%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T23:03:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4015115786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:03:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:25 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 12414/3213 objects degraded (386.368%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1817310722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:26.229 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:26.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:26.635 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:26.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:26.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:26.770 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:26.868 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:26.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:26.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:26.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4015115786' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:03:26.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[180979]: osdmap e405: 8 total, 8 up, 6 in 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4015115786' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[187743]: osdmap e405: 8 total, 8 up, 6 in 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:03:26.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:26 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:26.972 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:26.998 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:27.050 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:27.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:27.114 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:03:27.114 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:03:27.114 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:03:27.114 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:03:27.114 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44617 10.2 e325/317 10:42f612af:::benchmark_data_smithi052_461816_object44616:head v 325'2757, mlcod=325'2755)", 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.230235+0000", 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 450.88228528100001, 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.097530256999999995, 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.115 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230235+0000", 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230234+0000", 2023-09-23T23:03:27.116 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230284+0000", 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0538999999999998e-05 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230294+0000", 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2580000000000001e-06 2023-09-23T23:03:27.117 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230301+0000", 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0979999999999996e-06 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230348+0000", 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6904999999999999e-05 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.118 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.230370+0000", 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2036999999999999e-05 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.231896+0000", 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015261840000000001 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327765+0000", 2023-09-23T23:03:27.119 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.095868996999999997 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44626 10.2 e325/317 10:4fdb9bab:::benchmark_data_smithi052_461816_object44625:head v 325'2758, mlcod=325'2755)", 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.234607+0000", 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 450.87791330800002, 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096288367, 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.120 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234607+0000", 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.121 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234606+0000", 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.234684+0000", 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8349999999999994e-05 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.122 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327708+0000", 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.093023764999999994 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327737+0000", 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9223000000000001e-05 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327860+0000", 2023-09-23T23:03:27.123 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122982 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327916+0000", 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5405999999999998e-05 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330864+0000", 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002948492 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.124 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.330895+0000", 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0898999999999997e-05 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:44632 10.b e325/317 10:d9603e28:::benchmark_data_smithi052_461816_object44631:head v 325'2853, mlcod=325'2851)", 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:55:56.237428+0000", 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 450.87509173000001, 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.096280273999999999, 2023-09-23T23:03:27.125 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237428+0000", 2023-09-23T23:03:27.126 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237427+0000", 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.237481+0000", 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3566000000000002e-05 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.127 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327715+0000", 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.090234522999999997 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327736+0000", 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0746000000000001e-05 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327869+0000", 2023-09-23T23:03:27.128 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013274200000000001 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.327926+0000", 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7645999999999997e-05 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333664+0000", 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057375569999999999 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.129 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:55:56.333708+0000", 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4566e-05 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71910 10.b e327/317 10:da00891c:::benchmark_data_smithi052_461816_object71909:head v 327'4562, mlcod=327'4560)", 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.838808+0000", 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.27371214300001, 2023-09-23T23:03:27.130 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070775292000000004, 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.131 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838808+0000", 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838807+0000", 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.132 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838881+0000", 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3671999999999995e-05 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838927+0000", 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6146999999999997e-05 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838934+0000", 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3769999999999998e-06 2023-09-23T23:03:27.133 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838979+0000", 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.494e-05 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.838998+0000", 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8782000000000001e-05 2023-09-23T23:03:27.134 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909541+0000", 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070543146000000001 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.909583+0000", 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1967e-05 2023-09-23T23:03:27.135 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71913 10.1 e327/317 10:868eb1a2:::benchmark_data_smithi052_461816_object71912:head v 327'4350, mlcod=327'4347)", 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.840731+0000", 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.27178881499998, 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.066217576, 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.136 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840731+0000", 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840730+0000", 2023-09-23T23:03:27.137 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.840777+0000", 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6655e-05 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T23:03:27.138 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061592056999999999 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902407+0000", 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7666000000000002e-05 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902538+0000", 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000131793 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902555+0000", 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6888000000000002e-05 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906899+0000", 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043436869999999997 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.906949+0000", 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9629999999999997e-05 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:71918 10.b e327/317 10:d9e3a463:::benchmark_data_smithi052_461816_object71917:head v 327'4563, mlcod=327'4560)", 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:56:17.844519+0000", 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 429.26800081599998, 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074161447000000005, 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.141 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844519+0000", 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844518+0000", 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.844600+0000", 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2000000000000001e-05 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902369+0000", 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.057768742999999997 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.902406+0000", 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7252999999999997e-05 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.904983+0000", 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025770709999999998 2023-09-23T23:03:27.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.905005+0000", 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2028e-05 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918640+0000", 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01363467 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.145 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:56:17.918681+0000", 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0522000000000003e-05 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104103 10.c e340/329 10:3a122c6e:::benchmark_data_smithi052_461816_object7302:head v 340'6509, mlcod=0'0)", 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.450864+0000", 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 372.66165583100002, 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083293366999999993, 2023-09-23T23:03:27.146 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450864+0000", 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450864+0000", 2023-09-23T23:03:27.147 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450862+0000", 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.148 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450879+0000", 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7056000000000001e-05 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450910+0000", 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0960000000000002e-05 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.149 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450929+0000", 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.853e-05 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.450997+0000", 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8225000000000005e-05 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.150 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451014+0000", 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6988999999999999e-05 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534114+0000", 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083099808999999997 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534157+0000", 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3625000000000003e-05 2023-09-23T23:03:27.151 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104104 10.c e340/329 10:3017b5bb:::benchmark_data_smithi052_461816_object7303:head v 340'6510, mlcod=0'0)", 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.451247+0000", 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 372.661273171, 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.088953937999999996, 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.152 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451247+0000", 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451247+0000", 2023-09-23T23:03:27.153 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451246+0000", 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451251+0000", 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3050000000000003e-06 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.154 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451258+0000", 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9550000000000003e-06 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451263+0000", 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1240000000000004e-06 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.155 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457336+0000", 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060733820000000004 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457354+0000", 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7779999999999999e-05 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540161+0000", 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.082807243000000003 2023-09-23T23:03:27.156 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540201+0000", 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9532e-05 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104107 10.4 e340/317 10:2627ee68:::benchmark_data_smithi052_461816_object7306:head v 340'6525, mlcod=340'6521)", 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.451379+0000", 2023-09-23T23:03:27.157 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 372.66114121300001, 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.082812793999999995, 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451379+0000", 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.158 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451379+0000", 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451378+0000", 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451384+0000", 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4760000000000003e-06 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.159 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451394+0000", 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9410000000000002e-06 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.451400+0000", 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.015e-06 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.160 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457323+0000", 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059230339999999998 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457338+0000", 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4263e-05 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534163+0000", 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.076825685000000005 2023-09-23T23:03:27.161 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.534191+0000", 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8124999999999999e-05 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104106 10.c e340/329 10:3b7c91e7:::benchmark_data_smithi052_461816_object7305:head v 340'6511, mlcod=0'0)", 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.457021+0000", 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 372.65549855900002, 2023-09-23T23:03:27.162 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.083208802999999998, 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457021+0000", 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457021+0000", 2023-09-23T23:03:27.163 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457020+0000", 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457029+0000", 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2579999999999995e-06 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.164 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457043+0000", 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4239e-05 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.457063+0000", 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9656999999999998e-05 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.165 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.463087+0000", 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060247390000000003 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.463117+0000", 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9611000000000002e-05 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540206+0000", 2023-09-23T23:03:27.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.077089550000000007 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540230+0000", 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3666999999999999e-05 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104110 10.c e340/329 10:3405d415:::benchmark_data_smithi052_461816_object7309:head v 340'6512, mlcod=0'0)", 2023-09-23T23:03:27.167 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:14.460318+0000", 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 372.65220227600003, 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.079936283999999996, 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460318+0000", 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460318+0000", 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460316+0000", 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460327+0000", 2023-09-23T23:03:27.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0478e-05 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460337+0000", 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5440000000000004e-06 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.460342+0000", 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2249999999999999e-06 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.520173+0000", 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.059831670000000003 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.520196+0000", 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2121e-05 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540232+0000", 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020036824000000002 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:14.540254+0000", 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1539000000000002e-05 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.172 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104853 10.c e340/329 10:3b1f5b6f:::benchmark_data_smithi052_461816_object8052:head v 340'6559, mlcod=0'0)", 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 371.18793861799998, 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080972484999999997, 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:03:27.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924581+0000", 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924585+0000", 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0080000000000004e-06 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924591+0000", 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.942e-06 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924595+0000", 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4329999999999997e-06 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924640+0000", 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4134999999999997e-05 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924664+0000", 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4360999999999999e-05 2023-09-23T23:03:27.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:16.005514+0000", 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080849657000000005 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:16.005554+0000", 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0201999999999999e-05 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.177 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104855 10.4 e340/317 10:2c6443b8:::benchmark_data_smithi052_461816_object8054:head v 340'6568, mlcod=340'6566)", 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924876+0000", 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 371.187644057, 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.075044514000000007, 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924876+0000", 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924876+0000", 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924875+0000", 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:27.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924881+0000", 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.57e-06 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924890+0000", 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1330000000000005e-06 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.180 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924896+0000", 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6810000000000003e-06 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924934+0000", 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7849000000000003e-05 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924952+0000", 2023-09-23T23:03:27.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8175000000000001e-05 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.999878+0000", 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.074926648999999998 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.999920+0000", 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1924e-05 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.182 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:104856 10.2 e340/317 10:4d1d554a:::benchmark_data_smithi052_461816_object8055:head v 340'6546, mlcod=340'6544)", 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 371.18754673500001, 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.070607910999999995, 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924973+0000", 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924978+0000", 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9599999999999999e-06 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924986+0000", 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.924992+0000", 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5219999999999998e-06 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.986453+0000", 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.061461233999999997 2023-09-23T23:03:27.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.986477+0000", 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3586000000000001e-05 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.995543+0000", 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090668009999999993 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:15.995581+0000", 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7732999999999999e-05 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113501 10.4 e344/343 10:2471dee2:::benchmark_data_smithi052_461816_object16700:head v 344'7088, mlcod=344'7086)", 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.300805+0000", 2023-09-23T23:03:27.188 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 354.81171464400001, 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.114329005, 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300805+0000", 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300805+0000", 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300804+0000", 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300813+0000", 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.312e-06 2023-09-23T23:03:27.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300829+0000", 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5485000000000002e-05 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300839+0000", 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0029e-05 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.300894+0000", 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5551999999999999e-05 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405126+0000", 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.104231453 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.415005+0000", 2023-09-23T23:03:27.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098791069999999998 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.415134+0000", 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012936600000000001 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113493 10.c e344/343 10:3a2fa64b:::benchmark_data_smithi052_461816_object16692:head v 344'7088, mlcod=0'0)", 2023-09-23T23:03:27.193 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.302714+0000", 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 354.80980633399997, 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.115107467, 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302714+0000", 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302714+0000", 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302713+0000", 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:27.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.302720+0000", 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9609999999999999e-06 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405115+0000", 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.10239490699999999 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405128+0000", 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3213e-05 2023-09-23T23:03:27.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405167+0000", 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8720000000000002e-05 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.405189+0000", 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2401000000000001e-05 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:27.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.417788+0000", 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012598971 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.417821+0000", 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3126000000000001e-05 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:27.198 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:03:27.199 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:26 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4015115786' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:03:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:26 smithi151 ceph-mon[159210]: osdmap e405: 8 total, 8 up, 6 in 2023-09-23T23:03:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:26 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:03:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:26 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:03:27.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:26 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:27.201 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:27.225 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:27.268 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:27.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:27.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:27.373 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:27.404 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:27.973 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:27.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:27.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:28.037 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:03:28.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[180979]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[180979]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[180979]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 1.2 MiB/s, 22 objects/s recovering 2023-09-23T23:03:28.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[180979]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[180979]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[180979]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[187743]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[187743]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[187743]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 1.2 MiB/s, 22 objects/s recovering 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[187743]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[187743]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:27 smithi052 ceph-mon[187743]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.077 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:28.102 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:28.144 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:28.148 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:27 smithi151 ceph-mon[159210]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.148 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:27 smithi151 ceph-mon[159210]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:27 smithi151 ceph-mon[159210]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 1.2 MiB/s, 22 objects/s recovering 2023-09-23T23:03:28.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:27 smithi151 ceph-mon[159210]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:27 smithi151 ceph-mon[159210]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.149 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:27 smithi151 ceph-mon[159210]: failed to encode map e405 with expected crc 2023-09-23T23:03:28.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:28.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:28.251 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:28.276 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:28.320 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:28.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:28.425 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:28.451 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:28.493 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:28.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:28.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:28.601 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:28.627 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:28.673 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:28.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:28.742 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:03:28.743 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T23:03:28.743 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T23:03:28.779 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:28.814 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:28.956 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:28.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:29.060 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:29.085 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:29.128 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:29.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:29.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:28 smithi151 ceph-mon[159210]: failed to encode map e405 with expected crc 2023-09-23T23:03:29.199 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:29.234 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:29.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:28 smithi052 ceph-mon[180979]: failed to encode map e405 with expected crc 2023-09-23T23:03:29.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:28 smithi052 ceph-mon[187743]: failed to encode map e405 with expected crc 2023-09-23T23:03:29.261 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:29.523 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:29.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:29.557 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:29.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:29.667 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:29.755 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:29.772 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:03:29.772 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:29.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:03:29.974 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:29.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:29.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:30.079 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:30.113 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:29 smithi052 ceph-mon[187743]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 1.2 MiB/s, 22 objects/s recovering 2023-09-23T23:03:30.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:29 smithi052 ceph-mon[180979]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 1.2 MiB/s, 22 objects/s recovering 2023-09-23T23:03:30.115 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:29 smithi151 ceph-mon[159210]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean, 2 active+recovery_wait+degraded+remapped; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 1.2 MiB/s, 22 objects/s recovering 2023-09-23T23:03:31.025 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:31.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:31.049 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:30 smithi052 ceph-mon[187743]: from='client.46817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:31.049 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 12176/72 objects degraded (16911.111%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:31.050 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:30 smithi052 ceph-mon[180979]: from='client.46817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:31.050 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 12176/72 objects degraded (16911.111%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:31.109 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1253,"stamp":"2023-09-23T23:03:28.638877+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":1091,"num_objects_missing":1091,"num_objects_degraded":12176,"num_objects_misplaced":22257,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":193858,"num_write_kb":6197518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20917,"num_bytes_recovered":1274756028,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100592,"ondisk_log_size":100592,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4111640,"kb_used_data":682664,"kb_used_omap":137,"kb_used_meta":3428790,"kb_avail":558359272,"statfs":{"total":575970213888,"available":571759894528,"internally_reserved":0,"allocated":699047936,"data_stored":690817604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3511081144},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-195362840,"num_objects":-2982,"num_object_clones":0,"num_object_copies":-8946,"num_objects_missing_on_primary":-218,"num_objects_missing":-218,"num_objects_degraded":-733,"num_objects_misplaced":-772,"num_objects_unfound":0,"num_objects_dirty":-2982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2982,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":228,"num_bytes_recovered":12320768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003537"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":416,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807927+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:03:04.807927+0000","last_peered":"2023-09-23T23:03:04.807927+0000","last_clean":"2023-09-23T23:03:04.807927+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:03:04.807927+0000","last_undegraded":"2023-09-23T23:03:04.807927+0000","last_fullsized":"2023-09-23T23:03:04.807927+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.878844+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:03:14.878844+0000","last_peered":"2023-09-23T23:03:14.878844+0000","last_clean":"2023-09-23T23:03:14.878844+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:03:14.878844+0000","last_undegraded":"2023-09-23T23:03:14.878844+0000","last_fullsized":"2023-09-23T23:03:14.878844+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":74,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873511+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:03:14.873511+0000","last_peered":"2023-09-23T23:03:14.873511+0000","last_clean":"2023-09-23T23:03:14.873511+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:03:14.873511+0000","last_undegraded":"2023-09-23T23:03:14.873511+0000","last_fullsized":"2023-09-23T23:03:14.873511+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":402,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.874095+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:03:14.874095+0000","last_peered":"2023-09-23T23:03:14.874095+0000","last_clean":"2023-09-23T23:03:14.874095+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:03:14.874095+0000","last_undegraded":"2023-09-23T23:03:14.874095+0000","last_fullsized":"2023-09-23T23:03:14.874095+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873217+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:03:14.873217+0000","last_peered":"2023-09-23T23:03:14.873217+0000","last_clean":"2023-09-23T23:03:14.873217+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:03:14.873217+0000","last_undegraded":"2023-09-23T23:03:14.873217+0000","last_fullsized":"2023-09-23T23:03:14.873217+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797679+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:03:04.797679+0000","last_peered":"2023-09-23T23:03:04.797679+0000","last_clean":"2023-09-23T23:03:04.797679+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:03:04.797679+0000","last_undegraded":"2023-09-23T23:03:04.797679+0000","last_fullsized":"2023-09-23T23:03:04.797679+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":339,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.821053+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:03:04.821053+0000","last_peered":"2023-09-23T23:03:04.821053+0000","last_clean":"2023-09-23T23:03:04.821053+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:03:04.821053+0000","last_undegraded":"2023-09-23T23:03:04.821053+0000","last_fullsized":"2023-09-23T23:03:04.821053+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873091+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:03:14.873091+0000","last_peered":"2023-09-23T23:03:14.873091+0000","last_clean":"2023-09-23T23:03:14.873091+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:03:14.873091+0000","last_undegraded":"2023-09-23T23:03:14.873091+0000","last_fullsized":"2023-09-23T23:03:14.873091+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":300,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820805+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:03:04.820805+0000","last_peered":"2023-09-23T23:03:04.820805+0000","last_clean":"2023-09-23T23:03:04.820805+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:03:04.820805+0000","last_undegraded":"2023-09-23T23:03:04.820805+0000","last_fullsized":"2023-09-23T23:03:04.820805+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":89,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873380+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:03:14.873380+0000","last_peered":"2023-09-23T23:03:14.873380+0000","last_clean":"2023-09-23T23:03:14.873380+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:03:14.873380+0000","last_undegraded":"2023-09-23T23:03:14.873380+0000","last_fullsized":"2023-09-23T23:03:14.873380+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":95,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.878771+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:03:14.878771+0000","last_peered":"2023-09-23T23:03:14.878771+0000","last_clean":"2023-09-23T23:03:14.878771+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:03:14.878771+0000","last_undegraded":"2023-09-23T23:03:14.878771+0000","last_fullsized":"2023-09-23T23:03:14.878771+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.879471+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:03:14.879471+0000","last_peered":"2023-09-23T23:03:14.879471+0000","last_clean":"2023-09-23T23:03:14.879471+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:03:14.879471+0000","last_undegraded":"2023-09-23T23:03:14.879471+0000","last_fullsized":"2023-09-23T23:03:14.879471+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":475,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807606+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:03:04.807606+0000","last_peered":"2023-09-23T23:03:04.807606+0000","last_clean":"2023-09-23T23:03:04.807606+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:03:04.807606+0000","last_undegraded":"2023-09-23T23:03:04.807606+0000","last_fullsized":"2023-09-23T23:03:04.807606+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":378,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813686+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:03:04.813686+0000","last_peered":"2023-09-23T23:03:04.813686+0000","last_clean":"2023-09-23T23:03:04.813686+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:03:04.813686+0000","last_undegraded":"2023-09-23T23:03:04.813686+0000","last_fullsized":"2023-09-23T23:03:04.813686+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":304,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813796+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:03:04.813796+0000","last_peered":"2023-09-23T23:03:04.813796+0000","last_clean":"2023-09-23T23:03:04.813796+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:03:04.813796+0000","last_undegraded":"2023-09-23T23:03:04.813796+0000","last_fullsized":"2023-09-23T23:03:04.813796+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.876560+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:03:14.876560+0000","last_peered":"2023-09-23T23:03:14.876560+0000","last_clean":"2023-09-23T23:03:14.876560+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:03:14.876560+0000","last_undegraded":"2023-09-23T23:03:14.876560+0000","last_fullsized":"2023-09-23T23:03:14.876560+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":395,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873732+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:03:14.873732+0000","last_peered":"2023-09-23T23:03:14.873732+0000","last_clean":"2023-09-23T23:03:14.873732+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:03:14.873732+0000","last_undegraded":"2023-09-23T23:03:14.873732+0000","last_fullsized":"2023-09-23T23:03:14.873732+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":304,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820660+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:03:04.820660+0000","last_peered":"2023-09-23T23:03:04.820660+0000","last_clean":"2023-09-23T23:03:04.820660+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:03:04.820660+0000","last_undegraded":"2023-09-23T23:03:04.820660+0000","last_fullsized":"2023-09-23T23:03:04.820660+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":289,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809860+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:03:04.809860+0000","last_peered":"2023-09-23T23:03:04.809860+0000","last_clean":"2023-09-23T23:03:04.809860+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:03:04.809860+0000","last_undegraded":"2023-09-23T23:03:04.809860+0000","last_fullsized":"2023-09-23T23:03:04.809860+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":419,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.874229+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:03:14.874229+0000","last_peered":"2023-09-23T23:03:14.874229+0000","last_clean":"2023-09-23T23:03:14.874229+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:03:14.874229+0000","last_undegraded":"2023-09-23T23:03:14.874229+0000","last_fullsized":"2023-09-23T23:03:14.874229+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":123,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795260+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:03:04.795260+0000","last_peered":"2023-09-23T23:03:04.795260+0000","last_clean":"2023-09-23T23:03:04.795260+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:03:04.795260+0000","last_undegraded":"2023-09-23T23:03:04.795260+0000","last_fullsized":"2023-09-23T23:03:04.795260+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":99,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.874638+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:03:14.874638+0000","last_peered":"2023-09-23T23:03:14.874638+0000","last_clean":"2023-09-23T23:03:14.874638+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:03:14.874638+0000","last_undegraded":"2023-09-23T23:03:14.874638+0000","last_fullsized":"2023-09-23T23:03:14.874638+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"404'12086","reported_seq":12195,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:23.124959+0000","last_change":"2023-09-23T22:58:08.022433+0000","last_active":"2023-09-23T23:03:23.124959+0000","last_peered":"2023-09-23T23:03:23.124959+0000","last_clean":"2023-09-23T23:03:23.124959+0000","last_became_active":"2023-09-23T22:58:06.776245+0000","last_became_peered":"2023-09-23T22:58:06.776245+0000","last_unstale":"2023-09-23T23:03:23.124959+0000","last_undegraded":"2023-09-23T23:03:23.124959+0000","last_fullsized":"2023-09-23T23:03:23.124959+0000","mapping_epoch":353,"log_start":"340'6600","ondisk_log_start":"340'6600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:32.777213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12086,"num_write_kb":386752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":288,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809675+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:03:04.809675+0000","last_peered":"2023-09-23T23:03:04.809675+0000","last_clean":"2023-09-23T23:03:04.809675+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:03:04.809675+0000","last_undegraded":"2023-09-23T23:03:04.809675+0000","last_fullsized":"2023-09-23T23:03:04.809675+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":274,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820249+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:03:04.820249+0000","last_peered":"2023-09-23T23:03:04.820249+0000","last_clean":"2023-09-23T23:03:04.820249+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:03:04.820249+0000","last_undegraded":"2023-09-23T23:03:04.820249+0000","last_fullsized":"2023-09-23T23:03:04.820249+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"404'11950","reported_seq":14387,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:23.129041+0000","last_change":"2023-09-23T22:58:24.590942+0000","last_active":"2023-09-23T23:03:23.129041+0000","last_peered":"2023-09-23T23:03:23.129041+0000","last_clean":"2023-09-23T22:55:52.122215+0000","last_became_active":"2023-09-23T22:58:24.570960+0000","last_became_peered":"2023-09-23T22:58:24.570960+0000","last_unstale":"2023-09-23T23:03:23.129041+0000","last_undegraded":"2023-09-23T22:58:24.441235+0000","last_fullsized":"2023-09-23T22:58:24.237560+0000","mapping_epoch":360,"log_start":"326'3300","ondisk_log_start":"326'3300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8650,"log_dups_size":0,"ondisk_log_size":8650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11950,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[5,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"5,6","objects":-3567}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":298,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808078+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:03:04.808078+0000","last_peered":"2023-09-23T23:03:04.808078+0000","last_clean":"2023-09-23T23:03:04.808078+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:03:04.808078+0000","last_undegraded":"2023-09-23T23:03:04.808078+0000","last_fullsized":"2023-09-23T23:03:04.808078+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":411,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809779+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:03:04.809779+0000","last_peered":"2023-09-23T23:03:04.809779+0000","last_clean":"2023-09-23T23:03:04.809779+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:03:04.809779+0000","last_undegraded":"2023-09-23T23:03:04.809779+0000","last_fullsized":"2023-09-23T23:03:04.809779+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"404'12072","reported_seq":12175,"reported_epoch":404,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:03:23.181896+0000","last_change":"2023-09-23T22:58:07.950319+0000","last_active":"2023-09-23T23:03:23.181896+0000","last_peered":"2023-09-23T23:03:23.181896+0000","last_clean":"2023-09-23T22:57:25.501000+0000","last_became_active":"2023-09-23T22:58:07.928850+0000","last_became_peered":"2023-09-23T22:58:07.928850+0000","last_unstale":"2023-09-23T23:03:23.181896+0000","last_undegraded":"2023-09-23T22:58:07.725959+0000","last_fullsized":"2023-09-23T23:03:23.181896+0000","mapping_epoch":354,"log_start":"340'6600","ondisk_log_start":"340'6600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:17.153923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12072,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,1],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":155,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820761+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:03:04.820761+0000","last_peered":"2023-09-23T23:03:04.820761+0000","last_clean":"2023-09-23T23:03:04.820761+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:03:04.820761+0000","last_undegraded":"2023-09-23T23:03:04.820761+0000","last_fullsized":"2023-09-23T23:03:04.820761+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":273,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820348+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:03:04.820348+0000","last_peered":"2023-09-23T23:03:04.820348+0000","last_clean":"2023-09-23T23:03:04.820348+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:03:04.820348+0000","last_undegraded":"2023-09-23T23:03:04.820348+0000","last_fullsized":"2023-09-23T23:03:04.820348+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"404'12174","reported_seq":12463,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:23.173431+0000","last_change":"2023-09-23T22:58:23.452962+0000","last_active":"2023-09-23T23:03:23.173431+0000","last_peered":"2023-09-23T23:03:23.173431+0000","last_clean":"2023-09-23T22:55:52.228935+0000","last_became_active":"2023-09-23T22:58:23.440648+0000","last_became_peered":"2023-09-23T22:58:23.440648+0000","last_unstale":"2023-09-23T23:03:23.173431+0000","last_undegraded":"2023-09-23T22:58:23.411973+0000","last_fullsized":"2023-09-23T22:58:23.234454+0000","mapping_epoch":359,"log_start":"341'6700","ondisk_log_start":"341'6700","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5474,"log_dups_size":0,"ondisk_log_size":5474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:26:19.577560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4696,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12174,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,5","objects":-3610}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":75,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.876485+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:03:14.876485+0000","last_peered":"2023-09-23T23:03:14.876485+0000","last_clean":"2023-09-23T23:03:14.876485+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:03:14.876485+0000","last_undegraded":"2023-09-23T23:03:14.876485+0000","last_fullsized":"2023-09-23T23:03:14.876485+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797096+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:03:04.797096+0000","last_peered":"2023-09-23T23:03:04.797096+0000","last_clean":"2023-09-23T23:03:04.797096+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:03:04.797096+0000","last_undegraded":"2023-09-23T23:03:04.797096+0000","last_fullsized":"2023-09-23T23:03:04.797096+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"404'12094","reported_seq":12203,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:23.173998+0000","last_change":"2023-09-23T22:58:06.789327+0000","last_active":"2023-09-23T23:03:23.173998+0000","last_peered":"2023-09-23T23:03:23.173998+0000","last_clean":"2023-09-23T22:57:33.564887+0000","last_became_active":"2023-09-23T22:58:06.775798+0000","last_became_peered":"2023-09-23T22:58:06.775798+0000","last_unstale":"2023-09-23T23:03:23.173998+0000","last_undegraded":"2023-09-23T22:58:06.747896+0000","last_fullsized":"2023-09-23T23:03:23.173998+0000","mapping_epoch":353,"log_start":"340'6400","ondisk_log_start":"340'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12094,"num_write_kb":387008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.875274+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:03:14.875274+0000","last_peered":"2023-09-23T23:03:14.875274+0000","last_clean":"2023-09-23T23:03:14.875274+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:03:14.875274+0000","last_undegraded":"2023-09-23T23:03:14.875274+0000","last_fullsized":"2023-09-23T23:03:14.875274+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":397,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807474+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:03:04.807474+0000","last_peered":"2023-09-23T23:03:04.807474+0000","last_clean":"2023-09-23T23:03:04.807474+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:03:04.807474+0000","last_undegraded":"2023-09-23T23:03:04.807474+0000","last_fullsized":"2023-09-23T23:03:04.807474+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"404'12202","reported_seq":12319,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:23.148741+0000","last_change":"2023-09-23T22:58:23.493113+0000","last_active":"2023-09-23T23:03:23.148741+0000","last_peered":"2023-09-23T23:03:23.148741+0000","last_clean":"2023-09-23T22:55:52.120390+0000","last_became_active":"2023-09-23T22:58:23.475045+0000","last_became_peered":"2023-09-23T22:58:23.475045+0000","last_unstale":"2023-09-23T23:03:23.148741+0000","last_undegraded":"2023-09-23T22:58:23.463410+0000","last_fullsized":"2023-09-23T22:58:23.238264+0000","mapping_epoch":359,"log_start":"340'6500","ondisk_log_start":"340'6500","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4713,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12202,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5","objects":-3619}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":187,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808133+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:03:04.808133+0000","last_peered":"2023-09-23T23:03:04.808133+0000","last_clean":"2023-09-23T23:03:04.808133+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:03:04.808133+0000","last_undegraded":"2023-09-23T23:03:04.808133+0000","last_fullsized":"2023-09-23T23:03:04.808133+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":414,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807541+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:03:04.807541+0000","last_peered":"2023-09-23T23:03:04.807541+0000","last_clean":"2023-09-23T23:03:04.807541+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:03:04.807541+0000","last_undegraded":"2023-09-23T23:03:04.807541+0000","last_fullsized":"2023-09-23T23:03:04.807541+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"404'12060","reported_seq":20903,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:23.069228+0000","last_change":"2023-09-23T22:58:07.815188+0000","last_active":"2023-09-23T23:03:23.069228+0000","last_peered":"2023-09-23T23:03:23.069228+0000","last_clean":"2023-09-23T22:56:19.652050+0000","last_became_active":"2023-09-23T22:58:07.794965+0000","last_became_peered":"2023-09-23T22:58:07.794965+0000","last_unstale":"2023-09-23T23:03:23.069228+0000","last_undegraded":"2023-09-23T22:58:07.783791+0000","last_fullsized":"2023-09-23T23:03:23.069228+0000","mapping_epoch":354,"log_start":"339'6300","ondisk_log_start":"339'6300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12060,"num_write_kb":385920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":427,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809515+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:03:04.809515+0000","last_peered":"2023-09-23T23:03:04.809515+0000","last_clean":"2023-09-23T23:03:04.809515+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:03:04.809515+0000","last_undegraded":"2023-09-23T23:03:04.809515+0000","last_fullsized":"2023-09-23T23:03:04.809515+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":405,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808148+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:03:04.808148+0000","last_peered":"2023-09-23T23:03:04.808148+0000","last_clean":"2023-09-23T23:03:04.808148+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:03:04.808148+0000","last_undegraded":"2023-09-23T23:03:04.808148+0000","last_fullsized":"2023-09-23T23:03:04.808148+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"404'11968","reported_seq":12077,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:03:23.047371+0000","last_change":"2023-09-23T22:58:06.801157+0000","last_active":"2023-09-23T23:03:23.047371+0000","last_peered":"2023-09-23T23:03:23.047371+0000","last_clean":"2023-09-23T22:57:33.598856+0000","last_became_active":"2023-09-23T22:58:06.778294+0000","last_became_peered":"2023-09-23T22:58:06.778294+0000","last_unstale":"2023-09-23T23:03:23.047371+0000","last_undegraded":"2023-09-23T22:58:06.747776+0000","last_fullsized":"2023-09-23T23:03:23.047371+0000","mapping_epoch":353,"log_start":"340'6300","ondisk_log_start":"340'6300","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5668,"log_dups_size":0,"ondisk_log_size":5668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11968,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":75,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873338+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:03:14.873338+0000","last_peered":"2023-09-23T23:03:14.873338+0000","last_clean":"2023-09-23T23:03:14.873338+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:03:14.873338+0000","last_undegraded":"2023-09-23T23:03:14.873338+0000","last_fullsized":"2023-09-23T23:03:14.873338+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":309,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.882494+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:03:14.882494+0000","last_peered":"2023-09-23T23:03:14.882494+0000","last_clean":"2023-09-23T23:03:14.882494+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:03:14.882494+0000","last_undegraded":"2023-09-23T23:03:14.882494+0000","last_fullsized":"2023-09-23T23:03:14.882494+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"404'12076","reported_seq":21371,"reported_epoch":404,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:03:23.201075+0000","last_change":"2023-09-23T23:01:18.261752+0000","last_active":"2023-09-23T23:03:23.201075+0000","last_peered":"2023-09-23T23:03:23.201075+0000","last_clean":"2023-09-23T22:56:19.635718+0000","last_became_active":"2023-09-23T22:58:07.933736+0000","last_became_peered":"2023-09-23T22:58:07.933736+0000","last_unstale":"2023-09-23T23:03:23.201075+0000","last_undegraded":"2023-09-23T23:03:23.201075+0000","last_fullsized":"2023-09-23T23:03:23.201075+0000","mapping_epoch":354,"log_start":"340'6400","ondisk_log_start":"340'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12076,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,1],"avail_no_missing":["4","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":313,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.879131+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:03:14.879131+0000","last_peered":"2023-09-23T23:03:14.879131+0000","last_clean":"2023-09-23T23:03:14.879131+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:03:14.879131+0000","last_undegraded":"2023-09-23T23:03:14.879131+0000","last_fullsized":"2023-09-23T23:03:14.879131+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820567+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:03:04.820567+0000","last_peered":"2023-09-23T23:03:04.820567+0000","last_clean":"2023-09-23T23:03:04.820567+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:03:04.820567+0000","last_undegraded":"2023-09-23T23:03:04.820567+0000","last_fullsized":"2023-09-23T23:03:04.820567+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820196+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:03:04.820196+0000","last_peered":"2023-09-23T23:03:04.820196+0000","last_clean":"2023-09-23T23:03:04.820196+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:03:04.820196+0000","last_undegraded":"2023-09-23T23:03:04.820196+0000","last_fullsized":"2023-09-23T23:03:04.820196+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"404'12124","reported_seq":12246,"reported_epoch":404,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T23:03:23.161354+0000","last_change":"2023-09-23T22:58:23.374138+0000","last_active":"2023-09-23T23:03:23.161354+0000","last_peered":"2023-09-23T23:03:23.161354+0000","last_clean":"2023-09-23T22:58:21.227804+0000","last_became_active":"2023-09-23T22:58:23.358904+0000","last_became_peered":"2023-09-23T22:58:23.358904+0000","last_unstale":"2023-09-23T23:03:23.161354+0000","last_undegraded":"2023-09-23T23:03:23.161354+0000","last_fullsized":"2023-09-23T23:03:23.161354+0000","mapping_epoch":359,"log_start":"340'6600","ondisk_log_start":"340'6600","created":317,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.393016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12124,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,6],"avail_no_missing":["0","2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":75,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795883+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:03:04.795883+0000","last_peered":"2023-09-23T23:03:04.795883+0000","last_clean":"2023-09-23T23:03:04.795883+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:03:04.795883+0000","last_undegraded":"2023-09-23T23:03:04.795883+0000","last_fullsized":"2023-09-23T23:03:04.795883+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":424,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813886+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:03:04.813886+0000","last_peered":"2023-09-23T23:03:04.813886+0000","last_clean":"2023-09-23T23:03:04.813886+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:03:04.813886+0000","last_undegraded":"2023-09-23T23:03:04.813886+0000","last_fullsized":"2023-09-23T23:03:04.813886+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":298,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.879203+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:03:14.879203+0000","last_peered":"2023-09-23T23:03:14.879203+0000","last_clean":"2023-09-23T23:03:14.879203+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:03:14.879203+0000","last_undegraded":"2023-09-23T23:03:14.879203+0000","last_fullsized":"2023-09-23T23:03:14.879203+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"404'12064","reported_seq":20248,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:23.140562+0000","last_change":"2023-09-23T23:01:18.346728+0000","last_active":"2023-09-23T23:03:23.140562+0000","last_peered":"2023-09-23T23:03:23.140562+0000","last_clean":"2023-09-23T22:55:52.106099+0000","last_became_active":"2023-09-23T22:58:24.486985+0000","last_became_peered":"2023-09-23T22:58:24.486985+0000","last_unstale":"2023-09-23T23:03:23.140562+0000","last_undegraded":"2023-09-23T23:01:18.346487+0000","last_fullsized":"2023-09-23T22:58:24.237849+0000","mapping_epoch":360,"log_start":"326'3400","ondisk_log_start":"326'3400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":8664,"log_dups_size":0,"ondisk_log_size":8664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12064,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4033,"num_bytes_recovered":234029056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":373,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.878954+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:03:14.878954+0000","last_peered":"2023-09-23T23:03:14.878954+0000","last_clean":"2023-09-23T23:03:14.878954+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:03:14.878954+0000","last_undegraded":"2023-09-23T23:03:14.878954+0000","last_fullsized":"2023-09-23T23:03:14.878954+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":250,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.878650+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:03:14.878650+0000","last_peered":"2023-09-23T23:03:14.878650+0000","last_clean":"2023-09-23T23:03:14.878650+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:03:14.878650+0000","last_undegraded":"2023-09-23T23:03:14.878650+0000","last_fullsized":"2023-09-23T23:03:14.878650+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.821095+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:03:04.821095+0000","last_peered":"2023-09-23T23:03:04.821095+0000","last_clean":"2023-09-23T23:03:04.821095+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:03:04.821095+0000","last_undegraded":"2023-09-23T23:03:04.821095+0000","last_fullsized":"2023-09-23T23:03:04.821095+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"404'12224","reported_seq":17107,"reported_epoch":404,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:25.426850+0000","last_change":"2023-09-23T23:01:18.421643+0000","last_active":"2023-09-23T23:03:25.426850+0000","last_peered":"2023-09-23T23:03:25.426850+0000","last_clean":"2023-09-23T22:56:19.646980+0000","last_became_active":"2023-09-23T22:58:26.210373+0000","last_became_peered":"2023-09-23T22:58:26.210373+0000","last_unstale":"2023-09-23T23:03:25.426850+0000","last_undegraded":"2023-09-23T22:58:26.059423+0000","last_fullsized":"2023-09-23T22:58:07.730672+0000","mapping_epoch":354,"log_start":"322'2200","ondisk_log_start":"322'2200","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":10024,"log_dups_size":0,"ondisk_log_size":10024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1089,"num_objects_missing":1089,"num_objects_degraded":3549,"num_objects_misplaced":3549,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12224,"num_write_kb":391168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2368,"num_bytes_recovered":152567808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[6,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,6","objects":-3549}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":387,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820675+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:03:04.820675+0000","last_peered":"2023-09-23T23:03:04.820675+0000","last_clean":"2023-09-23T23:03:04.820675+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:03:04.820675+0000","last_undegraded":"2023-09-23T23:03:04.820675+0000","last_fullsized":"2023-09-23T23:03:04.820675+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":501,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:04.796492+0000","last_change":"2023-09-23T22:58:24.260644+0000","last_active":"2023-09-23T23:03:04.796492+0000","last_peered":"2023-09-23T23:03:04.796492+0000","last_clean":"2023-09-23T22:58:21.227810+0000","last_became_active":"2023-09-23T22:58:24.259948+0000","last_became_peered":"2023-09-23T22:58:24.259948+0000","last_unstale":"2023-09-23T23:03:04.796492+0000","last_undegraded":"2023-09-23T22:58:24.231878+0000","last_fullsized":"2023-09-23T22:58:24.231342+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:13.890005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":75,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873918+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:03:14.873918+0000","last_peered":"2023-09-23T23:03:14.873918+0000","last_clean":"2023-09-23T23:03:14.873918+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:03:14.873918+0000","last_undegraded":"2023-09-23T23:03:14.873918+0000","last_fullsized":"2023-09-23T23:03:14.873918+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807801+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:03:04.807801+0000","last_peered":"2023-09-23T23:03:04.807801+0000","last_clean":"2023-09-23T23:03:04.807801+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:03:04.807801+0000","last_undegraded":"2023-09-23T23:03:04.807801+0000","last_fullsized":"2023-09-23T23:03:04.807801+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"404'12084","reported_seq":12198,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:23.184453+0000","last_change":"2023-09-23T22:58:24.453313+0000","last_active":"2023-09-23T23:03:23.184453+0000","last_peered":"2023-09-23T23:03:23.184453+0000","last_clean":"2023-09-23T22:55:52.216995+0000","last_became_active":"2023-09-23T22:58:24.434373+0000","last_became_peered":"2023-09-23T22:58:24.434373+0000","last_unstale":"2023-09-23T23:03:23.184453+0000","last_undegraded":"2023-09-23T22:58:24.258665+0000","last_fullsized":"2023-09-23T22:58:24.230123+0000","mapping_epoch":360,"log_start":"340'6600","ondisk_log_start":"340'6600","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:42.678602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12084,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-3507}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":395,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795343+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:03:04.795343+0000","last_peered":"2023-09-23T23:03:04.795343+0000","last_clean":"2023-09-23T23:03:04.795343+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:03:04.795343+0000","last_undegraded":"2023-09-23T23:03:04.795343+0000","last_fullsized":"2023-09-23T23:03:04.795343+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":366,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.795757+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:03:04.795757+0000","last_peered":"2023-09-23T23:03:04.795757+0000","last_clean":"2023-09-23T23:03:04.795757+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:03:04.795757+0000","last_undegraded":"2023-09-23T23:03:04.795757+0000","last_fullsized":"2023-09-23T23:03:04.795757+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":326,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.798533+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:03:04.798533+0000","last_peered":"2023-09-23T23:03:04.798533+0000","last_clean":"2023-09-23T23:03:04.798533+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:03:04.798533+0000","last_undegraded":"2023-09-23T23:03:04.798533+0000","last_fullsized":"2023-09-23T23:03:04.798533+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"404'12204","reported_seq":12332,"reported_epoch":404,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T23:03:23.163026+0000","last_change":"2023-09-23T22:58:07.773488+0000","last_active":"2023-09-23T23:03:23.163026+0000","last_peered":"2023-09-23T23:03:23.163026+0000","last_clean":"2023-09-23T22:57:25.495850+0000","last_became_active":"2023-09-23T22:58:07.755728+0000","last_became_peered":"2023-09-23T22:58:07.755728+0000","last_unstale":"2023-09-23T23:03:23.163026+0000","last_undegraded":"2023-09-23T22:58:07.742600+0000","last_fullsized":"2023-09-23T23:03:23.163026+0000","mapping_epoch":354,"log_start":"340'6500","ondisk_log_start":"340'6500","created":317,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12204,"num_write_kb":390465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,6],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":341,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.797161+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:03:04.797161+0000","last_peered":"2023-09-23T23:03:04.797161+0000","last_clean":"2023-09-23T23:03:04.797161+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:03:04.797161+0000","last_undegraded":"2023-09-23T23:03:04.797161+0000","last_fullsized":"2023-09-23T23:03:04.797161+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":250,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.879285+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:03:14.879285+0000","last_peered":"2023-09-23T23:03:14.879285+0000","last_clean":"2023-09-23T23:03:14.879285+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:03:14.879285+0000","last_undegraded":"2023-09-23T23:03:14.879285+0000","last_fullsized":"2023-09-23T23:03:14.879285+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":217,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.820765+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:03:04.820765+0000","last_peered":"2023-09-23T23:03:04.820765+0000","last_clean":"2023-09-23T23:03:04.820765+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:03:04.820765+0000","last_undegraded":"2023-09-23T23:03:04.820765+0000","last_fullsized":"2023-09-23T23:03:04.820765+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"404'12146","reported_seq":22662,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:23.191699+0000","last_change":"2023-09-23T22:58:24.516220+0000","last_active":"2023-09-23T23:03:23.191699+0000","last_peered":"2023-09-23T23:03:23.191699+0000","last_clean":"2023-09-23T22:55:52.230686+0000","last_became_active":"2023-09-23T22:58:24.492807+0000","last_became_peered":"2023-09-23T22:58:24.492807+0000","last_unstale":"2023-09-23T23:03:23.191699+0000","last_undegraded":"2023-09-23T22:58:24.408242+0000","last_fullsized":"2023-09-23T22:58:24.239693+0000","mapping_epoch":360,"log_start":"340'6400","ondisk_log_start":"340'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12146,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4099,"num_bytes_recovered":268632064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,5","objects":-1385}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":99,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.874868+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:03:14.874868+0000","last_peered":"2023-09-23T23:03:14.874868+0000","last_clean":"2023-09-23T23:03:14.874868+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:03:14.874868+0000","last_undegraded":"2023-09-23T23:03:14.874868+0000","last_fullsized":"2023-09-23T23:03:14.874868+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809691+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:03:04.809691+0000","last_peered":"2023-09-23T23:03:04.809691+0000","last_clean":"2023-09-23T23:03:04.809691+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:03:04.809691+0000","last_undegraded":"2023-09-23T23:03:04.809691+0000","last_fullsized":"2023-09-23T23:03:04.809691+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T23:03:26.228918+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:03:26.228918+0000","last_peered":"2023-09-23T23:03:26.228918+0000","last_clean":"2023-09-23T23:03:26.228918+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:03:26.228918+0000","last_undegraded":"2023-09-23T23:03:26.228918+0000","last_fullsized":"2023-09-23T23:03:26.228918+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"404'12070","reported_seq":12608,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:03:22.992433+0000","last_change":"2023-09-23T22:58:24.609898+0000","last_active":"2023-09-23T23:03:22.992433+0000","last_peered":"2023-09-23T23:03:22.992433+0000","last_clean":"2023-09-23T22:55:52.124242+0000","last_became_active":"2023-09-23T22:58:24.588818+0000","last_became_peered":"2023-09-23T22:58:24.588818+0000","last_unstale":"2023-09-23T23:03:22.992433+0000","last_undegraded":"2023-09-23T22:58:24.467823+0000","last_fullsized":"2023-09-23T22:58:24.241929+0000","mapping_epoch":360,"log_start":"340'6400","ondisk_log_start":"340'6400","created":317,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:55:22.547749+0000","last_clean_scrub_stamp":"2023-09-23T22:55:22.547749+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4648,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12070,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[4,6],"avail_no_missing":["4","3","6"],"object_location_counts":[{"shards":"4,6","objects":-3551}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":362,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.882628+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:03:14.882628+0000","last_peered":"2023-09-23T23:03:14.882628+0000","last_clean":"2023-09-23T23:03:14.882628+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:03:14.882628+0000","last_undegraded":"2023-09-23T23:03:14.882628+0000","last_fullsized":"2023-09-23T23:03:14.882628+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":308,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.798488+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:03:04.798488+0000","last_peered":"2023-09-23T23:03:04.798488+0000","last_clean":"2023-09-23T23:03:04.798488+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:03:04.798488+0000","last_undegraded":"2023-09-23T23:03:04.798488+0000","last_fullsized":"2023-09-23T23:03:04.798488+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":378,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.809630+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:03:04.809630+0000","last_peered":"2023-09-23T23:03:04.809630+0000","last_clean":"2023-09-23T23:03:04.809630+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:03:04.809630+0000","last_undegraded":"2023-09-23T23:03:04.809630+0000","last_fullsized":"2023-09-23T23:03:04.809630+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":343,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.999363+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:03:04.999363+0000","last_peered":"2023-09-23T23:03:04.999363+0000","last_clean":"2023-09-23T23:03:04.999363+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:03:04.999363+0000","last_undegraded":"2023-09-23T23:03:04.999363+0000","last_fullsized":"2023-09-23T23:03:04.999363+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:21.374844+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:03:21.374844+0000","last_peered":"2023-09-23T23:03:21.374844+0000","last_clean":"2023-09-23T23:03:21.374844+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:03:21.374844+0000","last_undegraded":"2023-09-23T23:03:21.374844+0000","last_fullsized":"2023-09-23T23:03:21.374844+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":213,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807401+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:03:04.807401+0000","last_peered":"2023-09-23T23:03:04.807401+0000","last_clean":"2023-09-23T23:03:04.807401+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:03:04.807401+0000","last_undegraded":"2023-09-23T23:03:04.807401+0000","last_fullsized":"2023-09-23T23:03:04.807401+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":427,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.813965+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:03:04.813965+0000","last_peered":"2023-09-23T23:03:04.813965+0000","last_clean":"2023-09-23T23:03:04.813965+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:03:04.813965+0000","last_undegraded":"2023-09-23T23:03:04.813965+0000","last_fullsized":"2023-09-23T23:03:04.813965+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":227,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807663+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:03:04.807663+0000","last_peered":"2023-09-23T23:03:04.807663+0000","last_clean":"2023-09-23T23:03:04.807663+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:03:04.807663+0000","last_undegraded":"2023-09-23T23:03:04.807663+0000","last_fullsized":"2023-09-23T23:03:04.807663+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":314,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.808047+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:03:04.808047+0000","last_peered":"2023-09-23T23:03:04.808047+0000","last_clean":"2023-09-23T23:03:04.808047+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:03:04.808047+0000","last_undegraded":"2023-09-23T23:03:04.808047+0000","last_fullsized":"2023-09-23T23:03:04.808047+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":99,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.874377+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:03:14.874377+0000","last_peered":"2023-09-23T23:03:14.874377+0000","last_clean":"2023-09-23T23:03:14.874377+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:03:14.874377+0000","last_undegraded":"2023-09-23T23:03:14.874377+0000","last_fullsized":"2023-09-23T23:03:14.874377+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":102,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T23:03:14.873634+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:03:14.873634+0000","last_peered":"2023-09-23T23:03:14.873634+0000","last_clean":"2023-09-23T23:03:14.873634+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:03:14.873634+0000","last_undegraded":"2023-09-23T23:03:14.873634+0000","last_fullsized":"2023-09-23T23:03:14.873634+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":347,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.798314+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:03:04.798314+0000","last_peered":"2023-09-23T23:03:04.798314+0000","last_clean":"2023-09-23T23:03:04.798314+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:03:04.798314+0000","last_undegraded":"2023-09-23T23:03:04.798314+0000","last_fullsized":"2023-09-23T23:03:04.798314+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":389,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-09-23T23:03:04.807307+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:03:04.807307+0000","last_peered":"2023-09-23T23:03:04.807307+0000","last_clean":"2023-09-23T23:03:04.807307+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:03:04.807307+0000","last_undegraded":"2023-09-23T23:03:04.807307+0000","last_fullsized":"2023-09-23T23:03:04.807307+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1091,"num_objects_missing":1091,"num_objects_degraded":12174,"num_objects_misplaced":22257,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193598,"num_write_kb":6195073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20882,"num_bytes_recovered":1274740736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1669070848,"data_stored":1669070848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100398,"ondisk_log_size":100398,"up":48,"acting":40,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430645,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626870,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253183,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":839784,"kb_used_data":97248,"kb_used_omap":9,"kb_used_meta":742518,"kb_avail":92905368,"statfs":{"total":95995035648,"available":95135096832,"internally_reserved":0,"allocated":99581952,"data_stored":98030193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":760339433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7769999999999999,"15min":0.82699999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4780000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.995,"5min":2.1040000000000001,"15min":1.272},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.266,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.36199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.55300000000000005,"15min":0.53600000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.5359999999999996,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.67000000000000004,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.1310000000000002,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":0.315}]},{"osd":3,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.56899999999999995,"15min":1.7969999999999999},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":87.787999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.53300000000000003,"15min":2.0499999999999998},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":1.24,"5min":3.109,"15min":89.861999999999995},"last":0.499}]},{"osd":4,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3129999999999997,"5min":3.218,"15min":2.637},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.29199999999999998},"max":{"1min":67.738,"5min":151.036,"15min":151.036},"last":0.43099999999999999},{"interface":"front","average":{"1min":5.7089999999999996,"5min":3.431,"15min":2.9260000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":67.670000000000002,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3300000000000001,"5min":1.6930000000000001,"15min":2.5699999999999998},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":68.259,"5min":68.259,"15min":116.098},"last":0.46899999999999997},{"interface":"front","average":{"1min":5.5549999999999997,"5min":1.8759999999999999,"15min":2.6970000000000001},"min":{"1min":0.38200000000000001,"5min":0.33600000000000002,"15min":0.315},"max":{"1min":70.057000000000002,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":2.0600000000000001,"15min":2.5259999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":67.349000000000004,"5min":67.349000000000004,"15min":79.311000000000007},"last":0.45500000000000002},{"interface":"front","average":{"1min":5.5979999999999999,"5min":1.7529999999999999,"15min":2.125},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":67.366,"5min":67.366,"15min":79.328999999999994},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.78900000000000003,"15min":1.552},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4139999999999999,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.94499999999999995,"15min":1.925},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.4929999999999999,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.54000000000000004}]}]},{"osd":0,"up_from":353,"seq":1516123455553,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":215768,"kb_used_data":6224,"kb_used_omap":37,"kb_used_meta":209498,"kb_avail":93529384,"statfs":{"total":95995035648,"available":95774089216,"internally_reserved":0,"allocated":6373376,"data_stored":4653608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":214526114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.83299999999999996,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.6970000000000001,"5min":8.234,"15min":8.234},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.01,"5min":0.89700000000000002,"15min":0.747},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7169999999999996,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.83399999999999996,"15min":0.79800000000000004},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.694,"5min":12.215,"15min":12.215},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.9199999999999999,"5min":5.9199999999999999,"15min":5.9199999999999999},"last":74.698999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.997,"15min":0.63500000000000001},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.1109999999999998,"5min":12.178000000000001,"15min":12.178000000000001},"last":74.744},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.93799999999999994,"15min":0.91800000000000004},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.8050000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":74.777000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":3.298,"15min":1.6719999999999999},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2690000000000001,"5min":183.148,"15min":183.148},"last":74.805000000000007},{"interface":"front","average":{"1min":0.98899999999999999,"5min":3.4169999999999998,"15min":1.585},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.9910000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.38}]},{"osd":5,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.069,"15min":0.80900000000000005},"min":{"1min":0.433,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.3090000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.026,"15min":1.0069999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.7080000000000002,"5min":7.774,"15min":7.774},"last":74.858000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.0860000000000001,"15min":0.91800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.0309999999999997,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1,"15min":0.81599999999999995},"min":{"1min":0.44900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.3410000000000002,"5min":6.375,"15min":6.375},"last":0.65300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.139,"15min":1.248},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.0579999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.432},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.1699999999999999,"15min":1.242},"min":{"1min":0.496,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.3719999999999999,"5min":6.343,"15min":6.343},"last":0.68400000000000005}]}]},{"osd":2,"up_from":359,"seq":1541893259326,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":632472,"kb_used_data":469008,"kb_used_omap":83,"kb_used_meta":163372,"kb_avail":93112680,"statfs":{"total":95995035648,"available":95347384320,"internally_reserved":0,"allocated":480264192,"data_stored":478612409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":167293381},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.2749999999999999,"15min":0.66100000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":74.241,"5min":74.241,"15min":74.241},"last":0.28899999999999998},{"interface":"front","average":{"1min":4.6699999999999999,"5min":1.4590000000000001,"15min":0.95799999999999996},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":74.430000000000007,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.31}]},{"osd":1,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.68500000000000005,"15min":0.67200000000000004},"min":{"1min":0.26800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9710000000000001,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.083,"15min":1.131},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.32500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":0.81599999999999995,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.9530000000000003,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.2849999999999999,"5min":0.94399999999999995,"15min":0.85899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8170000000000002,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.53400000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.90000000000000002,"15min":0.67600000000000005},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.5359999999999996,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.96099999999999997,"15min":0.86899999999999999},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.8739999999999997,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.499}]},{"osd":5,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.0580000000000001,"15min":0.98899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.232,"5min":0.93999999999999995,"15min":0.76700000000000002},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.24,"15min":1.0600000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9359999999999999,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91700000000000004,"15min":0.70999999999999996},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.903,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1870000000000001,"15min":1.056},"min":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.5659999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.093,"15min":1.1499999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":5.9139999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":1.2270000000000001}]}]},{"osd":3,"up_from":94,"seq":403726926130,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":515116,"kb_used_data":4708,"kb_used_omap":7,"kb_used_meta":510392,"kb_avail":93230036,"statfs":{"total":95995035648,"available":95467556864,"internally_reserved":0,"allocated":4820992,"data_stored":3197748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":522642024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3780000000000001,"15min":1.0049999999999999},"min":{"1min":0.22,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.3019999999999996,"5min":122.289,"15min":122.289},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.0150000000000001,"15min":0.94299999999999995},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":8.9849999999999994,"5min":122.041,"15min":122.041},"last":0.44400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.87,"15min":2.2480000000000002},"min":{"1min":0.22,"5min":0.187,"15min":0.187},"max":{"1min":4.6100000000000003,"5min":8.6280000000000001,"15min":123.866},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.2929999999999999,"5min":0.90200000000000002,"15min":2.1539999999999999},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1710000000000003,"5min":6.8010000000000002,"15min":122.265},"last":0.56899999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70699999999999996,"15min":0.71499999999999997},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.28,"5min":1.097,"15min":0.86099999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.7919999999999998,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.67600000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.1970000000000001,"15min":1.966},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.6459999999999999,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.222,"5min":2.3119999999999998,"15min":1.9199999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2149999999999999,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.98699999999999999,"15min":1.403},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2530000000000001,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.0640000000000001,"15min":1.744},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":5.149,"5min":10.246,"15min":11.951000000000001},"last":0.84799999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.264,"15min":1.748},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":6.2830000000000004,"5min":9.2520000000000007,"15min":14.097},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.091,"5min":1.181,"15min":1.869},"min":{"1min":0.378,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":5.1970000000000001,"5min":9.2810000000000006,"15min":12.887},"last":0.91400000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95499999999999996,"15min":1.6850000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.119,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.6799999999999997,"5min":8.5830000000000002,"15min":12.32},"last":0.61299999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019327,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005080,"kb_used_data":101712,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92740072,"statfs":{"total":95995035648,"available":94965833728,"internally_reserved":0,"allocated":104153088,"data_stored":103299341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":7.6189999999999998,"5min":4.1950000000000003,"15min":1.7130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":156.375,"5min":156.375,"15min":156.375},"last":0.23100000000000001},{"interface":"front","average":{"1min":7.4009999999999998,"5min":4.335,"15min":1.8420000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":157.02199999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":0.97399999999999998,"15min":3.29},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8739999999999997,"5min":7.0640000000000001,"15min":132.304},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.90400000000000003,"15min":3.2410000000000001},"min":{"1min":0.308,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.181,"5min":10.361000000000001,"15min":132.453},"last":1.663}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.117,"15min":1.0529999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8380000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":2.5089999999999999},{"interface":"front","average":{"1min":1.653,"5min":1.0389999999999999,"15min":0.82499999999999996},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.071,"15min":2.79},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6189999999999998,"5min":10.337999999999999,"15min":164.453},"last":2.3769999999999998},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.2330000000000001,"15min":2.8170000000000002},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.32500000000000001},"max":{"1min":7.2519999999999998,"5min":7.2519999999999998,"15min":167.05199999999999},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:02:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.169,"15min":1.7589999999999999},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.9870000000000001,"5min":12.961,"15min":148.61600000000001},"last":0.318},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.92300000000000004,"15min":1.829},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":8.7810000000000006,"5min":8.7810000000000006,"15min":151.905},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.1870000000000001,"15min":1.2889999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":6.7050000000000001,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.014,"15min":1.1779999999999999},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.7409999999999997,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.429,"15min":1.2549999999999999},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1050000000000004,"5min":12.84,"15min":12.84},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2450000000000001,"15min":1.1240000000000001},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.2210000000000001,"5min":10.587,"15min":10.587},"last":0.40699999999999997}]}]},{"osd":5,"up_from":36,"seq":154618823101,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903420,"kb_used_data":3764,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":92841732,"statfs":{"total":95995035648,"available":95069933568,"internally_reserved":0,"allocated":3854336,"data_stored":3024305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.786,"15min":0.999},"min":{"1min":0.23799999999999999,"5min":0.221,"15min":0.221},"max":{"1min":23.346,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.9690000000000001,"15min":1.377},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":23.428999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.57099999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.76500000000000001,"15min":2.1429999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.9649999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.202,"15min":2.6389999999999998},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":2.3079999999999998,"5min":10.353,"15min":110.89},"last":0.58299999999999996}]},{"osd":2,"last update":"Sat Sep 23 23:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.997,"15min":0.86599999999999999},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.601,"5min":6.601,"15min":6.601},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.997,"15min":0.995},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6139999999999999,"5min":4.899,"15min":4.899},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0600000000000001,"15min":3.6230000000000002},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0899999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.2090000000000001,"15min":3.6680000000000001},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3420000000000001,"5min":10.965,"15min":108.15600000000001},"last":0.59699999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":1.298,"15min":1.552},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":1.321,"5min":10.603,"15min":115.06699999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92300000000000004,"15min":1.482},"min":{"1min":0.34599999999999997,"5min":0.253,"15min":0.253},"max":{"1min":1.764,"5min":9.5890000000000004,"15min":114.908},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.169,"15min":1.256},"min":{"1min":0.30499999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.77300000000000002,"5min":7.4630000000000001,"15min":12.615},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.90400000000000003,"15min":1.1100000000000001},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.0569999999999999,"5min":5.4909999999999997,"15min":12.74},"last":0.441}]},{"osd":7,"last update":"Sat Sep 23 23:03:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.04,"15min":1.0860000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.4770000000000001,"5min":9.5980000000000008,"15min":12.23},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.444,"15min":1.2090000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.8500000000000001,"5min":10.638999999999999,"15min":11.112},"last":0.52200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":917504,"data_stored":917504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":94306304,"data_stored":94306304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":475463680,"data_stored":475463680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":65536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":100270080,"data_stored":100270080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":998047744,"data_stored":998047744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:03:31.115 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:03:31.130 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:31.179 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:30 smithi151 ceph-mon[159210]: from='client.46817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 12176/72 objects degraded (16911.111%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:31.782 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:31.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:31.886 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:31.912 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:31.954 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:31.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:32.059 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:32.064 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:31 smithi052 ceph-mon[187743]: pgmap v1254: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 493 KiB/s, 23 objects/s recovering 2023-09-23T23:03:32.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:31 smithi052 ceph-mon[180979]: pgmap v1254: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 493 KiB/s, 23 objects/s recovering 2023-09-23T23:03:32.096 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:32.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:31 smithi151 ceph-mon[159210]: pgmap v1254: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 493 KiB/s, 23 objects/s recovering 2023-09-23T23:03:32.381 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:32.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:32.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:32.486 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:32.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:32.553 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:32.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:32.658 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:32.690 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:32.695 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:32.747 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:32.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:32.786 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:03:32.787 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:03:32.787 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T23:03:32.787 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:03:32.809 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:32.853 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:32.879 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:32.899 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:03:32.923 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:32.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:32.965 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/662343827' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:32.965 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:32 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:32.965 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/662343827' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:32.965 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:32 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:33.031 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:33.060 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/662343827' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:32 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:33.672 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:03:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:03:33] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.33.4" 2023-09-23T23:03:33.689 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:03:33.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: pgmap v1255: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 493 KiB/s, 23 objects/s recovering 2023-09-23T23:03:33.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: failed to encode map e406 with expected crc 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: osdmap e406: 8 total, 8 up, 6 in 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: failed to encode map e406 with expected crc 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: failed to encode map e406 with expected crc 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/995937984' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: pgmap v1255: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 493 KiB/s, 23 objects/s recovering 2023-09-23T23:03:33.948 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: failed to encode map e406 with expected crc 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: osdmap e406: 8 total, 8 up, 6 in 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: failed to encode map e406 with expected crc 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: failed to encode map e406 with expected crc 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/995937984' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:33.949 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:33 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:33.982 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_4' removed 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: pgmap v1255: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 12176/72 objects degraded (16911.111%); 22257/72 objects misplaced (30912.500%); 493 KiB/s, 23 objects/s recovering 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: failed to encode map e406 with expected crc 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: osdmap e406: 8 total, 8 up, 6 in 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: failed to encode map e406 with expected crc 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: failed to encode map e406 with expected crc 2023-09-23T23:03:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/995937984' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:34.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:33 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:03:34.521 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:34.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:34.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:34.662 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:34.861 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T23:03:34.862 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T23:03:34.862 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T23:03:34.862 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: osdmap e407: 8 total, 8 up, 6 in 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: failed to encode map e407 with expected crc 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:35.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: osdmap e407: 8 total, 8 up, 6 in 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: failed to encode map e407 with expected crc 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:35.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:35.190 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:35.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: failed to encode map e406 with expected crc 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: osdmap e407: 8 total, 8 up, 6 in 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: failed to encode map e407 with expected crc 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:35.294 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:35.324 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:35.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[180979]: failed to encode map e407 with expected crc 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[180979]: pgmap v1258: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 12176/72 objects degraded (16911.111%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[180979]: failed to encode map e408 with expected crc 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[180979]: osdmap e408: 8 total, 8 up, 6 in 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[180979]: failed to encode map e407 with expected crc 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[187743]: failed to encode map e407 with expected crc 2023-09-23T23:03:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[187743]: pgmap v1258: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T23:03:35.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 12176/72 objects degraded (16911.111%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:35.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[187743]: failed to encode map e408 with expected crc 2023-09-23T23:03:35.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[187743]: osdmap e408: 8 total, 8 up, 6 in 2023-09-23T23:03:35.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:35 smithi052 ceph-mon[187743]: failed to encode map e407 with expected crc 2023-09-23T23:03:36.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:36.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:36.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:36.182 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:36.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:35 smithi151 ceph-mon[159210]: failed to encode map e407 with expected crc 2023-09-23T23:03:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:35 smithi151 ceph-mon[159210]: pgmap v1258: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T23:03:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 12176/72 objects degraded (16911.111%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T23:03:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:35 smithi151 ceph-mon[159210]: failed to encode map e408 with expected crc 2023-09-23T23:03:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:35 smithi151 ceph-mon[159210]: osdmap e408: 8 total, 8 up, 6 in 2023-09-23T23:03:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:35 smithi151 ceph-mon[159210]: failed to encode map e407 with expected crc 2023-09-23T23:03:36.214 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:36.247 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:36.262 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:36.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:36.322 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:03:36.367 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:36.394 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:36.439 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:36.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:36.545 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:36.571 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:36.585 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:36.612 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:36.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:36.680 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:03:36.700 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:03:36.719 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:36.723 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:03:36.745 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:36.745 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:03:36.767 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:03:36.786 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:36.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:36.791 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:03:36.813 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:03:36.832 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:03:36.833 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.833 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.834 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.834 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.834 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.834 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.834 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.834 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:03:36.891 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:36.892 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:36.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:37.121 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:37.121 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1749,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":409,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":72},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":4210376704,"bytes_avail":571759837184,"bytes_total":575970213888,"degraded_objects":2,"degraded_total":72,"degraded_ratio":0.027777777777777776},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T22:59:22.549533+0000","services":{}},"progress_events":{"4e6b7cfb-1a29-45be-a285-3520f904eac8":{"message":"Global Recovery Event (7m)\n [======================......] (remaining: 97s)","progress":0.82022470235824585,"add_to_ceph_s":true},"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[180979]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[180979]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[180979]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[180979]: failed to encode map e409 with expected crc 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[180979]: osdmap e409: 8 total, 8 up, 6 in 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2449650321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:37.158 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[187743]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[187743]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[187743]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[187743]: failed to encode map e409 with expected crc 2023-09-23T23:03:37.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[187743]: osdmap e409: 8 total, 8 up, 6 in 2023-09-23T23:03:37.159 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2449650321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:37.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:36 smithi151 ceph-mon[159210]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:36 smithi151 ceph-mon[159210]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:36 smithi151 ceph-mon[159210]: failed to encode map e408 with expected crc 2023-09-23T23:03:37.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:36 smithi151 ceph-mon[159210]: failed to encode map e409 with expected crc 2023-09-23T23:03:37.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:36 smithi151 ceph-mon[159210]: osdmap e409: 8 total, 8 up, 6 in 2023-09-23T23:03:37.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2449650321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:38.155 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:37 smithi052 ceph-mon[180979]: pgmap v1261: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T23:03:38.155 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:37 smithi052 ceph-mon[180979]: failed to encode map e409 with expected crc 2023-09-23T23:03:38.155 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/657791563' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:03:38.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:37 smithi052 ceph-mon[187743]: pgmap v1261: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T23:03:38.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:37 smithi052 ceph-mon[187743]: failed to encode map e409 with expected crc 2023-09-23T23:03:38.155 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/657791563' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:03:38.196 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:38.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:38.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:37 smithi151 ceph-mon[159210]: pgmap v1261: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T23:03:38.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:37 smithi151 ceph-mon[159210]: failed to encode map e409 with expected crc 2023-09-23T23:03:38.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/657791563' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:03:38.300 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:38.329 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:38.372 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:38.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:38.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:38.481 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:38.511 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:39.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:38 smithi151 ceph-mon[159210]: pgmap v1262: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T23:03:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:38 smithi052 ceph-mon[187743]: pgmap v1262: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T23:03:39.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:38 smithi052 ceph-mon[180979]: pgmap v1262: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T23:03:39.501 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:03:39.501 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:03:39.563 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T23:03:39.564 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:03:39.664 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:39.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:39.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:39.770 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:39.799 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:40.082 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:03:40.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:03:40.733 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:40.735 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:03:40.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-09-23T23:03:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:40.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:40.810 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:03:40.838 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:40.870 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:40.918 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:40.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:40.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:41.026 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:41.054 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:41.098 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:41.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:41.132 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:41.207 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:41.233 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:41.276 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:41.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:41.381 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:41.384 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:41.407 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:41.449 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:41.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:41.554 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:41.587 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:41.647 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:41.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:41.752 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:41.792 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:41.800 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:41.883 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:03:41.884 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_5' created 2023-09-23T23:03:41.906 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:41 smithi052 ceph-mon[180979]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-09-23T23:03:41.906 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:41 smithi052 ceph-mon[187743]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-09-23T23:03:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:41 smithi151 ceph-mon[159210]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-09-23T23:03:42.403 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:42.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:42.507 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:42.540 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:42.588 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:42.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:42.693 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:42.723 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:42.929 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:42.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:42.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:42.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4111919561' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2680058822' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4111919561' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: osdmap e410: 8 total, 8 up, 6 in 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: failed to encode map e410 with expected crc 2023-09-23T23:03:42.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: failed to encode map e410 with expected crc 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[187743]: failed to encode map e410 with expected crc 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4111919561' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2680058822' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4111919561' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T23:03:42.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:03:42.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: osdmap e410: 8 total, 8 up, 6 in 2023-09-23T23:03:42.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: failed to encode map e410 with expected crc 2023-09-23T23:03:42.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: failed to encode map e410 with expected crc 2023-09-23T23:03:42.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:42 smithi052 ceph-mon[180979]: failed to encode map e410 with expected crc 2023-09-23T23:03:43.033 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:43.063 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:43.110 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4111919561' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2680058822' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4111919561' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:03:43.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: osdmap e410: 8 total, 8 up, 6 in 2023-09-23T23:03:43.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: failed to encode map e410 with expected crc 2023-09-23T23:03:43.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: failed to encode map e410 with expected crc 2023-09-23T23:03:43.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:42 smithi151 ceph-mon[159210]: failed to encode map e410 with expected crc 2023-09-23T23:03:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:43.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:43.179 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T23:03:43.219 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:43.250 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:43.502 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:43.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:43.606 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:43.635 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:43.682 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:43.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:03:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:03:43] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.33.4" 2023-09-23T23:03:43.787 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:43.815 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:43.858 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:43.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:43 smithi151 ceph-mon[159210]: failed to encode map e410 with expected crc 2023-09-23T23:03:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:43 smithi151 ceph-mon[159210]: pgmap v1265: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:43 smithi151 ceph-mon[159210]: failed to encode map e411 with expected crc 2023-09-23T23:03:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:43 smithi151 ceph-mon[159210]: failed to encode map e411 with expected crc 2023-09-23T23:03:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:43 smithi151 ceph-mon[159210]: osdmap e411: 8 total, 8 up, 6 in 2023-09-23T23:03:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:43 smithi151 ceph-mon[159210]: failed to encode map e411 with expected crc 2023-09-23T23:03:43.964 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:43.989 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:44.032 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:44.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:44.137 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:44.162 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[187743]: failed to encode map e410 with expected crc 2023-09-23T23:03:44.162 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[187743]: pgmap v1265: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[187743]: failed to encode map e411 with expected crc 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[187743]: failed to encode map e411 with expected crc 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[187743]: osdmap e411: 8 total, 8 up, 6 in 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[187743]: failed to encode map e411 with expected crc 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[180979]: failed to encode map e410 with expected crc 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[180979]: pgmap v1265: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[180979]: failed to encode map e411 with expected crc 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[180979]: failed to encode map e411 with expected crc 2023-09-23T23:03:44.163 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[180979]: osdmap e411: 8 total, 8 up, 6 in 2023-09-23T23:03:44.164 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:43 smithi052 ceph-mon[180979]: failed to encode map e411 with expected crc 2023-09-23T23:03:44.166 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:44.754 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:44.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:44.859 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:44.891 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:44.940 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:44.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:45.045 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:45.087 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:44 smithi052 ceph-mon[180979]: failed to encode map e412 with expected crc 2023-09-23T23:03:45.087 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:44 smithi052 ceph-mon[180979]: osdmap e412: 8 total, 8 up, 6 in 2023-09-23T23:03:45.088 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:44 smithi052 ceph-mon[180979]: failed to encode map e412 with expected crc 2023-09-23T23:03:45.088 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:44 smithi052 ceph-mon[187743]: failed to encode map e412 with expected crc 2023-09-23T23:03:45.088 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:44 smithi052 ceph-mon[187743]: osdmap e412: 8 total, 8 up, 6 in 2023-09-23T23:03:45.088 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:44 smithi052 ceph-mon[187743]: failed to encode map e412 with expected crc 2023-09-23T23:03:45.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:45.145 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:03:45.167 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:03:45.188 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:03:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:44 smithi151 ceph-mon[159210]: failed to encode map e412 with expected crc 2023-09-23T23:03:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:44 smithi151 ceph-mon[159210]: osdmap e412: 8 total, 8 up, 6 in 2023-09-23T23:03:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:44 smithi151 ceph-mon[159210]: failed to encode map e412 with expected crc 2023-09-23T23:03:45.202 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:45.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:45.211 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:03:45.231 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:03:45.267 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:03:45.269 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:03:45.288 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:03:45.289 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.290 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:03:45.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:45.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:45.447 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:45.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:45.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:45.551 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:45.581 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:45.896 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:113718 10.f e344/343 10:f7ba7a2d:::benchmark_data_smithi052_461816_object16917:head v 344'7098, mlcod=0'0)", 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:57:32.801170+0000", 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 372.62987207999998, 2023-09-23T23:03:45.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017963037000000001, 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801170+0000", 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801170+0000", 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801169+0000", 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801179+0000", 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8689999999999999e-06 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801196+0000", 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6954999999999999e-05 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801206+0000", 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0044000000000001e-05 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801266+0000", 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0256e-05 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.801291+0000", 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5378999999999999e-05 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.819085+0000", 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017794152000000001 2023-09-23T23:03:45.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:57:32.819133+0000", 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7685999999999997e-05 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:119634 10.d e373/354 10:b8a57e34:::benchmark_data_smithi052_461816_object22833:head v 373'7541, mlcod=344'7206)", 2023-09-23T23:03:45.903 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 284.17206197600001, 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11753303800000001, 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258980+0000", 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.258986+0000", 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9780000000000002e-06 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371535+0000", 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.112549314 2023-09-23T23:03:45.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371552+0000", 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7370000000000001e-05 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371649+0000", 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6151999999999997e-05 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.371672+0000", 2023-09-23T23:03:45.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3422e-05 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376456+0000", 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047836850000000002 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:59:01.376513+0000", 2023-09-23T23:03:45.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7488999999999998e-05 2023-09-23T23:03:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141431 10.c e386/354 10:3edfacc6:::benchmark_data_smithi052_461816_object44630:head v 386'8756, mlcod=338'6097)", 2023-09-23T23:03:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.787594+0000", 2023-09-23T23:03:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 183.64344856700001, 2023-09-23T23:03:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.064963310999999996, 2023-09-23T23:03:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.911 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787594+0000", 2023-09-23T23:03:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787594+0000", 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787593+0000", 2023-09-23T23:03:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787599+0000", 2023-09-23T23:03:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4389999999999999e-06 2023-09-23T23:03:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787620+0000", 2023-09-23T23:03:45.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0981000000000001e-05 2023-09-23T23:03:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787629+0000", 2023-09-23T23:03:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5459999999999997e-06 2023-09-23T23:03:45.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787683+0000", 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.41e-05 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.787706+0000", 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2459e-05 2023-09-23T23:03:45.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852517+0000", 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.064811553999999993 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852557+0000", 2023-09-23T23:03:45.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9502000000000002e-05 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141433 10.d e386/354 10:b300ffcd:::benchmark_data_smithi052_461816_object44632:head v 386'8896, mlcod=344'7206)", 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 183.63890790400001, 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.063758645000000003, 2023-09-23T23:03:45.919 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:03:45.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792134+0000", 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.792139+0000", 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.857e-06 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852291+0000", 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.060151688000000002 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852300+0000", 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.132e-06 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.853137+0000", 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00083692499999999997 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.853157+0000", 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9819000000000001e-05 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855854+0000", 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026973240000000001 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855893+0000", 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8544000000000002e-05 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:141434 10.5 e386/359 10:ac9ab671:::benchmark_data_smithi052_461816_object44633:head v 386'8906, mlcod=323'2477)", 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 183.61511175199999, 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.039674503, 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:03:45.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815930+0000", 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.815935+0000", 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1019999999999996e-06 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852242+0000", 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.036307219000000002 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852253+0000", 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1071e-05 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852543+0000", 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00028993500000000001 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.852567+0000", 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3705000000000001e-05 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855565+0000", 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029977020000000001 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:41.855605+0000", 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0085e-05 2023-09-23T23:03:45.929 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:146903 10.f e388/360 10:ff31cbf4:::benchmark_data_smithi052_461816_object50102:head v 388'9186, mlcod=323'2484)", 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:02.433106+0000", 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 162.997936428, 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015857768000000001, 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.930 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433106+0000", 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433106+0000", 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433105+0000", 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433110+0000", 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1499999999999998e-06 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433121+0000", 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0342e-05 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.433126+0000", 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4079999999999997e-06 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.437133+0000", 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040065090000000001 2023-09-23T23:03:45.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.437167+0000", 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4368e-05 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.448916+0000", 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011748827 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:02.448964+0000", 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7559000000000003e-05 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:150994 10.7 e389/360 10:ef49469c:::benchmark_data_smithi052_461816_object54193:head v 389'9284, mlcod=323'2408)", 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:16.053798+0000", 2023-09-23T23:03:45.935 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 149.37724451700001, 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015287967, 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053798+0000", 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053798+0000", 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053797+0000", 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053806+0000", 2023-09-23T23:03:45.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5589999999999994e-06 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053826+0000", 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9704999999999999e-05 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053835+0000", 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7420000000000003e-06 2023-09-23T23:03:45.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053896+0000", 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1141000000000003e-05 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.053923+0000", 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.707e-05 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.069025+0000", 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015102546 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:16.069086+0000", 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0239999999999999e-05 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.940 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151618 10.4 e390/354 10:21546296:::benchmark_data_smithi052_461816_object54817:head v 390'9478, mlcod=344'7124)", 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 147.68136513900001, 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065726571999999997, 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749677+0000", 2023-09-23T23:03:45.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749682+0000", 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9110000000000001e-06 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749708+0000", 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6335000000000001e-05 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749717+0000", 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0129999999999999e-06 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749774+0000", 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7306000000000003e-05 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.749799+0000", 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4269000000000001e-05 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815362+0000", 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.065563124 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815404+0000", 2023-09-23T23:03:45.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.18e-05 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:151620 10.5 e390/359 10:a897b5ac:::benchmark_data_smithi052_461816_object54819:head v 390'9515, mlcod=323'2477)", 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:17.802451+0000", 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 147.62859124100001, 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017011258000000001, 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.946 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802451+0000", 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802451+0000", 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802450+0000", 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.802457+0000", 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5320000000000003e-06 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815234+0000", 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.012777011 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815243+0000", 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1640000000000007e-06 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815317+0000", 2023-09-23T23:03:45.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4259999999999997e-05 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.815342+0000", 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4425000000000001e-05 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.819411+0000", 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00406909 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:17.819462+0000", 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1418000000000001e-05 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:152376 10.4 e390/354 10:20a7f37c:::benchmark_data_smithi052_461816_object55575:head v 390'9534, mlcod=344'7124)", 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:19.623703+0000", 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 145.807338751, 2023-09-23T23:03:45.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017693904999999999, 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623703+0000", 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623703+0000", 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623702+0000", 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623713+0000", 2023-09-23T23:03:45.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0216e-05 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623735+0000", 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2589e-05 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623745+0000", 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4730000000000006e-06 2023-09-23T23:03:45.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623814+0000", 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9177000000000005e-05 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.623837+0000", 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3247e-05 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.641367+0000", 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.017529718999999999 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:19.641397+0000", 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0599000000000003e-05 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.956 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:164594 10.5 e394/359 10:aabbab10:::benchmark_data_smithi052_461816_object67793:head v 394'10342, mlcod=323'2477)", 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:55.145148+0000", 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 110.28589372499999, 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015181661000000001, 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.145148+0000", 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.145148+0000", 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.145147+0000", 2023-09-23T23:03:45.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.145157+0000", 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1370000000000008e-06 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.145172+0000", 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5688e-05 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.145180+0000", 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.515e-06 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.151927+0000", 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0067469549999999998 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.151972+0000", 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4855000000000003e-05 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.160269+0000", 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0082969489999999996 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:55.160330+0000", 2023-09-23T23:03:45.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1527999999999998e-05 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:170504 10.3 e396/359 10:c1196ee9:::benchmark_data_smithi052_461816_object73703:head v 396'10764, mlcod=323'2482)", 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:10.901446+0000", 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 94.529596548000001, 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018595283000000001, 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.962 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.901446+0000", 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.901446+0000", 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.901444+0000", 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.901460+0000", 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6206e-05 2023-09-23T23:03:45.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.901482+0000", 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1705999999999999e-05 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.901494+0000", 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2619000000000001e-05 2023-09-23T23:03:45.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.903199+0000", 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017043869999999999 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.903226+0000", 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7623e-05 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.919990+0000", 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016763745 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:10.920041+0000", 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0652000000000002e-05 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:177017 10.5 e399/359 10:a24aba47:::benchmark_data_smithi052_461816_object80216:head v 399'11104, mlcod=323'2477)", 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:02:35.338909+0000", 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 70.092133583999995, 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01589016, 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.338909+0000", 2023-09-23T23:03:45.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.338909+0000", 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.338907+0000", 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.338918+0000", 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.042e-05 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.338943+0000", 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4932000000000001e-05 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.338953+0000", 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0342e-05 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.339040+0000", 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7235000000000001e-05 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.339078+0000", 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7814000000000001e-05 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.354757+0000", 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015679440999999999 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:02:35.354799+0000", 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1245000000000002e-05 2023-09-23T23:03:45.972 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:187476 10.7 e403/360 10:e668fdb5:::benchmark_data_smithi052_461816_object90675:head v 403'11549, mlcod=323'2408)", 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:06.999056+0000", 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 38.431986588999997, 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.046054049, 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.973 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999056+0000", 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999056+0000", 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999050+0000", 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999948 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999064+0000", 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3495e-05 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999075+0000", 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1669e-05 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999082+0000", 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4300000000000003e-06 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999129+0000", 2023-09-23T23:03:45.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7336000000000002e-05 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:06.999150+0000", 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0494000000000001e-05 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.045075+0000", 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.045925295999999997 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.045110+0000", 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4684000000000002e-05 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:187477 10.d e403/354 10:b5f29e47:::benchmark_data_smithi052_461816_object90676:head v 403'11808, mlcod=344'7206)", 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:07.017705+0000", 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 38.413337339000002, 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.027036402000000001, 2023-09-23T23:03:45.978 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.017705+0000", 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.017705+0000", 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.017703+0000", 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.017719+0000", 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6379999999999999e-05 2023-09-23T23:03:45.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.042235+0000", 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.024515499999999999 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.042248+0000", 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3407000000000001e-05 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.042325+0000", 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6798000000000006e-05 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.042341+0000", 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6297e-05 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.044703+0000", 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023618990000000002 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:07.044741+0000", 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7931e-05 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.983 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:190162 10.d e403/354 10:bccf6115:::benchmark_data_smithi052_461816_object93361:head v 403'11986, mlcod=344'7206)", 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:14.090443+0000", 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.340599364999999, 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018443675, 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.090443+0000", 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.090443+0000", 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.090442+0000", 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.090448+0000", 2023-09-23T23:03:45.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2859999999999996e-06 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.090460+0000", 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1219999999999999e-05 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.090468+0000", 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.977e-06 2023-09-23T23:03:45.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.098347+0000", 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.007879736 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.098367+0000", 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9984999999999999e-05 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.108843+0000", 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010475744 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.108886+0000", 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3421000000000003e-05 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.988 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:190381 10.7 e403/360 10:ec2c31bc:::benchmark_data_smithi052_461816_object93580:head v 403'11746, mlcod=323'2408)", 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:14.466496+0000", 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 30.964545799, 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015161958999999999, 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466496+0000", 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466496+0000", 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466495+0000", 2023-09-23T23:03:45.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466510+0000", 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5119e-05 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466536+0000", 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5925999999999999e-05 2023-09-23T23:03:45.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466549+0000", 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.27e-05 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466621+0000", 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1750999999999998e-05 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.466652+0000", 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1056999999999998e-05 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.481605+0000", 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014953892 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.481658+0000", 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2862000000000003e-05 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:191116 10.f e404/360 10:ff956662:::benchmark_data_smithi052_461816_object94315:head v 404'11923, mlcod=323'2484)", 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:16.308464+0000", 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 29.122578172000001, 2023-09-23T23:03:45.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022096581000000001, 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.308464+0000", 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.308464+0000", 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.308463+0000", 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.308472+0000", 2023-09-23T23:03:45.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6710000000000005e-06 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.308488+0000", 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6059e-05 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.308496+0000", 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7600000000000008e-06 2023-09-23T23:03:45.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.310000+0000", 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015033010000000001 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.310022+0000", 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2283999999999999e-05 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.330513+0000", 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020491465 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:16.330561+0000", 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7162e-05 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:45.999 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:03:46.000 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:03:46.000 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:03:46.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[187743]: pgmap v1268: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:46.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[187743]: failed to encode map e412 with expected crc 2023-09-23T23:03:46.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[187743]: failed to encode map e412 with expected crc 2023-09-23T23:03:46.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4145161576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:46.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[180979]: pgmap v1268: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:46.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[180979]: failed to encode map e412 with expected crc 2023-09-23T23:03:46.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[180979]: failed to encode map e412 with expected crc 2023-09-23T23:03:46.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4145161576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:45 smithi151 ceph-mon[159210]: pgmap v1268: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.9 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:45 smithi151 ceph-mon[159210]: failed to encode map e412 with expected crc 2023-09-23T23:03:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:45 smithi151 ceph-mon[159210]: failed to encode map e412 with expected crc 2023-09-23T23:03:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4145161576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:46.357 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:46.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:46.374 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:46.375 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:03:46.461 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:46.489 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:46.533 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:46.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:46.637 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:46.665 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:46.770 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:03:47.100 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:47.113 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:47.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:47.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:47.130 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:03:47.219 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:47.250 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:47.343 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1268,"stamp":"2023-09-23T23:03:44.643238+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":605660,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":228,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4040780,"kb_used_data":575708,"kb_used_omap":137,"kb_used_meta":3464886,"kb_avail":558430132,"statfs":{"total":575970213888,"available":571832455168,"internally_reserved":0,"allocated":589524992,"data_stored":581151823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3548043448},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001307"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":421,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313822+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:03:35.313822+0000","last_peered":"2023-09-23T23:03:35.313822+0000","last_clean":"2023-09-23T23:03:35.313822+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:03:35.313822+0000","last_undegraded":"2023-09-23T23:03:35.313822+0000","last_fullsized":"2023-09-23T23:03:35.313822+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":79,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488789+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:03:35.488789+0000","last_peered":"2023-09-23T23:03:35.488789+0000","last_clean":"2023-09-23T23:03:35.488789+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:03:35.488789+0000","last_undegraded":"2023-09-23T23:03:35.488789+0000","last_fullsized":"2023-09-23T23:03:35.488789+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":78,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488424+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:03:35.488424+0000","last_peered":"2023-09-23T23:03:35.488424+0000","last_clean":"2023-09-23T23:03:35.488424+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:03:35.488424+0000","last_undegraded":"2023-09-23T23:03:35.488424+0000","last_fullsized":"2023-09-23T23:03:35.488424+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":407,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:36.281489+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:03:36.281489+0000","last_peered":"2023-09-23T23:03:36.281489+0000","last_clean":"2023-09-23T23:03:36.281489+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:03:36.281489+0000","last_undegraded":"2023-09-23T23:03:36.281489+0000","last_fullsized":"2023-09-23T23:03:36.281489+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488346+0000","last_change":"2023-09-23T22:58:06.746441+0000","last_active":"2023-09-23T23:03:35.488346+0000","last_peered":"2023-09-23T23:03:35.488346+0000","last_clean":"2023-09-23T23:03:35.488346+0000","last_became_active":"2023-09-23T22:58:06.745727+0000","last_became_peered":"2023-09-23T22:58:06.745727+0000","last_unstale":"2023-09-23T23:03:35.488346+0000","last_undegraded":"2023-09-23T23:03:35.488346+0000","last_fullsized":"2023-09-23T23:03:35.488346+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:12.801102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":80,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.311224+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:03:35.311224+0000","last_peered":"2023-09-23T23:03:35.311224+0000","last_clean":"2023-09-23T23:03:35.311224+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:03:35.311224+0000","last_undegraded":"2023-09-23T23:03:35.311224+0000","last_fullsized":"2023-09-23T23:03:35.311224+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":344,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312912+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:03:35.312912+0000","last_peered":"2023-09-23T23:03:35.312912+0000","last_clean":"2023-09-23T23:03:35.312912+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:03:35.312912+0000","last_undegraded":"2023-09-23T23:03:35.312912+0000","last_fullsized":"2023-09-23T23:03:35.312912+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488217+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:03:35.488217+0000","last_peered":"2023-09-23T23:03:35.488217+0000","last_clean":"2023-09-23T23:03:35.488217+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:03:35.488217+0000","last_undegraded":"2023-09-23T23:03:35.488217+0000","last_fullsized":"2023-09-23T23:03:35.488217+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":305,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312776+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:03:35.312776+0000","last_peered":"2023-09-23T23:03:35.312776+0000","last_clean":"2023-09-23T23:03:35.312776+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:03:35.312776+0000","last_undegraded":"2023-09-23T23:03:35.312776+0000","last_fullsized":"2023-09-23T23:03:35.312776+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":93,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488337+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:03:35.488337+0000","last_peered":"2023-09-23T23:03:35.488337+0000","last_clean":"2023-09-23T23:03:35.488337+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:03:35.488337+0000","last_undegraded":"2023-09-23T23:03:35.488337+0000","last_fullsized":"2023-09-23T23:03:35.488337+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":99,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488772+0000","last_change":"2023-09-23T22:58:06.746958+0000","last_active":"2023-09-23T23:03:35.488772+0000","last_peered":"2023-09-23T23:03:35.488772+0000","last_clean":"2023-09-23T23:03:35.488772+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:03:35.488772+0000","last_undegraded":"2023-09-23T23:03:35.488772+0000","last_fullsized":"2023-09-23T23:03:35.488772+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.883604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314527+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:03:35.314527+0000","last_peered":"2023-09-23T23:03:35.314527+0000","last_clean":"2023-09-23T23:03:35.314527+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:03:35.314527+0000","last_undegraded":"2023-09-23T23:03:35.314527+0000","last_fullsized":"2023-09-23T23:03:35.314527+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":480,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313612+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:03:35.313612+0000","last_peered":"2023-09-23T23:03:35.313612+0000","last_clean":"2023-09-23T23:03:35.313612+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:03:35.313612+0000","last_undegraded":"2023-09-23T23:03:35.313612+0000","last_fullsized":"2023-09-23T23:03:35.313612+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":383,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314398+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:03:35.314398+0000","last_peered":"2023-09-23T23:03:35.314398+0000","last_clean":"2023-09-23T23:03:35.314398+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:03:35.314398+0000","last_undegraded":"2023-09-23T23:03:35.314398+0000","last_fullsized":"2023-09-23T23:03:35.314398+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":309,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314486+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:03:35.314486+0000","last_peered":"2023-09-23T23:03:35.314486+0000","last_clean":"2023-09-23T23:03:35.314486+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:03:35.314486+0000","last_undegraded":"2023-09-23T23:03:35.314486+0000","last_fullsized":"2023-09-23T23:03:35.314486+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":90,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488707+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:03:35.488707+0000","last_peered":"2023-09-23T23:03:35.488707+0000","last_clean":"2023-09-23T23:03:35.488707+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:03:35.488707+0000","last_undegraded":"2023-09-23T23:03:35.488707+0000","last_fullsized":"2023-09-23T23:03:35.488707+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":399,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488335+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:03:35.488335+0000","last_peered":"2023-09-23T23:03:35.488335+0000","last_clean":"2023-09-23T23:03:35.488335+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:03:35.488335+0000","last_undegraded":"2023-09-23T23:03:35.488335+0000","last_fullsized":"2023-09-23T23:03:35.488335+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312842+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:03:35.312842+0000","last_peered":"2023-09-23T23:03:35.312842+0000","last_clean":"2023-09-23T23:03:35.312842+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:03:35.312842+0000","last_undegraded":"2023-09-23T23:03:35.312842+0000","last_fullsized":"2023-09-23T23:03:35.312842+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":294,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314039+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:03:35.314039+0000","last_peered":"2023-09-23T23:03:35.314039+0000","last_clean":"2023-09-23T23:03:35.314039+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:03:35.314039+0000","last_undegraded":"2023-09-23T23:03:35.314039+0000","last_fullsized":"2023-09-23T23:03:35.314039+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":423,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488755+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:03:35.488755+0000","last_peered":"2023-09-23T23:03:35.488755+0000","last_clean":"2023-09-23T23:03:35.488755+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:03:35.488755+0000","last_undegraded":"2023-09-23T23:03:35.488755+0000","last_fullsized":"2023-09-23T23:03:35.488755+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":128,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.310399+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:03:35.310399+0000","last_peered":"2023-09-23T23:03:35.310399+0000","last_clean":"2023-09-23T23:03:35.310399+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:03:35.310399+0000","last_undegraded":"2023-09-23T23:03:35.310399+0000","last_fullsized":"2023-09-23T23:03:35.310399+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488762+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:03:35.488762+0000","last_peered":"2023-09-23T23:03:35.488762+0000","last_clean":"2023-09-23T23:03:35.488762+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:03:35.488762+0000","last_undegraded":"2023-09-23T23:03:35.488762+0000","last_fullsized":"2023-09-23T23:03:35.488762+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":293,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314472+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:03:35.314472+0000","last_peered":"2023-09-23T23:03:35.314472+0000","last_clean":"2023-09-23T23:03:35.314472+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:03:35.314472+0000","last_undegraded":"2023-09-23T23:03:35.314472+0000","last_fullsized":"2023-09-23T23:03:35.314472+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312503+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:03:35.312503+0000","last_peered":"2023-09-23T23:03:35.312503+0000","last_clean":"2023-09-23T23:03:35.312503+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:03:35.312503+0000","last_undegraded":"2023-09-23T23:03:35.312503+0000","last_fullsized":"2023-09-23T23:03:35.312503+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":303,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314371+0000","last_change":"2023-09-23T22:58:06.761966+0000","last_active":"2023-09-23T23:03:35.314371+0000","last_peered":"2023-09-23T23:03:35.314371+0000","last_clean":"2023-09-23T23:03:35.314371+0000","last_became_active":"2023-09-23T22:58:06.761633+0000","last_became_peered":"2023-09-23T22:58:06.761633+0000","last_unstale":"2023-09-23T23:03:35.314371+0000","last_undegraded":"2023-09-23T23:03:35.314371+0000","last_fullsized":"2023-09-23T23:03:35.314371+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":416,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313959+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:03:35.313959+0000","last_peered":"2023-09-23T23:03:35.313959+0000","last_clean":"2023-09-23T23:03:35.313959+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:03:35.313959+0000","last_undegraded":"2023-09-23T23:03:35.313959+0000","last_fullsized":"2023-09-23T23:03:35.313959+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":160,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312877+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:03:35.312877+0000","last_peered":"2023-09-23T23:03:35.312877+0000","last_clean":"2023-09-23T23:03:35.312877+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:03:35.312877+0000","last_undegraded":"2023-09-23T23:03:35.312877+0000","last_fullsized":"2023-09-23T23:03:35.312877+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":278,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312634+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:03:35.312634+0000","last_peered":"2023-09-23T23:03:35.312634+0000","last_clean":"2023-09-23T23:03:35.312634+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:03:35.312634+0000","last_undegraded":"2023-09-23T23:03:35.312634+0000","last_fullsized":"2023-09-23T23:03:35.312634+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":79,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488706+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:03:35.488706+0000","last_peered":"2023-09-23T23:03:35.488706+0000","last_clean":"2023-09-23T23:03:35.488706+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:03:35.488706+0000","last_undegraded":"2023-09-23T23:03:35.488706+0000","last_fullsized":"2023-09-23T23:03:35.488706+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":309,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.310885+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:03:35.310885+0000","last_peered":"2023-09-23T23:03:35.310885+0000","last_clean":"2023-09-23T23:03:35.310885+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:03:35.310885+0000","last_undegraded":"2023-09-23T23:03:35.310885+0000","last_fullsized":"2023-09-23T23:03:35.310885+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":98,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488471+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:03:35.488471+0000","last_peered":"2023-09-23T23:03:35.488471+0000","last_clean":"2023-09-23T23:03:35.488471+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:03:35.488471+0000","last_undegraded":"2023-09-23T23:03:35.488471+0000","last_fullsized":"2023-09-23T23:03:35.488471+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":402,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313513+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:03:35.313513+0000","last_peered":"2023-09-23T23:03:35.313513+0000","last_clean":"2023-09-23T23:03:35.313513+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:03:35.313513+0000","last_undegraded":"2023-09-23T23:03:35.313513+0000","last_fullsized":"2023-09-23T23:03:35.313513+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":192,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313971+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:03:35.313971+0000","last_peered":"2023-09-23T23:03:35.313971+0000","last_clean":"2023-09-23T23:03:35.313971+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:03:35.313971+0000","last_undegraded":"2023-09-23T23:03:35.313971+0000","last_fullsized":"2023-09-23T23:03:35.313971+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":419,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314143+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:03:35.314143+0000","last_peered":"2023-09-23T23:03:35.314143+0000","last_clean":"2023-09-23T23:03:35.314143+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:03:35.314143+0000","last_undegraded":"2023-09-23T23:03:35.314143+0000","last_fullsized":"2023-09-23T23:03:35.314143+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":432,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313766+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:03:35.313766+0000","last_peered":"2023-09-23T23:03:35.313766+0000","last_clean":"2023-09-23T23:03:35.313766+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:03:35.313766+0000","last_undegraded":"2023-09-23T23:03:35.313766+0000","last_fullsized":"2023-09-23T23:03:35.313766+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":410,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314365+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:03:35.314365+0000","last_peered":"2023-09-23T23:03:35.314365+0000","last_clean":"2023-09-23T23:03:35.314365+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:03:35.314365+0000","last_undegraded":"2023-09-23T23:03:35.314365+0000","last_fullsized":"2023-09-23T23:03:35.314365+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488480+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:03:35.488480+0000","last_peered":"2023-09-23T23:03:35.488480+0000","last_clean":"2023-09-23T23:03:35.488480+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:03:35.488480+0000","last_undegraded":"2023-09-23T23:03:35.488480+0000","last_fullsized":"2023-09-23T23:03:35.488480+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":313,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.315363+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:03:35.315363+0000","last_peered":"2023-09-23T23:03:35.315363+0000","last_clean":"2023-09-23T23:03:35.315363+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:03:35.315363+0000","last_undegraded":"2023-09-23T23:03:35.315363+0000","last_fullsized":"2023-09-23T23:03:35.315363+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":317,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.315272+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:03:35.315272+0000","last_peered":"2023-09-23T23:03:35.315272+0000","last_clean":"2023-09-23T23:03:35.315272+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:03:35.315272+0000","last_undegraded":"2023-09-23T23:03:35.315272+0000","last_fullsized":"2023-09-23T23:03:35.315272+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":319,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312766+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:03:35.312766+0000","last_peered":"2023-09-23T23:03:35.312766+0000","last_clean":"2023-09-23T23:03:35.312766+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:03:35.312766+0000","last_undegraded":"2023-09-23T23:03:35.312766+0000","last_fullsized":"2023-09-23T23:03:35.312766+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312582+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:03:35.312582+0000","last_peered":"2023-09-23T23:03:35.312582+0000","last_clean":"2023-09-23T23:03:35.312582+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:03:35.312582+0000","last_undegraded":"2023-09-23T23:03:35.312582+0000","last_fullsized":"2023-09-23T23:03:35.312582+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":80,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.311102+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:03:35.311102+0000","last_peered":"2023-09-23T23:03:35.311102+0000","last_clean":"2023-09-23T23:03:35.311102+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:03:35.311102+0000","last_undegraded":"2023-09-23T23:03:35.311102+0000","last_fullsized":"2023-09-23T23:03:35.311102+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":429,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314560+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:03:35.314560+0000","last_peered":"2023-09-23T23:03:35.314560+0000","last_clean":"2023-09-23T23:03:35.314560+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:03:35.314560+0000","last_undegraded":"2023-09-23T23:03:35.314560+0000","last_fullsized":"2023-09-23T23:03:35.314560+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":302,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.315351+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:03:35.315351+0000","last_peered":"2023-09-23T23:03:35.315351+0000","last_clean":"2023-09-23T23:03:35.315351+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:03:35.315351+0000","last_undegraded":"2023-09-23T23:03:35.315351+0000","last_fullsized":"2023-09-23T23:03:35.315351+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":377,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314385+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:03:35.314385+0000","last_peered":"2023-09-23T23:03:35.314385+0000","last_clean":"2023-09-23T23:03:35.314385+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:03:35.314385+0000","last_undegraded":"2023-09-23T23:03:35.314385+0000","last_fullsized":"2023-09-23T23:03:35.314385+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":254,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.315270+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:03:35.315270+0000","last_peered":"2023-09-23T23:03:35.315270+0000","last_clean":"2023-09-23T23:03:35.315270+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:03:35.315270+0000","last_undegraded":"2023-09-23T23:03:35.315270+0000","last_fullsized":"2023-09-23T23:03:35.315270+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":319,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312939+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:03:35.312939+0000","last_peered":"2023-09-23T23:03:35.312939+0000","last_clean":"2023-09-23T23:03:35.312939+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:03:35.312939+0000","last_undegraded":"2023-09-23T23:03:35.312939+0000","last_fullsized":"2023-09-23T23:03:35.312939+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":392,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312691+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:03:35.312691+0000","last_peered":"2023-09-23T23:03:35.312691+0000","last_clean":"2023-09-23T23:03:35.312691+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:03:35.312691+0000","last_undegraded":"2023-09-23T23:03:35.312691+0000","last_fullsized":"2023-09-23T23:03:35.312691+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":521,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T23:03:36.322526+0000","last_change":"2023-09-23T23:03:36.322526+0000","last_active":"2023-09-23T23:03:36.322526+0000","last_peered":"2023-09-23T23:03:36.322526+0000","last_clean":"2023-09-23T23:03:36.322526+0000","last_became_active":"2023-09-23T23:03:36.322065+0000","last_became_peered":"2023-09-23T23:03:36.322065+0000","last_unstale":"2023-09-23T23:03:36.322526+0000","last_undegraded":"2023-09-23T23:03:36.322526+0000","last_fullsized":"2023-09-23T23:03:36.322526+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:04.195321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T23:03:36.315854+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:03:36.315854+0000","last_peered":"2023-09-23T23:03:36.315854+0000","last_clean":"2023-09-23T23:03:36.315854+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:03:36.315854+0000","last_undegraded":"2023-09-23T23:03:36.315854+0000","last_fullsized":"2023-09-23T23:03:36.315854+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":319,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314248+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:03:35.314248+0000","last_peered":"2023-09-23T23:03:35.314248+0000","last_clean":"2023-09-23T23:03:35.314248+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:03:35.314248+0000","last_undegraded":"2023-09-23T23:03:35.314248+0000","last_fullsized":"2023-09-23T23:03:35.314248+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":400,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.310391+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:03:35.310391+0000","last_peered":"2023-09-23T23:03:35.310391+0000","last_clean":"2023-09-23T23:03:35.310391+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:03:35.310391+0000","last_undegraded":"2023-09-23T23:03:35.310391+0000","last_fullsized":"2023-09-23T23:03:35.310391+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":371,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.310506+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:03:35.310506+0000","last_peered":"2023-09-23T23:03:35.310506+0000","last_clean":"2023-09-23T23:03:35.310506+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:03:35.310506+0000","last_undegraded":"2023-09-23T23:03:35.310506+0000","last_fullsized":"2023-09-23T23:03:35.310506+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":331,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.311101+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:03:35.311101+0000","last_peered":"2023-09-23T23:03:35.311101+0000","last_clean":"2023-09-23T23:03:35.311101+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:03:35.311101+0000","last_undegraded":"2023-09-23T23:03:35.311101+0000","last_fullsized":"2023-09-23T23:03:35.311101+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":346,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.310963+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:03:35.310963+0000","last_peered":"2023-09-23T23:03:35.310963+0000","last_clean":"2023-09-23T23:03:35.310963+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:03:35.310963+0000","last_undegraded":"2023-09-23T23:03:35.310963+0000","last_fullsized":"2023-09-23T23:03:35.310963+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":254,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.315429+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:03:35.315429+0000","last_peered":"2023-09-23T23:03:35.315429+0000","last_clean":"2023-09-23T23:03:35.315429+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:03:35.315429+0000","last_undegraded":"2023-09-23T23:03:35.315429+0000","last_fullsized":"2023-09-23T23:03:35.315429+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":222,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.312722+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:03:35.312722+0000","last_peered":"2023-09-23T23:03:35.312722+0000","last_clean":"2023-09-23T23:03:35.312722+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:03:35.312722+0000","last_undegraded":"2023-09-23T23:03:35.312722+0000","last_fullsized":"2023-09-23T23:03:35.312722+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":103,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488869+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:03:35.488869+0000","last_peered":"2023-09-23T23:03:35.488869+0000","last_clean":"2023-09-23T23:03:35.488869+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:03:35.488869+0000","last_undegraded":"2023-09-23T23:03:35.488869+0000","last_fullsized":"2023-09-23T23:03:35.488869+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":117,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314433+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:03:35.314433+0000","last_peered":"2023-09-23T23:03:35.314433+0000","last_clean":"2023-09-23T23:03:35.314433+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:03:35.314433+0000","last_undegraded":"2023-09-23T23:03:35.314433+0000","last_fullsized":"2023-09-23T23:03:35.314433+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":95,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488338+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:03:35.488338+0000","last_peered":"2023-09-23T23:03:35.488338+0000","last_clean":"2023-09-23T23:03:35.488338+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:03:35.488338+0000","last_undegraded":"2023-09-23T23:03:35.488338+0000","last_fullsized":"2023-09-23T23:03:35.488338+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:03:41.870679+0000","last_change":"2023-09-23T23:03:41.870679+0000","last_active":"2023-09-23T23:03:41.870679+0000","last_peered":"2023-09-23T23:03:41.870679+0000","last_clean":"2023-09-23T23:03:41.870679+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:41.870679+0000","last_undegraded":"2023-09-23T23:03:41.870679+0000","last_fullsized":"2023-09-23T23:03:41.870679+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":366,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.315603+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:03:35.315603+0000","last_peered":"2023-09-23T23:03:35.315603+0000","last_clean":"2023-09-23T23:03:35.315603+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:03:35.315603+0000","last_undegraded":"2023-09-23T23:03:35.315603+0000","last_fullsized":"2023-09-23T23:03:35.315603+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":313,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.311063+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:03:35.311063+0000","last_peered":"2023-09-23T23:03:35.311063+0000","last_clean":"2023-09-23T23:03:35.311063+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:03:35.311063+0000","last_undegraded":"2023-09-23T23:03:35.311063+0000","last_fullsized":"2023-09-23T23:03:35.311063+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":383,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313838+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:03:35.313838+0000","last_peered":"2023-09-23T23:03:35.313838+0000","last_clean":"2023-09-23T23:03:35.313838+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:03:35.313838+0000","last_undegraded":"2023-09-23T23:03:35.313838+0000","last_fullsized":"2023-09-23T23:03:35.313838+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":348,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.311073+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:03:35.311073+0000","last_peered":"2023-09-23T23:03:35.311073+0000","last_clean":"2023-09-23T23:03:35.311073+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:03:35.311073+0000","last_undegraded":"2023-09-23T23:03:35.311073+0000","last_fullsized":"2023-09-23T23:03:35.311073+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":158,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314969+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:03:35.314969+0000","last_peered":"2023-09-23T23:03:35.314969+0000","last_clean":"2023-09-23T23:03:35.314969+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:03:35.314969+0000","last_undegraded":"2023-09-23T23:03:35.314969+0000","last_fullsized":"2023-09-23T23:03:35.314969+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":218,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313429+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:03:35.313429+0000","last_peered":"2023-09-23T23:03:35.313429+0000","last_clean":"2023-09-23T23:03:35.313429+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:03:35.313429+0000","last_undegraded":"2023-09-23T23:03:35.313429+0000","last_fullsized":"2023-09-23T23:03:35.313429+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":432,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314639+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:03:35.314639+0000","last_peered":"2023-09-23T23:03:35.314639+0000","last_clean":"2023-09-23T23:03:35.314639+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:03:35.314639+0000","last_undegraded":"2023-09-23T23:03:35.314639+0000","last_fullsized":"2023-09-23T23:03:35.314639+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":232,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.314173+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:03:35.314173+0000","last_peered":"2023-09-23T23:03:35.314173+0000","last_clean":"2023-09-23T23:03:35.314173+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:03:35.314173+0000","last_undegraded":"2023-09-23T23:03:35.314173+0000","last_fullsized":"2023-09-23T23:03:35.314173+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":319,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313897+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:03:35.313897+0000","last_peered":"2023-09-23T23:03:35.313897+0000","last_clean":"2023-09-23T23:03:35.313897+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:03:35.313897+0000","last_undegraded":"2023-09-23T23:03:35.313897+0000","last_fullsized":"2023-09-23T23:03:35.313897+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":104,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488867+0000","last_change":"2023-09-23T22:58:24.248682+0000","last_active":"2023-09-23T23:03:35.488867+0000","last_peered":"2023-09-23T23:03:35.488867+0000","last_clean":"2023-09-23T23:03:35.488867+0000","last_became_active":"2023-09-23T22:58:24.248303+0000","last_became_peered":"2023-09-23T22:58:24.248303+0000","last_unstale":"2023-09-23T23:03:35.488867+0000","last_undegraded":"2023-09-23T23:03:35.488867+0000","last_fullsized":"2023-09-23T23:03:35.488867+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.072367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":106,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.488599+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:03:35.488599+0000","last_peered":"2023-09-23T23:03:35.488599+0000","last_clean":"2023-09-23T23:03:35.488599+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:03:35.488599+0000","last_undegraded":"2023-09-23T23:03:35.488599+0000","last_fullsized":"2023-09-23T23:03:35.488599+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":352,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.311535+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:03:35.311535+0000","last_peered":"2023-09-23T23:03:35.311535+0000","last_clean":"2023-09-23T23:03:35.311535+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:03:35.311535+0000","last_undegraded":"2023-09-23T23:03:35.311535+0000","last_fullsized":"2023-09-23T23:03:35.311535+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":394,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T23:03:35.313318+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:03:35.313318+0000","last_peered":"2023-09-23T23:03:35.313318+0000","last_clean":"2023-09-23T23:03:35.313318+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:03:35.313318+0000","last_undegraded":"2023-09-23T23:03:35.313318+0000","last_fullsized":"2023-09-23T23:03:35.313318+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430648,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626873,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253186,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842604,"kb_used_data":82020,"kb_used_omap":9,"kb_used_meta":760566,"kb_avail":92902548,"statfs":{"total":95995035648,"available":95132209152,"internally_reserved":0,"allocated":83988480,"data_stored":82408862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":778820585},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7769999999999999,"15min":0.82699999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4780000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.995,"5min":2.1040000000000001,"15min":1.272},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.266,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.113,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9630000000000001,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.56899999999999995,"15min":1.7969999999999999},"min":{"1min":0.28699999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":87.787999999999997},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.53300000000000003,"15min":2.0499999999999998},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":1.24,"5min":3.109,"15min":89.861999999999995},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3129999999999997,"5min":3.218,"15min":2.637},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.29199999999999998},"max":{"1min":67.738,"5min":151.036,"15min":151.036},"last":0.56799999999999995},{"interface":"front","average":{"1min":5.7089999999999996,"5min":3.431,"15min":2.9260000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":67.670000000000002,"5min":151.06899999999999,"15min":151.06899999999999},"last":0.70899999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3300000000000001,"5min":1.6930000000000001,"15min":2.5699999999999998},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":68.259,"5min":68.259,"15min":116.098},"last":0.57799999999999996},{"interface":"front","average":{"1min":5.5549999999999997,"5min":1.8759999999999999,"15min":2.6970000000000001},"min":{"1min":0.38200000000000001,"5min":0.33600000000000002,"15min":0.315},"max":{"1min":70.057000000000002,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":5.7270000000000003,"5min":2.0600000000000001,"15min":2.5259999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":67.349000000000004,"5min":67.349000000000004,"15min":79.311000000000007},"last":0.62},{"interface":"front","average":{"1min":5.5979999999999999,"5min":1.7529999999999999,"15min":2.125},"min":{"1min":0.34999999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":67.366,"5min":67.366,"15min":79.328999999999994},"last":0.81399999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.78900000000000003,"15min":1.552},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4139999999999999,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.94499999999999995,"15min":1.925},"min":{"1min":0.40200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.4929999999999999,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.68400000000000005}]}]},{"osd":0,"up_from":353,"seq":1516123455556,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":214944,"kb_used_data":5400,"kb_used_omap":37,"kb_used_meta":209498,"kb_avail":93530208,"statfs":{"total":95995035648,"available":95774932992,"internally_reserved":0,"allocated":5529600,"data_stored":3777911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":214526114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.83299999999999996,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.6970000000000001,"5min":8.234,"15min":8.234},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.01,"5min":0.89700000000000002,"15min":0.747},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7169999999999996,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.373}]},{"osd":2,"last update":"Sat Sep 23 23:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.96799999999999997,"15min":0.84199999999999997},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":7.3369999999999997,"5min":12.215,"15min":12.215},"last":0.26400000000000001},{"interface":"front","average":{"1min":5.1340000000000003,"5min":1.7609999999999999,"15min":1.03},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":74.698999999999998,"5min":74.698999999999998,"15min":74.698999999999998},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.997,"15min":0.63500000000000001},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.1109999999999998,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.93799999999999994,"15min":0.91800000000000004},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.8050000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":3.298,"15min":1.6719999999999999},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2690000000000001,"5min":183.148,"15min":183.148},"last":0.495},{"interface":"front","average":{"1min":0.98899999999999999,"5min":3.4169999999999998,"15min":1.585},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.9910000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.069,"15min":0.80900000000000005},"min":{"1min":0.433,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.3090000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.026,"15min":1.0069999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.7080000000000002,"5min":7.774,"15min":7.774},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.0860000000000001,"15min":0.91800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.0309999999999997,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1,"15min":0.81599999999999995},"min":{"1min":0.44900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.3410000000000002,"5min":6.375,"15min":6.375},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.139,"15min":1.248},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.0579999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.1699999999999999,"15min":1.242},"min":{"1min":0.496,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.3719999999999999,"5min":6.343,"15min":6.343},"last":0.55100000000000005}]}]},{"osd":2,"up_from":359,"seq":1541893259329,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":552916,"kb_used_data":389452,"kb_used_omap":83,"kb_used_meta":163372,"kb_avail":93192236,"statfs":{"total":95995035648,"available":95428849664,"internally_reserved":0,"allocated":398798848,"data_stored":397119298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":167293381},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":6.2400000000000002,"5min":2.452,"15min":1.054},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.735999999999997,"5min":74.241,"15min":74.241},"last":0.36899999999999999},{"interface":"front","average":{"1min":6.3899999999999997,"5min":2.5960000000000001,"15min":1.337},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":55.969999999999999,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.433}]},{"osd":1,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.73099999999999998,"15min":0.68700000000000006},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.2080000000000002,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.056,"5min":1.0629999999999999,"15min":1.1240000000000001},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.397,"5min":11.397,"15min":11.397},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82299999999999995,"15min":0.749},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.2750000000000004,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.871,"15min":0.83499999999999996},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.41799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.92300000000000004,"15min":0.68400000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.1360000000000001,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.63},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.96799999999999997,"15min":0.871},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2430000000000003,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.99299999999999999,"15min":0.96699999999999997},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.3090000000000002,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.93500000000000005,"15min":0.76500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.1680000000000001,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.1919999999999999,"15min":1.044},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.968,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.94099999999999995,"15min":0.71799999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.9079999999999999,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.1559999999999999,"15min":1.046},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.7869999999999999,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.0449999999999999,"15min":1.1339999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3049999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.68400000000000005}]}]},{"osd":3,"up_from":94,"seq":403726926133,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533168,"kb_used_data":4712,"kb_used_omap":7,"kb_used_meta":528440,"kb_avail":93211984,"statfs":{"total":95995035648,"available":95449071616,"internally_reserved":0,"allocated":4825088,"data_stored":3173630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":541123176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3780000000000001,"15min":1.0049999999999999},"min":{"1min":0.22,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.3019999999999996,"5min":122.289,"15min":122.289},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.0150000000000001,"15min":0.94299999999999995},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":8.9849999999999994,"5min":122.041,"15min":122.041},"last":0.33500000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.87,"15min":2.2480000000000002},"min":{"1min":0.22,"5min":0.187,"15min":0.187},"max":{"1min":4.6100000000000003,"5min":8.6280000000000001,"15min":123.866},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.2929999999999999,"5min":0.90200000000000002,"15min":2.1539999999999999},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1710000000000003,"5min":6.8010000000000002,"15min":122.265},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 23:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.64200000000000002,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78400000000000003,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.127,"15min":0.871},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":2.9159999999999999,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.31}]},{"osd":4,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.1970000000000001,"15min":1.966},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.6459999999999999,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.222,"5min":2.3119999999999998,"15min":1.9199999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2149999999999999,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.98699999999999999,"15min":1.403},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2530000000000001,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.432},{"interface":"front","average":{"1min":1.163,"5min":1.0640000000000001,"15min":1.744},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":5.149,"5min":10.246,"15min":11.951000000000001},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.264,"15min":1.748},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":6.2830000000000004,"5min":9.2520000000000007,"15min":14.097},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.091,"5min":1.181,"15min":1.869},"min":{"1min":0.378,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":5.1970000000000001,"5min":9.2810000000000006,"15min":12.887},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95499999999999996,"15min":1.6850000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.119,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.6799999999999997,"5min":8.5830000000000002,"15min":12.32},"last":0.51500000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019330,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993676,"kb_used_data":90308,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92751476,"statfs":{"total":95995035648,"available":94977511424,"internally_reserved":0,"allocated":92475392,"data_stored":91608107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":7.6189999999999998,"5min":4.1950000000000003,"15min":1.7130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":156.375,"5min":156.375,"15min":156.375},"last":0.43099999999999999},{"interface":"front","average":{"1min":7.4009999999999998,"5min":4.335,"15min":1.8420000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":157.02199999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":1.395}]},{"osd":1,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":0.97399999999999998,"15min":3.29},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8739999999999997,"5min":7.0640000000000001,"15min":132.304},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.90400000000000003,"15min":3.2410000000000001},"min":{"1min":0.308,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.181,"5min":10.361000000000001,"15min":132.453},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.155,"15min":1.0649999999999999},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.2949999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":1.254},{"interface":"front","average":{"1min":1.329,"5min":1.161,"15min":0.86499999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.3799999999999999,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":1.1919999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.071,"15min":2.79},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6189999999999998,"5min":10.337999999999999,"15min":164.453},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.2330000000000001,"15min":2.8170000000000002},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.32500000000000001},"max":{"1min":7.2519999999999998,"5min":7.2519999999999998,"15min":167.05199999999999},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.036,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.214},"max":{"1min":1.492,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.92400000000000004,"15min":1.835},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":5.9409999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.1870000000000001,"15min":1.2889999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":6.7050000000000001,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.014,"15min":1.1779999999999999},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.7409999999999997,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.37}]},{"osd":7,"last update":"Sat Sep 23 23:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":1.2749999999999999,"15min":1.256},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.28399999999999997},"max":{"1min":4.0990000000000002,"5min":11.045999999999999,"15min":12.84},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.1379999999999999,"15min":1.1319999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":3.1960000000000002,"5min":10.587,"15min":10.587},"last":0.46800000000000003}]}]},{"osd":5,"up_from":36,"seq":154618823104,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903472,"kb_used_data":3816,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":92841680,"statfs":{"total":95995035648,"available":95069880320,"internally_reserved":0,"allocated":3907584,"data_stored":3064015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.786,"15min":0.999},"min":{"1min":0.23799999999999999,"5min":0.221,"15min":0.221},"max":{"1min":23.346,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.9690000000000001,"15min":1.377},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":23.428999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.49099999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.76500000000000001,"15min":2.1429999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.9649999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.622},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.202,"15min":2.6389999999999998},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":2.3079999999999998,"5min":10.353,"15min":110.89},"last":0.46899999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0589999999999999,"15min":0.88700000000000001},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9450000000000003,"5min":6.601,"15min":6.601},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.012,"5min":1.0009999999999999,"15min":0.997},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.9530000000000003,"5min":4.9530000000000003,"15min":4.9530000000000003},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0600000000000001,"15min":3.6230000000000002},"min":{"1min":0.32400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0899999999999999,"5min":9.2710000000000008,"15min":108.58799999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.2090000000000001,"15min":3.6680000000000001},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3420000000000001,"5min":10.965,"15min":108.15600000000001},"last":0.68200000000000005}]},{"osd":4,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":1.298,"15min":1.552},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":1.321,"5min":10.603,"15min":115.06699999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.92300000000000004,"15min":1.482},"min":{"1min":0.34599999999999997,"5min":0.253,"15min":0.253},"max":{"1min":1.764,"5min":9.5890000000000004,"15min":114.908},"last":0.67300000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.169,"15min":1.256},"min":{"1min":0.30499999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":0.77300000000000002,"5min":7.4630000000000001,"15min":12.615},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.90400000000000003,"15min":1.1100000000000001},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.0569999999999999,"5min":5.4909999999999997,"15min":12.74},"last":0.73299999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:03:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.04,"15min":1.0860000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.4770000000000001,"5min":9.5980000000000008,"15min":12.23},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.444,"15min":1.2090000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.8500000000000001,"5min":10.638999999999999,"15min":11.112},"last":0.53300000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:03:47.351 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:47.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:47.453 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:47.460 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:47.480 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:47.510 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:03:47.528 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:47.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:47.634 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:47.663 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:48.062 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:47 smithi151 ceph-mon[159210]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:48.063 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:47 smithi151 ceph-mon[159210]: pgmap v1269: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:48.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:47 smithi052 ceph-mon[180979]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:48.113 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:47 smithi052 ceph-mon[180979]: pgmap v1269: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:48.114 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:47 smithi052 ceph-mon[187743]: from='client.46865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:48.114 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:47 smithi052 ceph-mon[187743]: pgmap v1269: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:48.155 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:03:48.316 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:48.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:48.420 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:48.449 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:48.611 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:48.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:48.715 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:48.746 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:48.978 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:48.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:49.079 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:03:49.079 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:03:49.080 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T23:03:49.080 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:03:49.082 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:49.111 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:49.357 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:49.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:49.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:49.462 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:49.489 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:49.552 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:49.776 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:03:49.784 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:49.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:49.890 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:49.916 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:49.960 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:49.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:50.065 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:50.090 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:49 smithi151 ceph-mon[159210]: pgmap v1270: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:50.090 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:50.090 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:50.091 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1481205538' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:50.091 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:49 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:50.091 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2404692476' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:50.092 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:50.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[187743]: pgmap v1270: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:50.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:50.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:50.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1481205538' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2404692476' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[180979]: pgmap v1270: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1481205538' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:03:50.104 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2404692476' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:03:50.140 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:50.142 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:03:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:50.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:50.253 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:50.279 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:50.352 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:03:50.373 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:50.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:50.478 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:50.505 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:50.532 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:03:50.534 INFO:teuthology.orchestra.run.smithi052.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T23:03:50.685 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:50.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:50.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:50.789 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:50.817 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:50.859 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:50.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 5]}]: dispatch 2023-09-23T23:03:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]: dispatch 2023-09-23T23:03:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]: dispatch 2023-09-23T23:03:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]: dispatch 2023-09-23T23:03:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]: dispatch 2023-09-23T23:03:50.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1e", "id": [3, 5]}]: dispatch 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: failed to encode map e413 with expected crc 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2404692476' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 5]}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1e", "id": [3, 5]}]': finished 2023-09-23T23:03:50.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: osdmap e413: 8 total, 8 up, 6 in 2023-09-23T23:03:50.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:50 smithi151 ceph-mon[159210]: failed to encode map e413 with expected crc 2023-09-23T23:03:50.963 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:50.990 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 5]}]: dispatch 2023-09-23T23:03:50.990 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]: dispatch 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]: dispatch 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]: dispatch 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]: dispatch 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1e", "id": [3, 5]}]: dispatch 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: failed to encode map e413 with expected crc 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2404692476' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 5]}]': finished 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]': finished 2023-09-23T23:03:50.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]': finished 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]': finished 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1e", "id": [3, 5]}]': finished 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: osdmap e413: 8 total, 8 up, 6 in 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[187743]: failed to encode map e413 with expected crc 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 5]}]: dispatch 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]: dispatch 2023-09-23T23:03:50.992 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]: dispatch 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]: dispatch 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]: dispatch 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1e", "id": [3, 5]}]: dispatch 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: failed to encode map e413 with expected crc 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2404692476' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 5]}]': finished 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]': finished 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]': finished 2023-09-23T23:03:50.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2023-09-23T23:03:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]': finished 2023-09-23T23:03:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1e", "id": [3, 5]}]': finished 2023-09-23T23:03:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: osdmap e413: 8 total, 8 up, 6 in 2023-09-23T23:03:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:50 smithi052 ceph-mon[180979]: failed to encode map e413 with expected crc 2023-09-23T23:03:50.994 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:51.650 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:03:51.651 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:03:51.651 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:03:51.651 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:03:51.651 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:03:51.797 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:51.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:51.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:51.898 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: failed to encode map e413 with expected crc 2023-09-23T23:03:51.901 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:51.930 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:51.939 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:03:51.952 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:51.961 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:03:51.972 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:51.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:51.986 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:03:52.007 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:03:52.026 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:03:52.044 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:03:52.062 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:03:52.076 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:52.081 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:03:52.082 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.085 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.085 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.085 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.085 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.085 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.085 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:03:52.106 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:52.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: pgmap v1272: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: osdmap e414: 8 total, 8 up, 6 in 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[187743]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: pgmap v1272: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:52.197 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.197 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: osdmap e414: 8 total, 8 up, 6 in 2023-09-23T23:03:52.197 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:51 smithi052 ceph-mon[180979]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: failed to encode map e413 with expected crc 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: pgmap v1272: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: osdmap e414: 8 total, 8 up, 6 in 2023-09-23T23:03:52.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:51 smithi151 ceph-mon[159210]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.208 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:52.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:52.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:52.327 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:03:52.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:52.385 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:52.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:52.495 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:52.526 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:52.682 DEBUG:teuthology.orchestra.run.smithi052:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T23:03:52.685 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T23:03:52.685 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T23:03:52.689 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:03:52.731 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:hints = 1 2023-09-23T23:03:52.731 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T23:03:52.732 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object prefix: benchmark_data_smithi052_606809 2023-09-23T23:03:52.732 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:03:52.732 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 0 0 0 0 0 0 - 0 2023-09-23T23:03:52.913 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:52 smithi052 ceph-mon[180979]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.914 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2390442008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:52.914 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:52 smithi052 ceph-mon[180979]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:52 smithi052 ceph-mon[187743]: failed to encode map e414 with expected crc 2023-09-23T23:03:52.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2390442008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:52.914 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:52 smithi052 ceph-mon[187743]: failed to encode map e414 with expected crc 2023-09-23T23:03:53.043 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:53.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:53.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:53.149 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:52 smithi151 ceph-mon[159210]: failed to encode map e414 with expected crc 2023-09-23T23:03:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2390442008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:03:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:52 smithi151 ceph-mon[159210]: failed to encode map e414 with expected crc 2023-09-23T23:03:53.205 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:03:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:03:53] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-09-23T23:03:53.732 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 1 16 2245 2229 139.296 139.312 0.00575881 0.00714542 2023-09-23T23:03:53.858 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:03:53.957 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:53.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:53.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:54.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:53 smithi052 ceph-mon[180979]: failed to encode map e414 with expected crc 2023-09-23T23:03:54.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:53 smithi052 ceph-mon[180979]: pgmap v1274: 89 pgs: 4 peering, 85 active+clean; 582 KiB data, 3.7 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:54.065 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:53 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:03:54.066 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:53 smithi052 ceph-mon[187743]: failed to encode map e414 with expected crc 2023-09-23T23:03:54.066 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:53 smithi052 ceph-mon[187743]: pgmap v1274: 89 pgs: 4 peering, 85 active+clean; 582 KiB data, 3.7 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:54.066 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:53 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:03:54.078 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:54.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:53 smithi151 ceph-mon[159210]: failed to encode map e414 with expected crc 2023-09-23T23:03:54.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:53 smithi151 ceph-mon[159210]: pgmap v1274: 89 pgs: 4 peering, 85 active+clean; 582 KiB data, 3.7 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:54.102 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:53 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:03:54.140 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:54.307 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:03:54.307 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:03:54.309 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:54.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:54.418 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:54.488 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:54.595 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:54.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:54.664 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:54.704 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:54.732 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 2 16 4203 4187 130.83 122.375 0.0018116 0.0072772 2023-09-23T23:03:54.734 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:54.815 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:54.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:54.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:54.888 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:03:54.901 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:03:54.918 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:03:54.951 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:55.109 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:03:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:55.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:03:55.177 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:03:55.177 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:54 smithi151 ceph-mon[159210]: pgmap v1275: 89 pgs: 4 peering, 85 active+clean; 582 KiB data, 3.7 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:55.178 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:54 smithi151 ceph-mon[159210]: from='client.36828 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:55.213 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:55.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:54 smithi052 ceph-mon[180979]: pgmap v1275: 89 pgs: 4 peering, 85 active+clean; 582 KiB data, 3.7 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:55.232 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:54 smithi052 ceph-mon[180979]: from='client.36828 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:55.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:54 smithi052 ceph-mon[187743]: pgmap v1275: 89 pgs: 4 peering, 85 active+clean; 582 KiB data, 3.7 GiB used, 533 GiB / 536 GiB avail 2023-09-23T23:03:55.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:54 smithi052 ceph-mon[187743]: from='client.36828 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:03:55.255 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:55.309 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:55.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:55.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:55.415 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:55.468 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:55.605 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:03:55.735 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 3 16 6344 6328 131.729 133.812 0.00688243 0.00757898 2023-09-23T23:03:56.067 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:03:56.161 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:56 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:56.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:56 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:56 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:03:56.507 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:56.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:56.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:56.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:56.672 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:56.735 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 4 16 8253 8237 128.62 119.312 0.00262962 0.00776116 2023-09-23T23:03:56.941 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:56.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:56.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:57.051 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:03:57.079 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:57.239 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:57 smithi151 ceph-mon[159210]: pgmap v1276: 89 pgs: 89 active+clean; 439 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 11 B/s, 0 objects/s recovering 2023-09-23T23:03:57.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:57 smithi052 ceph-mon[180979]: pgmap v1276: 89 pgs: 89 active+clean; 439 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 11 B/s, 0 objects/s recovering 2023-09-23T23:03:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:57 smithi052 ceph-mon[187743]: pgmap v1276: 89 pgs: 89 active+clean; 439 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 11 B/s, 0 objects/s recovering 2023-09-23T23:03:57.432 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:03:57.432 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:03:57.734 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 5 16 9870 9854 123.109 101.062 0.0012383 0.00810103 2023-09-23T23:03:58.006 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:03:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:58.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:03:58.114 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:03:58.147 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:58.233 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:03:58.262 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:03:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:58.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:03:58.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:58 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T23:03:58.307 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:58 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T23:03:58.337 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1275,"stamp":"2023-09-23T23:03:54.645841+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":605660,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3870124,"kb_used_data":405052,"kb_used_omap":137,"kb_used_meta":3464886,"kb_avail":558600788,"statfs":{"total":575970213888,"available":572007206912,"internally_reserved":0,"allocated":414773248,"data_stored":406346975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3548043448},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002076"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":424,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885224+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:03:42.885224+0000","last_peered":"2023-09-23T23:03:42.885224+0000","last_clean":"2023-09-23T23:03:42.885224+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:03:42.885224+0000","last_undegraded":"2023-09-23T23:03:42.885224+0000","last_fullsized":"2023-09-23T23:03:42.885224+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624481+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:03:44.624481+0000","last_peered":"2023-09-23T23:03:44.624481+0000","last_clean":"2023-09-23T23:03:44.624481+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:03:44.624481+0000","last_undegraded":"2023-09-23T23:03:44.624481+0000","last_fullsized":"2023-09-23T23:03:44.624481+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623875+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:03:44.623875+0000","last_peered":"2023-09-23T23:03:44.623875+0000","last_clean":"2023-09-23T23:03:44.623875+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:03:44.623875+0000","last_undegraded":"2023-09-23T23:03:44.623875+0000","last_fullsized":"2023-09-23T23:03:44.623875+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":411,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624314+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:03:44.624314+0000","last_peered":"2023-09-23T23:03:44.624314+0000","last_clean":"2023-09-23T23:03:44.624314+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:03:44.624314+0000","last_undegraded":"2023-09-23T23:03:44.624314+0000","last_fullsized":"2023-09-23T23:03:44.624314+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2,"reported_epoch":413,"state":"peering","last_fresh":"2023-09-23T22:58:06.722013+0000","last_change":"2023-09-23T22:58:05.726467+0000","last_active":"2023-09-23T22:58:05.726298+0000","last_peered":"2023-09-23T22:58:04.718744+0000","last_clean":"2023-09-23T22:57:27.503438+0000","last_became_active":"2023-09-23T22:57:34.620880+0000","last_became_peered":"2023-09-23T22:57:34.620880+0000","last_unstale":"2023-09-23T22:58:06.722013+0000","last_undegraded":"2023-09-23T22:58:06.722013+0000","last_fullsized":"2023-09-23T22:58:06.722013+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:32.789048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.888755+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:03:42.888755+0000","last_peered":"2023-09-23T23:03:42.888755+0000","last_clean":"2023-09-23T23:03:42.888755+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:03:42.888755+0000","last_undegraded":"2023-09-23T23:03:42.888755+0000","last_fullsized":"2023-09-23T23:03:42.888755+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":347,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.883217+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:03:42.883217+0000","last_peered":"2023-09-23T23:03:42.883217+0000","last_clean":"2023-09-23T23:03:42.883217+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:03:42.883217+0000","last_undegraded":"2023-09-23T23:03:42.883217+0000","last_fullsized":"2023-09-23T23:03:42.883217+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623294+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:03:44.623294+0000","last_peered":"2023-09-23T23:03:44.623294+0000","last_clean":"2023-09-23T23:03:44.623294+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:03:44.623294+0000","last_undegraded":"2023-09-23T23:03:44.623294+0000","last_fullsized":"2023-09-23T23:03:44.623294+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":308,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882423+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:03:42.882423+0000","last_peered":"2023-09-23T23:03:42.882423+0000","last_clean":"2023-09-23T23:03:42.882423+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:03:42.882423+0000","last_undegraded":"2023-09-23T23:03:42.882423+0000","last_fullsized":"2023-09-23T23:03:42.882423+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":97,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623779+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:03:44.623779+0000","last_peered":"2023-09-23T23:03:44.623779+0000","last_clean":"2023-09-23T23:03:44.623779+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:03:44.623779+0000","last_undegraded":"2023-09-23T23:03:44.623779+0000","last_fullsized":"2023-09-23T23:03:44.623779+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":104,"reported_epoch":413,"state":"peering","last_fresh":"2023-09-23T23:03:50.527742+0000","last_change":"2023-09-23T23:03:50.527913+0000","last_active":"2023-09-23T23:03:50.527742+0000","last_peered":"2023-09-23T23:03:44.624438+0000","last_clean":"2023-09-23T23:03:44.624438+0000","last_became_active":"2023-09-23T22:58:06.746756+0000","last_became_peered":"2023-09-23T22:58:06.746756+0000","last_unstale":"2023-09-23T23:03:50.527742+0000","last_undegraded":"2023-09-23T23:03:50.527742+0000","last_fullsized":"2023-09-23T23:03:50.527742+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:55.741151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4,5],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":159,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.886608+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:03:43.886608+0000","last_peered":"2023-09-23T23:03:43.886608+0000","last_clean":"2023-09-23T23:03:43.886608+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:03:43.886608+0000","last_undegraded":"2023-09-23T23:03:43.886608+0000","last_fullsized":"2023-09-23T23:03:43.886608+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":483,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886299+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:03:42.886299+0000","last_peered":"2023-09-23T23:03:42.886299+0000","last_clean":"2023-09-23T23:03:42.886299+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:03:42.886299+0000","last_undegraded":"2023-09-23T23:03:42.886299+0000","last_fullsized":"2023-09-23T23:03:42.886299+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":386,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886081+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:03:42.886081+0000","last_peered":"2023-09-23T23:03:42.886081+0000","last_clean":"2023-09-23T23:03:42.886081+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:03:42.886081+0000","last_undegraded":"2023-09-23T23:03:42.886081+0000","last_fullsized":"2023-09-23T23:03:42.886081+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":312,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886118+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:03:42.886118+0000","last_peered":"2023-09-23T23:03:42.886118+0000","last_clean":"2023-09-23T23:03:42.886118+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:03:42.886118+0000","last_undegraded":"2023-09-23T23:03:42.886118+0000","last_fullsized":"2023-09-23T23:03:42.886118+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":94,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623405+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:03:44.623405+0000","last_peered":"2023-09-23T23:03:44.623405+0000","last_clean":"2023-09-23T23:03:44.623405+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:03:44.623405+0000","last_undegraded":"2023-09-23T23:03:44.623405+0000","last_fullsized":"2023-09-23T23:03:44.623405+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":403,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623938+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:03:44.623938+0000","last_peered":"2023-09-23T23:03:44.623938+0000","last_clean":"2023-09-23T23:03:44.623938+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:03:44.623938+0000","last_undegraded":"2023-09-23T23:03:44.623938+0000","last_fullsized":"2023-09-23T23:03:44.623938+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":312,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882933+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:03:42.882933+0000","last_peered":"2023-09-23T23:03:42.882933+0000","last_clean":"2023-09-23T23:03:42.882933+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:03:42.882933+0000","last_undegraded":"2023-09-23T23:03:42.882933+0000","last_fullsized":"2023-09-23T23:03:42.882933+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":297,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885915+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:03:42.885915+0000","last_peered":"2023-09-23T23:03:42.885915+0000","last_clean":"2023-09-23T23:03:42.885915+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:03:42.885915+0000","last_undegraded":"2023-09-23T23:03:42.885915+0000","last_fullsized":"2023-09-23T23:03:42.885915+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":427,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623470+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:03:44.623470+0000","last_peered":"2023-09-23T23:03:44.623470+0000","last_clean":"2023-09-23T23:03:44.623470+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:03:44.623470+0000","last_undegraded":"2023-09-23T23:03:44.623470+0000","last_fullsized":"2023-09-23T23:03:44.623470+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":131,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886701+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:03:42.886701+0000","last_peered":"2023-09-23T23:03:42.886701+0000","last_clean":"2023-09-23T23:03:42.886701+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:03:42.886701+0000","last_undegraded":"2023-09-23T23:03:42.886701+0000","last_fullsized":"2023-09-23T23:03:42.886701+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":107,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623588+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:03:44.623588+0000","last_peered":"2023-09-23T23:03:44.623588+0000","last_clean":"2023-09-23T23:03:44.623588+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:03:44.623588+0000","last_undegraded":"2023-09-23T23:03:44.623588+0000","last_fullsized":"2023-09-23T23:03:44.623588+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":10,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.132142+0000","last_change":"2023-09-23T23:03:43.131702+0000","last_active":"2023-09-23T23:03:43.132142+0000","last_peered":"2023-09-23T23:03:43.132142+0000","last_clean":"2023-09-23T23:03:43.132142+0000","last_became_active":"2023-09-23T23:03:43.131323+0000","last_became_peered":"2023-09-23T23:03:43.131323+0000","last_unstale":"2023-09-23T23:03:43.132142+0000","last_undegraded":"2023-09-23T23:03:43.132142+0000","last_fullsized":"2023-09-23T23:03:43.132142+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":296,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886652+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:03:42.886652+0000","last_peered":"2023-09-23T23:03:42.886652+0000","last_clean":"2023-09-23T23:03:42.886652+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:03:42.886652+0000","last_undegraded":"2023-09-23T23:03:42.886652+0000","last_fullsized":"2023-09-23T23:03:42.886652+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":282,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882480+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:03:42.882480+0000","last_peered":"2023-09-23T23:03:42.882480+0000","last_clean":"2023-09-23T23:03:42.882480+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:03:42.882480+0000","last_undegraded":"2023-09-23T23:03:42.882480+0000","last_fullsized":"2023-09-23T23:03:42.882480+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":11,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.889310+0000","last_change":"2023-09-23T23:03:43.132133+0000","last_active":"2023-09-23T23:03:43.889310+0000","last_peered":"2023-09-23T23:03:43.889310+0000","last_clean":"2023-09-23T23:03:43.889310+0000","last_became_active":"2023-09-23T23:03:43.131741+0000","last_became_peered":"2023-09-23T23:03:43.131741+0000","last_unstale":"2023-09-23T23:03:43.889310+0000","last_undegraded":"2023-09-23T23:03:43.889310+0000","last_fullsized":"2023-09-23T23:03:43.889310+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:36.498718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":2,"reported_epoch":413,"state":"peering","last_fresh":"2023-09-23T22:44:54.857508+0000","last_change":"2023-09-23T23:03:51.378109+0000","last_active":"2023-09-23T23:03:51.378109+0000","last_peered":"2023-09-23T22:44:54.857508+0000","last_clean":"2023-09-23T22:43:31.842625+0000","last_became_active":"2023-09-23T22:44:54.856337+0000","last_became_peered":"2023-09-23T22:44:54.856337+0000","last_unstale":"2023-09-23T22:44:54.857508+0000","last_undegraded":"2023-09-23T22:44:54.857508+0000","last_fullsized":"2023-09-23T22:44:54.857508+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.921280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":419,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885817+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:03:42.885817+0000","last_peered":"2023-09-23T23:03:42.885817+0000","last_clean":"2023-09-23T23:03:42.885817+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:03:42.885817+0000","last_undegraded":"2023-09-23T23:03:42.885817+0000","last_fullsized":"2023-09-23T23:03:42.885817+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":10,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.889757+0000","last_change":"2023-09-23T23:03:42.889536+0000","last_active":"2023-09-23T23:03:42.889757+0000","last_peered":"2023-09-23T23:03:42.889757+0000","last_clean":"2023-09-23T23:03:42.889757+0000","last_became_active":"2023-09-23T23:03:42.889213+0000","last_became_peered":"2023-09-23T23:03:42.889213+0000","last_unstale":"2023-09-23T23:03:42.889757+0000","last_undegraded":"2023-09-23T23:03:42.889757+0000","last_fullsized":"2023-09-23T23:03:42.889757+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":163,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.883402+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:03:42.883402+0000","last_peered":"2023-09-23T23:03:42.883402+0000","last_clean":"2023-09-23T23:03:42.883402+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:03:42.883402+0000","last_undegraded":"2023-09-23T23:03:42.883402+0000","last_fullsized":"2023-09-23T23:03:42.883402+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":281,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882515+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:03:42.882515+0000","last_peered":"2023-09-23T23:03:42.882515+0000","last_clean":"2023-09-23T23:03:42.882515+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:03:42.882515+0000","last_undegraded":"2023-09-23T23:03:42.882515+0000","last_fullsized":"2023-09-23T23:03:42.882515+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.377608+0000","last_change":"2023-09-23T23:03:43.136203+0000","last_active":"2023-09-23T23:03:46.377608+0000","last_peered":"2023-09-23T23:03:46.377608+0000","last_clean":"2023-09-23T23:03:46.377608+0000","last_became_active":"2023-09-23T23:03:43.135918+0000","last_became_peered":"2023-09-23T23:03:43.135918+0000","last_unstale":"2023-09-23T23:03:46.377608+0000","last_undegraded":"2023-09-23T23:03:46.377608+0000","last_fullsized":"2023-09-23T23:03:46.377608+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:11.372248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":83,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624233+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:03:44.624233+0000","last_peered":"2023-09-23T23:03:44.624233+0000","last_clean":"2023-09-23T23:03:44.624233+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:03:44.624233+0000","last_undegraded":"2023-09-23T23:03:44.624233+0000","last_fullsized":"2023-09-23T23:03:44.624233+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":312,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.888002+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:03:42.888002+0000","last_peered":"2023-09-23T23:03:42.888002+0000","last_clean":"2023-09-23T23:03:42.888002+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:03:42.888002+0000","last_undegraded":"2023-09-23T23:03:42.888002+0000","last_fullsized":"2023-09-23T23:03:42.888002+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":10,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.132143+0000","last_change":"2023-09-23T23:03:43.131843+0000","last_active":"2023-09-23T23:03:43.132143+0000","last_peered":"2023-09-23T23:03:43.132143+0000","last_clean":"2023-09-23T23:03:43.132143+0000","last_became_active":"2023-09-23T23:03:43.131420+0000","last_became_peered":"2023-09-23T23:03:43.131420+0000","last_unstale":"2023-09-23T23:03:43.132143+0000","last_undegraded":"2023-09-23T23:03:43.132143+0000","last_fullsized":"2023-09-23T23:03:43.132143+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":102,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624097+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:03:44.624097+0000","last_peered":"2023-09-23T23:03:44.624097+0000","last_clean":"2023-09-23T23:03:44.624097+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:03:44.624097+0000","last_undegraded":"2023-09-23T23:03:44.624097+0000","last_fullsized":"2023-09-23T23:03:44.624097+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":405,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886212+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:03:42.886212+0000","last_peered":"2023-09-23T23:03:42.886212+0000","last_clean":"2023-09-23T23:03:42.886212+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:03:42.886212+0000","last_undegraded":"2023-09-23T23:03:42.886212+0000","last_fullsized":"2023-09-23T23:03:42.886212+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.377578+0000","last_change":"2023-09-23T23:03:42.889763+0000","last_active":"2023-09-23T23:03:46.377578+0000","last_peered":"2023-09-23T23:03:46.377578+0000","last_clean":"2023-09-23T23:03:46.377578+0000","last_became_active":"2023-09-23T23:03:42.889481+0000","last_became_peered":"2023-09-23T23:03:42.889481+0000","last_unstale":"2023-09-23T23:03:46.377578+0000","last_undegraded":"2023-09-23T23:03:46.377578+0000","last_fullsized":"2023-09-23T23:03:46.377578+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:57.841265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":195,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885402+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:03:42.885402+0000","last_peered":"2023-09-23T23:03:42.885402+0000","last_clean":"2023-09-23T23:03:42.885402+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:03:42.885402+0000","last_undegraded":"2023-09-23T23:03:42.885402+0000","last_fullsized":"2023-09-23T23:03:42.885402+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":422,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886533+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:03:42.886533+0000","last_peered":"2023-09-23T23:03:42.886533+0000","last_clean":"2023-09-23T23:03:42.886533+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:03:42.886533+0000","last_undegraded":"2023-09-23T23:03:42.886533+0000","last_fullsized":"2023-09-23T23:03:42.886533+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.282830+0000","last_change":"2023-09-23T23:03:42.889334+0000","last_active":"2023-09-23T23:03:46.282830+0000","last_peered":"2023-09-23T23:03:46.282830+0000","last_clean":"2023-09-23T23:03:46.282830+0000","last_became_active":"2023-09-23T23:03:42.889046+0000","last_became_peered":"2023-09-23T23:03:42.889046+0000","last_unstale":"2023-09-23T23:03:46.282830+0000","last_undegraded":"2023-09-23T23:03:46.282830+0000","last_fullsized":"2023-09-23T23:03:46.282830+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:44.256081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":435,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885566+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:03:42.885566+0000","last_peered":"2023-09-23T23:03:42.885566+0000","last_clean":"2023-09-23T23:03:42.885566+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:03:42.885566+0000","last_undegraded":"2023-09-23T23:03:42.885566+0000","last_fullsized":"2023-09-23T23:03:42.885566+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":413,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886628+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:03:42.886628+0000","last_peered":"2023-09-23T23:03:42.886628+0000","last_clean":"2023-09-23T23:03:42.886628+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:03:42.886628+0000","last_undegraded":"2023-09-23T23:03:42.886628+0000","last_fullsized":"2023-09-23T23:03:42.886628+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.282812+0000","last_change":"2023-09-23T23:03:43.132462+0000","last_active":"2023-09-23T23:03:46.282812+0000","last_peered":"2023-09-23T23:03:46.282812+0000","last_clean":"2023-09-23T23:03:46.282812+0000","last_became_active":"2023-09-23T23:03:43.132267+0000","last_became_peered":"2023-09-23T23:03:43.132267+0000","last_unstale":"2023-09-23T23:03:46.282812+0000","last_undegraded":"2023-09-23T23:03:46.282812+0000","last_fullsized":"2023-09-23T23:03:46.282812+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:49.697445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":83,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624034+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:03:44.624034+0000","last_peered":"2023-09-23T23:03:44.624034+0000","last_clean":"2023-09-23T23:03:44.624034+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:03:44.624034+0000","last_undegraded":"2023-09-23T23:03:44.624034+0000","last_fullsized":"2023-09-23T23:03:44.624034+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":317,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.887301+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:03:43.887301+0000","last_peered":"2023-09-23T23:03:43.887301+0000","last_clean":"2023-09-23T23:03:43.887301+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:03:43.887301+0000","last_undegraded":"2023-09-23T23:03:43.887301+0000","last_fullsized":"2023-09-23T23:03:43.887301+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.377647+0000","last_change":"2023-09-23T23:03:43.132020+0000","last_active":"2023-09-23T23:03:46.377647+0000","last_peered":"2023-09-23T23:03:46.377647+0000","last_clean":"2023-09-23T23:03:46.377647+0000","last_became_active":"2023-09-23T23:03:43.131692+0000","last_became_peered":"2023-09-23T23:03:43.131692+0000","last_unstale":"2023-09-23T23:03:46.377647+0000","last_undegraded":"2023-09-23T23:03:46.377647+0000","last_fullsized":"2023-09-23T23:03:46.377647+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:13.311701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":321,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.886743+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:03:43.886743+0000","last_peered":"2023-09-23T23:03:43.886743+0000","last_clean":"2023-09-23T23:03:43.886743+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:03:43.886743+0000","last_undegraded":"2023-09-23T23:03:43.886743+0000","last_fullsized":"2023-09-23T23:03:43.886743+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":322,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882830+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:03:42.882830+0000","last_peered":"2023-09-23T23:03:42.882830+0000","last_clean":"2023-09-23T23:03:42.882830+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:03:42.882830+0000","last_undegraded":"2023-09-23T23:03:42.882830+0000","last_fullsized":"2023-09-23T23:03:42.882830+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882407+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:03:42.882407+0000","last_peered":"2023-09-23T23:03:42.882407+0000","last_clean":"2023-09-23T23:03:42.882407+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:03:42.882407+0000","last_undegraded":"2023-09-23T23:03:42.882407+0000","last_fullsized":"2023-09-23T23:03:42.882407+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.282920+0000","last_change":"2023-09-23T23:03:43.132707+0000","last_active":"2023-09-23T23:03:46.282920+0000","last_peered":"2023-09-23T23:03:46.282920+0000","last_clean":"2023-09-23T23:03:46.282920+0000","last_became_active":"2023-09-23T23:03:43.132372+0000","last_became_peered":"2023-09-23T23:03:43.132372+0000","last_unstale":"2023-09-23T23:03:46.282920+0000","last_undegraded":"2023-09-23T23:03:46.282920+0000","last_fullsized":"2023-09-23T23:03:46.282920+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.951582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":83,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886877+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:03:42.886877+0000","last_peered":"2023-09-23T23:03:42.886877+0000","last_clean":"2023-09-23T23:03:42.886877+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:03:42.886877+0000","last_undegraded":"2023-09-23T23:03:42.886877+0000","last_fullsized":"2023-09-23T23:03:42.886877+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":432,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886247+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:03:42.886247+0000","last_peered":"2023-09-23T23:03:42.886247+0000","last_clean":"2023-09-23T23:03:42.886247+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:03:42.886247+0000","last_undegraded":"2023-09-23T23:03:42.886247+0000","last_fullsized":"2023-09-23T23:03:42.886247+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":306,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.886669+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:03:43.886669+0000","last_peered":"2023-09-23T23:03:43.886669+0000","last_clean":"2023-09-23T23:03:43.886669+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:03:43.886669+0000","last_undegraded":"2023-09-23T23:03:43.886669+0000","last_fullsized":"2023-09-23T23:03:43.886669+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.282767+0000","last_change":"2023-09-23T23:03:42.889306+0000","last_active":"2023-09-23T23:03:46.282767+0000","last_peered":"2023-09-23T23:03:46.282767+0000","last_clean":"2023-09-23T23:03:46.282767+0000","last_became_active":"2023-09-23T23:03:42.888948+0000","last_became_peered":"2023-09-23T23:03:42.888948+0000","last_unstale":"2023-09-23T23:03:46.282767+0000","last_undegraded":"2023-09-23T23:03:46.282767+0000","last_fullsized":"2023-09-23T23:03:46.282767+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:56.393948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":381,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.886643+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:03:43.886643+0000","last_peered":"2023-09-23T23:03:43.886643+0000","last_clean":"2023-09-23T23:03:43.886643+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:03:43.886643+0000","last_undegraded":"2023-09-23T23:03:43.886643+0000","last_fullsized":"2023-09-23T23:03:43.886643+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":257,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882241+0000","last_change":"2023-09-23T22:58:24.260979+0000","last_active":"2023-09-23T23:03:42.882241+0000","last_peered":"2023-09-23T23:03:42.882241+0000","last_clean":"2023-09-23T23:03:42.882241+0000","last_became_active":"2023-09-23T22:58:24.260632+0000","last_became_peered":"2023-09-23T22:58:24.260632+0000","last_unstale":"2023-09-23T23:03:42.882241+0000","last_undegraded":"2023-09-23T23:03:42.882241+0000","last_fullsized":"2023-09-23T23:03:42.882241+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:27.732891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":322,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.883290+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:03:42.883290+0000","last_peered":"2023-09-23T23:03:42.883290+0000","last_clean":"2023-09-23T23:03:42.883290+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:03:42.883290+0000","last_undegraded":"2023-09-23T23:03:42.883290+0000","last_fullsized":"2023-09-23T23:03:42.883290+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.282883+0000","last_change":"2023-09-23T23:03:43.131943+0000","last_active":"2023-09-23T23:03:46.282883+0000","last_peered":"2023-09-23T23:03:46.282883+0000","last_clean":"2023-09-23T23:03:46.282883+0000","last_became_active":"2023-09-23T23:03:43.131668+0000","last_became_peered":"2023-09-23T23:03:43.131668+0000","last_unstale":"2023-09-23T23:03:46.282883+0000","last_undegraded":"2023-09-23T23:03:46.282883+0000","last_fullsized":"2023-09-23T23:03:46.282883+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:16.104887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":395,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882827+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:03:42.882827+0000","last_peered":"2023-09-23T23:03:42.882827+0000","last_clean":"2023-09-23T23:03:42.882827+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:03:42.882827+0000","last_undegraded":"2023-09-23T23:03:42.882827+0000","last_fullsized":"2023-09-23T23:03:42.882827+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":524,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.887079+0000","last_change":"2023-09-23T23:03:36.322526+0000","last_active":"2023-09-23T23:03:42.887079+0000","last_peered":"2023-09-23T23:03:42.887079+0000","last_clean":"2023-09-23T23:03:42.887079+0000","last_became_active":"2023-09-23T23:03:36.322065+0000","last_became_peered":"2023-09-23T23:03:36.322065+0000","last_unstale":"2023-09-23T23:03:42.887079+0000","last_undegraded":"2023-09-23T23:03:42.887079+0000","last_fullsized":"2023-09-23T23:03:42.887079+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:04.195321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":84,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624170+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:03:44.624170+0000","last_peered":"2023-09-23T23:03:44.624170+0000","last_clean":"2023-09-23T23:03:44.624170+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:03:44.624170+0000","last_undegraded":"2023-09-23T23:03:44.624170+0000","last_fullsized":"2023-09-23T23:03:44.624170+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":322,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886693+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:03:42.886693+0000","last_peered":"2023-09-23T23:03:42.886693+0000","last_clean":"2023-09-23T23:03:42.886693+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:03:42.886693+0000","last_undegraded":"2023-09-23T23:03:42.886693+0000","last_fullsized":"2023-09-23T23:03:42.886693+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.282849+0000","last_change":"2023-09-23T23:03:43.133127+0000","last_active":"2023-09-23T23:03:46.282849+0000","last_peered":"2023-09-23T23:03:46.282849+0000","last_clean":"2023-09-23T23:03:46.282849+0000","last_became_active":"2023-09-23T23:03:43.132709+0000","last_became_peered":"2023-09-23T23:03:43.132709+0000","last_unstale":"2023-09-23T23:03:46.282849+0000","last_undegraded":"2023-09-23T23:03:46.282849+0000","last_fullsized":"2023-09-23T23:03:46.282849+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:35.116957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":403,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886794+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:03:42.886794+0000","last_peered":"2023-09-23T23:03:42.886794+0000","last_clean":"2023-09-23T23:03:42.886794+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:03:42.886794+0000","last_undegraded":"2023-09-23T23:03:42.886794+0000","last_fullsized":"2023-09-23T23:03:42.886794+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":374,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.887485+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:03:42.887485+0000","last_peered":"2023-09-23T23:03:42.887485+0000","last_clean":"2023-09-23T23:03:42.887485+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:03:42.887485+0000","last_undegraded":"2023-09-23T23:03:42.887485+0000","last_fullsized":"2023-09-23T23:03:42.887485+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":334,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.888913+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:03:42.888913+0000","last_peered":"2023-09-23T23:03:42.888913+0000","last_clean":"2023-09-23T23:03:42.888913+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:03:42.888913+0000","last_undegraded":"2023-09-23T23:03:42.888913+0000","last_fullsized":"2023-09-23T23:03:42.888913+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.377628+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:03:46.377628+0000","last_peered":"2023-09-23T23:03:46.377628+0000","last_clean":"2023-09-23T23:03:46.377628+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:03:46.377628+0000","last_undegraded":"2023-09-23T23:03:46.377628+0000","last_fullsized":"2023-09-23T23:03:46.377628+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":349,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.888507+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:03:42.888507+0000","last_peered":"2023-09-23T23:03:42.888507+0000","last_clean":"2023-09-23T23:03:42.888507+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:03:42.888507+0000","last_undegraded":"2023-09-23T23:03:42.888507+0000","last_fullsized":"2023-09-23T23:03:42.888507+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.886985+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:03:43.886985+0000","last_peered":"2023-09-23T23:03:43.886985+0000","last_clean":"2023-09-23T23:03:43.886985+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:03:43.886985+0000","last_undegraded":"2023-09-23T23:03:43.886985+0000","last_fullsized":"2023-09-23T23:03:43.886985+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":225,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.882883+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:03:42.882883+0000","last_peered":"2023-09-23T23:03:42.882883+0000","last_clean":"2023-09-23T23:03:42.882883+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:03:42.882883+0000","last_undegraded":"2023-09-23T23:03:42.882883+0000","last_fullsized":"2023-09-23T23:03:42.882883+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":12,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:46.377658+0000","last_change":"2023-09-23T23:03:43.132731+0000","last_active":"2023-09-23T23:03:46.377658+0000","last_peered":"2023-09-23T23:03:46.377658+0000","last_clean":"2023-09-23T23:03:46.377658+0000","last_became_active":"2023-09-23T23:03:43.132451+0000","last_became_peered":"2023-09-23T23:03:43.132451+0000","last_unstale":"2023-09-23T23:03:46.377658+0000","last_undegraded":"2023-09-23T23:03:46.377658+0000","last_fullsized":"2023-09-23T23:03:46.377658+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:51.837853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":107,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.624419+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:03:44.624419+0000","last_peered":"2023-09-23T23:03:44.624419+0000","last_clean":"2023-09-23T23:03:44.624419+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:03:44.624419+0000","last_undegraded":"2023-09-23T23:03:44.624419+0000","last_fullsized":"2023-09-23T23:03:44.624419+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":120,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886642+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:03:42.886642+0000","last_peered":"2023-09-23T23:03:42.886642+0000","last_clean":"2023-09-23T23:03:42.886642+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:03:42.886642+0000","last_undegraded":"2023-09-23T23:03:42.886642+0000","last_fullsized":"2023-09-23T23:03:42.886642+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":99,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623751+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:03:44.623751+0000","last_peered":"2023-09-23T23:03:44.623751+0000","last_clean":"2023-09-23T23:03:44.623751+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:03:44.623751+0000","last_undegraded":"2023-09-23T23:03:44.623751+0000","last_fullsized":"2023-09-23T23:03:44.623751+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":10,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.132238+0000","last_change":"2023-09-23T23:03:43.131930+0000","last_active":"2023-09-23T23:03:43.132238+0000","last_peered":"2023-09-23T23:03:43.132238+0000","last_clean":"2023-09-23T23:03:43.132238+0000","last_became_active":"2023-09-23T23:03:43.131552+0000","last_became_peered":"2023-09-23T23:03:43.131552+0000","last_unstale":"2023-09-23T23:03:43.132238+0000","last_undegraded":"2023-09-23T23:03:43.132238+0000","last_fullsized":"2023-09-23T23:03:43.132238+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":370,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.887074+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:03:43.887074+0000","last_peered":"2023-09-23T23:03:43.887074+0000","last_clean":"2023-09-23T23:03:43.887074+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:03:43.887074+0000","last_undegraded":"2023-09-23T23:03:43.887074+0000","last_fullsized":"2023-09-23T23:03:43.887074+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":316,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.888830+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:03:42.888830+0000","last_peered":"2023-09-23T23:03:42.888830+0000","last_clean":"2023-09-23T23:03:42.888830+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:03:42.888830+0000","last_undegraded":"2023-09-23T23:03:42.888830+0000","last_fullsized":"2023-09-23T23:03:42.888830+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":386,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885668+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:03:42.885668+0000","last_peered":"2023-09-23T23:03:42.885668+0000","last_clean":"2023-09-23T23:03:42.885668+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:03:42.885668+0000","last_undegraded":"2023-09-23T23:03:42.885668+0000","last_fullsized":"2023-09-23T23:03:42.885668+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":351,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.888139+0000","last_change":"2023-09-23T22:58:06.749758+0000","last_active":"2023-09-23T23:03:42.888139+0000","last_peered":"2023-09-23T23:03:42.888139+0000","last_clean":"2023-09-23T23:03:42.888139+0000","last_became_active":"2023-09-23T22:58:06.749131+0000","last_became_peered":"2023-09-23T22:58:06.749131+0000","last_unstale":"2023-09-23T23:03:42.888139+0000","last_undegraded":"2023-09-23T23:03:42.888139+0000","last_fullsized":"2023-09-23T23:03:42.888139+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:35.319461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:43.887064+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:03:43.887064+0000","last_peered":"2023-09-23T23:03:43.887064+0000","last_clean":"2023-09-23T23:03:43.887064+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:03:43.887064+0000","last_undegraded":"2023-09-23T23:03:43.887064+0000","last_fullsized":"2023-09-23T23:03:43.887064+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":221,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886207+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:03:42.886207+0000","last_peered":"2023-09-23T23:03:42.886207+0000","last_clean":"2023-09-23T23:03:42.886207+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:03:42.886207+0000","last_undegraded":"2023-09-23T23:03:42.886207+0000","last_fullsized":"2023-09-23T23:03:42.886207+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":435,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886229+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:03:42.886229+0000","last_peered":"2023-09-23T23:03:42.886229+0000","last_clean":"2023-09-23T23:03:42.886229+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:03:42.886229+0000","last_undegraded":"2023-09-23T23:03:42.886229+0000","last_fullsized":"2023-09-23T23:03:42.886229+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":235,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.886544+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:03:42.886544+0000","last_peered":"2023-09-23T23:03:42.886544+0000","last_clean":"2023-09-23T23:03:42.886544+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:03:42.886544+0000","last_undegraded":"2023-09-23T23:03:42.886544+0000","last_fullsized":"2023-09-23T23:03:42.886544+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":322,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885312+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:03:42.885312+0000","last_peered":"2023-09-23T23:03:42.885312+0000","last_clean":"2023-09-23T23:03:42.885312+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:03:42.885312+0000","last_undegraded":"2023-09-23T23:03:42.885312+0000","last_fullsized":"2023-09-23T23:03:42.885312+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":3,"reported_epoch":413,"state":"peering","last_fresh":"2023-09-23T23:03:50.519925+0000","last_change":"2023-09-23T23:03:50.519925+0000","last_active":"2023-09-23T22:28:41.643121+0000","last_peered":"2023-09-23T22:28:41.643121+0000","last_clean":"2023-09-23T22:28:41.643121+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:03:50.519925+0000","last_undegraded":"2023-09-23T23:03:50.519925+0000","last_fullsized":"2023-09-23T23:03:50.519925+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:19.005121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":110,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T23:03:44.623310+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:03:44.623310+0000","last_peered":"2023-09-23T23:03:44.623310+0000","last_clean":"2023-09-23T23:03:44.623310+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:03:44.623310+0000","last_undegraded":"2023-09-23T23:03:44.623310+0000","last_fullsized":"2023-09-23T23:03:44.623310+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":355,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.889218+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:03:42.889218+0000","last_peered":"2023-09-23T23:03:42.889218+0000","last_clean":"2023-09-23T23:03:42.889218+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:03:42.889218+0000","last_undegraded":"2023-09-23T23:03:42.889218+0000","last_fullsized":"2023-09-23T23:03:42.889218+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":397,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T23:03:42.885072+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:03:42.885072+0000","last_peered":"2023-09-23T23:03:42.885072+0000","last_clean":"2023-09-23T23:03:42.885072+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:03:42.885072+0000","last_undegraded":"2023-09-23T23:03:42.885072+0000","last_fullsized":"2023-09-23T23:03:42.885072+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":15292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430650,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626875,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253188,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":825368,"kb_used_data":64784,"kb_used_omap":9,"kb_used_meta":760566,"kb_avail":92919784,"statfs":{"total":95995035648,"available":95149858816,"internally_reserved":0,"allocated":66338816,"data_stored":64748291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":778820585},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7769999999999999,"15min":0.82699999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4780000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.378},{"interface":"front","average":{"1min":0.995,"5min":2.1040000000000001,"15min":1.272},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.266,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.113,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9630000000000001,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":0.498}]},{"osd":3,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.65400000000000003,"15min":1.675},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.20200000000000001},"max":{"1min":2.8879999999999999,"5min":3.7250000000000001,"15min":87.787999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.64800000000000002,"15min":1.887},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":5.1989999999999998,"5min":5.1989999999999998,"15min":89.861999999999995},"last":0.434}]},{"osd":4,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.8540000000000001,"15min":2.3149999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.004,"5min":67.738,"15min":151.036},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.0800000000000001,"15min":2.6589999999999998},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.5590000000000002,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.823,"15min":2.2890000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":8.7159999999999993,"5min":68.259,"15min":116.098},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.915,"15min":2.331},"min":{"1min":0.39900000000000002,"5min":0.378,"15min":0.315},"max":{"1min":3.04,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.60499999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":2.1230000000000002,"15min":2.2450000000000001},"min":{"1min":0.38100000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":2.6030000000000002,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.7529999999999999,"15min":1.8080000000000001},"min":{"1min":0.40000000000000002,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":2.1920000000000002,"5min":67.366,"15min":67.366},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.879,"15min":1.4570000000000001},"min":{"1min":0.48599999999999999,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":3.069,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.006,"15min":1.8720000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33100000000000002},"max":{"1min":2.6360000000000001,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.53200000000000003}]}]},{"osd":0,"up_from":353,"seq":1516123455558,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":214980,"kb_used_data":5436,"kb_used_omap":37,"kb_used_meta":209498,"kb_avail":93530172,"statfs":{"total":95995035648,"available":95774896128,"internally_reserved":0,"allocated":5566464,"data_stored":3803466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":214526114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.83299999999999996,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.6970000000000001,"5min":8.234,"15min":8.234},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.89700000000000002,"15min":0.747},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7169999999999996,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.96799999999999997,"15min":0.84199999999999997},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":7.3369999999999997,"5min":12.215,"15min":12.215},"last":0.40300000000000002},{"interface":"front","average":{"1min":5.1340000000000003,"5min":1.7609999999999999,"15min":1.03},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":74.698999999999998,"5min":74.698999999999998,"15min":74.698999999999998},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.997,"15min":0.63500000000000001},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.1109999999999998,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.93799999999999994,"15min":0.91800000000000004},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.8050000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":3.298,"15min":1.6719999999999999},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2690000000000001,"5min":183.148,"15min":183.148},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.98899999999999999,"5min":3.4169999999999998,"15min":1.585},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.9910000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.069,"15min":0.80900000000000005},"min":{"1min":0.433,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.3090000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.026,"15min":1.0069999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.7080000000000002,"5min":7.774,"15min":7.774},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.0860000000000001,"15min":0.91800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.0309999999999997,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1,"15min":0.81599999999999995},"min":{"1min":0.44900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.3410000000000002,"5min":6.375,"15min":6.375},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.139,"15min":1.248},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.0579999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.1699999999999999,"15min":1.242},"min":{"1min":0.496,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.3719999999999999,"5min":6.343,"15min":6.343},"last":0.52300000000000002}]}]},{"osd":2,"up_from":359,"seq":1541893259331,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":418564,"kb_used_data":255100,"kb_used_omap":83,"kb_used_meta":163372,"kb_avail":93326588,"statfs":{"total":95995035648,"available":95566426112,"internally_reserved":0,"allocated":261222400,"data_stored":259528226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":167293381},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":6.2400000000000002,"5min":2.452,"15min":1.054},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.735999999999997,"5min":74.241,"15min":74.241},"last":0.30599999999999999},{"interface":"front","average":{"1min":6.3899999999999997,"5min":2.5960000000000001,"15min":1.337},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":55.969999999999999,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.41399999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.73099999999999998,"15min":0.68700000000000006},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.2080000000000002,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.0629999999999999,"15min":1.1240000000000001},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.397,"5min":11.397,"15min":11.397},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82299999999999995,"15min":0.749},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.2750000000000004,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.377},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.871,"15min":0.83499999999999996},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.90000000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.92300000000000004,"15min":0.68400000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.1360000000000001,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.96799999999999997,"15min":0.871},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2430000000000003,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.745}]},{"osd":5,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.99299999999999999,"15min":0.96699999999999997},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.3090000000000002,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.93500000000000005,"15min":0.76500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.1680000000000001,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.81799999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.1919999999999999,"15min":1.044},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.968,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.94099999999999995,"15min":0.71799999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.9079999999999999,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.76600000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.1559999999999999,"15min":1.046},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.7869999999999999,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.0449999999999999,"15min":1.1339999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3049999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":1.0009999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926135,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533216,"kb_used_data":4760,"kb_used_omap":7,"kb_used_meta":528440,"kb_avail":93211936,"statfs":{"total":95995035648,"available":95449022464,"internally_reserved":0,"allocated":4874240,"data_stored":3208158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":541123176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3780000000000001,"15min":1.0049999999999999},"min":{"1min":0.22,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.3019999999999996,"5min":122.289,"15min":122.289},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.0150000000000001,"15min":0.94299999999999995},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":8.9849999999999994,"5min":122.041,"15min":122.041},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.87,"15min":2.2480000000000002},"min":{"1min":0.22,"5min":0.187,"15min":0.187},"max":{"1min":4.6100000000000003,"5min":8.6280000000000001,"15min":123.866},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.2929999999999999,"5min":0.90200000000000002,"15min":2.1539999999999999},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1710000000000003,"5min":6.8010000000000002,"15min":122.265},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.64200000000000002,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78400000000000003,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.127,"15min":0.871},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":2.9159999999999999,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":2.1970000000000001,"15min":1.966},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":4.6459999999999999,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.222,"5min":2.3119999999999998,"15min":1.9199999999999999},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.2149999999999999,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.98699999999999999,"15min":1.403},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2530000000000001,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.163,"5min":1.0640000000000001,"15min":1.744},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":5.149,"5min":10.246,"15min":11.951000000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.264,"15min":1.748},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":6.2830000000000004,"5min":9.2520000000000007,"15min":14.097},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.091,"5min":1.181,"15min":1.869},"min":{"1min":0.378,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":5.1970000000000001,"5min":9.2810000000000006,"15min":12.887},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.95499999999999996,"15min":1.6850000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.119,"15min":1.5960000000000001},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":4.6799999999999997,"5min":8.5830000000000002,"15min":12.32},"last":0.54100000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019332,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":974500,"kb_used_data":71132,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92770652,"statfs":{"total":95995035648,"available":94997147648,"internally_reserved":0,"allocated":72839168,"data_stored":71971063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":7.6189999999999998,"5min":4.1950000000000003,"15min":1.7130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":156.375,"5min":156.375,"15min":156.375},"last":0.61499999999999999},{"interface":"front","average":{"1min":7.4009999999999998,"5min":4.335,"15min":1.8420000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":157.02199999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.64500000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":0.97399999999999998,"15min":3.29},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8739999999999997,"5min":7.0640000000000001,"15min":132.304},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.90400000000000003,"15min":3.2410000000000001},"min":{"1min":0.308,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.181,"5min":10.361000000000001,"15min":132.453},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.155,"15min":1.0649999999999999},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.2949999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.329,"5min":1.161,"15min":0.86499999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.3799999999999999,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.68700000000000006}]},{"osd":3,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.071,"15min":2.79},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6189999999999998,"5min":10.337999999999999,"15min":164.453},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.2330000000000001,"15min":2.8170000000000002},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.32500000000000001},"max":{"1min":7.2519999999999998,"5min":7.2519999999999998,"15min":167.05199999999999},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.036,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.214},"max":{"1min":1.492,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.92400000000000004,"15min":1.835},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":5.9409999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:03:08 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.1870000000000001,"15min":1.2889999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":6.7050000000000001,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.014,"15min":1.1779999999999999},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.7409999999999997,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.81899999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":1.2749999999999999,"15min":1.256},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.28399999999999997},"max":{"1min":4.0990000000000002,"5min":11.045999999999999,"15min":12.84},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.1379999999999999,"15min":1.1319999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":3.1960000000000002,"5min":10.587,"15min":10.587},"last":0.67700000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823106,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903496,"kb_used_data":3840,"kb_used_omap":0,"kb_used_meta":899648,"kb_avail":92841656,"statfs":{"total":95995035648,"available":95069855744,"internally_reserved":0,"allocated":3932160,"data_stored":3087771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921239552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.786,"15min":0.999},"min":{"1min":0.23799999999999999,"5min":0.221,"15min":0.221},"max":{"1min":23.346,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.9690000000000001,"15min":1.377},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":23.428999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.57099999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.753,"15min":1.984},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.0979999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.627},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.1419999999999999,"15min":2.4209999999999998},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":1.9510000000000001,"5min":7.1219999999999999,"15min":110.89},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0589999999999999,"15min":0.88700000000000001},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9450000000000003,"5min":6.601,"15min":6.601},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.012,"5min":1.0009999999999999,"15min":0.997},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.9530000000000003,"5min":4.9530000000000003,"15min":4.9530000000000003},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.1359999999999999,"15min":2.7440000000000002},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.6629999999999998,"5min":9.2710000000000008,"15min":98.641000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.2330000000000001,"15min":2.7919999999999998},"min":{"1min":0.35799999999999998,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":3.6789999999999998,"5min":10.965,"15min":98.906000000000006},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.0640000000000001,"15min":1.5549999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23300000000000001},"max":{"1min":1.107,"5min":7.0380000000000003,"15min":115.06699999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.97699999999999998,"15min":1.502},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.253},"max":{"1min":2.5419999999999998,"5min":9.5890000000000004,"15min":114.908},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.2490000000000001,"15min":1.2110000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":3.0600000000000001,"5min":7.4630000000000001,"15min":11.121},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.879,"15min":1.0529999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.2370000000000001,"5min":5.4909999999999997,"15min":12.137},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:03:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.04,"15min":1.0860000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.4770000000000001,"5min":9.5980000000000008,"15min":12.23},"last":0.437},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.444,"15min":1.2090000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.8500000000000001,"5min":10.638999999999999,"15min":11.112},"last":0.61399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:03:58.343 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:03:58.369 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:03:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:58 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T23:03:58.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:58.561 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:03:58.613 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:03:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:58.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:03:58.659 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:03:58.728 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:03:58.735 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 6 16 11524 11508 119.819 103.375 0.0122296 0.00833994 2023-09-23T23:03:58.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:58.998 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:03:59.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:59.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:03:59.107 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:59.141 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:59.207 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:59.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:03:59.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:03:59 smithi052 ceph-mon[180979]: pgmap v1277: 89 pgs: 89 active+clean; 439 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 54 MiB/s wr, 862 op/s; 11 B/s, 0 objects/s recovering 2023-09-23T23:03:59.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:03:59 smithi052 ceph-mon[187743]: pgmap v1277: 89 pgs: 89 active+clean; 439 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 54 MiB/s wr, 862 op/s; 11 B/s, 0 objects/s recovering 2023-09-23T23:03:59.311 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:03:59.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:59.381 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:03:59.383 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:03:59 smithi151 ceph-mon[159210]: pgmap v1277: 89 pgs: 89 active+clean; 439 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 54 MiB/s wr, 862 op/s; 11 B/s, 0 objects/s recovering 2023-09-23T23:03:59.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:59.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:03:59.485 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:03:59.514 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:59.563 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:03:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:59.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:03:59.667 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:03:59.695 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:03:59.735 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 7 16 12888 12872 114.873 85.25 0.00670559 0.00870014 2023-09-23T23:03:59.946 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:03:59.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:03:59.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:00.050 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:00.078 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:00.125 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:00.127 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:04:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:00.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1341740949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:00.230 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:00.273 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:00.371 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:00.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:00.466 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:04:00.466 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:04:00.475 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:00.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1341740949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:00.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:00.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1341740949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:00.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:00.506 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:00.662 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:00.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:00.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:00.735 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 8 15 14365 14350 112.061 92.375 0.00662915 0.00891417 2023-09-23T23:04:00.765 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:00.831 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:01.121 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:01.292 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:01.430 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:01 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:01.430 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:01 smithi151 ceph-mon[159210]: failed to encode map e415 with expected crc 2023-09-23T23:04:01.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:01 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:04:01.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:01 smithi151 ceph-mon[159210]: osdmap e415: 8 total, 8 up, 6 in 2023-09-23T23:04:01.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:01 smithi151 ceph-mon[159210]: failed to encode map e415 with expected crc 2023-09-23T23:04:01.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:01 smithi151 ceph-mon[159210]: pgmap v1279: 89 pgs: 89 active+clean; 599 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 9 B/s, 0 objects/s recovering 2023-09-23T23:04:01.450 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:04:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[180979]: failed to encode map e415 with expected crc 2023-09-23T23:04:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:04:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[180979]: osdmap e415: 8 total, 8 up, 6 in 2023-09-23T23:04:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[180979]: failed to encode map e415 with expected crc 2023-09-23T23:04:01.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[180979]: pgmap v1279: 89 pgs: 89 active+clean; 599 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 9 B/s, 0 objects/s recovering 2023-09-23T23:04:01.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[187743]: failed to encode map e415 with expected crc 2023-09-23T23:04:01.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:04:01.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[187743]: osdmap e415: 8 total, 8 up, 6 in 2023-09-23T23:04:01.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[187743]: failed to encode map e415 with expected crc 2023-09-23T23:04:01.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:01 smithi052 ceph-mon[187743]: pgmap v1279: 89 pgs: 89 active+clean; 599 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s; 9 B/s, 0 objects/s recovering 2023-09-23T23:04:01.517 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:01.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:01.625 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:01.655 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:01.710 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:01.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:01.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 9 16 15890 15874 110.183 95.25 0.00656561 0.00906736 2023-09-23T23:04:01.817 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:01.848 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:01.883 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:01.908 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:01.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:01.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:02.016 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:02.046 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:02.146 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:02.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:02.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:02.251 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:02.277 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:04:02.317 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:04:02.329 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:02.361 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:04:02.414 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:04:02.452 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:04:02.486 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:04:02.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/556230120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:02.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/556230120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:02.521 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:04:02.553 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:04:02.557 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.557 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.558 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:04:02.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/556230120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:02.736 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 10 16 17373 17357 108.433 92.6875 0.00242703 0.00910349 2023-09-23T23:04:02.864 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:02.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:02.974 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:03.005 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:03.256 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:03.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:03.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:03.360 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:03.387 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:03.431 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:03.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:03.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:03.516 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:03 smithi151 ceph-mon[159210]: pgmap v1280: 89 pgs: 89 active+clean; 915 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:04:03.535 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:03.568 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:03.648 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:04:03.672 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:03.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:03.717 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:04:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:04:03] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-09-23T23:04:03.717 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:03 smithi052 ceph-mon[187743]: pgmap v1280: 89 pgs: 89 active+clean; 915 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:04:03.718 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:03 smithi052 ceph-mon[180979]: pgmap v1280: 89 pgs: 89 active+clean; 915 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:04:03.733 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:04:03.733 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:04:03.733 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:04:03.733 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:04:03.733 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.733 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73740 10.7 10:e8cc0162:::benchmark_data_smithi052_461816_object73739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202451+0000", 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 463.52760643699997, 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2554468189999999, 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73740 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.734 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202451+0000", 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202451+0000", 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.735 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202449+0000", 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000173795 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202623+0000", 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2417e-05 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.736 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202635+0000", 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5763000000000001e-05 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202651+0000", 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8651000000000001e-05 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202749+0000", 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.145e-05 2023-09-23T23:04:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202761+0000", 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2256656210000001 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.428426+0000", 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014574740000000001 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.738 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.429884+0000", 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000133375 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.430017+0000", 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010129305999999999 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.440147+0000", 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011353296000000001 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451500+0000", 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003660782 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.455161+0000", 2023-09-23T23:04:03.740 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3852e-05 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.455205+0000", 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026931049999999999 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.457898+0000", 2023-09-23T23:04:03.741 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2554468189999999 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73741 10.7 10:e8bfd85c:::benchmark_data_smithi052_461816_object73740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202698+0000", 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 463.52735884200001, 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2575494979999999, 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.742 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73741 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202698+0000", 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.743 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202698+0000", 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202698+0000", 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0708000000000003e-05 2023-09-23T23:04:03.744 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202748+0000", 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1382e-05 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202760+0000", 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0619999999999997e-06 2023-09-23T23:04:03.745 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202766+0000", 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000388203 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203154+0000", 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.893e-06 2023-09-23T23:04:03.746 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203164+0000", 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2287006620000001 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.431865+0000", 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000108812 2023-09-23T23:04:03.747 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.431973+0000", 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000142452 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.432116+0000", 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011844664 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.748 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.443960+0000", 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075977429999999997 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451558+0000", 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0064268040000000004 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.749 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.457985+0000", 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0398000000000001e-05 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.458035+0000", 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022125040000000001 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.750 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.460248+0000", 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2575494979999999 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73749 10.7 10:ec9af002:::benchmark_data_smithi052_461816_object73748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T23:04:03.751 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202791+0000", 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 463.52726579099999, 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.263480012, 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73749 2023-09-23T23:04:03.752 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202791+0000", 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202791+0000", 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.753 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202791+0000", 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4153999999999999e-05 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202825+0000", 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0713000000000001e-05 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202836+0000", 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0301e-05 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202846+0000", 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00077028199999999995 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203616+0000", 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9519999999999997e-06 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203623+0000", 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2305048009999999 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.434128+0000", 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012141099999999999 2023-09-23T23:04:03.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.434249+0000", 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000146086 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.434396+0000", 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095770060000000008 2023-09-23T23:04:03.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.443973+0000", 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076323290000000002 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451605+0000", 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012022138999999999 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.463627+0000", 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2588000000000004e-05 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.463700+0000", 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0025718580000000002 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.466271+0000", 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.263480012 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:73751 10.7 10:e00325f7:::benchmark_data_smithi052_461816_object73750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e329)", 2023-09-23T23:04:03.760 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:56:20.202864+0000", 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 463.52719297200002, 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2747165890000001, 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 73751 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202864+0000", 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202864+0000", 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202863+0000", 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6653000000000003e-05 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202900+0000", 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0098000000000001e-05 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202910+0000", 2023-09-23T23:04:03.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1009999999999997e-06 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.202916+0000", 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010056469999999999 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203922+0000", 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0539999999999997e-06 2023-09-23T23:04:03.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:20.203929+0000", 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.232469193 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.436398+0000", 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013066400000000001 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.436529+0000", 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012086 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.436650+0000", 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073337649999999999 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.443984+0000", 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076664180000000004 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.451650+0000", 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022855486000000001 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.474505+0000", 2023-09-23T23:04:03.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1630000000000005e-05 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.474567+0000", 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030137919999999999 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:56:21.477581+0000", 2023-09-23T23:04:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2747165890000001 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97333 10.7 10:e37b5792:::benchmark_data_smithi052_461816_object532:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.854177+0000", 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 419.87588006099998, 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94848450200000001, 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.769 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97333 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854177+0000", 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854177+0000", 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854175+0000", 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8010000000000004e-06 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854183+0000", 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5661000000000001e-05 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854199+0000", 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.114e-06 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854206+0000", 2023-09-23T23:04:03.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6054000000000001e-05 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854272+0000", 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3147e-05 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854285+0000", 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92368130199999998 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.777966+0000", 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013291099999999999 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.778099+0000", 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014655700000000001 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.778246+0000", 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012212475 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.790458+0000", 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099334079999999995 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800392+0000", 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7689999999999998e-05 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800449+0000", 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8202000000000003e-05 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800508+0000", 2023-09-23T23:04:03.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021539710000000002 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.802662+0000", 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94848450200000001 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97340 10.9 10:92897f12:::benchmark_data_smithi052_461816_object539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.860204+0000", 2023-09-23T23:04:03.777 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 419.86985303, 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93248454700000005, 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97340 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860204+0000", 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860204+0000", 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860202+0000", 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.465e-06 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860211+0000", 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.808e-05 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860229+0000", 2023-09-23T23:04:03.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860238+0000", 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0139000000000002e-05 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860288+0000", 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0019999999999995e-06 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860297+0000", 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90190687999999997 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.762204+0000", 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002049875 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.764253+0000", 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0403999999999997e-05 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.764344+0000", 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0083045500000000008 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.772648+0000", 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019917872 2023-09-23T23:04:03.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792566+0000", 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5818000000000001e-05 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792612+0000", 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6701999999999998e-05 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792689+0000", 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93248454700000005 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97348 10.9 10:9297810a:::benchmark_data_smithi052_461816_object547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.871378+0000", 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 419.85867946600001, 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92148298399999995, 2023-09-23T23:04:03.785 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97348 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871378+0000", 2023-09-23T23:04:03.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871378+0000", 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871376+0000", 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8349999999999999e-06 2023-09-23T23:04:03.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871386+0000", 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5957999999999999e-05 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871401+0000", 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9549999999999992e-06 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871409+0000", 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000133244 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871543+0000", 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0589e-05 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871553+0000", 2023-09-23T23:04:03.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.89562549300000005 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767179+0000", 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000122841 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767302+0000", 2023-09-23T23:04:03.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013168399999999999 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767433+0000", 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0078544730000000007 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.775288+0000", 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017446303999999999 2023-09-23T23:04:03.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792734+0000", 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5383999999999998e-05 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792779+0000", 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1309000000000004e-05 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792861+0000", 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92148298399999995 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97368 10.9 10:977fa808:::benchmark_data_smithi052_461816_object567:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.885379+0000", 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 419.84467837, 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.908468681, 2023-09-23T23:04:03.793 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97368 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885379+0000", 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885379+0000", 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885377+0000", 2023-09-23T23:04:03.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1136e-05 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885388+0000", 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1736e-05 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885409+0000", 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9409999999999996e-06 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885418+0000", 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073186099999999999 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.886150+0000", 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9389999999999992e-06 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.886160+0000", 2023-09-23T23:04:03.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88392771000000003 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770088+0000", 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010628699999999999 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770194+0000", 2023-09-23T23:04:03.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100766 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770295+0000", 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076737589999999996 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.777969+0000", 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015755661000000001 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793724+0000", 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0615000000000003e-05 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793765+0000", 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2626999999999993e-05 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793848+0000", 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.908468681 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97377 10.7 10:e47c8c4a:::benchmark_data_smithi052_461816_object576:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.898849+0000", 2023-09-23T23:04:03.801 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 419.83120780899998, 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90806386800000005, 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97377 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898849+0000", 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898849+0000", 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898847+0000", 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0633e-05 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898858+0000", 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8763e-05 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.804 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898877+0000", 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0770000000000008e-06 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898885+0000", 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2481e-05 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898947+0000", 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9250000000000007e-06 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898957+0000", 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88245452400000002 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781412+0000", 2023-09-23T23:04:03.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013464799999999999 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781547+0000", 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017924 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781726+0000", 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016204418000000002 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.797930+0000", 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00481702 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.802747+0000", 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001920106 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.804667+0000", 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8881000000000001e-05 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.804726+0000", 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021871030000000001 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.806913+0000", 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90806386800000005 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114190 10.a 10:552a2e4c:::benchmark_data_smithi052_461816_object17389:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637641+0000", 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 390.09241601100001, 2023-09-23T23:04:03.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.055871596, 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114190 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637641+0000", 2023-09-23T23:04:03.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637641+0000", 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637640+0000", 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7300000000000002e-06 2023-09-23T23:04:03.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637646+0000", 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1294999999999999e-05 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637657+0000", 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6370000000000004e-06 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637663+0000", 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3239000000000003e-05 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637716+0000", 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0421000000000001e-05 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637726+0000", 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0540947350000001 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691821+0000", 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00169184 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.693513+0000", 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.055871596 2023-09-23T23:04:03.815 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114194 10.8 10.1b16bda8 (undecoded) ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 390.092380101, 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058402421, 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:04:03.816 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114194 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9859999999999999e-06 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637680+0000", 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.246e-06 2023-09-23T23:04:03.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637686+0000", 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0130000000000002e-06 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637689+0000", 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6356000000000001e-05 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637735+0000", 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1199999999999996e-06 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637742+0000", 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058305386 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.696048+0000", 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1797000000000001e-05 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.696080+0000", 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058402421 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.821 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114198 10.5 10.b52bfca5 (undecoded) ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637701+0000", 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 390.09235654100002, 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0533679819999999, 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114198 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637701+0000", 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637701+0000", 2023-09-23T23:04:03.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637700+0000", 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6850000000000001e-06 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637703+0000", 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4110000000000004e-06 2023-09-23T23:04:03.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637708+0000", 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.89e-06 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637712+0000", 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0080000000000002e-05 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637762+0000", 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0999999999999993e-06 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637771+0000", 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0532533829999999 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691025+0000", 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3807000000000003e-05 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691069+0000", 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0533679819999999 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.827 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114194 10.8 10:15bd68d8:::benchmark_data_smithi052_461816_object17393:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e352)", 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733139+0000", 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99691830900002, 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.051601287, 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114194 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733139+0000", 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733139+0000", 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999919 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733131+0000", 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6708999999999999e-05 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733148+0000", 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.889e-05 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733167+0000", 2023-09-23T23:04:03.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9740000000000008e-06 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733176+0000", 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00030421400000000003 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733480+0000", 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0818999999999999e-05 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733491+0000", 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.047172094 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.780663+0000", 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040775170000000001 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.784740+0000", 2023-09-23T23:04:03.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.051601287 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114211 10.8 10:14a263e5:::benchmark_data_smithi052_461816_object17410:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99686309600003, 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083107408, 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.833 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114211 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9090000000000002e-06 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733197+0000", 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0261000000000001e-05 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733208+0000", 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6239999999999999e-06 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733213+0000", 2023-09-23T23:04:03.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045623199999999999 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733670+0000", 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0780000000000001e-06 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733676+0000", 2023-09-23T23:04:03.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.05109607 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.784772+0000", 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00058614799999999999 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.785358+0000", 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015455199999999999 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.785512+0000", 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017875975999999998 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803388+0000", 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012748966 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816137+0000", 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0511999999999999e-05 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816198+0000", 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103699 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816302+0000", 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083107408 2023-09-23T23:04:03.840 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114213 10.8 10:18638f4e:::benchmark_data_smithi052_461816_object17412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733219+0000", 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99683827699999, 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0833211110000001, 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.841 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114213 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733219+0000", 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733219+0000", 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733218+0000", 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.574e-06 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733222+0000", 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6910000000000003e-06 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733231+0000", 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2459999999999997e-06 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733235+0000", 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000902928 2023-09-23T23:04:03.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734138+0000", 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3409999999999997e-06 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734145+0000", 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.054099326 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788245+0000", 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139839 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788384+0000", 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013002399999999999 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788514+0000", 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014879720000000001 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803394+0000", 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012988685999999999 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816383+0000", 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0869999999999999e-05 2023-09-23T23:04:03.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816434+0000", 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106334 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816540+0000", 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0833211110000001 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114214 10.8 10:12e4921e:::benchmark_data_smithi052_461816_object17413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733241+0000", 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99681658100002, 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083505803, 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.849 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114214 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733241+0000", 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733241+0000", 2023-09-23T23:04:03.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733240+0000", 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9979999999999999e-06 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733243+0000", 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3750000000000003e-06 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733252+0000", 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4340000000000002e-06 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733256+0000", 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001189924 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734446+0000", 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5579999999999997e-06 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734452+0000", 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0568490829999999 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791302+0000", 2023-09-23T23:04:03.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124091 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791426+0000", 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019142000000000001 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791617+0000", 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011782971999999999 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803400+0000", 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013200309 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816600+0000", 2023-09-23T23:04:03.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7296999999999998e-05 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816648+0000", 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8789000000000006e-05 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816746+0000", 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083505803 2023-09-23T23:04:03.856 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114223 10.8 10:1be79ee9:::benchmark_data_smithi052_461816_object17422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99679690099998, 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0836756359999999, 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.857 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114223 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9550000000000001e-06 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733263+0000", 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9780000000000005e-06 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733271+0000", 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4569999999999998e-06 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733275+0000", 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001796401 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735072+0000", 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7419999999999997e-06 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735079+0000", 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0592689369999999 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794348+0000", 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129245 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794478+0000", 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016843500000000001 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794646+0000", 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021672981000000001 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816319+0000", 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047669500000000002 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816796+0000", 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5318000000000002e-05 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816841+0000", 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4890999999999997e-05 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816936+0000", 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0836756359999999 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114229 10.8 10:11c38b0a:::benchmark_data_smithi052_461816_object17428:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733280+0000", 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99677740800001, 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083835967, 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114229 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.866 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733280+0000", 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733280+0000", 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733279+0000", 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3809999999999999e-06 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733283+0000", 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2919999999999996e-06 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733291+0000", 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8060000000000002e-06 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733296+0000", 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022410709999999999 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735537+0000", 2023-09-23T23:04:03.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7049999999999994e-06 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735547+0000", 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0619144519999999 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797461+0000", 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124339 2023-09-23T23:04:03.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797585+0000", 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013691500000000001 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797722+0000", 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018607162999999999 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816330+0000", 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00064679199999999998 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816976+0000", 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5336000000000001e-05 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817022+0000", 2023-09-23T23:04:03.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4147000000000005e-05 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817116+0000", 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083835967 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114239 10.8 10:1ab2d4f2:::benchmark_data_smithi052_461816_object17438:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.873 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99675707900002, 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0839988570000001, 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114239 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.101e-06 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733303+0000", 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9850000000000003e-06 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733309+0000", 2023-09-23T23:04:03.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0110000000000002e-06 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733313+0000", 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023753369999999999 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735688+0000", 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5669999999999995e-06 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735697+0000", 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.064783783 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800481+0000", 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011374699999999999 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800594+0000", 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012472500000000001 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800719+0000", 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015619889 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816339+0000", 2023-09-23T23:04:03.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00081903599999999998 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817158+0000", 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5899999999999998e-05 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817204+0000", 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5204000000000005e-05 2023-09-23T23:04:03.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817299+0000", 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0839988570000001 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114244 10.8 10:16611384:::benchmark_data_smithi052_461816_object17443:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733317+0000", 2023-09-23T23:04:03.881 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 357.99674056800001, 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0845525629999999, 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114244 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733317+0000", 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733317+0000", 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733316+0000", 2023-09-23T23:04:03.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.531e-06 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733319+0000", 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6490000000000008e-06 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733326+0000", 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0069999999999999e-06 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733330+0000", 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031344089999999999 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.736465+0000", 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1509999999999998e-06 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.736472+0000", 2023-09-23T23:04:03.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0669546030000001 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803427+0000", 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114894 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803542+0000", 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129417 2023-09-23T23:04:03.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803671+0000", 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013664033000000001 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817335+0000", 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037598800000000001 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817711+0000", 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1017999999999998e-05 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817772+0000", 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7250999999999994e-05 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817869+0000", 2023-09-23T23:04:03.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0845525629999999 2023-09-23T23:04:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:03.889 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:03.889 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:04:03.889 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:03.890 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 11 16 18437 18421 104.621 66.5 0.00642786 0.00955021 2023-09-23T23:04:03.966 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:04.601 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:04:04.645 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:04.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:04.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 12 16 19917 19901 103.611 92.5 0.017279 0.00964425 2023-09-23T23:04:04.754 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:04.815 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:04.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:04.950 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:04.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:04.990 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:04.996 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:04.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:04.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:05.099 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:05.130 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:05.136 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:05.229 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:05.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:05.333 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:05.362 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:05.565 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:05.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:05.669 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:05.694 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:05 smithi151 ceph-mon[159210]: pgmap v1281: 89 pgs: 89 active+clean; 915 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:04:05.694 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:05.697 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:05.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 13 16 21372 21356 102.636 90.9375 0.00211234 0.00970363 2023-09-23T23:04:05.739 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:05.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:05.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:05 smithi052 ceph-mon[187743]: pgmap v1281: 89 pgs: 89 active+clean; 915 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:04:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:05.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:05 smithi052 ceph-mon[180979]: pgmap v1281: 89 pgs: 89 active+clean; 915 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 8 B/s, 0 objects/s recovering 2023-09-23T23:04:05.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:05.780 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:04:05.781 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1778,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":415,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":6,"osd_in_since":1695509845,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":14660,"data_bytes":959780768,"bytes_used":6625452032,"bytes_avail":569344761856,"bytes_total":575970213888,"recovering_objects_per_sec":0,"recovering_bytes_per_sec":8,"recovering_keys_per_sec":0,"num_objects_recovered":3,"num_bytes_recovered":90,"num_keys_recovered":0,"write_bytes_sec":95867072,"read_op_per_sec":0,"write_op_per_sec":1462},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":31,"modified":"2023-09-23T23:03:50.644975+0000","services":{}},"progress_events":{"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T23:04:05.843 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:05.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:06.216 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:06.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:06.323 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:06.357 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:06.463 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:06.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:06.567 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:06.640 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:04:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:04:06.640 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T23:04:06.640 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:04:06.657 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:06.660 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:04:06.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2543741692' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:04:06.737 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 14 16 22842 22826 101.866 91.875 0.0106726 0.00980582 2023-09-23T23:04:06.985 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2543741692' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:04:06.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2543741692' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:04:06.997 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:07.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:07.102 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:07.134 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:07.423 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:04:07.424 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:07.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:07.529 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:07.554 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:07.600 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:07.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:07.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:07.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:07 smithi151 ceph-mon[159210]: pgmap v1282: 89 pgs: 89 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:04:07.706 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:07.738 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:07.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 15 16 23708 23692 98.6724 54.125 0.0098431 0.0101279 2023-09-23T23:04:07.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:07 smithi052 ceph-mon[180979]: pgmap v1282: 89 pgs: 89 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:04:07.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:07 smithi052 ceph-mon[187743]: pgmap v1282: 89 pgs: 89 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:04:07.794 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:07.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:07.817 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:07.899 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:07.973 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:08.080 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:04:08.080 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:04:08.109 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:08.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:08.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:08.222 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:08.256 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:08.357 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:08.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:08.462 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:08.488 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:08.530 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:08.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:08.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:08.611 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:08.633 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:08.661 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:08.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 16 16 24797 24781 96.7595 68.0625 0.0115879 0.0103272 2023-09-23T23:04:08.900 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:08.966 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:08.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:09.069 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:09.146 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:09.421 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:09.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:09.472 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:04:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:04:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:04:09.472 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:04:09.473 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:04:09.532 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:09.611 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:09.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:09 smithi052 ceph-mon[180979]: pgmap v1283: 89 pgs: 89 active+clean; 1.3 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:04:09.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3788118657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:09.732 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:09.733 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:09 smithi052 ceph-mon[187743]: pgmap v1283: 89 pgs: 89 active+clean; 1.3 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:04:09.733 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3788118657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:09.733 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:09.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 17 16 26141 26125 96.0087 84 0.0222659 0.0104059 2023-09-23T23:04:09.744 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:04:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:09 smithi151 ceph-mon[159210]: pgmap v1283: 89 pgs: 89 active+clean; 1.3 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T23:04:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3788118657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:10.341 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:04:10.733 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:10.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 18 16 27146 27130 94.1647 62.8125 0.00648484 0.0106124 2023-09-23T23:04:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:04:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:04:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T23:04:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[187743]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.997 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: failed to encode map e416 with expected crc 2023-09-23T23:04:10.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:10 smithi052 ceph-mon[180979]: failed to encode map e416 with expected crc 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: failed to encode map e416 with expected crc 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: failed to encode map e416 with expected crc 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: failed to encode map e416 with expected crc 2023-09-23T23:04:11.004 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:10 smithi151 ceph-mon[159210]: failed to encode map e416 with expected crc 2023-09-23T23:04:11.737 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:11 smithi052 ceph-mon[180979]: pgmap v1285: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T23:04:11.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 19 16 28429 28413 93.4289 80.1875 0.00190576 0.01069 2023-09-23T23:04:11.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:11 smithi052 ceph-mon[187743]: pgmap v1285: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T23:04:12.022 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:11 smithi151 ceph-mon[159210]: pgmap v1285: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T23:04:12.044 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:12.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:12.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:12.157 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:12.204 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115752 10.b 10:dd4f5f0e:::benchmark_data_smithi052_461816_object18951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.460 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 330.40416839099998, 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.395328154, 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115752 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.461 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052721+0000", 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:12.462 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052719+0000", 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9670000000000001e-06 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052727+0000", 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4453e-05 2023-09-23T23:04:12.463 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052742+0000", 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5230000000000001e-06 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.052748+0000", 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1202455760000001 2023-09-23T23:04:12.464 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.172994+0000", 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010631813 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.183626+0000", 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011626968 2023-09-23T23:04:12.465 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.195253+0000", 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.037761408000000003 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.233014+0000", 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00097050000000000001 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.466 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.233984+0000", 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029044180000000002 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.236889+0000", 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.182741982 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.467 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.419631+0000", 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043122619999999999 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.423943+0000", 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024105932 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.448049+0000", 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.395328154 2023-09-23T23:04:12.468 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115897 10.b 10:da71a67f:::benchmark_data_smithi052_461816_object19096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 329.49769238599998, 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511318728, 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.469 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115897 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.470 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959197+0000", 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959194+0000", 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4834000000000001e-05 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959209+0000", 2023-09-23T23:04:12.471 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6504999999999999e-05 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959226+0000", 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.585e-06 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:42.959234+0000", 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.341365068 2023-09-23T23:04:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.300599+0000", 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024846097000000001 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.325445+0000", 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035146149000000002 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.473 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.360592+0000", 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.066987158000000005 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.427579+0000", 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050410359999999996 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.474 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.432620+0000", 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0016353240000000001 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.434255+0000", 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046578970000000003 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.438913+0000", 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054771630000000002 2023-09-23T23:04:12.475 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.444390+0000", 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.026125426 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.470516+0000", 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511318728 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.476 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115903 10.b 10:df88d60c:::benchmark_data_smithi052_461816_object19102:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 329.454559045, 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5939529189999999, 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.477 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115903 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.478 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002330+0000", 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002328+0000", 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5429999999999999e-06 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002338+0000", 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4139e-05 2023-09-23T23:04:12.479 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002352+0000", 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2870000000000001e-06 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.002358+0000", 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4692954739999999 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.480 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.471654+0000", 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013024641999999999 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.484678+0000", 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024981533 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.481 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.509660+0000", 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.059660008 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.569320+0000", 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073572899999999996 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.570056+0000", 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021077549999999998 2023-09-23T23:04:12.482 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.572163+0000", 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0027725240000000002 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.574936+0000", 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034689930000000001 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.483 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.578405+0000", 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017878277000000001 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.596283+0000", 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5939529189999999 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.484 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:115918 10.b 10:d203a882:::benchmark_data_smithi052_461816_object19117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 329.336152981, 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511800968, 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 115918 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.485 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120736+0000", 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.486 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120735+0000", 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.841e-06 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120743+0000", 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3618000000000001e-05 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.487 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120756+0000", 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6620000000000002e-06 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.120762+0000", 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4106586059999999 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.531421+0000", 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012043214 2023-09-23T23:04:12.488 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.543464+0000", 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012684575999999999 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.556148+0000", 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016711621999999999 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.489 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.572860+0000", 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.025699692999999999 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.598560+0000", 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002245637 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.490 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.600805+0000", 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003157188 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.603963+0000", 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049459949999999999 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.608909+0000", 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023628764999999999 2023-09-23T23:04:12.491 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.632537+0000", 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.511800968 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116102 10.b 10:d60ae293:::benchmark_data_smithi052_461816_object19301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:04:12.492 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 328.50301051000002, 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.473893417, 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116102 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.493 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953879+0000", 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999967 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953875+0000", 2023-09-23T23:04:12.494 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4865000000000001e-05 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953890+0000", 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1577999999999999e-05 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953912+0000", 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.075e-05 2023-09-23T23:04:12.495 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.953923+0000", 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2057103410000001 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.159633+0000", 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0091687120000000007 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.496 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.168802+0000", 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011821033999999999 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.180623+0000", 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018815996000000001 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.199439+0000", 2023-09-23T23:04:12.497 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022259570000000002 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.201665+0000", 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021258484000000001 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.222923+0000", 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.186530428 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.498 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.409453+0000", 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034981790000000001 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.412952+0000", 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014820543 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.499 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.427772+0000", 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.473893417 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116106 10.b 10:dc0e4ffc:::benchmark_data_smithi052_461816_object19305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 328.472163593, 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4803338619999999, 2023-09-23T23:04:12.500 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116106 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.501 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984726+0000", 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984724+0000", 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0258000000000001e-05 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.502 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984734+0000", 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8243000000000001e-05 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984752+0000", 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0596e-05 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.503 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:43.984763+0000", 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.238889275 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.223652+0000", 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012527140000000001 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.236179+0000", 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015698178 2023-09-23T23:04:12.504 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.251877+0000", 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.043569531000000002 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.295447+0000", 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020602350000000001 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.505 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.297507+0000", 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020648390000000002 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.299572+0000", 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.134241317 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.506 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.433813+0000", 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068885409999999998 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.440702+0000", 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024357711000000001 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.507 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.465060+0000", 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4803338619999999 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116119 10.b 10:dbf3fcdb:::benchmark_data_smithi052_461816_object19318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e367)", 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:04:12.509 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 328.379100816, 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4178687299999999, 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116119 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.510 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077788+0000", 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.511 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077785+0000", 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5787e-05 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077801+0000", 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3853e-05 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077825+0000", 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1316e-05 2023-09-23T23:04:12.512 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:44.077836+0000", 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.185146891 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.262983+0000", 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095509050000000002 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.513 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.272534+0000", 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011856833000000001 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.284391+0000", 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017265624 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.514 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.301657+0000", 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002050713 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.303707+0000", 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12684677899999999 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.515 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.430554+0000", 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035632981000000001 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.466187+0000", 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056081689999999997 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.471795+0000", 2023-09-23T23:04:12.516 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023861869000000001 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.495657+0000", 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4178687299999999 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.517 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116267 10.b 10:d6c817ba:::benchmark_data_smithi052_461816_object19466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 327.33251379900003, 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4895586060000001, 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116267 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.518 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124375+0000", 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.519 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124373+0000", 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0097e-05 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124383+0000", 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6015000000000002e-05 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124399+0000", 2023-09-23T23:04:12.520 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4009999999999997e-06 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.124408+0000", 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.33608481 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.460493+0000", 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019478128000000001 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.521 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.479971+0000", 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023908400999999999 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.503879+0000", 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.084884822999999998 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.522 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.588764+0000", 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029924790000000001 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.591756+0000", 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022099440000000001 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.593966+0000", 2023-09-23T23:04:12.523 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00070443600000000002 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.594671+0000", 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035467929999999999 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.598218+0000", 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015716463 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.524 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.613934+0000", 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4895586060000001 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116332 10.b 10:dc130431:::benchmark_data_smithi052_461816_object19531:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:04:12.525 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 326.852688118, 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001, 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116332 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.526 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604199+0000", 2023-09-23T23:04:12.527 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1632e-05 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604211+0000", 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7592999999999999e-05 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604228+0000", 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0749999999999998e-06 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.528 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604236+0000", 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.223173936 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.827410+0000", 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015809969 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.529 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.843220+0000", 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013161054 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.856381+0000", 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035988506000000003 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.892370+0000", 2023-09-23T23:04:12.530 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.027739362 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.920109+0000", 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020550299999999998 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922164+0000", 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.28725175800000002 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.531 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.209416+0000", 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0092034360000000006 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.218619+0000", 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030490359000000002 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.532 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.249110+0000", 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116347 10.b 10:db51aae1:::benchmark_data_smithi052_461816_object19546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 326.80235823300001, 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001, 2023-09-23T23:04:12.533 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116347 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.534 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654529+0000", 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5270000000000001e-06 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.535 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654537+0000", 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5132e-05 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654552+0000", 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5509999999999999e-06 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654557+0000", 2023-09-23T23:04:12.536 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2683612470000001 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922919+0000", 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0088446870000000004 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.931763+0000", 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011420990000000001 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.537 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.943184+0000", 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040857233999999999 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.984042+0000", 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022559659999999999 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.538 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.986298+0000", 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002071696 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.988369+0000", 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26508044400000003 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.253450+0000", 2023-09-23T23:04:12.539 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055181509999999998 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.258968+0000", 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017719586999999998 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.276687+0000", 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.540 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116354 10.b 10:dc4e8e26:::benchmark_data_smithi052_461816_object19553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 326.7341788, 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.590831154, 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.541 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116354 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.542 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722710+0000", 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722708+0000", 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1059e-05 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722720+0000", 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8337000000000001e-05 2023-09-23T23:04:12.543 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722738+0000", 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0482999999999999e-05 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.722748+0000", 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2310586699999999 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.544 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.953807+0000", 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089079880000000004 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.962715+0000", 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011595375 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.545 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.974310+0000", 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016259483000000002 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.990570+0000", 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002797819 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.993368+0000", 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034075324999999997 2023-09-23T23:04:12.546 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.027443+0000", 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.252294817 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.279738+0000", 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053108809999999999 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.547 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.285049+0000", 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.028492905999999998 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.313542+0000", 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.590831154 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.548 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116358 10.b 10:d271a06f:::benchmark_data_smithi052_461816_object19557:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 326.69889546399997, 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.585204461, 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.549 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116358 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757994+0000", 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.550 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.757992+0000", 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7521e-05 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758009+0000", 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3963e-05 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.551 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758023+0000", 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0438999999999999e-05 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.758034+0000", 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2363462219999999 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.994380+0000", 2023-09-23T23:04:12.552 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089635630000000008 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.003343+0000", 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014499898000000001 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.017843+0000", 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.042956156000000002 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.553 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.060800+0000", 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00068733800000000001 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.061487+0000", 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024376810000000001 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.554 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.063925+0000", 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25300748499999998 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.316932+0000", 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039558340000000001 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.320888+0000", 2023-09-23T23:04:12.555 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022310436999999999 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.343198+0000", 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.585204461 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116372 10.b 10:ddfaedb1:::benchmark_data_smithi052_461816_object19571:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.556 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 326.61536301000001, 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.532253965, 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116372 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.557 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841526+0000", 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.558 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841525+0000", 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5699999999999999e-06 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841534+0000", 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6439999999999998e-05 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841551+0000", 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9019999999999999e-06 2023-09-23T23:04:12.559 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.841558+0000", 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1866856050000001 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.028244+0000", 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099314550000000005 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.560 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.038176+0000", 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011378353000000001 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.049554+0000", 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015845318000000001 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.561 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.065399+0000", 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062175360000000001 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.071617+0000", 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068547419999999996 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.078471+0000", 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26989495099999999 2023-09-23T23:04:12.562 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.348366+0000", 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045601740000000002 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.352927+0000", 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020853618000000001 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.563 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.373780+0000", 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.532253965 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116379 10.b 10:d308aafa:::benchmark_data_smithi052_461816_object19578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 326.49808539399999, 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4501070380000001, 2023-09-23T23:04:12.564 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116379 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:04:12.565 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958804+0000", 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958802+0000", 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0233e-05 2023-09-23T23:04:12.566 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958812+0000", 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5444999999999999e-05 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958828+0000", 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6100000000000006e-06 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.567 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.958836+0000", 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.12189563 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.080732+0000", 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021823031 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.102555+0000", 2023-09-23T23:04:12.568 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018619785 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.121175+0000", 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22298790399999999 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.344162+0000", 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035032671000000001 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.569 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.379195+0000", 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068400680000000004 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.386035+0000", 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003642756 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.570 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.389678+0000", 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004166154 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.393844+0000", 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015066784 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.408911+0000", 2023-09-23T23:04:12.571 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4501070380000001 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123112 10.b 10:d1224097:::benchmark_data_smithi052_461816_object26311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 295.21632455399998, 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.389065591, 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.572 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123112 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.573 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240565+0000", 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240563+0000", 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5110000000000003e-06 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240572+0000", 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5494e-05 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240597+0000", 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.776e-06 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.240604+0000", 2023-09-23T23:04:12.575 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.150625987 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.391230+0000", 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014517555 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.405747+0000", 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011751549 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.417499+0000", 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.041869322 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.459368+0000", 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001986991 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.461355+0000", 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.046885463000000002 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.508241+0000", 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.076663249000000003 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.584904+0000", 2023-09-23T23:04:12.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011116469 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.596020+0000", 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.033609914999999997 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.629630+0000", 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.389065591 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.579 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123247 10.b 10:df1d14f6:::benchmark_data_smithi052_461816_object26446:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 294.54706912699999, 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4117894469999999, 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.580 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123247 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.581 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909820+0000", 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909818+0000", 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1479999999999995e-06 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909827+0000", 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4456e-05 2023-09-23T23:04:12.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909842+0000", 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0669999999999997e-06 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.909848+0000", 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2526637350000001 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.162512+0000", 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016769646999999999 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.179281+0000", 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015743411999999998 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.584 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.195025+0000", 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.080734345999999998 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.275759+0000", 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053276499999999997 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.281087+0000", 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046729550000000003 2023-09-23T23:04:12.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.285760+0000", 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012794950000000001 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.287039+0000", 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011935753 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.586 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.298975+0000", 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.022634743999999998 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.321610+0000", 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4117894469999999 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.587 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123254 10.b 10:d637fe9a:::benchmark_data_smithi052_461816_object26453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 294.49101344899998, 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.394393993, 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123254 2023-09-23T23:04:12.588 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965876+0000", 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965874+0000", 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7100000000000002e-06 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965884+0000", 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4491999999999999e-05 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965898+0000", 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8060000000000001e-06 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:17.965907+0000", 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2457147019999999 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.211622+0000", 2023-09-23T23:04:12.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014958720999999999 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.226581+0000", 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023973563 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.250554+0000", 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.072382668999999997 2023-09-23T23:04:12.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.322937+0000", 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045649990000000001 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.327502+0000", 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005907359 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.333409+0000", 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006139238 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.339548+0000", 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055687870000000004 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.345117+0000", 2023-09-23T23:04:12.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015152598 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.360270+0000", 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.394393993 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:123279 10.b 10:dfdd8540:::benchmark_data_smithi052_461816_object26478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-09-23T23:04:12.595 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 294.34228750800003, 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4014525230000001, 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 123279 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114602+0000", 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114600+0000", 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0774999999999999e-05 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114610+0000", 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7652999999999999e-05 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114628+0000", 2023-09-23T23:04:12.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0809999999999994e-06 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:18.114636+0000", 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2466315459999999 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.361268+0000", 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017441438 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.378709+0000", 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031131333000000001 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.409840+0000", 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.072885458 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.482726+0000", 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024013120000000001 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.485127+0000", 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026478410000000002 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.487775+0000", 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012406679999999999 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.489016+0000", 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.007715375 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.496731+0000", 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019323248000000001 2023-09-23T23:04:12.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:59:19.516054+0000", 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4014525230000001 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:173316 10.b 10:d6e586c9:::benchmark_data_smithi052_461816_object76515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e397)", 2023-09-23T23:04:12.603 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:02:21.438591+0000", 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 111.018297811, 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.466487418, 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 173316 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438591+0000", 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438591+0000", 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438590+0000", 2023-09-23T23:04:12.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9659999999999999e-06 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438597+0000", 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1036000000000001e-05 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438608+0000", 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2539999999999998e-06 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.438614+0000", 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014370368999999999 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.452984+0000", 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019723929999999998 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:04:12.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:21.454957+0000", 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.72160893000000004 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.176566+0000", 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037524379999999999 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.180318+0000", 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.665614394 2023-09-23T23:04:12.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.845932+0000", 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0025265259999999999 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.848459+0000", 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006650647 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.855110+0000", 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.036248412000000001 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.891358+0000", 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001098896 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.610 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.892457+0000", 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010248480000000001 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.893482+0000", 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001292353 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.894774+0000", 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018811380000000001 2023-09-23T23:04:12.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.896655+0000", 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0084236209999999992 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:22.905079+0000", 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.466487418 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.612 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:181396 10.b 10:dc4ff8eb:::benchmark_data_smithi052_461816_object84595:head [delete] snapc 0=[] ondisk+write+known_if_redirected e401)", 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:02:50.110952+0000", 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 82.345937734000003, 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4397370709999999, 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:12.613 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 181396 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.110952+0000", 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.110952+0000", 2023-09-23T23:04:12.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.110950+0000", 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5180000000000001e-06 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.110957+0000", 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3292000000000001e-05 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.110970+0000", 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.339e-06 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.110975+0000", 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021578311999999999 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.132554+0000", 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001119578 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.133673+0000", 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.64624405799999995 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.779917+0000", 2023-09-23T23:04:12.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023239300000000001 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:50.782241+0000", 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.63375716100000001 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.415998+0000", 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018033120000000001 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.417802+0000", 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050576170000000004 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.422859+0000", 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.026280320999999999 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.449140+0000", 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015221991000000001 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.464362+0000", 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00070645599999999995 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.465068+0000", 2023-09-23T23:04:12.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.079739596999999995 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.544808+0000", 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001074073 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.545882+0000", 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048069660000000002 2023-09-23T23:04:12.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:02:51.550689+0000", 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4397370709999999 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:04:12.622 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:04:12.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T23:04:12.739526+0000 min lat: 0.00102195 max lat: 0.329795 avg lat: 0.0108759 2023-09-23T23:04:12.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:04:12.740 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 20 16 29428 29412 91.8795 62.4375 0.00971982 0.0108759 2023-09-23T23:04:12.810 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:12.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:12.918 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:12.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:13.132 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:13.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:13.239 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:13.276 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:13.303 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:04:13.328 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:13.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:13.435 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:13.451 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:04:13.490 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:04:13.511 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:13.532 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:04:13.576 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:04:13.627 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:04:13.664 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:04:13.688 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:13.709 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:04:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:04:13] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.33.4" 2023-09-23T23:04:13.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:13.723 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:13.724 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:04:13.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 21 16 30679 30663 91.2272 78.1875 0.0129209 0.0109549 2023-09-23T23:04:13.773 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:04:13.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:13.841 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:13.939 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:13.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:13.982 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:13 smithi052 ceph-mon[187743]: pgmap v1286: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:04:13.982 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/455272379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:13.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:13 smithi052 ceph-mon[180979]: pgmap v1286: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:04:13.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/455272379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:14.043 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:14.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:14.130 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:13 smithi151 ceph-mon[159210]: pgmap v1286: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:04:14.130 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/455272379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:14.183 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:04:14.190 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:04:14.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:14.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:14.339 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:14.372 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:14.423 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:14.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:14.526 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:14.553 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:14.605 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:14.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:14.664 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:04:14.708 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:14.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 22 16 31692 31676 89.9585 63.3125 0.00725592 0.0110859 2023-09-23T23:04:14.743 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:15.404 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:15.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:15.486 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:04:15.486 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:04:15.511 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:15.541 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:15.739 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 23 16 32639 32623 88.6207 59.1875 0.00203886 0.0112244 2023-09-23T23:04:15.741 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:15.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:15.845 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:15.874 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:15.923 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:15.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:16.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:16.092 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:15 smithi052 ceph-mon[180979]: from='client.36849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:16.092 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:15 smithi052 ceph-mon[180979]: pgmap v1287: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:04:16.092 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:15 smithi052 ceph-mon[187743]: from='client.36849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:16.092 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:15 smithi052 ceph-mon[187743]: pgmap v1287: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:04:16.121 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1286,"stamp":"2023-09-23T23:04:12.654684+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1807947680,"num_objects":27602,"num_object_clones":0,"num_object_copies":82806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27602,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":27838,"num_write_kb":1767437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":605750,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27772,"ondisk_log_size":27772,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8850928,"kb_used_data":5317888,"kb_used_omap":137,"kb_used_meta":3532854,"kb_avail":553619984,"statfs":{"total":575970213888,"available":566906863616,"internally_reserved":0,"allocated":5445517312,"data_stored":5437057557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3617642680},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,2,1,2,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":848166912,"num_objects":12942,"num_object_clones":0,"num_object_copies":38826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12942,"num_write_kb":828288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005976"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":428,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184472+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:04:00.184472+0000","last_peered":"2023-09-23T23:04:00.184472+0000","last_clean":"2023-09-23T23:04:00.184472+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:04:00.184472+0000","last_undegraded":"2023-09-23T23:04:00.184472+0000","last_fullsized":"2023-09-23T23:04:00.184472+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182188+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:04:00.182188+0000","last_peered":"2023-09-23T23:04:00.182188+0000","last_clean":"2023-09-23T23:04:00.182188+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:04:00.182188+0000","last_undegraded":"2023-09-23T23:04:00.182188+0000","last_fullsized":"2023-09-23T23:04:00.182188+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181537+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:04:00.181537+0000","last_peered":"2023-09-23T23:04:00.181537+0000","last_clean":"2023-09-23T23:04:00.181537+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:04:00.181537+0000","last_undegraded":"2023-09-23T23:04:00.181537+0000","last_fullsized":"2023-09-23T23:04:00.181537+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":414,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182338+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:04:00.182338+0000","last_peered":"2023-09-23T23:04:00.182338+0000","last_clean":"2023-09-23T23:04:00.182338+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:04:00.182338+0000","last_undegraded":"2023-09-23T23:04:00.182338+0000","last_fullsized":"2023-09-23T23:04:00.182338+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.179449+0000","last_change":"2023-09-23T23:03:51.528123+0000","last_active":"2023-09-23T23:04:00.179449+0000","last_peered":"2023-09-23T23:04:00.179449+0000","last_clean":"2023-09-23T23:04:00.179449+0000","last_became_active":"2023-09-23T23:03:51.527770+0000","last_became_peered":"2023-09-23T23:03:51.527770+0000","last_unstale":"2023-09-23T23:04:00.179449+0000","last_undegraded":"2023-09-23T23:04:00.179449+0000","last_fullsized":"2023-09-23T23:04:00.179449+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:32.789048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.297398+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:04:00.297398+0000","last_peered":"2023-09-23T23:04:00.297398+0000","last_clean":"2023-09-23T23:04:00.297398+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:04:00.297398+0000","last_undegraded":"2023-09-23T23:04:00.297398+0000","last_fullsized":"2023-09-23T23:04:00.297398+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":351,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.125016+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:04:00.125016+0000","last_peered":"2023-09-23T23:04:00.125016+0000","last_clean":"2023-09-23T23:04:00.125016+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:04:00.125016+0000","last_undegraded":"2023-09-23T23:04:00.125016+0000","last_fullsized":"2023-09-23T23:04:00.125016+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":95,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181170+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:04:00.181170+0000","last_peered":"2023-09-23T23:04:00.181170+0000","last_clean":"2023-09-23T23:04:00.181170+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:04:00.181170+0000","last_undegraded":"2023-09-23T23:04:00.181170+0000","last_fullsized":"2023-09-23T23:04:00.181170+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":312,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124969+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:04:00.124969+0000","last_peered":"2023-09-23T23:04:00.124969+0000","last_clean":"2023-09-23T23:04:00.124969+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:04:00.124969+0000","last_undegraded":"2023-09-23T23:04:00.124969+0000","last_fullsized":"2023-09-23T23:04:00.124969+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":100,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181396+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:04:00.181396+0000","last_peered":"2023-09-23T23:04:00.181396+0000","last_clean":"2023-09-23T23:04:00.181396+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:04:00.181396+0000","last_undegraded":"2023-09-23T23:04:00.181396+0000","last_fullsized":"2023-09-23T23:04:00.181396+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":121,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182104+0000","last_change":"2023-09-23T23:03:51.536661+0000","last_active":"2023-09-23T23:04:00.182104+0000","last_peered":"2023-09-23T23:04:00.182104+0000","last_clean":"2023-09-23T23:04:00.182104+0000","last_became_active":"2023-09-23T23:03:51.532360+0000","last_became_peered":"2023-09-23T23:03:51.532360+0000","last_unstale":"2023-09-23T23:04:00.182104+0000","last_undegraded":"2023-09-23T23:04:00.182104+0000","last_fullsized":"2023-09-23T23:04:00.182104+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:55.428059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":162,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.180307+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:04:00.180307+0000","last_peered":"2023-09-23T23:04:00.180307+0000","last_clean":"2023-09-23T23:04:00.180307+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:04:00.180307+0000","last_undegraded":"2023-09-23T23:04:00.180307+0000","last_fullsized":"2023-09-23T23:04:00.180307+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":487,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184725+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:04:00.184725+0000","last_peered":"2023-09-23T23:04:00.184725+0000","last_clean":"2023-09-23T23:04:00.184725+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:04:00.184725+0000","last_undegraded":"2023-09-23T23:04:00.184725+0000","last_fullsized":"2023-09-23T23:04:00.184725+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":390,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184554+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:04:00.184554+0000","last_peered":"2023-09-23T23:04:00.184554+0000","last_clean":"2023-09-23T23:04:00.184554+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:04:00.184554+0000","last_undegraded":"2023-09-23T23:04:00.184554+0000","last_fullsized":"2023-09-23T23:04:00.184554+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":316,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184658+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:04:00.184658+0000","last_peered":"2023-09-23T23:04:00.184658+0000","last_clean":"2023-09-23T23:04:00.184658+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:04:00.184658+0000","last_undegraded":"2023-09-23T23:04:00.184658+0000","last_fullsized":"2023-09-23T23:04:00.184658+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":97,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181669+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:04:00.181669+0000","last_peered":"2023-09-23T23:04:00.181669+0000","last_clean":"2023-09-23T23:04:00.181669+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:04:00.181669+0000","last_undegraded":"2023-09-23T23:04:00.181669+0000","last_fullsized":"2023-09-23T23:04:00.181669+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":406,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181966+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:04:00.181966+0000","last_peered":"2023-09-23T23:04:00.181966+0000","last_clean":"2023-09-23T23:04:00.181966+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:04:00.181966+0000","last_undegraded":"2023-09-23T23:04:00.181966+0000","last_fullsized":"2023-09-23T23:04:00.181966+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":316,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124839+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:04:00.124839+0000","last_peered":"2023-09-23T23:04:00.124839+0000","last_clean":"2023-09-23T23:04:00.124839+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:04:00.124839+0000","last_undegraded":"2023-09-23T23:04:00.124839+0000","last_fullsized":"2023-09-23T23:04:00.124839+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":301,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184509+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:04:00.184509+0000","last_peered":"2023-09-23T23:04:00.184509+0000","last_clean":"2023-09-23T23:04:00.184509+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:04:00.184509+0000","last_undegraded":"2023-09-23T23:04:00.184509+0000","last_fullsized":"2023-09-23T23:04:00.184509+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":430,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182468+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:04:00.182468+0000","last_peered":"2023-09-23T23:04:00.182468+0000","last_clean":"2023-09-23T23:04:00.182468+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:04:00.182468+0000","last_undegraded":"2023-09-23T23:04:00.182468+0000","last_fullsized":"2023-09-23T23:04:00.182468+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":135,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.297133+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:04:00.297133+0000","last_peered":"2023-09-23T23:04:00.297133+0000","last_clean":"2023-09-23T23:04:00.297133+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:04:00.297133+0000","last_undegraded":"2023-09-23T23:04:00.297133+0000","last_fullsized":"2023-09-23T23:04:00.297133+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":110,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182683+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:04:00.182683+0000","last_peered":"2023-09-23T23:04:00.182683+0000","last_clean":"2023-09-23T23:04:00.182683+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:04:00.182683+0000","last_undegraded":"2023-09-23T23:04:00.182683+0000","last_fullsized":"2023-09-23T23:04:00.182683+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"416'1656","reported_seq":1670,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:10.327672+0000","last_change":"2023-09-23T23:03:43.131702+0000","last_active":"2023-09-23T23:04:10.327672+0000","last_peered":"2023-09-23T23:04:10.327672+0000","last_clean":"2023-09-23T23:04:10.327672+0000","last_became_active":"2023-09-23T23:03:43.131323+0000","last_became_peered":"2023-09-23T23:03:43.131323+0000","last_unstale":"2023-09-23T23:04:10.327672+0000","last_undegraded":"2023-09-23T23:04:10.327672+0000","last_fullsized":"2023-09-23T23:04:10.327672+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1656,"log_dups_size":0,"ondisk_log_size":1656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108527616,"num_objects":1656,"num_object_clones":0,"num_object_copies":4968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1656,"num_write_kb":105984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":300,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184989+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:04:00.184989+0000","last_peered":"2023-09-23T23:04:00.184989+0000","last_clean":"2023-09-23T23:04:00.184989+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:04:00.184989+0000","last_undegraded":"2023-09-23T23:04:00.184989+0000","last_fullsized":"2023-09-23T23:04:00.184989+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":286,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124607+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:04:00.124607+0000","last_peered":"2023-09-23T23:04:00.124607+0000","last_clean":"2023-09-23T23:04:00.124607+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:04:00.124607+0000","last_undegraded":"2023-09-23T23:04:00.124607+0000","last_fullsized":"2023-09-23T23:04:00.124607+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"416'1663","reported_seq":1677,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:10.281193+0000","last_change":"2023-09-23T23:03:43.132133+0000","last_active":"2023-09-23T23:04:10.281193+0000","last_peered":"2023-09-23T23:04:10.281193+0000","last_clean":"2023-09-23T23:04:10.281193+0000","last_became_active":"2023-09-23T23:03:43.131741+0000","last_became_peered":"2023-09-23T23:03:43.131741+0000","last_unstale":"2023-09-23T23:04:10.281193+0000","last_undegraded":"2023-09-23T23:04:10.281193+0000","last_fullsized":"2023-09-23T23:04:10.281193+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1663,"log_dups_size":0,"ondisk_log_size":1663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:36.498718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108986368,"num_objects":1663,"num_object_clones":0,"num_object_copies":4989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1663,"num_write_kb":106432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":17,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.179781+0000","last_change":"2023-09-23T23:03:51.533976+0000","last_active":"2023-09-23T23:04:00.179781+0000","last_peered":"2023-09-23T23:04:00.179781+0000","last_clean":"2023-09-23T23:04:00.179781+0000","last_became_active":"2023-09-23T23:03:51.528701+0000","last_became_peered":"2023-09-23T23:03:51.528701+0000","last_unstale":"2023-09-23T23:04:00.179781+0000","last_undegraded":"2023-09-23T23:04:00.179781+0000","last_fullsized":"2023-09-23T23:04:00.179781+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.921280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":423,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184509+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:04:00.184509+0000","last_peered":"2023-09-23T23:04:00.184509+0000","last_clean":"2023-09-23T23:04:00.184509+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:04:00.184509+0000","last_undegraded":"2023-09-23T23:04:00.184509+0000","last_fullsized":"2023-09-23T23:04:00.184509+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"416'1667","reported_seq":1681,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:10.374012+0000","last_change":"2023-09-23T23:03:42.889536+0000","last_active":"2023-09-23T23:04:10.374012+0000","last_peered":"2023-09-23T23:04:10.374012+0000","last_clean":"2023-09-23T23:04:10.374012+0000","last_became_active":"2023-09-23T23:03:42.889213+0000","last_became_peered":"2023-09-23T23:03:42.889213+0000","last_unstale":"2023-09-23T23:04:10.374012+0000","last_undegraded":"2023-09-23T23:04:10.374012+0000","last_fullsized":"2023-09-23T23:04:10.374012+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1667,"log_dups_size":0,"ondisk_log_size":1667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109248512,"num_objects":1667,"num_object_clones":0,"num_object_copies":5001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1667,"num_write_kb":106688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":167,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124976+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:04:00.124976+0000","last_peered":"2023-09-23T23:04:00.124976+0000","last_clean":"2023-09-23T23:04:00.124976+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:04:00.124976+0000","last_undegraded":"2023-09-23T23:04:00.124976+0000","last_fullsized":"2023-09-23T23:04:00.124976+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":285,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124583+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:04:00.124583+0000","last_peered":"2023-09-23T23:04:00.124583+0000","last_clean":"2023-09-23T23:04:00.124583+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:04:00.124583+0000","last_undegraded":"2023-09-23T23:04:00.124583+0000","last_fullsized":"2023-09-23T23:04:00.124583+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"416'1730","reported_seq":1745,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.347923+0000","last_change":"2023-09-23T23:03:43.136203+0000","last_active":"2023-09-23T23:04:11.347923+0000","last_peered":"2023-09-23T23:04:11.347923+0000","last_clean":"2023-09-23T23:04:11.347923+0000","last_became_active":"2023-09-23T23:03:43.135918+0000","last_became_peered":"2023-09-23T23:03:43.135918+0000","last_unstale":"2023-09-23T23:04:11.347923+0000","last_undegraded":"2023-09-23T23:04:11.347923+0000","last_fullsized":"2023-09-23T23:04:11.347923+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1730,"log_dups_size":0,"ondisk_log_size":1730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:11.372248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113377280,"num_objects":1730,"num_object_clones":0,"num_object_copies":5190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1730,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730024+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:04:09.730024+0000","last_peered":"2023-09-23T23:04:09.730024+0000","last_clean":"2023-09-23T23:04:09.730024+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:04:09.730024+0000","last_undegraded":"2023-09-23T23:04:09.730024+0000","last_fullsized":"2023-09-23T23:04:09.730024+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":316,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.297820+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:04:00.297820+0000","last_peered":"2023-09-23T23:04:00.297820+0000","last_clean":"2023-09-23T23:04:00.297820+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:04:00.297820+0000","last_undegraded":"2023-09-23T23:04:00.297820+0000","last_fullsized":"2023-09-23T23:04:00.297820+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"416'1676","reported_seq":1690,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:10.344183+0000","last_change":"2023-09-23T23:03:43.131843+0000","last_active":"2023-09-23T23:04:10.344183+0000","last_peered":"2023-09-23T23:04:10.344183+0000","last_clean":"2023-09-23T23:04:10.344183+0000","last_became_active":"2023-09-23T23:03:43.131420+0000","last_became_peered":"2023-09-23T23:03:43.131420+0000","last_unstale":"2023-09-23T23:04:10.344183+0000","last_undegraded":"2023-09-23T23:04:10.344183+0000","last_fullsized":"2023-09-23T23:04:10.344183+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1676,"log_dups_size":0,"ondisk_log_size":1676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109838336,"num_objects":1676,"num_object_clones":0,"num_object_copies":5028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1676,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":106,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731481+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:04:09.731481+0000","last_peered":"2023-09-23T23:04:09.731481+0000","last_clean":"2023-09-23T23:04:09.731481+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:04:09.731481+0000","last_undegraded":"2023-09-23T23:04:09.731481+0000","last_fullsized":"2023-09-23T23:04:09.731481+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":409,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184702+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:04:00.184702+0000","last_peered":"2023-09-23T23:04:00.184702+0000","last_clean":"2023-09-23T23:04:00.184702+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:04:00.184702+0000","last_undegraded":"2023-09-23T23:04:00.184702+0000","last_fullsized":"2023-09-23T23:04:00.184702+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"416'1756","reported_seq":1771,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.360432+0000","last_change":"2023-09-23T23:03:42.889763+0000","last_active":"2023-09-23T23:04:11.360432+0000","last_peered":"2023-09-23T23:04:11.360432+0000","last_clean":"2023-09-23T23:04:11.360432+0000","last_became_active":"2023-09-23T23:03:42.889481+0000","last_became_peered":"2023-09-23T23:03:42.889481+0000","last_unstale":"2023-09-23T23:04:11.360432+0000","last_undegraded":"2023-09-23T23:04:11.360432+0000","last_fullsized":"2023-09-23T23:04:11.360432+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1756,"log_dups_size":0,"ondisk_log_size":1756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:57.841265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115081216,"num_objects":1756,"num_object_clones":0,"num_object_copies":5268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1756,"num_write_kb":112384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184526+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:04:00.184526+0000","last_peered":"2023-09-23T23:04:00.184526+0000","last_clean":"2023-09-23T23:04:00.184526+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:04:00.184526+0000","last_undegraded":"2023-09-23T23:04:00.184526+0000","last_fullsized":"2023-09-23T23:04:00.184526+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":426,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184860+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:04:00.184860+0000","last_peered":"2023-09-23T23:04:00.184860+0000","last_clean":"2023-09-23T23:04:00.184860+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:04:00.184860+0000","last_undegraded":"2023-09-23T23:04:00.184860+0000","last_fullsized":"2023-09-23T23:04:00.184860+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"416'1801","reported_seq":1816,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.286436+0000","last_change":"2023-09-23T23:03:42.889334+0000","last_active":"2023-09-23T23:04:11.286436+0000","last_peered":"2023-09-23T23:04:11.286436+0000","last_clean":"2023-09-23T23:04:11.286436+0000","last_became_active":"2023-09-23T23:03:42.889046+0000","last_became_peered":"2023-09-23T23:03:42.889046+0000","last_unstale":"2023-09-23T23:04:11.286436+0000","last_undegraded":"2023-09-23T23:04:11.286436+0000","last_fullsized":"2023-09-23T23:04:11.286436+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1801,"log_dups_size":0,"ondisk_log_size":1801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:44.256081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118030336,"num_objects":1801,"num_object_clones":0,"num_object_copies":5403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1801,"num_write_kb":115264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":439,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184339+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:04:00.184339+0000","last_peered":"2023-09-23T23:04:00.184339+0000","last_clean":"2023-09-23T23:04:00.184339+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:04:00.184339+0000","last_undegraded":"2023-09-23T23:04:00.184339+0000","last_fullsized":"2023-09-23T23:04:00.184339+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":417,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184905+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:04:00.184905+0000","last_peered":"2023-09-23T23:04:00.184905+0000","last_clean":"2023-09-23T23:04:00.184905+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:04:00.184905+0000","last_undegraded":"2023-09-23T23:04:00.184905+0000","last_fullsized":"2023-09-23T23:04:00.184905+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"416'1733","reported_seq":1747,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.277088+0000","last_change":"2023-09-23T23:03:43.132462+0000","last_active":"2023-09-23T23:04:11.277088+0000","last_peered":"2023-09-23T23:04:11.277088+0000","last_clean":"2023-09-23T23:04:11.277088+0000","last_became_active":"2023-09-23T23:03:43.132267+0000","last_became_peered":"2023-09-23T23:03:43.132267+0000","last_unstale":"2023-09-23T23:04:11.277088+0000","last_undegraded":"2023-09-23T23:04:11.277088+0000","last_fullsized":"2023-09-23T23:04:11.277088+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1733,"log_dups_size":0,"ondisk_log_size":1733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:49.697445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113573888,"num_objects":1733,"num_object_clones":0,"num_object_copies":5199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1733,"num_write_kb":110912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181564+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:04:00.181564+0000","last_peered":"2023-09-23T23:04:00.181564+0000","last_clean":"2023-09-23T23:04:00.181564+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:04:00.181564+0000","last_undegraded":"2023-09-23T23:04:00.181564+0000","last_fullsized":"2023-09-23T23:04:00.181564+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":320,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.180839+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:04:00.180839+0000","last_peered":"2023-09-23T23:04:00.180839+0000","last_clean":"2023-09-23T23:04:00.180839+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:04:00.180839+0000","last_undegraded":"2023-09-23T23:04:00.180839+0000","last_fullsized":"2023-09-23T23:04:00.180839+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"416'1759","reported_seq":1774,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.357578+0000","last_change":"2023-09-23T23:03:43.132020+0000","last_active":"2023-09-23T23:04:11.357578+0000","last_peered":"2023-09-23T23:04:11.357578+0000","last_clean":"2023-09-23T23:04:11.357578+0000","last_became_active":"2023-09-23T23:03:43.131692+0000","last_became_peered":"2023-09-23T23:03:43.131692+0000","last_unstale":"2023-09-23T23:04:11.357578+0000","last_undegraded":"2023-09-23T23:04:11.357578+0000","last_fullsized":"2023-09-23T23:04:11.357578+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1759,"log_dups_size":0,"ondisk_log_size":1759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:13.311701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115277824,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1759,"num_write_kb":112576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":324,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.179553+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:04:00.179553+0000","last_peered":"2023-09-23T23:04:00.179553+0000","last_clean":"2023-09-23T23:04:00.179553+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:04:00.179553+0000","last_undegraded":"2023-09-23T23:04:00.179553+0000","last_fullsized":"2023-09-23T23:04:00.179553+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":326,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124858+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:04:00.124858+0000","last_peered":"2023-09-23T23:04:00.124858+0000","last_clean":"2023-09-23T23:04:00.124858+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:04:00.124858+0000","last_undegraded":"2023-09-23T23:04:00.124858+0000","last_fullsized":"2023-09-23T23:04:00.124858+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124530+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:04:00.124530+0000","last_peered":"2023-09-23T23:04:00.124530+0000","last_clean":"2023-09-23T23:04:00.124530+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:04:00.124530+0000","last_undegraded":"2023-09-23T23:04:00.124530+0000","last_fullsized":"2023-09-23T23:04:00.124530+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"416'1737","reported_seq":1752,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.280204+0000","last_change":"2023-09-23T23:03:43.132707+0000","last_active":"2023-09-23T23:04:11.280204+0000","last_peered":"2023-09-23T23:04:11.280204+0000","last_clean":"2023-09-23T23:04:11.280204+0000","last_became_active":"2023-09-23T23:03:43.132372+0000","last_became_peered":"2023-09-23T23:03:43.132372+0000","last_unstale":"2023-09-23T23:04:11.280204+0000","last_undegraded":"2023-09-23T23:04:11.280204+0000","last_fullsized":"2023-09-23T23:04:11.280204+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1737,"log_dups_size":0,"ondisk_log_size":1737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.951582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836032,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1737,"num_write_kb":111168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":87,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.297472+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:04:00.297472+0000","last_peered":"2023-09-23T23:04:00.297472+0000","last_clean":"2023-09-23T23:04:00.297472+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:04:00.297472+0000","last_undegraded":"2023-09-23T23:04:00.297472+0000","last_fullsized":"2023-09-23T23:04:00.297472+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":436,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184768+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:04:00.184768+0000","last_peered":"2023-09-23T23:04:00.184768+0000","last_clean":"2023-09-23T23:04:00.184768+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:04:00.184768+0000","last_undegraded":"2023-09-23T23:04:00.184768+0000","last_fullsized":"2023-09-23T23:04:00.184768+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":309,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.179477+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:04:00.179477+0000","last_peered":"2023-09-23T23:04:00.179477+0000","last_clean":"2023-09-23T23:04:00.179477+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:04:00.179477+0000","last_undegraded":"2023-09-23T23:04:00.179477+0000","last_fullsized":"2023-09-23T23:04:00.179477+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"416'1789","reported_seq":1804,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.273952+0000","last_change":"2023-09-23T23:03:42.889306+0000","last_active":"2023-09-23T23:04:11.273952+0000","last_peered":"2023-09-23T23:04:11.273952+0000","last_clean":"2023-09-23T23:04:11.273952+0000","last_became_active":"2023-09-23T23:03:42.888948+0000","last_became_peered":"2023-09-23T23:03:42.888948+0000","last_unstale":"2023-09-23T23:04:11.273952+0000","last_undegraded":"2023-09-23T23:04:11.273952+0000","last_fullsized":"2023-09-23T23:04:11.273952+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1789,"log_dups_size":0,"ondisk_log_size":1789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:56.393948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117243904,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1789,"num_write_kb":114496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":384,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.180250+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:04:00.180250+0000","last_peered":"2023-09-23T23:04:00.180250+0000","last_clean":"2023-09-23T23:04:00.180250+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:04:00.180250+0000","last_undegraded":"2023-09-23T23:04:00.180250+0000","last_fullsized":"2023-09-23T23:04:00.180250+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124485+0000","last_change":"2023-09-23T23:03:51.532583+0000","last_active":"2023-09-23T23:04:00.124485+0000","last_peered":"2023-09-23T23:04:00.124485+0000","last_clean":"2023-09-23T23:04:00.124485+0000","last_became_active":"2023-09-23T23:03:51.532300+0000","last_became_peered":"2023-09-23T23:03:51.532300+0000","last_unstale":"2023-09-23T23:04:00.124485+0000","last_undegraded":"2023-09-23T23:04:00.124485+0000","last_fullsized":"2023-09-23T23:04:00.124485+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":326,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.125042+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:04:00.125042+0000","last_peered":"2023-09-23T23:04:00.125042+0000","last_clean":"2023-09-23T23:04:00.125042+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:04:00.125042+0000","last_undegraded":"2023-09-23T23:04:00.125042+0000","last_fullsized":"2023-09-23T23:04:00.125042+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"416'1775","reported_seq":1790,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.264537+0000","last_change":"2023-09-23T23:03:43.131943+0000","last_active":"2023-09-23T23:04:11.264537+0000","last_peered":"2023-09-23T23:04:11.264537+0000","last_clean":"2023-09-23T23:04:11.264537+0000","last_became_active":"2023-09-23T23:03:43.131668+0000","last_became_peered":"2023-09-23T23:03:43.131668+0000","last_unstale":"2023-09-23T23:04:11.264537+0000","last_undegraded":"2023-09-23T23:04:11.264537+0000","last_fullsized":"2023-09-23T23:04:11.264537+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1775,"log_dups_size":0,"ondisk_log_size":1775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:16.104887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1775,"num_write_kb":113600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":399,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124715+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:04:00.124715+0000","last_peered":"2023-09-23T23:04:00.124715+0000","last_clean":"2023-09-23T23:04:00.124715+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:04:00.124715+0000","last_undegraded":"2023-09-23T23:04:00.124715+0000","last_fullsized":"2023-09-23T23:04:00.124715+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":529,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:05.303216+0000","last_change":"2023-09-23T23:03:36.322526+0000","last_active":"2023-09-23T23:04:05.303216+0000","last_peered":"2023-09-23T23:04:05.303216+0000","last_clean":"2023-09-23T23:04:05.303216+0000","last_became_active":"2023-09-23T23:03:36.322065+0000","last_became_peered":"2023-09-23T23:03:36.322065+0000","last_unstale":"2023-09-23T23:04:05.303216+0000","last_undegraded":"2023-09-23T23:04:05.303216+0000","last_fullsized":"2023-09-23T23:04:05.303216+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:04.195321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":87,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182334+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:04:00.182334+0000","last_peered":"2023-09-23T23:04:00.182334+0000","last_clean":"2023-09-23T23:04:00.182334+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:04:00.182334+0000","last_undegraded":"2023-09-23T23:04:00.182334+0000","last_fullsized":"2023-09-23T23:04:00.182334+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":326,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184935+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:04:00.184935+0000","last_peered":"2023-09-23T23:04:00.184935+0000","last_clean":"2023-09-23T23:04:00.184935+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:04:00.184935+0000","last_undegraded":"2023-09-23T23:04:00.184935+0000","last_fullsized":"2023-09-23T23:04:00.184935+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"416'1708","reported_seq":1723,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.257972+0000","last_change":"2023-09-23T23:03:43.133127+0000","last_active":"2023-09-23T23:04:11.257972+0000","last_peered":"2023-09-23T23:04:11.257972+0000","last_clean":"2023-09-23T23:04:11.257972+0000","last_became_active":"2023-09-23T23:03:43.132709+0000","last_became_peered":"2023-09-23T23:03:43.132709+0000","last_unstale":"2023-09-23T23:04:11.257972+0000","last_undegraded":"2023-09-23T23:04:11.257972+0000","last_fullsized":"2023-09-23T23:04:11.257972+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1708,"log_dups_size":0,"ondisk_log_size":1708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:35.116957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111935488,"num_objects":1708,"num_object_clones":0,"num_object_copies":5124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1708,"num_write_kb":109312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":408,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:05.303194+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:04:05.303194+0000","last_peered":"2023-09-23T23:04:05.303194+0000","last_clean":"2023-09-23T23:04:05.303194+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:04:05.303194+0000","last_undegraded":"2023-09-23T23:04:05.303194+0000","last_fullsized":"2023-09-23T23:04:05.303194+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":378,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.295741+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:04:00.295741+0000","last_peered":"2023-09-23T23:04:00.295741+0000","last_clean":"2023-09-23T23:04:00.295741+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:04:00.295741+0000","last_undegraded":"2023-09-23T23:04:00.295741+0000","last_fullsized":"2023-09-23T23:04:00.295741+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":338,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.298624+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:04:00.298624+0000","last_peered":"2023-09-23T23:04:00.298624+0000","last_clean":"2023-09-23T23:04:00.298624+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:04:00.298624+0000","last_undegraded":"2023-09-23T23:04:00.298624+0000","last_fullsized":"2023-09-23T23:04:00.298624+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"416'1702","reported_seq":1717,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.344924+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:04:11.344924+0000","last_peered":"2023-09-23T23:04:11.344924+0000","last_clean":"2023-09-23T23:04:11.344924+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:04:11.344924+0000","last_undegraded":"2023-09-23T23:04:11.344924+0000","last_fullsized":"2023-09-23T23:04:11.344924+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1702,"log_dups_size":0,"ondisk_log_size":1702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111542272,"num_objects":1702,"num_object_clones":0,"num_object_copies":5106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1702,"num_write_kb":108928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":353,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.297094+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:04:00.297094+0000","last_peered":"2023-09-23T23:04:00.297094+0000","last_clean":"2023-09-23T23:04:00.297094+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:04:00.297094+0000","last_undegraded":"2023-09-23T23:04:00.297094+0000","last_fullsized":"2023-09-23T23:04:00.297094+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":261,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.180688+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:04:00.180688+0000","last_peered":"2023-09-23T23:04:00.180688+0000","last_clean":"2023-09-23T23:04:00.180688+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:04:00.180688+0000","last_undegraded":"2023-09-23T23:04:00.180688+0000","last_fullsized":"2023-09-23T23:04:00.180688+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":229,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.124770+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:04:00.124770+0000","last_peered":"2023-09-23T23:04:00.124770+0000","last_clean":"2023-09-23T23:04:00.124770+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:04:00.124770+0000","last_undegraded":"2023-09-23T23:04:00.124770+0000","last_fullsized":"2023-09-23T23:04:00.124770+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"416'1749","reported_seq":1764,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.379187+0000","last_change":"2023-09-23T23:03:43.132731+0000","last_active":"2023-09-23T23:04:11.379187+0000","last_peered":"2023-09-23T23:04:11.379187+0000","last_clean":"2023-09-23T23:04:11.379187+0000","last_became_active":"2023-09-23T23:03:43.132451+0000","last_became_peered":"2023-09-23T23:03:43.132451+0000","last_unstale":"2023-09-23T23:04:11.379187+0000","last_undegraded":"2023-09-23T23:04:11.379187+0000","last_fullsized":"2023-09-23T23:04:11.379187+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1749,"log_dups_size":0,"ondisk_log_size":1749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:51.837853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114622464,"num_objects":1749,"num_object_clones":0,"num_object_copies":5247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1749,"num_write_kb":111936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":111,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.182829+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:04:00.182829+0000","last_peered":"2023-09-23T23:04:00.182829+0000","last_clean":"2023-09-23T23:04:00.182829+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:04:00.182829+0000","last_undegraded":"2023-09-23T23:04:00.182829+0000","last_fullsized":"2023-09-23T23:04:00.182829+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184995+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:04:00.184995+0000","last_peered":"2023-09-23T23:04:00.184995+0000","last_clean":"2023-09-23T23:04:00.184995+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:04:00.184995+0000","last_undegraded":"2023-09-23T23:04:00.184995+0000","last_fullsized":"2023-09-23T23:04:00.184995+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":102,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181313+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:04:00.181313+0000","last_peered":"2023-09-23T23:04:00.181313+0000","last_clean":"2023-09-23T23:04:00.181313+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:04:00.181313+0000","last_undegraded":"2023-09-23T23:04:00.181313+0000","last_fullsized":"2023-09-23T23:04:00.181313+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"416'1677","reported_seq":1691,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:10.345869+0000","last_change":"2023-09-23T23:03:43.131930+0000","last_active":"2023-09-23T23:04:10.345869+0000","last_peered":"2023-09-23T23:04:10.345869+0000","last_clean":"2023-09-23T23:04:10.345869+0000","last_became_active":"2023-09-23T23:03:43.131552+0000","last_became_peered":"2023-09-23T23:03:43.131552+0000","last_unstale":"2023-09-23T23:04:10.345869+0000","last_undegraded":"2023-09-23T23:04:10.345869+0000","last_fullsized":"2023-09-23T23:04:10.345869+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":1677,"log_dups_size":0,"ondisk_log_size":1677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109903872,"num_objects":1677,"num_object_clones":0,"num_object_copies":5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1677,"num_write_kb":107328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":373,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.180297+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:04:00.180297+0000","last_peered":"2023-09-23T23:04:00.180297+0000","last_clean":"2023-09-23T23:04:00.180297+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:04:00.180297+0000","last_undegraded":"2023-09-23T23:04:00.180297+0000","last_fullsized":"2023-09-23T23:04:00.180297+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":320,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.298544+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:04:00.298544+0000","last_peered":"2023-09-23T23:04:00.298544+0000","last_clean":"2023-09-23T23:04:00.298544+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:04:00.298544+0000","last_undegraded":"2023-09-23T23:04:00.298544+0000","last_fullsized":"2023-09-23T23:04:00.298544+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":390,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184401+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:04:00.184401+0000","last_peered":"2023-09-23T23:04:00.184401+0000","last_clean":"2023-09-23T23:04:00.184401+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:04:00.184401+0000","last_undegraded":"2023-09-23T23:04:00.184401+0000","last_fullsized":"2023-09-23T23:04:00.184401+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":364,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.297965+0000","last_change":"2023-09-23T23:03:51.529572+0000","last_active":"2023-09-23T23:04:00.297965+0000","last_peered":"2023-09-23T23:04:00.297965+0000","last_clean":"2023-09-23T23:04:00.297965+0000","last_became_active":"2023-09-23T23:03:51.529215+0000","last_became_peered":"2023-09-23T23:03:51.529215+0000","last_unstale":"2023-09-23T23:04:00.297965+0000","last_undegraded":"2023-09-23T23:04:00.297965+0000","last_fullsized":"2023-09-23T23:04:00.297965+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":165,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.180148+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:04:00.180148+0000","last_peered":"2023-09-23T23:04:00.180148+0000","last_clean":"2023-09-23T23:04:00.180148+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:04:00.180148+0000","last_undegraded":"2023-09-23T23:04:00.180148+0000","last_fullsized":"2023-09-23T23:04:00.180148+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":225,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184619+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:04:00.184619+0000","last_peered":"2023-09-23T23:04:00.184619+0000","last_clean":"2023-09-23T23:04:00.184619+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:04:00.184619+0000","last_undegraded":"2023-09-23T23:04:00.184619+0000","last_fullsized":"2023-09-23T23:04:00.184619+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":439,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184808+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:04:00.184808+0000","last_peered":"2023-09-23T23:04:00.184808+0000","last_clean":"2023-09-23T23:04:00.184808+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:04:00.184808+0000","last_undegraded":"2023-09-23T23:04:00.184808+0000","last_fullsized":"2023-09-23T23:04:00.184808+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":239,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184832+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:04:00.184832+0000","last_peered":"2023-09-23T23:04:00.184832+0000","last_clean":"2023-09-23T23:04:00.184832+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:04:00.184832+0000","last_undegraded":"2023-09-23T23:04:00.184832+0000","last_fullsized":"2023-09-23T23:04:00.184832+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":326,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184591+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:04:00.184591+0000","last_peered":"2023-09-23T23:04:00.184591+0000","last_clean":"2023-09-23T23:04:00.184591+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:04:00.184591+0000","last_undegraded":"2023-09-23T23:04:00.184591+0000","last_fullsized":"2023-09-23T23:04:00.184591+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.179944+0000","last_change":"2023-09-23T23:03:51.529901+0000","last_active":"2023-09-23T23:04:00.179944+0000","last_peered":"2023-09-23T23:04:00.179944+0000","last_clean":"2023-09-23T23:04:00.179944+0000","last_became_active":"2023-09-23T23:03:51.529442+0000","last_became_peered":"2023-09-23T23:03:51.529442+0000","last_unstale":"2023-09-23T23:04:00.179944+0000","last_undegraded":"2023-09-23T23:04:00.179944+0000","last_fullsized":"2023-09-23T23:04:00.179944+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:19.005121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.181665+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:04:00.181665+0000","last_peered":"2023-09-23T23:04:00.181665+0000","last_clean":"2023-09-23T23:04:00.181665+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:04:00.181665+0000","last_undegraded":"2023-09-23T23:04:00.181665+0000","last_fullsized":"2023-09-23T23:04:00.181665+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":359,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.298073+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:04:00.298073+0000","last_peered":"2023-09-23T23:04:00.298073+0000","last_clean":"2023-09-23T23:04:00.298073+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:04:00.298073+0000","last_undegraded":"2023-09-23T23:04:00.298073+0000","last_fullsized":"2023-09-23T23:04:00.298073+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":401,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T23:04:00.184382+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:04:00.184382+0000","last_peered":"2023-09-23T23:04:00.184382+0000","last_clean":"2023-09-23T23:04:00.184382+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:04:00.184382+0000","last_undegraded":"2023-09-23T23:04:00.184382+0000","last_fullsized":"2023-09-23T23:04:00.184382+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1807351808,"num_objects":27578,"num_object_clones":0,"num_object_copies":82734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27578,"num_write_kb":1764992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5461245952,"data_stored":5461245952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27578,"ondisk_log_size":27578,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15382,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430654,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626879,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253192,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179692,"kb_used_data":1413348,"kb_used_omap":9,"kb_used_meta":766326,"kb_avail":91565460,"statfs":{"total":95995035648,"available":93763031040,"internally_reserved":0,"allocated":1447268352,"data_stored":1445674079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":784718825},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.7769999999999999,"15min":0.82699999999999996},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4780000000000002,"5min":83.664000000000001,"15min":83.664000000000001},"last":0.247},{"interface":"front","average":{"1min":0.995,"5min":2.1040000000000001,"15min":1.272},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.266,"5min":83.700999999999993,"15min":83.700999999999993},"last":0.26800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.113,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":50.774999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9630000000000001,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":52.167999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.65400000000000003,"15min":1.675},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.20200000000000001},"max":{"1min":2.8879999999999999,"5min":3.7250000000000001,"15min":87.787999999999997},"last":51.094999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.64800000000000002,"15min":1.887},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":5.1989999999999998,"5min":5.1989999999999998,"15min":89.861999999999995},"last":50.969000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.8540000000000001,"15min":2.3149999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.004,"5min":67.738,"15min":151.036},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.0800000000000001,"15min":2.6589999999999998},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.5590000000000002,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.823,"15min":2.2890000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":8.7159999999999993,"5min":68.259,"15min":116.098},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.915,"15min":2.331},"min":{"1min":0.39900000000000002,"5min":0.378,"15min":0.315},"max":{"1min":3.04,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":2.1230000000000002,"15min":2.2450000000000001},"min":{"1min":0.38100000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":2.6030000000000002,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.7529999999999999,"15min":1.8080000000000001},"min":{"1min":0.40000000000000002,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":2.1920000000000002,"5min":67.366,"15min":67.366},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.879,"15min":1.4570000000000001},"min":{"1min":0.48599999999999999,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":3.069,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.006,"15min":1.8720000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33100000000000002},"max":{"1min":2.6360000000000001,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.45900000000000002}]}]},{"osd":0,"up_from":353,"seq":1516123455562,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275312,"kb_used_data":1055912,"kb_used_omap":37,"kb_used_meta":219354,"kb_avail":92469840,"statfs":{"total":95995035648,"available":94689116160,"internally_reserved":0,"allocated":1081253888,"data_stored":1079480204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":224618658},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.83299999999999996,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.6970000000000001,"5min":8.234,"15min":8.234},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.01,"5min":0.89700000000000002,"15min":0.747},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7169999999999996,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Sep 23 23:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.96799999999999997,"15min":0.84199999999999997},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":7.3369999999999997,"5min":12.215,"15min":12.215},"last":0.69999999999999996},{"interface":"front","average":{"1min":5.1340000000000003,"5min":1.7609999999999999,"15min":1.03},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":74.698999999999998,"5min":74.698999999999998,"15min":74.698999999999998},"last":0.75600000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.997,"15min":0.63500000000000001},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.1109999999999998,"5min":12.178000000000001,"15min":12.178000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.93799999999999994,"15min":0.91800000000000004},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.8050000000000002,"5min":6.8170000000000002,"15min":6.8170000000000002},"last":0.86099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":3.298,"15min":1.6719999999999999},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2690000000000001,"5min":183.148,"15min":183.148},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.98899999999999999,"5min":3.4169999999999998,"15min":1.585},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.9910000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.621}]},{"osd":5,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.069,"15min":0.80900000000000005},"min":{"1min":0.433,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.3090000000000002,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.026,"15min":1.0069999999999999},"min":{"1min":0.36299999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.7080000000000002,"5min":7.774,"15min":7.774},"last":0.83199999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.0860000000000001,"15min":0.91800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.0309999999999997,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1,"15min":0.81599999999999995},"min":{"1min":0.44900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.3410000000000002,"5min":6.375,"15min":6.375},"last":0.77300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.139,"15min":1.248},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.0579999999999998,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.1699999999999999,"15min":1.242},"min":{"1min":0.496,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.3719999999999999,"5min":6.343,"15min":6.343},"last":0.84699999999999998}]}]},{"osd":2,"up_from":359,"seq":1541893259335,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1093224,"kb_used_data":889696,"kb_used_omap":83,"kb_used_meta":203436,"kb_avail":92651928,"statfs":{"total":95995035648,"available":94875574272,"internally_reserved":0,"allocated":911048704,"data_stored":909343597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":208318917},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":6.2400000000000002,"5min":2.452,"15min":1.054},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.735999999999997,"5min":74.241,"15min":74.241},"last":0.33500000000000002},{"interface":"front","average":{"1min":6.3899999999999997,"5min":2.5960000000000001,"15min":1.337},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":55.969999999999999,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.29999999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.73099999999999998,"15min":0.68700000000000006},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.2080000000000002,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.0629999999999999,"15min":1.1240000000000001},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.397,"5min":11.397,"15min":11.397},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82299999999999995,"15min":0.749},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.2750000000000004,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.871,"15min":0.83499999999999996},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.92300000000000004,"15min":0.68400000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.1360000000000001,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.96799999999999997,"15min":0.871},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2430000000000003,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.99299999999999999,"15min":0.96699999999999997},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.3090000000000002,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":139.405},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.93500000000000005,"15min":0.76500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.1680000000000001,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":139.42099999999999}]},{"osd":6,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.1919999999999999,"15min":1.044},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.968,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.624},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.94099999999999995,"15min":0.71799999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.9079999999999999,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.67600000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.1559999999999999,"15min":1.046},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.7869999999999999,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.0449999999999999,"15min":1.1339999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3049999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":0.55700000000000005}]}]},{"osd":3,"up_from":94,"seq":403726926139,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1513536,"kb_used_data":976888,"kb_used_omap":7,"kb_used_meta":536632,"kb_avail":92231616,"statfs":{"total":95995035648,"available":94445174784,"internally_reserved":0,"allocated":1000333312,"data_stored":998659859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":549511784},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3780000000000001,"15min":1.0049999999999999},"min":{"1min":0.22,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.3019999999999996,"5min":122.289,"15min":122.289},"last":5.7130000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.0150000000000001,"15min":0.94299999999999995},"min":{"1min":0.22500000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":8.9849999999999994,"5min":122.041,"15min":122.041},"last":1.0469999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.145,"15min":2.2130000000000001},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.187},"max":{"1min":15.025,"5min":15.025,"15min":123.866},"last":10.102},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0289999999999999,"15min":2.1219999999999999},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.718,"5min":9.718,"15min":122.265},"last":8.907}]},{"osd":2,"last update":"Sat Sep 23 23:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.64200000000000002,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78400000000000003,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":2.3540000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.127,"15min":0.871},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":2.9159999999999999,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.92400000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.3559999999999999,"15min":1.9450000000000001},"min":{"1min":0.311,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":3.8450000000000002,"5min":87.046999999999997,"15min":87.046999999999997},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.97099999999999997,"5min":2.411,"15min":1.9079999999999999},"min":{"1min":0.34699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8490000000000002,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.119,"15min":1.407},"min":{"1min":0.373,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":4.9059999999999997,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.085,"15min":1.6140000000000001},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":0.92000000000000004,"5min":10.246,"15min":11.951000000000001},"last":0.70899999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.3420000000000001,"15min":1.706},"min":{"1min":0.40899999999999997,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":4.8840000000000003,"5min":9.2520000000000007,"15min":14.097},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.198,"15min":1.742},"min":{"1min":0.41299999999999998,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":2.6949999999999998,"5min":9.2810000000000006,"15min":12.887},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.006,"15min":1.5489999999999999},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":3.3260000000000001,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.391,"5min":1.2869999999999999,"15min":1.575},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":6.2050000000000001,"5min":8.5830000000000002,"15min":12.32},"last":0.64500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019336,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1363712,"kb_used_data":452152,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":92381440,"statfs":{"total":95995035648,"available":94598594560,"internally_reserved":0,"allocated":463003648,"data_stored":462139177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:13 2023","interfaces":[{"interface":"back","average":{"1min":7.6189999999999998,"5min":4.1950000000000003,"15min":1.7130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":156.375,"5min":156.375,"15min":156.375},"last":0.56399999999999995},{"interface":"front","average":{"1min":7.4009999999999998,"5min":4.335,"15min":1.8420000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":157.02199999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.81399999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.196,"15min":3.3199999999999998},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":28.917000000000002,"5min":28.917000000000002,"15min":132.304},"last":1.214},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.147,"15min":3.302},"min":{"1min":0.28000000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.731000000000002,"5min":28.731000000000002,"15min":132.453},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.155,"15min":1.0649999999999999},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.2949999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":4.3410000000000002},{"interface":"front","average":{"1min":1.329,"5min":1.161,"15min":0.86499999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.3799999999999999,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":1.331}]},{"osd":3,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.083,"15min":2.6869999999999998},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.278,"5min":10.337999999999999,"15min":164.453},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.2569999999999999,"15min":2.7530000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":3.4689999999999999,"5min":7.2519999999999998,"15min":167.05199999999999},"last":3.4689999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.036,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.214},"max":{"1min":1.492,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.92400000000000004,"15min":1.835},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":5.9409999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.317}]},{"osd":6,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.208,"15min":1.294},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":5.8399999999999999,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.96999999999999997,"15min":1.161},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":1.252,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.439}]},{"osd":7,"last update":"Sat Sep 23 23:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":1.2749999999999999,"15min":1.256},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.28399999999999997},"max":{"1min":4.0990000000000002,"5min":11.045999999999999,"15min":12.84},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.1379999999999999,"15min":1.1319999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":3.1960000000000002,"5min":10.587,"15min":10.587},"last":0.52700000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823110,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1425452,"kb_used_data":529892,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92319700,"statfs":{"total":95995035648,"available":94535372800,"internally_reserved":0,"allocated":542609408,"data_stored":541760641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.786,"15min":0.999},"min":{"1min":0.23799999999999999,"5min":0.221,"15min":0.221},"max":{"1min":23.346,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.9690000000000001,"15min":1.377},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":23.428999999999998,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.623}]},{"osd":1,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.753,"15min":1.984},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.0979999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.1419999999999999,"15min":2.4209999999999998},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":1.9510000000000001,"5min":7.1219999999999999,"15min":110.89},"last":0.753}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0589999999999999,"15min":0.88700000000000001},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9450000000000003,"5min":6.601,"15min":6.601},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.0009999999999999,"15min":0.997},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.9530000000000003,"5min":4.9530000000000003,"15min":4.9530000000000003},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.1359999999999999,"15min":2.7440000000000002},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.6629999999999998,"5min":9.2710000000000008,"15min":98.641000000000005},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.2330000000000001,"15min":2.7919999999999998},"min":{"1min":0.35799999999999998,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":3.6789999999999998,"5min":10.965,"15min":98.906000000000006},"last":0.74199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.0640000000000001,"15min":1.5549999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23300000000000001},"max":{"1min":1.107,"5min":7.0380000000000003,"15min":115.06699999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.97699999999999998,"15min":1.502},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.253},"max":{"1min":2.5419999999999998,"5min":9.5890000000000004,"15min":114.908},"last":1.1499999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.2490000000000001,"15min":1.2110000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":3.0600000000000001,"5min":7.4630000000000001,"15min":11.121},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.879,"15min":1.0529999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.2370000000000001,"5min":5.4909999999999997,"15min":12.137},"last":0.91800000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:03:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.04,"15min":1.0860000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.4770000000000001,"5min":9.5980000000000008,"15min":12.23},"last":1.048},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.444,"15min":1.2090000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.8500000000000001,"5min":10.638999999999999,"15min":11.112},"last":1.034}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1091371008,"data_stored":1091371008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1432092672,"data_stored":1432092672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":918290432,"data_stored":918290432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1032454144,"data_stored":1032454144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":436273152,"data_stored":436273152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":550764544,"data_stored":550764544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:04:16.142 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:15 smithi151 ceph-mon[159210]: from='client.36849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:15 smithi151 ceph-mon[159210]: pgmap v1287: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T23:04:16.273 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:16.276 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:16.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:16.389 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:16.390 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:04:16.461 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:16.518 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:16.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:16.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:16.706 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:16.744 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 24 16 33928 33912 88.269 80.5625 0.0188669 0.0113225 2023-09-23T23:04:16.842 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:17.076 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:17.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:17.184 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:17.254 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:17.318 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:17.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:17.427 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:17.454 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:17.499 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:17.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:17.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:17.602 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:17.629 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:17.674 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:17.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:17.744 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 25 16 34789 34773 86.8908 53.8125 0.289573 0.0113546 2023-09-23T23:04:17.777 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:17.804 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:17.912 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:04:18.032 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:17 smithi151 ceph-mon[159210]: pgmap v1288: 89 pgs: 89 active+clean; 2.0 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:04:18.168 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:18.178 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:17 smithi052 ceph-mon[180979]: pgmap v1288: 89 pgs: 89 active+clean; 2.0 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:04:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:18.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:18.180 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:17 smithi052 ceph-mon[187743]: pgmap v1288: 89 pgs: 89 active+clean; 2.0 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:04:18.281 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:18.330 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:18.385 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:18.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:18.481 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:04:18.481 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:04:18.489 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:18.548 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:18.744 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 26 16 35904 35888 86.2291 69.6875 0.00798928 0.0115923 2023-09-23T23:04:18.844 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:18.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:18.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:18.953 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:19.017 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:19.081 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:19.126 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:19.128 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:04:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:19.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:19.237 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:19.245 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:04:19.339 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:19.471 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:19.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:19.579 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:19.612 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:19.643 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:04:19.674 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:19.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:19.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 27 16 36976 36960 85.5159 67 0.0101447 0.0116881 2023-09-23T23:04:19.779 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:19.837 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:04:19.853 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:19.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:19 smithi151 ceph-mon[159210]: pgmap v1289: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:04:19.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2496178714' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:04:19.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:19.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:20.061 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[180979]: pgmap v1289: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:04:20.061 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2496178714' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:04:20.061 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:20.061 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:20.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[187743]: pgmap v1289: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T23:04:20.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2496178714' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:04:20.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:20.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:20.143 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:20.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:20.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:20.250 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:20.313 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:20.522 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:20.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:20.627 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:20.735 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:20.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 28 16 37975 37959 84.6918 62.4375 0.0133257 0.0118013 2023-09-23T23:04:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2496178714' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:04:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T23:04:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[187743]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2496178714' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: failed to encode map e417 with expected crc 2023-09-23T23:04:20.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:20 smithi052 ceph-mon[180979]: failed to encode map e417 with expected crc 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2496178714' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: failed to encode map e417 with expected crc 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: failed to encode map e417 with expected crc 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: failed to encode map e417 with expected crc 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: failed to encode map e417 with expected crc 2023-09-23T23:04:21.061 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:20 smithi151 ceph-mon[159210]: failed to encode map e417 with expected crc 2023-09-23T23:04:21.184 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:21.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:21.290 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:21.306 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:04:21.306 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:04:21.318 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:21.683 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:21.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:21.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 29 16 39051 39035 84.0903 67.25 0.0141042 0.0118855 2023-09-23T23:04:21.792 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:21.829 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:21.875 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:21.991 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:21.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:21 smithi052 ceph-mon[180979]: pgmap v1291: 89 pgs: 89 active+clean; 2.1 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:04:21.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:21 smithi052 ceph-mon[187743]: pgmap v1291: 89 pgs: 89 active+clean; 2.1 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:04:22.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:22.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:22.099 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:21 smithi151 ceph-mon[159210]: pgmap v1291: 89 pgs: 89 active+clean; 2.1 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T23:04:22.102 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:22.120 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:04:22.185 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:22.394 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:22.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:22.506 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:22.572 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:04:22.615 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:04:22.633 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:22.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:04:22.692 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:04:22.731 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:04:22.744 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 30 16 40031 40015 83.329 61.25 0.0101292 0.0119941 2023-09-23T23:04:22.774 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:04:22.819 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:04:22.871 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:04:22.877 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.877 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.877 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.877 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.878 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.878 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.878 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.878 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:04:22.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1260264301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:22.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1260264301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:23.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1260264301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:23.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:04:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:04:23] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.33.4" 2023-09-23T23:04:23.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:04:23.705 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:23.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:23.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:23.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 31 16 41144 41128 82.8849 69.5625 0.00296566 0.0120506 2023-09-23T23:04:23.814 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:23.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:23.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:23 smithi052 ceph-mon[187743]: pgmap v1292: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:04:23.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:23 smithi052 ceph-mon[180979]: pgmap v1292: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:04:24.070 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:24.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:24.177 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:24.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:23 smithi151 ceph-mon[159210]: pgmap v1292: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:04:24.209 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:24.253 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:24.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:24.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:24.388 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:24.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 32 16 41823 41807 81.6212 42.4375 0.0155592 0.0122437 2023-09-23T23:04:24.862 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:24.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:24.971 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:24.985 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:25.042 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:25.134 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:25.344 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:25.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:25.450 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:25.479 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:25.524 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:25.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:25.630 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:25.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:25.706 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:25.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:25.750 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 33 16 42834 42818 81.0626 63.1875 0.0157101 0.0123308 2023-09-23T23:04:25.812 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:25.835 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:25 smithi052 ceph-mon[180979]: pgmap v1293: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:04:25.839 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:25.884 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:25.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:25.991 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:26.017 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:26.058 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:26.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:26.072 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:04:26.074 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:04:26.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:25 smithi052 ceph-mon[187743]: pgmap v1293: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:04:26.166 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:26.190 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:25 smithi151 ceph-mon[159210]: pgmap v1293: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T23:04:26.193 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:26.244 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:26.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:26.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:26.350 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:26.448 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:26.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 34 16 43750 43734 80.3623 57.25 0.0210552 0.0124368 2023-09-23T23:04:26.807 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:26.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:26.848 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:04:26.914 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:26.945 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:27.052 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:27.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:27.156 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:26 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:27.224 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:26 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:27.225 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:26 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:04:27.238 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:27.745 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 35 16 44699 44683 79.7607 59.3125 0.00188211 0.0125237 2023-09-23T23:04:27.859 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:27 smithi052 ceph-mon[180979]: from='client.36864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:27.859 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:27 smithi052 ceph-mon[180979]: pgmap v1294: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:04:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:27 smithi151 ceph-mon[159210]: from='client.36864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:27 smithi151 ceph-mon[159210]: pgmap v1294: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:04:28.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:27 smithi052 ceph-mon[187743]: from='client.36864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:28.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:27 smithi052 ceph-mon[187743]: pgmap v1294: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:04:28.699 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:28.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:28.728 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1293,"stamp":"2023-09-23T23:04:24.658110+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2512721824,"num_objects":38356,"num_object_clones":0,"num_object_copies":115068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38356,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":38592,"num_write_kb":2455693,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":605750,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38526,"ondisk_log_size":38526,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10965552,"kb_used_data":7374272,"kb_used_omap":137,"kb_used_meta":3591094,"kb_avail":551505360,"statfs":{"total":575970213888,"available":564741488640,"internally_reserved":0,"allocated":7551254528,"data_stored":7542779967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3677280440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":704774144,"num_objects":10754,"num_object_clones":0,"num_object_copies":32262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":688256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002830"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":429,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726259+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:04:09.726259+0000","last_peered":"2023-09-23T23:04:09.726259+0000","last_clean":"2023-09-23T23:04:09.726259+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:04:09.726259+0000","last_undegraded":"2023-09-23T23:04:09.726259+0000","last_fullsized":"2023-09-23T23:04:09.726259+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":87,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731168+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:04:09.731168+0000","last_peered":"2023-09-23T23:04:09.731168+0000","last_clean":"2023-09-23T23:04:09.731168+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:04:09.731168+0000","last_undegraded":"2023-09-23T23:04:09.731168+0000","last_fullsized":"2023-09-23T23:04:09.731168+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730953+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:04:09.730953+0000","last_peered":"2023-09-23T23:04:09.730953+0000","last_clean":"2023-09-23T23:04:09.730953+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:04:09.730953+0000","last_undegraded":"2023-09-23T23:04:09.730953+0000","last_fullsized":"2023-09-23T23:04:09.730953+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":415,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730274+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:04:09.730274+0000","last_peered":"2023-09-23T23:04:09.730274+0000","last_clean":"2023-09-23T23:04:09.730274+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:04:09.730274+0000","last_undegraded":"2023-09-23T23:04:09.730274+0000","last_fullsized":"2023-09-23T23:04:09.730274+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.727489+0000","last_change":"2023-09-23T23:03:51.528123+0000","last_active":"2023-09-23T23:04:09.727489+0000","last_peered":"2023-09-23T23:04:09.727489+0000","last_clean":"2023-09-23T23:04:09.727489+0000","last_became_active":"2023-09-23T23:03:51.527770+0000","last_became_peered":"2023-09-23T23:03:51.527770+0000","last_unstale":"2023-09-23T23:04:09.727489+0000","last_undegraded":"2023-09-23T23:04:09.727489+0000","last_fullsized":"2023-09-23T23:04:09.727489+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:32.789048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806123+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:04:19.806123+0000","last_peered":"2023-09-23T23:04:19.806123+0000","last_clean":"2023-09-23T23:04:19.806123+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:04:19.806123+0000","last_undegraded":"2023-09-23T23:04:19.806123+0000","last_fullsized":"2023-09-23T23:04:19.806123+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":352,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737385+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:04:09.737385+0000","last_peered":"2023-09-23T23:04:09.737385+0000","last_clean":"2023-09-23T23:04:09.737385+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:04:09.737385+0000","last_undegraded":"2023-09-23T23:04:09.737385+0000","last_fullsized":"2023-09-23T23:04:09.737385+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":96,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731504+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:04:09.731504+0000","last_peered":"2023-09-23T23:04:09.731504+0000","last_clean":"2023-09-23T23:04:09.731504+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:04:09.731504+0000","last_undegraded":"2023-09-23T23:04:09.731504+0000","last_fullsized":"2023-09-23T23:04:09.731504+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":313,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737233+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:04:09.737233+0000","last_peered":"2023-09-23T23:04:09.737233+0000","last_clean":"2023-09-23T23:04:09.737233+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:04:09.737233+0000","last_undegraded":"2023-09-23T23:04:09.737233+0000","last_fullsized":"2023-09-23T23:04:09.737233+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730877+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:04:09.730877+0000","last_peered":"2023-09-23T23:04:09.730877+0000","last_clean":"2023-09-23T23:04:09.730877+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:04:09.730877+0000","last_undegraded":"2023-09-23T23:04:09.730877+0000","last_fullsized":"2023-09-23T23:04:09.730877+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":122,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731175+0000","last_change":"2023-09-23T23:03:51.536661+0000","last_active":"2023-09-23T23:04:09.731175+0000","last_peered":"2023-09-23T23:04:09.731175+0000","last_clean":"2023-09-23T23:04:09.731175+0000","last_became_active":"2023-09-23T23:03:51.532360+0000","last_became_peered":"2023-09-23T23:03:51.532360+0000","last_unstale":"2023-09-23T23:04:09.731175+0000","last_undegraded":"2023-09-23T23:04:09.731175+0000","last_fullsized":"2023-09-23T23:04:09.731175+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:55.428059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":163,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.727355+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:04:09.727355+0000","last_peered":"2023-09-23T23:04:09.727355+0000","last_clean":"2023-09-23T23:04:09.727355+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:04:09.727355+0000","last_undegraded":"2023-09-23T23:04:09.727355+0000","last_fullsized":"2023-09-23T23:04:09.727355+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":488,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726524+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:04:09.726524+0000","last_peered":"2023-09-23T23:04:09.726524+0000","last_clean":"2023-09-23T23:04:09.726524+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:04:09.726524+0000","last_undegraded":"2023-09-23T23:04:09.726524+0000","last_fullsized":"2023-09-23T23:04:09.726524+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":391,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726242+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:04:09.726242+0000","last_peered":"2023-09-23T23:04:09.726242+0000","last_clean":"2023-09-23T23:04:09.726242+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:04:09.726242+0000","last_undegraded":"2023-09-23T23:04:09.726242+0000","last_fullsized":"2023-09-23T23:04:09.726242+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":317,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726297+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:04:09.726297+0000","last_peered":"2023-09-23T23:04:09.726297+0000","last_clean":"2023-09-23T23:04:09.726297+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:04:09.726297+0000","last_undegraded":"2023-09-23T23:04:09.726297+0000","last_fullsized":"2023-09-23T23:04:09.726297+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730125+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:04:09.730125+0000","last_peered":"2023-09-23T23:04:09.730125+0000","last_clean":"2023-09-23T23:04:09.730125+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:04:09.730125+0000","last_undegraded":"2023-09-23T23:04:09.730125+0000","last_fullsized":"2023-09-23T23:04:09.730125+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":407,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731266+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:04:09.731266+0000","last_peered":"2023-09-23T23:04:09.731266+0000","last_clean":"2023-09-23T23:04:09.731266+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:04:09.731266+0000","last_undegraded":"2023-09-23T23:04:09.731266+0000","last_fullsized":"2023-09-23T23:04:09.731266+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":317,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737118+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:04:09.737118+0000","last_peered":"2023-09-23T23:04:09.737118+0000","last_clean":"2023-09-23T23:04:09.737118+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:04:09.737118+0000","last_undegraded":"2023-09-23T23:04:09.737118+0000","last_fullsized":"2023-09-23T23:04:09.737118+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":302,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726448+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:04:09.726448+0000","last_peered":"2023-09-23T23:04:09.726448+0000","last_clean":"2023-09-23T23:04:09.726448+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:04:09.726448+0000","last_undegraded":"2023-09-23T23:04:09.726448+0000","last_fullsized":"2023-09-23T23:04:09.726448+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":431,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730398+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:04:09.730398+0000","last_peered":"2023-09-23T23:04:09.730398+0000","last_clean":"2023-09-23T23:04:09.730398+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:04:09.730398+0000","last_undegraded":"2023-09-23T23:04:09.730398+0000","last_fullsized":"2023-09-23T23:04:09.730398+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":136,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.719955+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:04:09.719955+0000","last_peered":"2023-09-23T23:04:09.719955+0000","last_clean":"2023-09-23T23:04:09.719955+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:04:09.719955+0000","last_undegraded":"2023-09-23T23:04:09.719955+0000","last_fullsized":"2023-09-23T23:04:09.719955+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":111,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730598+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:04:09.730598+0000","last_peered":"2023-09-23T23:04:09.730598+0000","last_clean":"2023-09-23T23:04:09.730598+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:04:09.730598+0000","last_undegraded":"2023-09-23T23:04:09.730598+0000","last_fullsized":"2023-09-23T23:04:09.730598+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"417'2346","reported_seq":2360,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.410405+0000","last_change":"2023-09-23T23:03:43.131702+0000","last_active":"2023-09-23T23:04:20.410405+0000","last_peered":"2023-09-23T23:04:20.410405+0000","last_clean":"2023-09-23T23:04:20.410405+0000","last_became_active":"2023-09-23T23:03:43.131323+0000","last_became_peered":"2023-09-23T23:03:43.131323+0000","last_unstale":"2023-09-23T23:04:20.410405+0000","last_undegraded":"2023-09-23T23:04:20.410405+0000","last_fullsized":"2023-09-23T23:04:20.410405+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2346,"log_dups_size":0,"ondisk_log_size":2346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153747456,"num_objects":2346,"num_object_clones":0,"num_object_copies":7038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2346,"num_write_kb":150144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":301,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726581+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:04:09.726581+0000","last_peered":"2023-09-23T23:04:09.726581+0000","last_clean":"2023-09-23T23:04:09.726581+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:04:09.726581+0000","last_undegraded":"2023-09-23T23:04:09.726581+0000","last_fullsized":"2023-09-23T23:04:09.726581+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":287,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.736868+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:04:09.736868+0000","last_peered":"2023-09-23T23:04:09.736868+0000","last_clean":"2023-09-23T23:04:09.736868+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:04:09.736868+0000","last_undegraded":"2023-09-23T23:04:09.736868+0000","last_fullsized":"2023-09-23T23:04:09.736868+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"417'2333","reported_seq":2349,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.246240+0000","last_change":"2023-09-23T23:03:43.132133+0000","last_active":"2023-09-23T23:04:20.246240+0000","last_peered":"2023-09-23T23:04:20.246240+0000","last_clean":"2023-09-23T23:04:20.246240+0000","last_became_active":"2023-09-23T23:03:43.131741+0000","last_became_peered":"2023-09-23T23:03:43.131741+0000","last_unstale":"2023-09-23T23:04:20.246240+0000","last_undegraded":"2023-09-23T23:04:20.246240+0000","last_fullsized":"2023-09-23T23:04:20.246240+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2333,"log_dups_size":0,"ondisk_log_size":2333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:36.498718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152895488,"num_objects":2333,"num_object_clones":0,"num_object_copies":6999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2333,"num_write_kb":149312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":18,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.727748+0000","last_change":"2023-09-23T23:03:51.533976+0000","last_active":"2023-09-23T23:04:09.727748+0000","last_peered":"2023-09-23T23:04:09.727748+0000","last_clean":"2023-09-23T23:04:09.727748+0000","last_became_active":"2023-09-23T23:03:51.528701+0000","last_became_peered":"2023-09-23T23:03:51.528701+0000","last_unstale":"2023-09-23T23:04:09.727748+0000","last_undegraded":"2023-09-23T23:04:09.727748+0000","last_fullsized":"2023-09-23T23:04:09.727748+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.921280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":424,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726353+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:04:09.726353+0000","last_peered":"2023-09-23T23:04:09.726353+0000","last_clean":"2023-09-23T23:04:09.726353+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:04:09.726353+0000","last_undegraded":"2023-09-23T23:04:09.726353+0000","last_fullsized":"2023-09-23T23:04:09.726353+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"417'2350","reported_seq":2365,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.288206+0000","last_change":"2023-09-23T23:03:42.889536+0000","last_active":"2023-09-23T23:04:20.288206+0000","last_peered":"2023-09-23T23:04:20.288206+0000","last_clean":"2023-09-23T23:04:20.288206+0000","last_became_active":"2023-09-23T23:03:42.889213+0000","last_became_peered":"2023-09-23T23:03:42.889213+0000","last_unstale":"2023-09-23T23:04:20.288206+0000","last_undegraded":"2023-09-23T23:04:20.288206+0000","last_fullsized":"2023-09-23T23:04:20.288206+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2350,"log_dups_size":0,"ondisk_log_size":2350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154009600,"num_objects":2350,"num_object_clones":0,"num_object_copies":7050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2350,"num_write_kb":150400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":168,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737234+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:04:09.737234+0000","last_peered":"2023-09-23T23:04:09.737234+0000","last_clean":"2023-09-23T23:04:09.737234+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:04:09.737234+0000","last_undegraded":"2023-09-23T23:04:09.737234+0000","last_fullsized":"2023-09-23T23:04:09.737234+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":286,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.736858+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:04:09.736858+0000","last_peered":"2023-09-23T23:04:09.736858+0000","last_clean":"2023-09-23T23:04:09.736858+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:04:09.736858+0000","last_undegraded":"2023-09-23T23:04:09.736858+0000","last_fullsized":"2023-09-23T23:04:09.736858+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"417'2396","reported_seq":2412,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.254441+0000","last_change":"2023-09-23T23:03:43.136203+0000","last_active":"2023-09-23T23:04:21.254441+0000","last_peered":"2023-09-23T23:04:21.254441+0000","last_clean":"2023-09-23T23:04:21.254441+0000","last_became_active":"2023-09-23T23:03:43.135918+0000","last_became_peered":"2023-09-23T23:03:43.135918+0000","last_unstale":"2023-09-23T23:04:21.254441+0000","last_undegraded":"2023-09-23T23:04:21.254441+0000","last_fullsized":"2023-09-23T23:04:21.254441+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2396,"log_dups_size":0,"ondisk_log_size":2396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:11.372248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157024256,"num_objects":2396,"num_object_clones":0,"num_object_copies":7188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2396,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":88,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.290124+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:04:11.290124+0000","last_peered":"2023-09-23T23:04:11.290124+0000","last_clean":"2023-09-23T23:04:11.290124+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:04:11.290124+0000","last_undegraded":"2023-09-23T23:04:11.290124+0000","last_fullsized":"2023-09-23T23:04:11.290124+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":317,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.720445+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:04:09.720445+0000","last_peered":"2023-09-23T23:04:09.720445+0000","last_clean":"2023-09-23T23:04:09.720445+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:04:09.720445+0000","last_undegraded":"2023-09-23T23:04:09.720445+0000","last_fullsized":"2023-09-23T23:04:09.720445+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"417'2366","reported_seq":2380,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.408924+0000","last_change":"2023-09-23T23:03:43.131843+0000","last_active":"2023-09-23T23:04:20.408924+0000","last_peered":"2023-09-23T23:04:20.408924+0000","last_clean":"2023-09-23T23:04:20.408924+0000","last_became_active":"2023-09-23T23:03:43.131420+0000","last_became_peered":"2023-09-23T23:03:43.131420+0000","last_unstale":"2023-09-23T23:04:20.408924+0000","last_undegraded":"2023-09-23T23:04:20.408924+0000","last_fullsized":"2023-09-23T23:04:20.408924+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2366,"log_dups_size":0,"ondisk_log_size":2366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155058176,"num_objects":2366,"num_object_clones":0,"num_object_copies":7098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2366,"num_write_kb":151424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":107,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:11.290104+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:04:11.290104+0000","last_peered":"2023-09-23T23:04:11.290104+0000","last_clean":"2023-09-23T23:04:11.290104+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:04:11.290104+0000","last_undegraded":"2023-09-23T23:04:11.290104+0000","last_fullsized":"2023-09-23T23:04:11.290104+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":410,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726430+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:04:09.726430+0000","last_peered":"2023-09-23T23:04:09.726430+0000","last_clean":"2023-09-23T23:04:09.726430+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:04:09.726430+0000","last_undegraded":"2023-09-23T23:04:09.726430+0000","last_fullsized":"2023-09-23T23:04:09.726430+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"417'2394","reported_seq":2410,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.239887+0000","last_change":"2023-09-23T23:03:42.889763+0000","last_active":"2023-09-23T23:04:21.239887+0000","last_peered":"2023-09-23T23:04:21.239887+0000","last_clean":"2023-09-23T23:04:21.239887+0000","last_became_active":"2023-09-23T23:03:42.889481+0000","last_became_peered":"2023-09-23T23:03:42.889481+0000","last_unstale":"2023-09-23T23:04:21.239887+0000","last_undegraded":"2023-09-23T23:04:21.239887+0000","last_fullsized":"2023-09-23T23:04:21.239887+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2394,"log_dups_size":0,"ondisk_log_size":2394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:57.841265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156893184,"num_objects":2394,"num_object_clones":0,"num_object_copies":7182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2394,"num_write_kb":153216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726356+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:04:09.726356+0000","last_peered":"2023-09-23T23:04:09.726356+0000","last_clean":"2023-09-23T23:04:09.726356+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:04:09.726356+0000","last_undegraded":"2023-09-23T23:04:09.726356+0000","last_fullsized":"2023-09-23T23:04:09.726356+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":427,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726481+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:04:09.726481+0000","last_peered":"2023-09-23T23:04:09.726481+0000","last_clean":"2023-09-23T23:04:09.726481+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:04:09.726481+0000","last_undegraded":"2023-09-23T23:04:09.726481+0000","last_fullsized":"2023-09-23T23:04:09.726481+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"417'2478","reported_seq":2494,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.256367+0000","last_change":"2023-09-23T23:03:42.889334+0000","last_active":"2023-09-23T23:04:21.256367+0000","last_peered":"2023-09-23T23:04:21.256367+0000","last_clean":"2023-09-23T23:04:21.256367+0000","last_became_active":"2023-09-23T23:03:42.889046+0000","last_became_peered":"2023-09-23T23:03:42.889046+0000","last_unstale":"2023-09-23T23:04:21.256367+0000","last_undegraded":"2023-09-23T23:04:21.256367+0000","last_fullsized":"2023-09-23T23:04:21.256367+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2478,"log_dups_size":0,"ondisk_log_size":2478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:44.256081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2478,"num_write_kb":158592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":440,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726131+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:04:09.726131+0000","last_peered":"2023-09-23T23:04:09.726131+0000","last_clean":"2023-09-23T23:04:09.726131+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:04:09.726131+0000","last_undegraded":"2023-09-23T23:04:09.726131+0000","last_fullsized":"2023-09-23T23:04:09.726131+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":418,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726657+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:04:09.726657+0000","last_peered":"2023-09-23T23:04:09.726657+0000","last_clean":"2023-09-23T23:04:09.726657+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:04:09.726657+0000","last_undegraded":"2023-09-23T23:04:09.726657+0000","last_fullsized":"2023-09-23T23:04:09.726657+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"417'2359","reported_seq":2375,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.249826+0000","last_change":"2023-09-23T23:03:43.132462+0000","last_active":"2023-09-23T23:04:21.249826+0000","last_peered":"2023-09-23T23:04:21.249826+0000","last_clean":"2023-09-23T23:04:21.249826+0000","last_became_active":"2023-09-23T23:03:43.132267+0000","last_became_peered":"2023-09-23T23:03:43.132267+0000","last_unstale":"2023-09-23T23:04:21.249826+0000","last_undegraded":"2023-09-23T23:04:21.249826+0000","last_fullsized":"2023-09-23T23:04:21.249826+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2359,"log_dups_size":0,"ondisk_log_size":2359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:49.697445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2359,"num_write_kb":150976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731874+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:04:09.731874+0000","last_peered":"2023-09-23T23:04:09.731874+0000","last_clean":"2023-09-23T23:04:09.731874+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:04:09.731874+0000","last_undegraded":"2023-09-23T23:04:09.731874+0000","last_fullsized":"2023-09-23T23:04:09.731874+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":321,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.728744+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:04:09.728744+0000","last_peered":"2023-09-23T23:04:09.728744+0000","last_clean":"2023-09-23T23:04:09.728744+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:04:09.728744+0000","last_undegraded":"2023-09-23T23:04:09.728744+0000","last_fullsized":"2023-09-23T23:04:09.728744+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"417'2423","reported_seq":2439,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.249029+0000","last_change":"2023-09-23T23:03:43.132020+0000","last_active":"2023-09-23T23:04:21.249029+0000","last_peered":"2023-09-23T23:04:21.249029+0000","last_clean":"2023-09-23T23:04:21.249029+0000","last_became_active":"2023-09-23T23:03:43.131692+0000","last_became_peered":"2023-09-23T23:03:43.131692+0000","last_unstale":"2023-09-23T23:04:21.249029+0000","last_undegraded":"2023-09-23T23:04:21.249029+0000","last_fullsized":"2023-09-23T23:04:21.249029+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2423,"log_dups_size":0,"ondisk_log_size":2423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:13.311701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158793728,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2423,"num_write_kb":155072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":325,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.728786+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:04:09.728786+0000","last_peered":"2023-09-23T23:04:09.728786+0000","last_clean":"2023-09-23T23:04:09.728786+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:04:09.728786+0000","last_undegraded":"2023-09-23T23:04:09.728786+0000","last_fullsized":"2023-09-23T23:04:09.728786+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":327,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737060+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:04:09.737060+0000","last_peered":"2023-09-23T23:04:09.737060+0000","last_clean":"2023-09-23T23:04:09.737060+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:04:09.737060+0000","last_undegraded":"2023-09-23T23:04:09.737060+0000","last_fullsized":"2023-09-23T23:04:09.737060+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":140,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.736814+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:04:09.736814+0000","last_peered":"2023-09-23T23:04:09.736814+0000","last_clean":"2023-09-23T23:04:09.736814+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:04:09.736814+0000","last_undegraded":"2023-09-23T23:04:09.736814+0000","last_fullsized":"2023-09-23T23:04:09.736814+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"417'2428","reported_seq":2443,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.438297+0000","last_change":"2023-09-23T23:03:43.132707+0000","last_active":"2023-09-23T23:04:21.438297+0000","last_peered":"2023-09-23T23:04:21.438297+0000","last_clean":"2023-09-23T23:04:21.438297+0000","last_became_active":"2023-09-23T23:03:43.132372+0000","last_became_peered":"2023-09-23T23:03:43.132372+0000","last_unstale":"2023-09-23T23:04:21.438297+0000","last_undegraded":"2023-09-23T23:04:21.438297+0000","last_fullsized":"2023-09-23T23:04:21.438297+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2428,"log_dups_size":0,"ondisk_log_size":2428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.951582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159121408,"num_objects":2428,"num_object_clones":0,"num_object_copies":7284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2428,"num_write_kb":155392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":89,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.805024+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:04:19.805024+0000","last_peered":"2023-09-23T23:04:19.805024+0000","last_clean":"2023-09-23T23:04:19.805024+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:04:19.805024+0000","last_undegraded":"2023-09-23T23:04:19.805024+0000","last_fullsized":"2023-09-23T23:04:19.805024+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":437,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726417+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:04:09.726417+0000","last_peered":"2023-09-23T23:04:09.726417+0000","last_clean":"2023-09-23T23:04:09.726417+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:04:09.726417+0000","last_undegraded":"2023-09-23T23:04:09.726417+0000","last_fullsized":"2023-09-23T23:04:09.726417+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":310,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.729229+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:04:09.729229+0000","last_peered":"2023-09-23T23:04:09.729229+0000","last_clean":"2023-09-23T23:04:09.729229+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:04:09.729229+0000","last_undegraded":"2023-09-23T23:04:09.729229+0000","last_fullsized":"2023-09-23T23:04:09.729229+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"417'2461","reported_seq":2477,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.226618+0000","last_change":"2023-09-23T23:03:42.889306+0000","last_active":"2023-09-23T23:04:21.226618+0000","last_peered":"2023-09-23T23:04:21.226618+0000","last_clean":"2023-09-23T23:04:21.226618+0000","last_became_active":"2023-09-23T23:03:42.888948+0000","last_became_peered":"2023-09-23T23:03:42.888948+0000","last_unstale":"2023-09-23T23:04:21.226618+0000","last_undegraded":"2023-09-23T23:04:21.226618+0000","last_fullsized":"2023-09-23T23:04:21.226618+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2461,"log_dups_size":0,"ondisk_log_size":2461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:56.393948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161284096,"num_objects":2461,"num_object_clones":0,"num_object_copies":7383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2461,"num_write_kb":157504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":385,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.727975+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:04:09.727975+0000","last_peered":"2023-09-23T23:04:09.727975+0000","last_clean":"2023-09-23T23:04:09.727975+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:04:09.727975+0000","last_undegraded":"2023-09-23T23:04:09.727975+0000","last_fullsized":"2023-09-23T23:04:09.727975+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.736774+0000","last_change":"2023-09-23T23:03:51.532583+0000","last_active":"2023-09-23T23:04:09.736774+0000","last_peered":"2023-09-23T23:04:09.736774+0000","last_clean":"2023-09-23T23:04:09.736774+0000","last_became_active":"2023-09-23T23:03:51.532300+0000","last_became_peered":"2023-09-23T23:03:51.532300+0000","last_unstale":"2023-09-23T23:04:09.736774+0000","last_undegraded":"2023-09-23T23:04:09.736774+0000","last_fullsized":"2023-09-23T23:04:09.736774+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":327,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737461+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:04:09.737461+0000","last_peered":"2023-09-23T23:04:09.737461+0000","last_clean":"2023-09-23T23:04:09.737461+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:04:09.737461+0000","last_undegraded":"2023-09-23T23:04:09.737461+0000","last_fullsized":"2023-09-23T23:04:09.737461+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"417'2474","reported_seq":2490,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.222736+0000","last_change":"2023-09-23T23:03:43.131943+0000","last_active":"2023-09-23T23:04:21.222736+0000","last_peered":"2023-09-23T23:04:21.222736+0000","last_clean":"2023-09-23T23:04:21.222736+0000","last_became_active":"2023-09-23T23:03:43.131668+0000","last_became_peered":"2023-09-23T23:03:43.131668+0000","last_unstale":"2023-09-23T23:04:21.222736+0000","last_undegraded":"2023-09-23T23:04:21.222736+0000","last_fullsized":"2023-09-23T23:04:21.222736+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2474,"log_dups_size":0,"ondisk_log_size":2474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:16.104887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162136064,"num_objects":2474,"num_object_clones":0,"num_object_copies":7422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2474,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":400,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737004+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:04:09.737004+0000","last_peered":"2023-09-23T23:04:09.737004+0000","last_clean":"2023-09-23T23:04:09.737004+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:04:09.737004+0000","last_undegraded":"2023-09-23T23:04:09.737004+0000","last_fullsized":"2023-09-23T23:04:09.737004+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":530,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.722297+0000","last_change":"2023-09-23T23:03:36.322526+0000","last_active":"2023-09-23T23:04:09.722297+0000","last_peered":"2023-09-23T23:04:09.722297+0000","last_clean":"2023-09-23T23:04:09.722297+0000","last_became_active":"2023-09-23T23:03:36.322065+0000","last_became_peered":"2023-09-23T23:03:36.322065+0000","last_unstale":"2023-09-23T23:04:09.722297+0000","last_undegraded":"2023-09-23T23:04:09.722297+0000","last_fullsized":"2023-09-23T23:04:09.722297+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:04.195321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731640+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:04:09.731640+0000","last_peered":"2023-09-23T23:04:09.731640+0000","last_clean":"2023-09-23T23:04:09.731640+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:04:09.731640+0000","last_undegraded":"2023-09-23T23:04:09.731640+0000","last_fullsized":"2023-09-23T23:04:09.731640+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726565+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:04:09.726565+0000","last_peered":"2023-09-23T23:04:09.726565+0000","last_clean":"2023-09-23T23:04:09.726565+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:04:09.726565+0000","last_undegraded":"2023-09-23T23:04:09.726565+0000","last_fullsized":"2023-09-23T23:04:09.726565+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"417'2396","reported_seq":2412,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.254514+0000","last_change":"2023-09-23T23:03:43.133127+0000","last_active":"2023-09-23T23:04:21.254514+0000","last_peered":"2023-09-23T23:04:21.254514+0000","last_clean":"2023-09-23T23:04:21.254514+0000","last_became_active":"2023-09-23T23:03:43.132709+0000","last_became_peered":"2023-09-23T23:03:43.132709+0000","last_unstale":"2023-09-23T23:04:21.254514+0000","last_undegraded":"2023-09-23T23:04:21.254514+0000","last_fullsized":"2023-09-23T23:04:21.254514+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2396,"log_dups_size":0,"ondisk_log_size":2396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:35.116957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157024256,"num_objects":2396,"num_object_clones":0,"num_object_copies":7188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2396,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":409,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.720048+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:04:09.720048+0000","last_peered":"2023-09-23T23:04:09.720048+0000","last_clean":"2023-09-23T23:04:09.720048+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:04:09.720048+0000","last_undegraded":"2023-09-23T23:04:09.720048+0000","last_fullsized":"2023-09-23T23:04:09.720048+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":380,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.805401+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:04:19.805401+0000","last_peered":"2023-09-23T23:04:19.805401+0000","last_clean":"2023-09-23T23:04:19.805401+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:04:19.805401+0000","last_undegraded":"2023-09-23T23:04:19.805401+0000","last_fullsized":"2023-09-23T23:04:19.805401+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":339,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.720958+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:04:09.720958+0000","last_peered":"2023-09-23T23:04:09.720958+0000","last_clean":"2023-09-23T23:04:09.720958+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:04:09.720958+0000","last_undegraded":"2023-09-23T23:04:09.720958+0000","last_fullsized":"2023-09-23T23:04:09.720958+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"417'2395","reported_seq":2411,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.246825+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:04:21.246825+0000","last_peered":"2023-09-23T23:04:21.246825+0000","last_clean":"2023-09-23T23:04:21.246825+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:04:21.246825+0000","last_undegraded":"2023-09-23T23:04:21.246825+0000","last_fullsized":"2023-09-23T23:04:21.246825+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2395,"log_dups_size":0,"ondisk_log_size":2395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156958720,"num_objects":2395,"num_object_clones":0,"num_object_copies":7185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2395,"num_write_kb":153280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":355,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.805803+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:04:19.805803+0000","last_peered":"2023-09-23T23:04:19.805803+0000","last_clean":"2023-09-23T23:04:19.805803+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:04:19.805803+0000","last_undegraded":"2023-09-23T23:04:19.805803+0000","last_fullsized":"2023-09-23T23:04:19.805803+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":262,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.729100+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:04:09.729100+0000","last_peered":"2023-09-23T23:04:09.729100+0000","last_clean":"2023-09-23T23:04:09.729100+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:04:09.729100+0000","last_undegraded":"2023-09-23T23:04:09.729100+0000","last_fullsized":"2023-09-23T23:04:09.729100+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.737092+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:04:09.737092+0000","last_peered":"2023-09-23T23:04:09.737092+0000","last_clean":"2023-09-23T23:04:09.737092+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:04:09.737092+0000","last_undegraded":"2023-09-23T23:04:09.737092+0000","last_fullsized":"2023-09-23T23:04:09.737092+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"417'2404","reported_seq":2420,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:21.203510+0000","last_change":"2023-09-23T23:03:43.132731+0000","last_active":"2023-09-23T23:04:21.203510+0000","last_peered":"2023-09-23T23:04:21.203510+0000","last_clean":"2023-09-23T23:04:21.203510+0000","last_became_active":"2023-09-23T23:03:43.132451+0000","last_became_peered":"2023-09-23T23:03:43.132451+0000","last_unstale":"2023-09-23T23:04:21.203510+0000","last_undegraded":"2023-09-23T23:04:21.203510+0000","last_fullsized":"2023-09-23T23:04:21.203510+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2404,"log_dups_size":0,"ondisk_log_size":2404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:51.837853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157548544,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2404,"num_write_kb":153856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":112,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.730744+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:04:09.730744+0000","last_peered":"2023-09-23T23:04:09.730744+0000","last_clean":"2023-09-23T23:04:09.730744+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:04:09.730744+0000","last_undegraded":"2023-09-23T23:04:09.730744+0000","last_fullsized":"2023-09-23T23:04:09.730744+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":125,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726696+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:04:09.726696+0000","last_peered":"2023-09-23T23:04:09.726696+0000","last_clean":"2023-09-23T23:04:09.726696+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:04:09.726696+0000","last_undegraded":"2023-09-23T23:04:09.726696+0000","last_fullsized":"2023-09-23T23:04:09.726696+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":103,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731643+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:04:09.731643+0000","last_peered":"2023-09-23T23:04:09.731643+0000","last_clean":"2023-09-23T23:04:09.731643+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:04:09.731643+0000","last_undegraded":"2023-09-23T23:04:09.731643+0000","last_fullsized":"2023-09-23T23:04:09.731643+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"417'2329","reported_seq":2344,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.288945+0000","last_change":"2023-09-23T23:03:43.131930+0000","last_active":"2023-09-23T23:04:20.288945+0000","last_peered":"2023-09-23T23:04:20.288945+0000","last_clean":"2023-09-23T23:04:20.288945+0000","last_became_active":"2023-09-23T23:03:43.131552+0000","last_became_peered":"2023-09-23T23:03:43.131552+0000","last_unstale":"2023-09-23T23:04:20.288945+0000","last_undegraded":"2023-09-23T23:04:20.288945+0000","last_fullsized":"2023-09-23T23:04:20.288945+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":2329,"log_dups_size":0,"ondisk_log_size":2329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152633344,"num_objects":2329,"num_object_clones":0,"num_object_copies":6987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2329,"num_write_kb":149056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":374,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.728398+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:04:09.728398+0000","last_peered":"2023-09-23T23:04:09.728398+0000","last_clean":"2023-09-23T23:04:09.728398+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:04:09.728398+0000","last_undegraded":"2023-09-23T23:04:09.728398+0000","last_fullsized":"2023-09-23T23:04:09.728398+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":321,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.720883+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:04:09.720883+0000","last_peered":"2023-09-23T23:04:09.720883+0000","last_clean":"2023-09-23T23:04:09.720883+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:04:09.720883+0000","last_undegraded":"2023-09-23T23:04:09.720883+0000","last_fullsized":"2023-09-23T23:04:09.720883+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":391,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726370+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:04:09.726370+0000","last_peered":"2023-09-23T23:04:09.726370+0000","last_clean":"2023-09-23T23:04:09.726370+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:04:09.726370+0000","last_undegraded":"2023-09-23T23:04:09.726370+0000","last_fullsized":"2023-09-23T23:04:09.726370+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":365,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.720529+0000","last_change":"2023-09-23T23:03:51.529572+0000","last_active":"2023-09-23T23:04:09.720529+0000","last_peered":"2023-09-23T23:04:09.720529+0000","last_clean":"2023-09-23T23:04:09.720529+0000","last_became_active":"2023-09-23T23:03:51.529215+0000","last_became_peered":"2023-09-23T23:03:51.529215+0000","last_unstale":"2023-09-23T23:04:09.720529+0000","last_undegraded":"2023-09-23T23:04:09.720529+0000","last_fullsized":"2023-09-23T23:04:09.720529+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":166,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.729173+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:04:09.729173+0000","last_peered":"2023-09-23T23:04:09.729173+0000","last_clean":"2023-09-23T23:04:09.729173+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:04:09.729173+0000","last_undegraded":"2023-09-23T23:04:09.729173+0000","last_fullsized":"2023-09-23T23:04:09.729173+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":226,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726406+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:04:09.726406+0000","last_peered":"2023-09-23T23:04:09.726406+0000","last_clean":"2023-09-23T23:04:09.726406+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:04:09.726406+0000","last_undegraded":"2023-09-23T23:04:09.726406+0000","last_fullsized":"2023-09-23T23:04:09.726406+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":440,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726585+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:04:09.726585+0000","last_peered":"2023-09-23T23:04:09.726585+0000","last_clean":"2023-09-23T23:04:09.726585+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:04:09.726585+0000","last_undegraded":"2023-09-23T23:04:09.726585+0000","last_fullsized":"2023-09-23T23:04:09.726585+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":240,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726540+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:04:09.726540+0000","last_peered":"2023-09-23T23:04:09.726540+0000","last_clean":"2023-09-23T23:04:09.726540+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:04:09.726540+0000","last_undegraded":"2023-09-23T23:04:09.726540+0000","last_fullsized":"2023-09-23T23:04:09.726540+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":327,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726282+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:04:09.726282+0000","last_peered":"2023-09-23T23:04:09.726282+0000","last_clean":"2023-09-23T23:04:09.726282+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:04:09.726282+0000","last_undegraded":"2023-09-23T23:04:09.726282+0000","last_fullsized":"2023-09-23T23:04:09.726282+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.727896+0000","last_change":"2023-09-23T23:03:51.529901+0000","last_active":"2023-09-23T23:04:09.727896+0000","last_peered":"2023-09-23T23:04:09.727896+0000","last_clean":"2023-09-23T23:04:09.727896+0000","last_became_active":"2023-09-23T23:03:51.529442+0000","last_became_peered":"2023-09-23T23:03:51.529442+0000","last_unstale":"2023-09-23T23:04:09.727896+0000","last_undegraded":"2023-09-23T23:04:09.727896+0000","last_fullsized":"2023-09-23T23:04:09.727896+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:19.005121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.731049+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:04:09.731049+0000","last_peered":"2023-09-23T23:04:09.731049+0000","last_clean":"2023-09-23T23:04:09.731049+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:04:09.731049+0000","last_undegraded":"2023-09-23T23:04:09.731049+0000","last_fullsized":"2023-09-23T23:04:09.731049+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":361,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806535+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:04:19.806535+0000","last_peered":"2023-09-23T23:04:19.806535+0000","last_clean":"2023-09-23T23:04:19.806535+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:04:19.806535+0000","last_undegraded":"2023-09-23T23:04:19.806535+0000","last_fullsized":"2023-09-23T23:04:19.806535+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":402,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T23:04:09.726168+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:04:09.726168+0000","last_peered":"2023-09-23T23:04:09.726168+0000","last_clean":"2023-09-23T23:04:09.726168+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:04:09.726168+0000","last_undegraded":"2023-09-23T23:04:09.726168+0000","last_fullsized":"2023-09-23T23:04:09.726168+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2512125952,"num_objects":38332,"num_object_clones":0,"num_object_copies":114996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":38332,"num_write_kb":2453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7584546816,"data_stored":7584546816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38332,"ondisk_log_size":38332,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15382,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430656,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626881,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253194,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2702584,"kb_used_data":1914096,"kb_used_omap":9,"kb_used_meta":788470,"kb_avail":91042568,"statfs":{"total":95995035648,"available":93227589632,"internally_reserved":0,"allocated":1960034304,"data_stored":1958436376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":807394281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:14 2023","interfaces":[{"interface":"back","average":{"1min":20.364999999999998,"5min":5.7789999999999999,"15min":2.161},"min":{"1min":0.224,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":279.30500000000001,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":20.523,"5min":6.0369999999999999,"15min":2.5830000000000002},"min":{"1min":0.26800000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":279.39800000000002,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.113,"5min":4.5359999999999996,"15min":4.5359999999999996},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.68600000000000005,"15min":0.71199999999999997},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9630000000000001,"5min":4.1310000000000002,"15min":4.1310000000000002},"last":1.8140000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.65400000000000003,"15min":1.675},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.20200000000000001},"max":{"1min":2.8879999999999999,"5min":3.7250000000000001,"15min":87.787999999999997},"last":1},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.64800000000000002,"15min":1.887},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":5.1989999999999998,"5min":5.1989999999999998,"15min":89.861999999999995},"last":2.0379999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.8540000000000001,"15min":2.3149999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.004,"5min":67.738,"15min":151.036},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.0800000000000001,"15min":2.6589999999999998},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.5590000000000002,"5min":67.670000000000002,"15min":151.06899999999999},"last":1.222}]},{"osd":5,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.823,"15min":2.2890000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":8.7159999999999993,"5min":68.259,"15min":116.098},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.915,"15min":2.331},"min":{"1min":0.39900000000000002,"5min":0.378,"15min":0.315},"max":{"1min":3.04,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":2.1230000000000002,"15min":2.2450000000000001},"min":{"1min":0.38100000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":2.6030000000000002,"5min":67.349000000000004,"15min":67.349000000000004},"last":1.258},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.7529999999999999,"15min":1.8080000000000001},"min":{"1min":0.40000000000000002,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":2.1920000000000002,"5min":67.366,"15min":67.366},"last":0.36299999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.879,"15min":1.4570000000000001},"min":{"1min":0.48599999999999999,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":3.069,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.006,"15min":1.8720000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33100000000000002},"max":{"1min":2.6360000000000001,"5min":3.9860000000000002,"15min":20.902999999999999},"last":1.2849999999999999}]}]},{"osd":0,"up_from":353,"seq":1516123455564,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1728444,"kb_used_data":1486900,"kb_used_omap":37,"kb_used_meta":241498,"kb_avail":92016708,"statfs":{"total":95995035648,"available":94225108992,"internally_reserved":0,"allocated":1522585600,"data_stored":1520808261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":247294114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.83699999999999997,"15min":0.63900000000000001},"min":{"1min":0.216,"5min":0.214,"15min":0.214},"max":{"1min":2.7570000000000001,"5min":8.234,"15min":8.234},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.86899999999999999,"15min":0.73699999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.867,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.96799999999999997,"15min":0.84199999999999997},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":7.3369999999999997,"5min":12.215,"15min":12.215},"last":0.318},{"interface":"front","average":{"1min":5.1340000000000003,"5min":1.7609999999999999,"15min":1.03},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":74.698999999999998,"5min":74.698999999999998,"15min":74.698999999999998},"last":0.69299999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":6.7220000000000004,"5min":2.25,"15min":1.052},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":74.744,"5min":74.744,"15min":74.744},"last":0.36299999999999999},{"interface":"front","average":{"1min":6.8479999999999999,"5min":2.125,"15min":1.3140000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":74.777000000000001,"5min":74.777000000000001,"15min":74.777000000000001},"last":0.72399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":4.7409999999999997,"5min":4.0750000000000002,"15min":1.931},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":74.805000000000007,"5min":183.148,"15min":183.148},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":3.4119999999999999,"15min":1.583},"min":{"1min":0.38,"5min":0.31,"15min":0.31},"max":{"1min":1.8320000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.0640000000000001,"15min":0.80700000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":2.2530000000000001,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":0.39400000000000002},{"interface":"front","average":{"1min":4.774,"5min":1.7809999999999999,"15min":1.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":74.858000000000004,"5min":74.858000000000004,"15min":74.858000000000004},"last":0.79800000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.0549999999999999,"15min":0.90800000000000003},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.893,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.0029999999999999,"15min":0.81699999999999995},"min":{"1min":0.46800000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.7149999999999999,"5min":6.375,"15min":6.375},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.0209999999999999,"15min":1.2090000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.917,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.0680000000000001,"15min":1.208},"min":{"1min":0.48799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.79,"5min":6.343,"15min":6.343},"last":0.48399999999999999}]}]},{"osd":2,"up_from":359,"seq":1541893259337,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1429684,"kb_used_data":1226156,"kb_used_omap":83,"kb_used_meta":203436,"kb_avail":92315468,"statfs":{"total":95995035648,"available":94531039232,"internally_reserved":0,"allocated":1255583744,"data_stored":1253874982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":208318917},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":6.2400000000000002,"5min":2.452,"15min":1.054},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.735999999999997,"5min":74.241,"15min":74.241},"last":0.53000000000000003},{"interface":"front","average":{"1min":6.3899999999999997,"5min":2.5960000000000001,"15min":1.337},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":55.969999999999999,"5min":74.430000000000007,"15min":74.430000000000007},"last":6.3079999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.73099999999999998,"15min":0.68700000000000006},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.2080000000000002,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":3.4420000000000002},{"interface":"front","average":{"1min":1.056,"5min":1.0629999999999999,"15min":1.1240000000000001},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.397,"5min":11.397,"15min":11.397},"last":2.9260000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82299999999999995,"15min":0.749},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.2750000000000004,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":6.3609999999999998},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.871,"15min":0.83499999999999996},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":2.8900000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.92300000000000004,"15min":0.68400000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.1360000000000001,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.96799999999999997,"15min":0.871},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2430000000000003,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":3.5299999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.99299999999999999,"15min":0.96699999999999997},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.3090000000000002,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.93500000000000005,"15min":0.76500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.1680000000000001,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.1919999999999999,"15min":1.044},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.968,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":1.222},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.94099999999999995,"15min":0.71799999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.9079999999999999,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.1559999999999999,"15min":1.046},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.7869999999999999,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.0449999999999999,"15min":1.1339999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3049999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":3.4929999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926141,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1936460,"kb_used_data":1381764,"kb_used_omap":7,"kb_used_meta":554680,"kb_avail":91808692,"statfs":{"total":95995035648,"available":94012100608,"internally_reserved":0,"allocated":1414926336,"data_stored":1413249228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":567992936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.5419999999999998,"15min":1.0600000000000001},"min":{"1min":0.25600000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.7130000000000001,"5min":122.289,"15min":122.289},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":2.04,"15min":0.95099999999999996},"min":{"1min":0.218,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":1.6839999999999999,"5min":122.041,"15min":122.041},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.145,"15min":2.2130000000000001},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.187},"max":{"1min":15.025,"5min":15.025,"15min":123.866},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0289999999999999,"15min":2.1219999999999999},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.718,"5min":9.718,"15min":122.265},"last":0.497}]},{"osd":2,"last update":"Sat Sep 23 23:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.64200000000000002,"15min":0.69299999999999995},"min":{"1min":0.25,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.78400000000000003,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":1.653},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.127,"15min":0.871},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":2.9159999999999999,"5min":8.6590000000000007,"15min":8.6590000000000007},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.3559999999999999,"15min":1.9450000000000001},"min":{"1min":0.311,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":3.8450000000000002,"5min":87.046999999999997,"15min":87.046999999999997},"last":1.9690000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":2.411,"15min":1.9079999999999999},"min":{"1min":0.34699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8490000000000002,"5min":87.066000000000003,"15min":87.066000000000003},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.119,"15min":1.407},"min":{"1min":0.373,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":4.9059999999999997,"5min":6.2530000000000001,"15min":11.803000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.085,"15min":1.6140000000000001},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":0.92000000000000004,"5min":10.246,"15min":11.951000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.3420000000000001,"15min":1.706},"min":{"1min":0.40899999999999997,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":4.8840000000000003,"5min":9.2520000000000007,"15min":14.097},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.198,"15min":1.742},"min":{"1min":0.41299999999999998,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":2.6949999999999998,"5min":9.2810000000000006,"15min":12.887},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.006,"15min":1.5489999999999999},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":3.3260000000000001,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.391,"5min":1.2869999999999999,"15min":1.575},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":6.2050000000000001,"5min":8.5830000000000002,"15min":12.32},"last":2.004}]}]},{"osd":4,"up_from":30,"seq":128849019338,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519752,"kb_used_data":612288,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92225400,"statfs":{"total":95995035648,"available":94438809600,"internally_reserved":0,"allocated":626982912,"data_stored":626118348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":4.1989999999999998,"15min":1.714},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.93400000000000005,"5min":156.375,"15min":156.375},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":4.351,"15min":1.847},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.5509999999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.495}]},{"osd":1,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.196,"15min":3.3199999999999998},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":28.917000000000002,"5min":28.917000000000002,"15min":132.304},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.147,"15min":3.302},"min":{"1min":0.28000000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.731000000000002,"5min":28.731000000000002,"15min":132.453},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.155,"15min":1.0649999999999999},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.2949999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":3.25},{"interface":"front","average":{"1min":1.329,"5min":1.161,"15min":0.86499999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.3799999999999999,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.083,"15min":2.6869999999999998},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.278,"5min":10.337999999999999,"15min":164.453},"last":3.3109999999999999},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.2569999999999999,"15min":2.7530000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":3.4689999999999999,"5min":7.2519999999999998,"15min":167.05199999999999},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 23:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.036,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.214},"max":{"1min":1.492,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.92400000000000004,"15min":1.835},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":5.9409999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.34000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.208,"15min":1.294},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":5.8399999999999999,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.373},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.96999999999999997,"15min":1.161},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":1.252,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":1.2749999999999999,"15min":1.256},"min":{"1min":0.433,"5min":0.34100000000000003,"15min":0.28399999999999997},"max":{"1min":4.0990000000000002,"5min":11.045999999999999,"15min":12.84},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.1379999999999999,"15min":1.1319999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":3.1960000000000002,"5min":10.587,"15min":10.587},"last":0.61299999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823112,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648628,"kb_used_data":753068,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92096524,"statfs":{"total":95995035648,"available":94306840576,"internally_reserved":0,"allocated":771141632,"data_stored":770292772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.8859999999999999,"15min":1.032},"min":{"1min":0.26900000000000002,"5min":0.221,"15min":0.221},"max":{"1min":7.7469999999999999,"5min":42.155000000000001,"15min":42.155000000000001},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.96,"15min":1.3740000000000001},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.7000000000000002,"5min":43.378999999999998,"15min":43.378999999999998},"last":1.341}]},{"osd":1,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.753,"15min":1.984},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.0979999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.1419999999999999,"15min":2.4209999999999998},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":1.9510000000000001,"5min":7.1219999999999999,"15min":110.89},"last":1.4059999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0589999999999999,"15min":0.88700000000000001},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9450000000000003,"5min":6.601,"15min":6.601},"last":0.88},{"interface":"front","average":{"1min":1.012,"5min":1.0009999999999999,"15min":0.997},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.9530000000000003,"5min":4.9530000000000003,"15min":4.9530000000000003},"last":0.873}]},{"osd":3,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.1359999999999999,"15min":2.7440000000000002},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.6629999999999998,"5min":9.2710000000000008,"15min":98.641000000000005},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.2330000000000001,"15min":2.7919999999999998},"min":{"1min":0.35799999999999998,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":3.6789999999999998,"5min":10.965,"15min":98.906000000000006},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.0640000000000001,"15min":1.5549999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23300000000000001},"max":{"1min":1.107,"5min":7.0380000000000003,"15min":115.06699999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.97699999999999998,"15min":1.502},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.253},"max":{"1min":2.5419999999999998,"5min":9.5890000000000004,"15min":114.908},"last":0.30599999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.2490000000000001,"15min":1.2110000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":3.0600000000000001,"5min":7.4630000000000001,"15min":11.121},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.879,"15min":1.0529999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.2370000000000001,"5min":5.4909999999999997,"15min":12.137},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 23:03:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.04,"15min":1.0860000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.4770000000000001,"5min":9.5980000000000008,"15min":12.23},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.444,"15min":1.2090000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.8500000000000001,"5min":10.638999999999999,"15min":11.112},"last":0.39900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1539702784,"data_stored":1539702784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1958477824,"data_stored":1958477824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1265893376,"data_stored":1265893376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1429078016,"data_stored":1429078016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":613416960,"data_stored":613416960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":777977856,"data_stored":777977856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:04:28.735 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:04:28.736 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:04:28.736 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T23:04:28.736 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:04:28.748 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 36 16 45505 45489 78.9387 50.375 0.003271 0.012658 2023-09-23T23:04:28.817 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:28.846 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:28.901 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:28.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:29.009 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:29.109 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:29.492 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:29.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:29.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:29.519 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:46151 11.6 11:6e5198b1:::benchmark_data_smithi052_606809_object46150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.504503+0000", 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.013906323, 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013941976999999999, 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:04:29.520 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 46151 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.504503+0000", 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.504503+0000", 2023-09-23T23:04:29.521 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.504501+0000", 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.504536+0000", 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5729999999999998e-05 2023-09-23T23:04:29.522 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.504558+0000", 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1418000000000001e-05 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.504568+0000", 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.039e-05 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.523 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.507036+0000", 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024679200000000002 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.507177+0000", 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014076900000000001 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:04:29.524 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.507312+0000", 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000135733 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509524+0000", 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022116649999999998 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.525 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:46149 11.f e417/410 11:f5a4f95b:::benchmark_data_smithi052_606809_object46148:head v 417'2862, mlcod=417'2859)", 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.509718+0000", 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0086912759999999995, 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0088238640000000007, 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509718+0000", 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.526 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509718+0000", 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509717+0000", 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509741+0000", 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3112000000000002e-05 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509756+0000", 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5633000000000001e-05 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509764+0000", 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6820000000000003e-06 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509828+0000", 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3910000000000003e-05 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.509848+0000", 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9879000000000001e-05 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.529 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:46147 11.e e417/410 11:7966c294:::benchmark_data_smithi052_606809_object46146:head v 417'2871, mlcod=417'2868)", 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.512265+0000", 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.006144549, 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063463160000000003, 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512265+0000", 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512265+0000", 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512264+0000", 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512300+0000", 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6298999999999999e-05 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512313+0000", 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2840000000000001e-05 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512321+0000", 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5719999999999997e-06 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512610+0000", 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000288726 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512629+0000", 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9202999999999999e-05 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:46139 11.1 e417/410 11:8fa47347:::benchmark_data_smithi052_606809_object46138:head v 417'2930, mlcod=417'2927)", 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.512348+0000", 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0060616330000000003, 2023-09-23T23:04:29.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063310090000000003, 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512348+0000", 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512348+0000", 2023-09-23T23:04:29.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512348+0000", 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512371+0000", 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3850999999999999e-05 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512379+0000", 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2880000000000004e-06 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512383+0000", 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6149999999999999e-06 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512452+0000", 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8677000000000007e-05 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512476+0000", 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4528e-05 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:46153 11.5 e417/410 11:a59a7774:::benchmark_data_smithi052_606809_object46152:head v 417'2880, mlcod=417'2878)", 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.512458+0000", 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0059520620000000002, 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062911950000000003, 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512458+0000", 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512458+0000", 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512457+0000", 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512488+0000", 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1551999999999999e-05 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512500+0000", 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1958e-05 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.512508+0000", 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6020000000000002e-06 2023-09-23T23:04:29.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.516383+0000", 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003875388 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.516412+0000", 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8796999999999999e-05 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.542 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:46152 11.2 e417/410 11:463a2458:::benchmark_data_smithi052_606809_object46151:head v 417'2879, mlcod=417'2876)", 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.514332+0000", 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0040773770000000001, 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0044754340000000004, 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.543 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514332+0000", 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514332+0000", 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.544 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514331+0000", 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514359+0000", 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7387000000000001e-05 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514371+0000", 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2152e-05 2023-09-23T23:04:29.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514378+0000", 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4410000000000004e-06 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514445+0000", 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7034999999999998e-05 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.514466+0000", 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0981000000000001e-05 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:46155 11.2 e417/410 11:4f47ff71:::benchmark_data_smithi052_606809_object46154:head v 417'2880, mlcod=417'2877)", 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:29.518313+0000", 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 9.7048999999999999e-05, 2023-09-23T23:04:29.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00055369499999999999, 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518313+0000", 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518313+0000", 2023-09-23T23:04:29.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518312+0000", 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518339+0000", 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7934000000000001e-05 2023-09-23T23:04:29.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518353+0000", 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3417e-05 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518361+0000", 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0090000000000007e-06 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518440+0000", 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9540000000000001e-05 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:29.518464+0000", 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3592999999999999e-05 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T23:04:29.551 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 7 2023-09-23T23:04:29.552 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:04:29.552 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:29.602 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:29.632 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:29.680 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:29.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:29.750 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 37 16 46323 46307 78.1871 51.125 0.00815291 0.0127854 2023-09-23T23:04:29.788 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:29.819 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:29.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:29 smithi052 ceph-mon[180979]: pgmap v1295: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:04:29.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:29.869 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:29.869 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:29.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:29.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:29 smithi151 ceph-mon[159210]: pgmap v1295: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:04:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:29.975 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:30.077 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:30.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:29 smithi052 ceph-mon[187743]: pgmap v1295: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T23:04:30.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:30.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:30.538 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:30.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:30.653 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:30.680 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:30.729 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:30.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:30.748 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 38 16 47231 47215 77.6231 56.75 0.0141932 0.0128762 2023-09-23T23:04:30.834 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:30.863 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:30.891 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:04:30.911 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:30.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2270390600' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:30 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:31.018 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:31.100 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:31.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2270390600' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:31.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:30 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:31.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2270390600' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:31.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:30 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:31.332 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:04:31.748 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 39 16 48205 48189 77.1936 60.875 0.0106432 0.0129459 2023-09-23T23:04:31.972 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:31.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:32.076 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: pgmap v1296: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[187743]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.102 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: pgmap v1296: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T23:04:32.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:32.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:32.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T23:04:32.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:31 smithi052 ceph-mon[180979]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.104 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: pgmap v1296: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T23:04:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T23:04:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:31 smithi151 ceph-mon[159210]: failed to encode map e418 with expected crc 2023-09-23T23:04:32.748 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T23:04:32.748773+0000 min lat: 0.00102195 max lat: 0.430607 avg lat: 0.0130617 2023-09-23T23:04:32.749 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:04:32.749 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 40 16 48906 48890 76.3593 43.8125 0.00777763 0.0130617 2023-09-23T23:04:33.149 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:33.271 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:33.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:33.390 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:33.417 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:33.462 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:33.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:33.497 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:04:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:04:33] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.33.4" 2023-09-23T23:04:33.566 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:33.593 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:33.639 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:33.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:33.746 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:33.760 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 41 16 49878 49862 75.9614 60.75 0.00598908 0.0131575 2023-09-23T23:04:33.853 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:33.861 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:34.079 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:34.091 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:33 smithi052 ceph-mon[187743]: pgmap v1298: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T23:04:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:34.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:34.092 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:33 smithi052 ceph-mon[180979]: pgmap v1298: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T23:04:34.144 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:04:34.186 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:04:34.192 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:34.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:33 smithi151 ceph-mon[159210]: pgmap v1298: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T23:04:34.229 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:04:34.251 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:34.273 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:04:34.323 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:04:34.361 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:04:34.403 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:04:34.443 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:04:34.454 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.455 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:04:34.758 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 42 16 50751 50735 75.452 54.5625 0.00310837 0.013231 2023-09-23T23:04:35.187 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:04:35.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2155281933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:35.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2155281933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:35.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:35.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:35.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2155281933' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:35.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:35.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:35.280 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:35.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:35.392 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:35.422 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:35.469 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:35.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:35.577 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:35.604 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:35.758 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 43 16 51474 51458 74.7484 45.1875 0.00946743 0.013372 2023-09-23T23:04:35.908 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:35 smithi052 ceph-mon[187743]: pgmap v1299: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T23:04:36.084 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:36.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:36.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:36.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:35 smithi052 ceph-mon[180979]: pgmap v1299: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T23:04:36.196 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:36.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:35 smithi151 ceph-mon[159210]: pgmap v1299: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T23:04:36.289 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:36.600 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:36.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:36.719 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:36.747 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:36.758 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 44 16 52386 52370 74.3451 57 0.0181722 0.013443 2023-09-23T23:04:36.794 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:36.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:36.899 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:36.987 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:37.177 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:04:37.183 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:04:37.195 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:37.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:37.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:37.301 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:37.329 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:37.376 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:37.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:37.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:37.482 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:37.567 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:37.759 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 45 16 53300 53284 73.9619 57.125 0.0164589 0.0135152 2023-09-23T23:04:38.035 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:38.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:38.047 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:04:38.098 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:37 smithi052 ceph-mon[187743]: pgmap v1300: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T23:04:38.099 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:37 smithi052 ceph-mon[180979]: pgmap v1300: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T23:04:38.143 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:38.171 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:38.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:37 smithi151 ceph-mon[159210]: pgmap v1300: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T23:04:38.508 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:38.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:38.617 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:38.645 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:38.698 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:38.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:38.763 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 46 16 54092 54076 73.4239 49.5 0.011923 0.0136122 2023-09-23T23:04:38.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:38.856 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:38.911 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1300,"stamp":"2023-09-23T23:04:36.661333+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3396016032,"num_objects":51834,"num_object_clones":0,"num_object_copies":155502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51834,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":52070,"num_write_kb":3318285,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":605750,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52004,"ondisk_log_size":52004,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13334512,"kb_used_data":9747328,"kb_used_omap":137,"kb_used_meta":3586998,"kb_avail":549136400,"statfs":{"total":575970213888,"available":562315673600,"internally_reserved":0,"allocated":9981263872,"data_stored":9972774505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":3673086136},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,3,6,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":567934976,"num_objects":8666,"num_object_clones":0,"num_object_copies":25998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8666,"num_write_kb":554624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002820"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":430,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819940+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:04:19.819940+0000","last_peered":"2023-09-23T23:04:19.819940+0000","last_clean":"2023-09-23T23:04:19.819940+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:04:19.819940+0000","last_undegraded":"2023-09-23T23:04:19.819940+0000","last_fullsized":"2023-09-23T23:04:19.819940+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":88,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.798839+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:04:19.798839+0000","last_peered":"2023-09-23T23:04:19.798839+0000","last_clean":"2023-09-23T23:04:19.798839+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:04:19.798839+0000","last_undegraded":"2023-09-23T23:04:19.798839+0000","last_fullsized":"2023-09-23T23:04:19.798839+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":87,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.801273+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:04:19.801273+0000","last_peered":"2023-09-23T23:04:19.801273+0000","last_clean":"2023-09-23T23:04:19.801273+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:04:19.801273+0000","last_undegraded":"2023-09-23T23:04:19.801273+0000","last_fullsized":"2023-09-23T23:04:19.801273+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":416,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.798989+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:04:19.798989+0000","last_peered":"2023-09-23T23:04:19.798989+0000","last_clean":"2023-09-23T23:04:19.798989+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:04:19.798989+0000","last_undegraded":"2023-09-23T23:04:19.798989+0000","last_fullsized":"2023-09-23T23:04:19.798989+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.807481+0000","last_change":"2023-09-23T23:03:51.528123+0000","last_active":"2023-09-23T23:04:19.807481+0000","last_peered":"2023-09-23T23:04:19.807481+0000","last_clean":"2023-09-23T23:04:19.807481+0000","last_became_active":"2023-09-23T23:03:51.527770+0000","last_became_peered":"2023-09-23T23:03:51.527770+0000","last_unstale":"2023-09-23T23:04:19.807481+0000","last_undegraded":"2023-09-23T23:04:19.807481+0000","last_fullsized":"2023-09-23T23:04:19.807481+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:32.789048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":90,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.308271+0000","last_change":"2023-09-23T22:58:06.763109+0000","last_active":"2023-09-23T23:04:20.308271+0000","last_peered":"2023-09-23T23:04:20.308271+0000","last_clean":"2023-09-23T23:04:20.308271+0000","last_became_active":"2023-09-23T22:58:06.762913+0000","last_became_peered":"2023-09-23T22:58:06.762913+0000","last_unstale":"2023-09-23T23:04:20.308271+0000","last_undegraded":"2023-09-23T23:04:20.308271+0000","last_fullsized":"2023-09-23T23:04:20.308271+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:27:32.573626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":353,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817293+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:04:19.817293+0000","last_peered":"2023-09-23T23:04:19.817293+0000","last_clean":"2023-09-23T23:04:19.817293+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:04:19.817293+0000","last_undegraded":"2023-09-23T23:04:19.817293+0000","last_fullsized":"2023-09-23T23:04:19.817293+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":98,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.899931+0000","last_change":"2023-09-23T22:58:24.254368+0000","last_active":"2023-09-23T23:04:30.899931+0000","last_peered":"2023-09-23T23:04:30.899931+0000","last_clean":"2023-09-23T23:04:30.899931+0000","last_became_active":"2023-09-23T22:58:24.253995+0000","last_became_peered":"2023-09-23T22:58:24.253995+0000","last_unstale":"2023-09-23T23:04:30.899931+0000","last_undegraded":"2023-09-23T23:04:30.899931+0000","last_fullsized":"2023-09-23T23:04:30.899931+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:03.968173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":314,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817079+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:04:19.817079+0000","last_peered":"2023-09-23T23:04:19.817079+0000","last_clean":"2023-09-23T23:04:19.817079+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:04:19.817079+0000","last_undegraded":"2023-09-23T23:04:19.817079+0000","last_fullsized":"2023-09-23T23:04:19.817079+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":103,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.900164+0000","last_change":"2023-09-23T22:58:06.747411+0000","last_active":"2023-09-23T23:04:30.900164+0000","last_peered":"2023-09-23T23:04:30.900164+0000","last_clean":"2023-09-23T23:04:30.900164+0000","last_became_active":"2023-09-23T22:58:06.747188+0000","last_became_peered":"2023-09-23T22:58:06.747188+0000","last_unstale":"2023-09-23T23:04:30.900164+0000","last_undegraded":"2023-09-23T23:04:30.900164+0000","last_fullsized":"2023-09-23T23:04:30.900164+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:02.553440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":123,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.798769+0000","last_change":"2023-09-23T23:03:51.536661+0000","last_active":"2023-09-23T23:04:19.798769+0000","last_peered":"2023-09-23T23:04:19.798769+0000","last_clean":"2023-09-23T23:04:19.798769+0000","last_became_active":"2023-09-23T23:03:51.532360+0000","last_became_peered":"2023-09-23T23:03:51.532360+0000","last_unstale":"2023-09-23T23:04:19.798769+0000","last_undegraded":"2023-09-23T23:04:19.798769+0000","last_fullsized":"2023-09-23T23:04:19.798769+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:55.428059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":164,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.811609+0000","last_change":"2023-09-23T22:58:24.248812+0000","last_active":"2023-09-23T23:04:19.811609+0000","last_peered":"2023-09-23T23:04:19.811609+0000","last_clean":"2023-09-23T23:04:19.811609+0000","last_became_active":"2023-09-23T22:58:24.248485+0000","last_became_peered":"2023-09-23T22:58:24.248485+0000","last_unstale":"2023-09-23T23:04:19.811609+0000","last_undegraded":"2023-09-23T23:04:19.811609+0000","last_fullsized":"2023-09-23T23:04:19.811609+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:43.033445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":489,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820395+0000","last_change":"2023-09-23T22:58:24.254285+0000","last_active":"2023-09-23T23:04:19.820395+0000","last_peered":"2023-09-23T23:04:19.820395+0000","last_clean":"2023-09-23T23:04:19.820395+0000","last_became_active":"2023-09-23T22:58:24.253823+0000","last_became_peered":"2023-09-23T22:58:24.253823+0000","last_unstale":"2023-09-23T23:04:19.820395+0000","last_undegraded":"2023-09-23T23:04:19.820395+0000","last_fullsized":"2023-09-23T23:04:19.820395+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":392,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820197+0000","last_change":"2023-09-23T22:56:20.208176+0000","last_active":"2023-09-23T23:04:19.820197+0000","last_peered":"2023-09-23T23:04:19.820197+0000","last_clean":"2023-09-23T23:04:19.820197+0000","last_became_active":"2023-09-23T22:56:20.207846+0000","last_became_peered":"2023-09-23T22:56:20.207846+0000","last_unstale":"2023-09-23T23:04:19.820197+0000","last_undegraded":"2023-09-23T23:04:19.820197+0000","last_fullsized":"2023-09-23T23:04:19.820197+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":318,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820219+0000","last_change":"2023-09-23T22:58:27.289031+0000","last_active":"2023-09-23T23:04:19.820219+0000","last_peered":"2023-09-23T23:04:19.820219+0000","last_clean":"2023-09-23T23:04:19.820219+0000","last_became_active":"2023-09-23T22:58:27.288705+0000","last_became_peered":"2023-09-23T22:58:27.288705+0000","last_unstale":"2023-09-23T23:04:19.820219+0000","last_undegraded":"2023-09-23T23:04:19.820219+0000","last_fullsized":"2023-09-23T23:04:19.820219+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:26.453462+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:04:26.453462+0000","last_peered":"2023-09-23T23:04:26.453462+0000","last_clean":"2023-09-23T23:04:26.453462+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:04:26.453462+0000","last_undegraded":"2023-09-23T23:04:26.453462+0000","last_fullsized":"2023-09-23T23:04:26.453462+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":409,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.900408+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:04:30.900408+0000","last_peered":"2023-09-23T23:04:30.900408+0000","last_clean":"2023-09-23T23:04:30.900408+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:04:30.900408+0000","last_undegraded":"2023-09-23T23:04:30.900408+0000","last_fullsized":"2023-09-23T23:04:30.900408+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":318,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817313+0000","last_change":"2023-09-23T22:58:06.767000+0000","last_active":"2023-09-23T23:04:19.817313+0000","last_peered":"2023-09-23T23:04:19.817313+0000","last_clean":"2023-09-23T23:04:19.817313+0000","last_became_active":"2023-09-23T22:58:06.766659+0000","last_became_peered":"2023-09-23T22:58:06.766659+0000","last_unstale":"2023-09-23T23:04:19.817313+0000","last_undegraded":"2023-09-23T23:04:19.817313+0000","last_fullsized":"2023-09-23T23:04:19.817313+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":303,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819879+0000","last_change":"2023-09-23T22:58:06.753148+0000","last_active":"2023-09-23T23:04:19.819879+0000","last_peered":"2023-09-23T23:04:19.819879+0000","last_clean":"2023-09-23T23:04:19.819879+0000","last_became_active":"2023-09-23T22:58:06.752825+0000","last_became_peered":"2023-09-23T22:58:06.752825+0000","last_unstale":"2023-09-23T23:04:19.819879+0000","last_undegraded":"2023-09-23T23:04:19.819879+0000","last_fullsized":"2023-09-23T23:04:19.819879+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":432,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.799129+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:04:19.799129+0000","last_peered":"2023-09-23T23:04:19.799129+0000","last_clean":"2023-09-23T23:04:19.799129+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:04:19.799129+0000","last_undegraded":"2023-09-23T23:04:19.799129+0000","last_fullsized":"2023-09-23T23:04:19.799129+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":137,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806137+0000","last_change":"2023-09-23T22:58:06.763323+0000","last_active":"2023-09-23T23:04:19.806137+0000","last_peered":"2023-09-23T23:04:19.806137+0000","last_clean":"2023-09-23T23:04:19.806137+0000","last_became_active":"2023-09-23T22:58:06.762863+0000","last_became_peered":"2023-09-23T22:58:06.762863+0000","last_unstale":"2023-09-23T23:04:19.806137+0000","last_undegraded":"2023-09-23T23:04:19.806137+0000","last_fullsized":"2023-09-23T23:04:19.806137+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:12.010419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":112,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.799438+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:04:19.799438+0000","last_peered":"2023-09-23T23:04:19.799438+0000","last_clean":"2023-09-23T23:04:19.799438+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:04:19.799438+0000","last_undegraded":"2023-09-23T23:04:19.799438+0000","last_fullsized":"2023-09-23T23:04:19.799438+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"418'3193","reported_seq":3209,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:35.414625+0000","last_change":"2023-09-23T23:03:43.131702+0000","last_active":"2023-09-23T23:04:35.414625+0000","last_peered":"2023-09-23T23:04:35.414625+0000","last_clean":"2023-09-23T23:04:35.414625+0000","last_became_active":"2023-09-23T23:03:43.131323+0000","last_became_peered":"2023-09-23T23:03:43.131323+0000","last_unstale":"2023-09-23T23:04:35.414625+0000","last_undegraded":"2023-09-23T23:04:35.414625+0000","last_fullsized":"2023-09-23T23:04:35.414625+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3193,"log_dups_size":0,"ondisk_log_size":3193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209256448,"num_objects":3193,"num_object_clones":0,"num_object_copies":9579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3193,"num_write_kb":204352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":302,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820835+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:04:19.820835+0000","last_peered":"2023-09-23T23:04:19.820835+0000","last_clean":"2023-09-23T23:04:19.820835+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:04:19.820835+0000","last_undegraded":"2023-09-23T23:04:19.820835+0000","last_fullsized":"2023-09-23T23:04:19.820835+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":288,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817982+0000","last_change":"2023-09-23T22:57:26.518101+0000","last_active":"2023-09-23T23:04:19.817982+0000","last_peered":"2023-09-23T23:04:19.817982+0000","last_clean":"2023-09-23T23:04:19.817982+0000","last_became_active":"2023-09-23T22:57:26.517651+0000","last_became_peered":"2023-09-23T22:57:26.517651+0000","last_unstale":"2023-09-23T23:04:19.817982+0000","last_undegraded":"2023-09-23T23:04:19.817982+0000","last_fullsized":"2023-09-23T23:04:19.817982+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"418'3163","reported_seq":3180,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:35.248493+0000","last_change":"2023-09-23T23:03:43.132133+0000","last_active":"2023-09-23T23:04:35.248493+0000","last_peered":"2023-09-23T23:04:35.248493+0000","last_clean":"2023-09-23T23:04:35.248493+0000","last_became_active":"2023-09-23T23:03:43.131741+0000","last_became_peered":"2023-09-23T23:03:43.131741+0000","last_unstale":"2023-09-23T23:04:35.248493+0000","last_undegraded":"2023-09-23T23:04:35.248493+0000","last_fullsized":"2023-09-23T23:04:35.248493+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3163,"log_dups_size":0,"ondisk_log_size":3163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:36.498718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207290368,"num_objects":3163,"num_object_clones":0,"num_object_copies":9489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3163,"num_write_kb":202432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":19,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.811394+0000","last_change":"2023-09-23T23:03:51.533976+0000","last_active":"2023-09-23T23:04:19.811394+0000","last_peered":"2023-09-23T23:04:19.811394+0000","last_clean":"2023-09-23T23:04:19.811394+0000","last_became_active":"2023-09-23T23:03:51.528701+0000","last_became_peered":"2023-09-23T23:03:51.528701+0000","last_unstale":"2023-09-23T23:04:19.811394+0000","last_undegraded":"2023-09-23T23:04:19.811394+0000","last_fullsized":"2023-09-23T23:04:19.811394+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.921280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":425,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819856+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:04:19.819856+0000","last_peered":"2023-09-23T23:04:19.819856+0000","last_clean":"2023-09-23T23:04:19.819856+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:04:19.819856+0000","last_undegraded":"2023-09-23T23:04:19.819856+0000","last_fullsized":"2023-09-23T23:04:19.819856+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"418'3212","reported_seq":3228,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:35.354508+0000","last_change":"2023-09-23T23:03:42.889536+0000","last_active":"2023-09-23T23:04:35.354508+0000","last_peered":"2023-09-23T23:04:35.354508+0000","last_clean":"2023-09-23T23:04:35.354508+0000","last_became_active":"2023-09-23T23:03:42.889213+0000","last_became_peered":"2023-09-23T23:03:42.889213+0000","last_unstale":"2023-09-23T23:04:35.354508+0000","last_undegraded":"2023-09-23T23:04:35.354508+0000","last_fullsized":"2023-09-23T23:04:35.354508+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3212,"log_dups_size":0,"ondisk_log_size":3212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210501632,"num_objects":3212,"num_object_clones":0,"num_object_copies":9636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3212,"num_write_kb":205568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":169,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819062+0000","last_change":"2023-09-23T22:57:26.614010+0000","last_active":"2023-09-23T23:04:19.819062+0000","last_peered":"2023-09-23T23:04:19.819062+0000","last_clean":"2023-09-23T23:04:19.819062+0000","last_became_active":"2023-09-23T22:57:26.526872+0000","last_became_peered":"2023-09-23T22:57:26.526872+0000","last_unstale":"2023-09-23T23:04:19.819062+0000","last_undegraded":"2023-09-23T23:04:19.819062+0000","last_fullsized":"2023-09-23T23:04:19.819062+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":287,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.816819+0000","last_change":"2023-09-23T22:58:06.767345+0000","last_active":"2023-09-23T23:04:19.816819+0000","last_peered":"2023-09-23T23:04:19.816819+0000","last_clean":"2023-09-23T23:04:19.816819+0000","last_became_active":"2023-09-23T22:58:06.766151+0000","last_became_peered":"2023-09-23T22:58:06.766151+0000","last_unstale":"2023-09-23T23:04:19.816819+0000","last_undegraded":"2023-09-23T23:04:19.816819+0000","last_fullsized":"2023-09-23T23:04:19.816819+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"418'3286","reported_seq":3303,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.398728+0000","last_change":"2023-09-23T23:03:43.136203+0000","last_active":"2023-09-23T23:04:36.398728+0000","last_peered":"2023-09-23T23:04:36.398728+0000","last_clean":"2023-09-23T23:04:36.398728+0000","last_became_active":"2023-09-23T23:03:43.135918+0000","last_became_peered":"2023-09-23T23:03:43.135918+0000","last_unstale":"2023-09-23T23:04:36.398728+0000","last_undegraded":"2023-09-23T23:04:36.398728+0000","last_fullsized":"2023-09-23T23:04:36.398728+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3286,"log_dups_size":0,"ondisk_log_size":3286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:11.372248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215351296,"num_objects":3286,"num_object_clones":0,"num_object_copies":9858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3286,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.798332+0000","last_change":"2023-09-23T22:57:26.525980+0000","last_active":"2023-09-23T23:04:19.798332+0000","last_peered":"2023-09-23T23:04:19.798332+0000","last_clean":"2023-09-23T23:04:19.798332+0000","last_became_active":"2023-09-23T22:57:26.525691+0000","last_became_peered":"2023-09-23T22:57:26.525691+0000","last_unstale":"2023-09-23T23:04:19.798332+0000","last_undegraded":"2023-09-23T23:04:19.798332+0000","last_fullsized":"2023-09-23T23:04:19.798332+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:43.131585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":318,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806230+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:04:19.806230+0000","last_peered":"2023-09-23T23:04:19.806230+0000","last_clean":"2023-09-23T23:04:19.806230+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:04:19.806230+0000","last_undegraded":"2023-09-23T23:04:19.806230+0000","last_fullsized":"2023-09-23T23:04:19.806230+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"418'3226","reported_seq":3242,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:35.400454+0000","last_change":"2023-09-23T23:03:43.131843+0000","last_active":"2023-09-23T23:04:35.400454+0000","last_peered":"2023-09-23T23:04:35.400454+0000","last_clean":"2023-09-23T23:04:35.400454+0000","last_became_active":"2023-09-23T23:03:43.131420+0000","last_became_peered":"2023-09-23T23:03:43.131420+0000","last_unstale":"2023-09-23T23:04:35.400454+0000","last_undegraded":"2023-09-23T23:04:35.400454+0000","last_fullsized":"2023-09-23T23:04:35.400454+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3226,"log_dups_size":0,"ondisk_log_size":3226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211419136,"num_objects":3226,"num_object_clones":0,"num_object_copies":9678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3226,"num_write_kb":206464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":108,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.797795+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:04:19.797795+0000","last_peered":"2023-09-23T23:04:19.797795+0000","last_clean":"2023-09-23T23:04:19.797795+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:04:19.797795+0000","last_undegraded":"2023-09-23T23:04:19.797795+0000","last_fullsized":"2023-09-23T23:04:19.797795+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":411,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820371+0000","last_change":"2023-09-23T22:57:27.782918+0000","last_active":"2023-09-23T23:04:19.820371+0000","last_peered":"2023-09-23T23:04:19.820371+0000","last_clean":"2023-09-23T23:04:19.820371+0000","last_became_active":"2023-09-23T22:57:27.780777+0000","last_became_peered":"2023-09-23T22:57:27.780777+0000","last_unstale":"2023-09-23T23:04:19.820371+0000","last_undegraded":"2023-09-23T23:04:19.820371+0000","last_fullsized":"2023-09-23T23:04:19.820371+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"418'3251","reported_seq":3268,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.349223+0000","last_change":"2023-09-23T23:03:42.889763+0000","last_active":"2023-09-23T23:04:36.349223+0000","last_peered":"2023-09-23T23:04:36.349223+0000","last_clean":"2023-09-23T23:04:36.349223+0000","last_became_active":"2023-09-23T23:03:42.889481+0000","last_became_peered":"2023-09-23T23:03:42.889481+0000","last_unstale":"2023-09-23T23:04:36.349223+0000","last_undegraded":"2023-09-23T23:04:36.349223+0000","last_fullsized":"2023-09-23T23:04:36.349223+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3251,"log_dups_size":0,"ondisk_log_size":3251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:57.841265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213057536,"num_objects":3251,"num_object_clones":0,"num_object_copies":9753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3251,"num_write_kb":208064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":201,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820047+0000","last_change":"2023-09-23T22:57:26.510867+0000","last_active":"2023-09-23T23:04:19.820047+0000","last_peered":"2023-09-23T23:04:19.820047+0000","last_clean":"2023-09-23T23:04:19.820047+0000","last_became_active":"2023-09-23T22:57:26.510510+0000","last_became_peered":"2023-09-23T22:57:26.510510+0000","last_unstale":"2023-09-23T23:04:19.820047+0000","last_undegraded":"2023-09-23T23:04:19.820047+0000","last_fullsized":"2023-09-23T23:04:19.820047+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":428,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820504+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:04:19.820504+0000","last_peered":"2023-09-23T23:04:19.820504+0000","last_clean":"2023-09-23T23:04:19.820504+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:04:19.820504+0000","last_undegraded":"2023-09-23T23:04:19.820504+0000","last_fullsized":"2023-09-23T23:04:19.820504+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"418'3323","reported_seq":3340,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.424624+0000","last_change":"2023-09-23T23:03:42.889334+0000","last_active":"2023-09-23T23:04:36.424624+0000","last_peered":"2023-09-23T23:04:36.424624+0000","last_clean":"2023-09-23T23:04:36.424624+0000","last_became_active":"2023-09-23T23:03:42.889046+0000","last_became_peered":"2023-09-23T23:03:42.889046+0000","last_unstale":"2023-09-23T23:04:36.424624+0000","last_undegraded":"2023-09-23T23:04:36.424624+0000","last_fullsized":"2023-09-23T23:04:36.424624+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3323,"log_dups_size":0,"ondisk_log_size":3323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:44.256081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217776128,"num_objects":3323,"num_object_clones":0,"num_object_copies":9969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3323,"num_write_kb":212672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":441,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819688+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:04:19.819688+0000","last_peered":"2023-09-23T23:04:19.819688+0000","last_clean":"2023-09-23T23:04:19.819688+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:04:19.819688+0000","last_undegraded":"2023-09-23T23:04:19.819688+0000","last_fullsized":"2023-09-23T23:04:19.819688+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":419,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820505+0000","last_change":"2023-09-23T22:57:26.525080+0000","last_active":"2023-09-23T23:04:19.820505+0000","last_peered":"2023-09-23T23:04:19.820505+0000","last_clean":"2023-09-23T23:04:19.820505+0000","last_became_active":"2023-09-23T22:57:26.524724+0000","last_became_peered":"2023-09-23T22:57:26.524724+0000","last_unstale":"2023-09-23T23:04:19.820505+0000","last_undegraded":"2023-09-23T23:04:19.820505+0000","last_fullsized":"2023-09-23T23:04:19.820505+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"418'3185","reported_seq":3201,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.421705+0000","last_change":"2023-09-23T23:03:43.132462+0000","last_active":"2023-09-23T23:04:36.421705+0000","last_peered":"2023-09-23T23:04:36.421705+0000","last_clean":"2023-09-23T23:04:36.421705+0000","last_became_active":"2023-09-23T23:03:43.132267+0000","last_became_peered":"2023-09-23T23:03:43.132267+0000","last_unstale":"2023-09-23T23:04:36.421705+0000","last_undegraded":"2023-09-23T23:04:36.421705+0000","last_fullsized":"2023-09-23T23:04:36.421705+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3185,"log_dups_size":0,"ondisk_log_size":3185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:49.697445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208732160,"num_objects":3185,"num_object_clones":0,"num_object_copies":9555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3185,"num_write_kb":203840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":89,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.797497+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:04:19.797497+0000","last_peered":"2023-09-23T23:04:19.797497+0000","last_clean":"2023-09-23T23:04:19.797497+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:04:19.797497+0000","last_undegraded":"2023-09-23T23:04:19.797497+0000","last_fullsized":"2023-09-23T23:04:19.797497+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":322,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.812861+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:04:19.812861+0000","last_peered":"2023-09-23T23:04:19.812861+0000","last_clean":"2023-09-23T23:04:19.812861+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:04:19.812861+0000","last_undegraded":"2023-09-23T23:04:19.812861+0000","last_fullsized":"2023-09-23T23:04:19.812861+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"418'3273","reported_seq":3290,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.382173+0000","last_change":"2023-09-23T23:03:43.132020+0000","last_active":"2023-09-23T23:04:36.382173+0000","last_peered":"2023-09-23T23:04:36.382173+0000","last_clean":"2023-09-23T23:04:36.382173+0000","last_became_active":"2023-09-23T23:03:43.131692+0000","last_became_peered":"2023-09-23T23:03:43.131692+0000","last_unstale":"2023-09-23T23:04:36.382173+0000","last_undegraded":"2023-09-23T23:04:36.382173+0000","last_fullsized":"2023-09-23T23:04:36.382173+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3273,"log_dups_size":0,"ondisk_log_size":3273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:13.311701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3273,"num_write_kb":209472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":326,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.811544+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:04:19.811544+0000","last_peered":"2023-09-23T23:04:19.811544+0000","last_clean":"2023-09-23T23:04:19.811544+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:04:19.811544+0000","last_undegraded":"2023-09-23T23:04:19.811544+0000","last_fullsized":"2023-09-23T23:04:19.811544+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":328,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817236+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:04:19.817236+0000","last_peered":"2023-09-23T23:04:19.817236+0000","last_clean":"2023-09-23T23:04:19.817236+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:04:19.817236+0000","last_undegraded":"2023-09-23T23:04:19.817236+0000","last_fullsized":"2023-09-23T23:04:19.817236+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":141,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.816784+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:04:19.816784+0000","last_peered":"2023-09-23T23:04:19.816784+0000","last_clean":"2023-09-23T23:04:19.816784+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:04:19.816784+0000","last_undegraded":"2023-09-23T23:04:19.816784+0000","last_fullsized":"2023-09-23T23:04:19.816784+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"418'3202","reported_seq":3219,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.426643+0000","last_change":"2023-09-23T23:03:43.132707+0000","last_active":"2023-09-23T23:04:36.426643+0000","last_peered":"2023-09-23T23:04:36.426643+0000","last_clean":"2023-09-23T23:04:36.426643+0000","last_became_active":"2023-09-23T23:03:43.132372+0000","last_became_peered":"2023-09-23T23:03:43.132372+0000","last_unstale":"2023-09-23T23:04:36.426643+0000","last_undegraded":"2023-09-23T23:04:36.426643+0000","last_fullsized":"2023-09-23T23:04:36.426643+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3202,"log_dups_size":0,"ondisk_log_size":3202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.951582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209846272,"num_objects":3202,"num_object_clones":0,"num_object_copies":9606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3202,"num_write_kb":204928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":90,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.308170+0000","last_change":"2023-09-23T22:58:06.757448+0000","last_active":"2023-09-23T23:04:20.308170+0000","last_peered":"2023-09-23T23:04:20.308170+0000","last_clean":"2023-09-23T23:04:20.308170+0000","last_became_active":"2023-09-23T22:58:06.757099+0000","last_became_peered":"2023-09-23T22:58:06.757099+0000","last_unstale":"2023-09-23T23:04:20.308170+0000","last_undegraded":"2023-09-23T23:04:20.308170+0000","last_fullsized":"2023-09-23T23:04:20.308170+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:25.839892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":438,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820290+0000","last_change":"2023-09-23T22:58:06.752460+0000","last_active":"2023-09-23T23:04:19.820290+0000","last_peered":"2023-09-23T23:04:19.820290+0000","last_clean":"2023-09-23T23:04:19.820290+0000","last_became_active":"2023-09-23T22:58:06.751954+0000","last_became_peered":"2023-09-23T22:58:06.751954+0000","last_unstale":"2023-09-23T23:04:19.820290+0000","last_undegraded":"2023-09-23T23:04:19.820290+0000","last_fullsized":"2023-09-23T23:04:19.820290+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":312,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.888469+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:04:30.888469+0000","last_peered":"2023-09-23T23:04:30.888469+0000","last_clean":"2023-09-23T23:04:30.888469+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:04:30.888469+0000","last_undegraded":"2023-09-23T23:04:30.888469+0000","last_fullsized":"2023-09-23T23:04:30.888469+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"418'3310","reported_seq":3327,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.336598+0000","last_change":"2023-09-23T23:03:42.889306+0000","last_active":"2023-09-23T23:04:36.336598+0000","last_peered":"2023-09-23T23:04:36.336598+0000","last_clean":"2023-09-23T23:04:36.336598+0000","last_became_active":"2023-09-23T23:03:42.888948+0000","last_became_peered":"2023-09-23T23:03:42.888948+0000","last_unstale":"2023-09-23T23:04:36.336598+0000","last_undegraded":"2023-09-23T23:04:36.336598+0000","last_fullsized":"2023-09-23T23:04:36.336598+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3310,"log_dups_size":0,"ondisk_log_size":3310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:31:56.393948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216924160,"num_objects":3310,"num_object_clones":0,"num_object_copies":9930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3310,"num_write_kb":211840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":387,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.888283+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:04:30.888283+0000","last_peered":"2023-09-23T23:04:30.888283+0000","last_clean":"2023-09-23T23:04:30.888283+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:04:30.888283+0000","last_undegraded":"2023-09-23T23:04:30.888283+0000","last_fullsized":"2023-09-23T23:04:30.888283+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.816710+0000","last_change":"2023-09-23T23:03:51.532583+0000","last_active":"2023-09-23T23:04:19.816710+0000","last_peered":"2023-09-23T23:04:19.816710+0000","last_clean":"2023-09-23T23:04:19.816710+0000","last_became_active":"2023-09-23T23:03:51.532300+0000","last_became_peered":"2023-09-23T23:03:51.532300+0000","last_unstale":"2023-09-23T23:04:19.816710+0000","last_undegraded":"2023-09-23T23:04:19.816710+0000","last_fullsized":"2023-09-23T23:04:19.816710+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":328,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817366+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:04:19.817366+0000","last_peered":"2023-09-23T23:04:19.817366+0000","last_clean":"2023-09-23T23:04:19.817366+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:04:19.817366+0000","last_undegraded":"2023-09-23T23:04:19.817366+0000","last_fullsized":"2023-09-23T23:04:19.817366+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"418'3304","reported_seq":3321,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.433738+0000","last_change":"2023-09-23T23:03:43.131943+0000","last_active":"2023-09-23T23:04:36.433738+0000","last_peered":"2023-09-23T23:04:36.433738+0000","last_clean":"2023-09-23T23:04:36.433738+0000","last_became_active":"2023-09-23T23:03:43.131668+0000","last_became_peered":"2023-09-23T23:03:43.131668+0000","last_unstale":"2023-09-23T23:04:36.433738+0000","last_undegraded":"2023-09-23T23:04:36.433738+0000","last_fullsized":"2023-09-23T23:04:36.433738+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3304,"log_dups_size":0,"ondisk_log_size":3304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:16.104887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216530944,"num_objects":3304,"num_object_clones":0,"num_object_copies":9912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3304,"num_write_kb":211456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":401,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.816946+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:04:19.816946+0000","last_peered":"2023-09-23T23:04:19.816946+0000","last_clean":"2023-09-23T23:04:19.816946+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:04:19.816946+0000","last_undegraded":"2023-09-23T23:04:19.816946+0000","last_fullsized":"2023-09-23T23:04:19.816946+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":531,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806556+0000","last_change":"2023-09-23T23:03:36.322526+0000","last_active":"2023-09-23T23:04:19.806556+0000","last_peered":"2023-09-23T23:04:19.806556+0000","last_clean":"2023-09-23T23:04:19.806556+0000","last_became_active":"2023-09-23T23:03:36.322065+0000","last_became_peered":"2023-09-23T23:03:36.322065+0000","last_unstale":"2023-09-23T23:04:19.806556+0000","last_undegraded":"2023-09-23T23:04:19.806556+0000","last_fullsized":"2023-09-23T23:04:19.806556+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:55:04.195321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.798224+0000","last_change":"2023-09-23T22:57:26.514329+0000","last_active":"2023-09-23T23:04:19.798224+0000","last_peered":"2023-09-23T23:04:19.798224+0000","last_clean":"2023-09-23T23:04:19.798224+0000","last_became_active":"2023-09-23T22:57:26.513968+0000","last_became_peered":"2023-09-23T22:57:26.513968+0000","last_unstale":"2023-09-23T23:04:19.798224+0000","last_undegraded":"2023-09-23T23:04:19.798224+0000","last_fullsized":"2023-09-23T23:04:19.798224+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:37.107651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":328,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820595+0000","last_change":"2023-09-23T22:58:06.752560+0000","last_active":"2023-09-23T23:04:19.820595+0000","last_peered":"2023-09-23T23:04:19.820595+0000","last_clean":"2023-09-23T23:04:19.820595+0000","last_became_active":"2023-09-23T22:58:06.752154+0000","last_became_peered":"2023-09-23T22:58:06.752154+0000","last_unstale":"2023-09-23T23:04:19.820595+0000","last_undegraded":"2023-09-23T23:04:19.820595+0000","last_fullsized":"2023-09-23T23:04:19.820595+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"418'3240","reported_seq":3257,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.426858+0000","last_change":"2023-09-23T23:03:43.133127+0000","last_active":"2023-09-23T23:04:36.426858+0000","last_peered":"2023-09-23T23:04:36.426858+0000","last_clean":"2023-09-23T23:04:36.426858+0000","last_became_active":"2023-09-23T23:03:43.132709+0000","last_became_peered":"2023-09-23T23:03:43.132709+0000","last_unstale":"2023-09-23T23:04:36.426858+0000","last_undegraded":"2023-09-23T23:04:36.426858+0000","last_fullsized":"2023-09-23T23:04:36.426858+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3240,"log_dups_size":0,"ondisk_log_size":3240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:35.116957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212336640,"num_objects":3240,"num_object_clones":0,"num_object_copies":9720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3240,"num_write_kb":207360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":410,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806230+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:04:19.806230+0000","last_peered":"2023-09-23T23:04:19.806230+0000","last_clean":"2023-09-23T23:04:19.806230+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:04:19.806230+0000","last_undegraded":"2023-09-23T23:04:19.806230+0000","last_fullsized":"2023-09-23T23:04:19.806230+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":381,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.308142+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:04:20.308142+0000","last_peered":"2023-09-23T23:04:20.308142+0000","last_clean":"2023-09-23T23:04:20.308142+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:04:20.308142+0000","last_undegraded":"2023-09-23T23:04:20.308142+0000","last_fullsized":"2023-09-23T23:04:20.308142+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":341,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.890607+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:04:30.890607+0000","last_peered":"2023-09-23T23:04:30.890607+0000","last_clean":"2023-09-23T23:04:30.890607+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:04:30.890607+0000","last_undegraded":"2023-09-23T23:04:30.890607+0000","last_fullsized":"2023-09-23T23:04:30.890607+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"418'3232","reported_seq":3249,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.405197+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:04:36.405197+0000","last_peered":"2023-09-23T23:04:36.405197+0000","last_clean":"2023-09-23T23:04:36.405197+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:04:36.405197+0000","last_undegraded":"2023-09-23T23:04:36.405197+0000","last_fullsized":"2023-09-23T23:04:36.405197+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3232,"log_dups_size":0,"ondisk_log_size":3232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211812352,"num_objects":3232,"num_object_clones":0,"num_object_copies":9696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3232,"num_write_kb":206848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":356,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.308222+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:04:20.308222+0000","last_peered":"2023-09-23T23:04:20.308222+0000","last_clean":"2023-09-23T23:04:20.308222+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:04:20.308222+0000","last_undegraded":"2023-09-23T23:04:20.308222+0000","last_fullsized":"2023-09-23T23:04:20.308222+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":264,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.888842+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:04:30.888842+0000","last_peered":"2023-09-23T23:04:30.888842+0000","last_clean":"2023-09-23T23:04:30.888842+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:04:30.888842+0000","last_undegraded":"2023-09-23T23:04:30.888842+0000","last_fullsized":"2023-09-23T23:04:30.888842+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":231,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.817549+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:04:19.817549+0000","last_peered":"2023-09-23T23:04:19.817549+0000","last_clean":"2023-09-23T23:04:19.817549+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:04:19.817549+0000","last_undegraded":"2023-09-23T23:04:19.817549+0000","last_fullsized":"2023-09-23T23:04:19.817549+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"418'3226","reported_seq":3243,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:36.395827+0000","last_change":"2023-09-23T23:03:43.132731+0000","last_active":"2023-09-23T23:04:36.395827+0000","last_peered":"2023-09-23T23:04:36.395827+0000","last_clean":"2023-09-23T23:04:36.395827+0000","last_became_active":"2023-09-23T23:03:43.132451+0000","last_became_peered":"2023-09-23T23:03:43.132451+0000","last_unstale":"2023-09-23T23:04:36.395827+0000","last_undegraded":"2023-09-23T23:04:36.395827+0000","last_fullsized":"2023-09-23T23:04:36.395827+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3226,"log_dups_size":0,"ondisk_log_size":3226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:51.837853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211419136,"num_objects":3226,"num_object_clones":0,"num_object_copies":9678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3226,"num_write_kb":206464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":113,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.799549+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:04:19.799549+0000","last_peered":"2023-09-23T23:04:19.799549+0000","last_clean":"2023-09-23T23:04:19.799549+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:04:19.799549+0000","last_undegraded":"2023-09-23T23:04:19.799549+0000","last_fullsized":"2023-09-23T23:04:19.799549+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820722+0000","last_change":"2023-09-23T22:57:26.524864+0000","last_active":"2023-09-23T23:04:19.820722+0000","last_peered":"2023-09-23T23:04:19.820722+0000","last_clean":"2023-09-23T23:04:19.820722+0000","last_became_active":"2023-09-23T22:57:26.524545+0000","last_became_peered":"2023-09-23T22:57:26.524545+0000","last_unstale":"2023-09-23T23:04:19.820722+0000","last_undegraded":"2023-09-23T23:04:19.820722+0000","last_fullsized":"2023-09-23T23:04:19.820722+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.797255+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:04:19.797255+0000","last_peered":"2023-09-23T23:04:19.797255+0000","last_clean":"2023-09-23T23:04:19.797255+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:04:19.797255+0000","last_undegraded":"2023-09-23T23:04:19.797255+0000","last_fullsized":"2023-09-23T23:04:19.797255+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"418'3184","reported_seq":3200,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:35.352709+0000","last_change":"2023-09-23T23:03:43.131930+0000","last_active":"2023-09-23T23:04:35.352709+0000","last_peered":"2023-09-23T23:04:35.352709+0000","last_clean":"2023-09-23T23:04:35.352709+0000","last_became_active":"2023-09-23T23:03:43.131552+0000","last_became_peered":"2023-09-23T23:03:43.131552+0000","last_unstale":"2023-09-23T23:04:35.352709+0000","last_undegraded":"2023-09-23T23:04:35.352709+0000","last_fullsized":"2023-09-23T23:04:35.352709+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":3184,"log_dups_size":0,"ondisk_log_size":3184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208666624,"num_objects":3184,"num_object_clones":0,"num_object_copies":9552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3184,"num_write_kb":203776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":375,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.812777+0000","last_change":"2023-09-23T22:58:24.248943+0000","last_active":"2023-09-23T23:04:19.812777+0000","last_peered":"2023-09-23T23:04:19.812777+0000","last_clean":"2023-09-23T23:04:19.812777+0000","last_became_active":"2023-09-23T22:58:24.248592+0000","last_became_peered":"2023-09-23T22:58:24.248592+0000","last_unstale":"2023-09-23T23:04:19.812777+0000","last_undegraded":"2023-09-23T23:04:19.812777+0000","last_fullsized":"2023-09-23T23:04:19.812777+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:23.958129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":323,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T23:04:30.890530+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:04:30.890530+0000","last_peered":"2023-09-23T23:04:30.890530+0000","last_clean":"2023-09-23T23:04:30.890530+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:04:30.890530+0000","last_undegraded":"2023-09-23T23:04:30.890530+0000","last_fullsized":"2023-09-23T23:04:30.890530+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":392,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819737+0000","last_change":"2023-09-23T22:58:06.785180+0000","last_active":"2023-09-23T23:04:19.819737+0000","last_peered":"2023-09-23T23:04:19.819737+0000","last_clean":"2023-09-23T23:04:19.819737+0000","last_became_active":"2023-09-23T22:58:06.784689+0000","last_became_peered":"2023-09-23T22:58:06.784689+0000","last_unstale":"2023-09-23T23:04:19.819737+0000","last_undegraded":"2023-09-23T23:04:19.819737+0000","last_fullsized":"2023-09-23T23:04:19.819737+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":366,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.806324+0000","last_change":"2023-09-23T23:03:51.529572+0000","last_active":"2023-09-23T23:04:19.806324+0000","last_peered":"2023-09-23T23:04:19.806324+0000","last_clean":"2023-09-23T23:04:19.806324+0000","last_became_active":"2023-09-23T23:03:51.529215+0000","last_became_peered":"2023-09-23T23:03:51.529215+0000","last_unstale":"2023-09-23T23:04:19.806324+0000","last_undegraded":"2023-09-23T23:04:19.806324+0000","last_fullsized":"2023-09-23T23:04:19.806324+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":167,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.811831+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:04:19.811831+0000","last_peered":"2023-09-23T23:04:19.811831+0000","last_clean":"2023-09-23T23:04:19.811831+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:04:19.811831+0000","last_undegraded":"2023-09-23T23:04:19.811831+0000","last_fullsized":"2023-09-23T23:04:19.811831+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":227,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820276+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:04:19.820276+0000","last_peered":"2023-09-23T23:04:19.820276+0000","last_clean":"2023-09-23T23:04:19.820276+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:04:19.820276+0000","last_undegraded":"2023-09-23T23:04:19.820276+0000","last_fullsized":"2023-09-23T23:04:19.820276+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":441,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820281+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:04:19.820281+0000","last_peered":"2023-09-23T23:04:19.820281+0000","last_clean":"2023-09-23T23:04:19.820281+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:04:19.820281+0000","last_undegraded":"2023-09-23T23:04:19.820281+0000","last_fullsized":"2023-09-23T23:04:19.820281+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":241,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820553+0000","last_change":"2023-09-23T22:56:20.290157+0000","last_active":"2023-09-23T23:04:19.820553+0000","last_peered":"2023-09-23T23:04:19.820553+0000","last_clean":"2023-09-23T23:04:19.820553+0000","last_became_active":"2023-09-23T22:56:20.210651+0000","last_became_peered":"2023-09-23T22:56:20.210651+0000","last_unstale":"2023-09-23T23:04:19.820553+0000","last_undegraded":"2023-09-23T23:04:19.820553+0000","last_fullsized":"2023-09-23T23:04:19.820553+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":328,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.820096+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:04:19.820096+0000","last_peered":"2023-09-23T23:04:19.820096+0000","last_clean":"2023-09-23T23:04:19.820096+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:04:19.820096+0000","last_undegraded":"2023-09-23T23:04:19.820096+0000","last_fullsized":"2023-09-23T23:04:19.820096+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":16,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.812853+0000","last_change":"2023-09-23T23:03:51.529901+0000","last_active":"2023-09-23T23:04:19.812853+0000","last_peered":"2023-09-23T23:04:19.812853+0000","last_clean":"2023-09-23T23:04:19.812853+0000","last_became_active":"2023-09-23T23:03:51.529442+0000","last_became_peered":"2023-09-23T23:03:51.529442+0000","last_unstale":"2023-09-23T23:04:19.812853+0000","last_undegraded":"2023-09-23T23:04:19.812853+0000","last_fullsized":"2023-09-23T23:04:19.812853+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:19.005121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.801394+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:04:19.801394+0000","last_peered":"2023-09-23T23:04:19.801394+0000","last_clean":"2023-09-23T23:04:19.801394+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:04:19.801394+0000","last_undegraded":"2023-09-23T23:04:19.801394+0000","last_fullsized":"2023-09-23T23:04:19.801394+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":362,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:20.308284+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:04:20.308284+0000","last_peered":"2023-09-23T23:04:20.308284+0000","last_clean":"2023-09-23T23:04:20.308284+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:04:20.308284+0000","last_undegraded":"2023-09-23T23:04:20.308284+0000","last_fullsized":"2023-09-23T23:04:20.308284+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":403,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T23:04:19.819684+0000","last_change":"2023-09-23T22:57:27.549645+0000","last_active":"2023-09-23T23:04:19.819684+0000","last_peered":"2023-09-23T23:04:19.819684+0000","last_clean":"2023-09-23T23:04:19.819684+0000","last_became_active":"2023-09-23T22:57:27.549273+0000","last_became_peered":"2023-09-23T22:57:27.549273+0000","last_unstale":"2023-09-23T23:04:19.819684+0000","last_undegraded":"2023-09-23T23:04:19.819684+0000","last_fullsized":"2023-09-23T23:04:19.819684+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3395420160,"num_objects":51810,"num_object_clones":0,"num_object_copies":155430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51810,"num_write_kb":3315840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10018488320,"data_stored":10018488320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":51810,"ondisk_log_size":51810,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":15382,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430659,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388626884,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":84,"seq":360777253196,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3126980,"kb_used_data":2338492,"kb_used_omap":9,"kb_used_meta":788470,"kb_avail":90618172,"statfs":{"total":95995035648,"available":92793008128,"internally_reserved":0,"allocated":2394615808,"data_stored":2393014225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":807394281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:14 2023","interfaces":[{"interface":"back","average":{"1min":20.364999999999998,"5min":5.7789999999999999,"15min":2.161},"min":{"1min":0.224,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":279.30500000000001,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.221},{"interface":"front","average":{"1min":20.523,"5min":6.0369999999999999,"15min":2.5830000000000002},"min":{"1min":0.26800000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":279.39800000000002,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.433}]},{"osd":2,"last update":"Sat Sep 23 23:04:31 2023","interfaces":[{"interface":"back","average":{"1min":17.202999999999999,"5min":3.8980000000000001,"15min":1.651},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":161.61699999999999,"5min":161.61699999999999,"15min":161.61699999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":16.652999999999999,"5min":3.871,"15min":1.774},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":162.61699999999999,"5min":162.61699999999999,"15min":162.61699999999999},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.65400000000000003,"15min":1.675},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.20200000000000001},"max":{"1min":2.8879999999999999,"5min":3.7250000000000001,"15min":87.787999999999997},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.64800000000000002,"15min":1.887},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.19500000000000001},"max":{"1min":5.1989999999999998,"5min":5.1989999999999998,"15min":89.861999999999995},"last":2.5169999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.8540000000000001,"15min":2.3149999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.004,"5min":67.738,"15min":151.036},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.0800000000000001,"15min":2.6589999999999998},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.5590000000000002,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.823,"15min":2.2890000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":8.7159999999999993,"5min":68.259,"15min":116.098},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.915,"15min":2.331},"min":{"1min":0.39900000000000002,"5min":0.378,"15min":0.315},"max":{"1min":3.04,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":2.1230000000000002,"15min":2.2450000000000001},"min":{"1min":0.38100000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":2.6030000000000002,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.7529999999999999,"15min":1.8080000000000001},"min":{"1min":0.40000000000000002,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":2.1920000000000002,"5min":67.366,"15min":67.366},"last":0.54500000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.879,"15min":1.4570000000000001},"min":{"1min":0.48599999999999999,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":3.069,"5min":4.1669999999999998,"15min":11.481999999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.006,"15min":1.8720000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33100000000000002},"max":{"1min":2.6360000000000001,"5min":3.9860000000000002,"15min":20.902999999999999},"last":0.63400000000000001}]}]},{"osd":0,"up_from":353,"seq":1516123455567,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2263240,"kb_used_data":2029888,"kb_used_omap":37,"kb_used_meta":233306,"kb_avail":91481912,"statfs":{"total":95995035648,"available":93677477888,"internally_reserved":0,"allocated":2078605312,"data_stored":2076824318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":238905506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.83699999999999997,"15min":0.63900000000000001},"min":{"1min":0.216,"5min":0.214,"15min":0.214},"max":{"1min":2.7570000000000001,"5min":8.234,"15min":8.234},"last":2.4060000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.86899999999999999,"15min":0.73699999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.867,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":2.371}]},{"osd":2,"last update":"Sat Sep 23 23:03:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.96799999999999997,"15min":0.84199999999999997},"min":{"1min":0.30099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":7.3369999999999997,"5min":12.215,"15min":12.215},"last":1.9430000000000001},{"interface":"front","average":{"1min":5.1340000000000003,"5min":1.7609999999999999,"15min":1.03},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":74.698999999999998,"5min":74.698999999999998,"15min":74.698999999999998},"last":0.64000000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":6.7220000000000004,"5min":2.25,"15min":1.052},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":74.744,"5min":74.744,"15min":74.744},"last":1.986},{"interface":"front","average":{"1min":6.8479999999999999,"5min":2.125,"15min":1.3140000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":74.777000000000001,"5min":74.777000000000001,"15min":74.777000000000001},"last":1.548}]},{"osd":4,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":4.7409999999999997,"5min":4.0750000000000002,"15min":1.931},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":74.805000000000007,"5min":183.148,"15min":183.148},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":3.4119999999999999,"15min":1.583},"min":{"1min":0.38,"5min":0.31,"15min":0.31},"max":{"1min":1.8320000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":2.246}]},{"osd":5,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.0640000000000001,"15min":0.80700000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":2.2530000000000001,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":2.2639999999999998},{"interface":"front","average":{"1min":4.774,"5min":1.7809999999999999,"15min":1.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":74.858000000000004,"5min":74.858000000000004,"15min":74.858000000000004},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.0549999999999999,"15min":0.90800000000000003},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.893,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":2.3010000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.0029999999999999,"15min":0.81699999999999995},"min":{"1min":0.46800000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.7149999999999999,"5min":6.375,"15min":6.375},"last":2.3140000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.0209999999999999,"15min":1.2090000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.917,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":2.3959999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.0680000000000001,"15min":1.208},"min":{"1min":0.48799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.79,"5min":6.343,"15min":6.343},"last":2.4580000000000002}]}]},{"osd":2,"up_from":359,"seq":1541893259340,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1871104,"kb_used_data":1663480,"kb_used_omap":83,"kb_used_meta":207532,"kb_avail":91874048,"statfs":{"total":95995035648,"available":94079025152,"internally_reserved":0,"allocated":1703403520,"data_stored":1701691103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":212513221},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":6.2400000000000002,"5min":2.452,"15min":1.054},"min":{"1min":0.247,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.735999999999997,"5min":74.241,"15min":74.241},"last":1.7849999999999999},{"interface":"front","average":{"1min":6.3899999999999997,"5min":2.5960000000000001,"15min":1.337},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":55.969999999999999,"5min":74.430000000000007,"15min":74.430000000000007},"last":5.4930000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.73099999999999998,"15min":0.68700000000000006},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.2080000000000002,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":5.4569999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.0629999999999999,"15min":1.1240000000000001},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.397,"5min":11.397,"15min":11.397},"last":6.7270000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82299999999999995,"15min":0.749},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.2750000000000004,"5min":5.9530000000000003,"15min":5.9530000000000003},"last":5.5389999999999997},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.871,"15min":0.83499999999999996},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":9.8170000000000002,"15min":9.8170000000000002},"last":6.4850000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.92300000000000004,"15min":0.68400000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.1360000000000001,"5min":7.1159999999999997,"15min":7.1159999999999997},"last":2.0070000000000001},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.96799999999999997,"15min":0.871},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2430000000000003,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":1.028}]},{"osd":5,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.99299999999999999,"15min":0.96699999999999997},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.3090000000000002,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":6.0979999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.93500000000000005,"15min":0.76500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.1680000000000001,"5min":7.0839999999999996,"15min":7.0839999999999996},"last":1.972}]},{"osd":6,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.1919999999999999,"15min":1.044},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.968,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":6.1420000000000003},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.94099999999999995,"15min":0.71799999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.9079999999999999,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":2.0840000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.1559999999999999,"15min":1.046},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.7869999999999999,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":2.0529999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.0449999999999999,"15min":1.1339999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.3049999999999997,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":1.0700000000000001}]}]},{"osd":3,"up_from":94,"seq":403726926144,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2426776,"kb_used_data":1867984,"kb_used_omap":7,"kb_used_meta":558776,"kb_avail":91318376,"statfs":{"total":95995035648,"available":93510017024,"internally_reserved":0,"allocated":1912815616,"data_stored":1911134853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":572187240},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.5419999999999998,"15min":1.0600000000000001},"min":{"1min":0.25600000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.7130000000000001,"5min":122.289,"15min":122.289},"last":3.7629999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":2.04,"15min":0.95099999999999996},"min":{"1min":0.218,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":1.6839999999999999,"5min":122.041,"15min":122.041},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.145,"15min":2.2130000000000001},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.187},"max":{"1min":15.025,"5min":15.025,"15min":123.866},"last":2.0760000000000001},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0289999999999999,"15min":2.1219999999999999},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.718,"5min":9.718,"15min":122.265},"last":1.8149999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":0.78200000000000003,"15min":0.73999999999999999},"min":{"1min":0.309,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":5.056,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.33500000000000002},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.4239999999999999,"15min":0.96999999999999997},"min":{"1min":0.29299999999999998,"5min":0.215,"15min":0.215},"max":{"1min":9.9789999999999992,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":2.2000000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.3559999999999999,"15min":1.9450000000000001},"min":{"1min":0.311,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":3.8450000000000002,"5min":87.046999999999997,"15min":87.046999999999997},"last":1.9890000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":2.411,"15min":1.9079999999999999},"min":{"1min":0.34699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8490000000000002,"5min":87.066000000000003,"15min":87.066000000000003},"last":1.7789999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.119,"15min":1.407},"min":{"1min":0.373,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":4.9059999999999997,"5min":6.2530000000000001,"15min":11.803000000000001},"last":1.851},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.085,"15min":1.6140000000000001},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":0.92000000000000004,"5min":10.246,"15min":11.951000000000001},"last":2.4380000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.3420000000000001,"15min":1.706},"min":{"1min":0.40899999999999997,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":4.8840000000000003,"5min":9.2520000000000007,"15min":14.097},"last":1.881},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.198,"15min":1.742},"min":{"1min":0.41299999999999998,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":2.6949999999999998,"5min":9.2810000000000006,"15min":12.887},"last":2.4830000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.006,"15min":1.5489999999999999},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":3.3260000000000001,"5min":5.2409999999999997,"15min":11.694000000000001},"last":2.5169999999999999},{"interface":"front","average":{"1min":1.391,"5min":1.2869999999999999,"15min":1.575},"min":{"1min":0.41299999999999998,"5min":0.37,"15min":0.33200000000000002},"max":{"1min":6.2050000000000001,"5min":8.5830000000000002,"15min":12.32},"last":2.0350000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019341,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1723408,"kb_used_data":820040,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92021744,"statfs":{"total":95995035648,"available":94230265856,"internally_reserved":0,"allocated":839720960,"data_stored":838856303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":4.1989999999999998,"15min":1.714},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.93400000000000005,"5min":156.375,"15min":156.375},"last":0.25},{"interface":"front","average":{"1min":0.67900000000000005,"5min":4.351,"15min":1.847},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.5509999999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.196,"15min":3.3199999999999998},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":28.917000000000002,"5min":28.917000000000002,"15min":132.304},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.147,"15min":3.302},"min":{"1min":0.28000000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.731000000000002,"5min":28.731000000000002,"15min":132.453},"last":1.627}]},{"osd":2,"last update":"Sat Sep 23 23:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.155,"15min":1.0649999999999999},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.2949999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":1.6599999999999999},{"interface":"front","average":{"1min":1.329,"5min":1.161,"15min":0.86499999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.3799999999999999,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.083,"15min":2.6869999999999998},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.278,"5min":10.337999999999999,"15min":164.453},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.2569999999999999,"15min":2.7530000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":3.4689999999999999,"5min":7.2519999999999998,"15min":167.05199999999999},"last":1.7250000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.036,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.214},"max":{"1min":1.492,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.92400000000000004,"15min":1.835},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":5.9409999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.208,"15min":1.294},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":5.8399999999999999,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.38},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.96999999999999997,"15min":1.161},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":1.252,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.222,"15min":1.252},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.28399999999999997},"max":{"1min":0.90900000000000003,"5min":11.045999999999999,"15min":12.84},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.96599999999999997,"15min":1.1140000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":0.71399999999999997,"5min":10.227,"15min":10.587},"last":0.439}]}]},{"osd":5,"up_from":36,"seq":154618823115,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1923004,"kb_used_data":1027444,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":91822148,"statfs":{"total":95995035648,"available":94025879552,"internally_reserved":0,"allocated":1052102656,"data_stored":1051253703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.8859999999999999,"15min":1.032},"min":{"1min":0.26900000000000002,"5min":0.221,"15min":0.221},"max":{"1min":7.7469999999999999,"5min":42.155000000000001,"15min":42.155000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.96,"15min":1.3740000000000001},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.7000000000000002,"5min":43.378999999999998,"15min":43.378999999999998},"last":2.0099999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.753,"15min":1.984},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":2.0979999999999999,"5min":3.6680000000000001,"15min":110.36499999999999},"last":4.0030000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.1419999999999999,"15min":2.4209999999999998},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":1.9510000000000001,"5min":7.1219999999999999,"15min":110.89},"last":0.27000000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.024,"15min":0.875},"min":{"1min":0.29999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.5549999999999999,"5min":6.601,"15min":6.601},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.94099999999999995,"15min":0.97699999999999998},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":1.9770000000000001,"5min":4.9530000000000003,"15min":4.9530000000000003},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.1359999999999999,"15min":2.7440000000000002},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.6629999999999998,"5min":9.2710000000000008,"15min":98.641000000000005},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.2330000000000001,"15min":2.7919999999999998},"min":{"1min":0.35799999999999998,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":3.6789999999999998,"5min":10.965,"15min":98.906000000000006},"last":1.276}]},{"osd":4,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.0640000000000001,"15min":1.5549999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23300000000000001},"max":{"1min":1.107,"5min":7.0380000000000003,"15min":115.06699999999999},"last":0.309},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.97699999999999998,"15min":1.502},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.253},"max":{"1min":2.5419999999999998,"5min":9.5890000000000004,"15min":114.908},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.2490000000000001,"15min":1.2110000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":3.0600000000000001,"5min":7.4630000000000001,"15min":11.121},"last":0.375},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.879,"15min":1.0529999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.2370000000000001,"5min":5.4909999999999997,"15min":12.137},"last":0.35699999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.94599999999999995,"15min":1.079},"min":{"1min":0.375,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.048,"5min":9.5980000000000008,"15min":12.23},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.2350000000000001,"15min":1.2030000000000001},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.2689999999999999,"5min":7.6719999999999997,"15min":11.112},"last":0.41599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2091974656,"data_stored":2091974656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2395471872,"data_stored":2395471872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1710424064,"data_stored":1710424064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1926299648,"data_stored":1926299648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":836435968,"data_stored":836435968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1057882112,"data_stored":1057882112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:04:38.917 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-09-23T23:04:38.954 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:39.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:38 smithi052 ceph-mon[180979]: from='client.46949 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:39.172 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:38 smithi052 ceph-mon[187743]: from='client.46949 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:39.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:38 smithi151 ceph-mon[159210]: from='client.46949 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:04:39.228 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:39.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:39.340 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:39.368 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:39.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:39.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:39.501 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:39.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:39.609 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:39.690 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:04:39.690 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:04:39.764 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 47 16 54851 54835 72.8695 47.4375 0.0334784 0.0137179 2023-09-23T23:04:39.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:39 smithi052 ceph-mon[180979]: pgmap v1301: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T23:04:39.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:39 smithi052 ceph-mon[187743]: pgmap v1301: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T23:04:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:39 smithi151 ceph-mon[159210]: pgmap v1301: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T23:04:40.764 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 48 16 55781 55765 72.5624 58.125 0.00386979 0.0137528 2023-09-23T23:04:40.991 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:41.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:41.103 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:41.130 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:41.181 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:41.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:41.287 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:41.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:41 smithi151 ceph-mon[159210]: pgmap v1302: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T23:04:41.722 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:41 smithi052 ceph-mon[180979]: pgmap v1302: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T23:04:41.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:41 smithi052 ceph-mon[187743]: pgmap v1302: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T23:04:41.745 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:04:41.764 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 49 16 56576 56560 72.0957 49.6875 0.0156204 0.0138652 2023-09-23T23:04:42.053 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:42.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:42.162 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:42.189 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:42.207 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:42.235 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:42.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:42.342 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:42.445 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:42.770 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 50 16 57411 57395 71.6939 52.1875 0.021353 0.0139425 2023-09-23T23:04:43.458 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:04:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:04:43] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.33.4" 2023-09-23T23:04:43.561 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:43.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:43.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:43.665 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:43.744 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:43.767 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 51 16 58175 58159 71.2246 47.75 0.015811 0.0140329 2023-09-23T23:04:43.835 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:43 smithi052 ceph-mon[187743]: pgmap v1303: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T23:04:43.835 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:43 smithi052 ceph-mon[180979]: pgmap v1303: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T23:04:44.122 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:44.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:44.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:43 smithi151 ceph-mon[159210]: pgmap v1303: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T23:04:44.228 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:44.254 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:44.296 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:44.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:44.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:44.401 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:44.430 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:44.478 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:44.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:44.585 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:44.669 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:44.773 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 52 16 58979 58963 70.8176 50.25 0.0219419 0.0141161 2023-09-23T23:04:44.783 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:04:44.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/905937260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:44.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:44.938 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/905937260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:44.938 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:45.173 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:45.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/905937260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:04:45.280 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:45.306 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:45.348 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:45.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:45.452 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:45.478 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:45.520 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:45.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:45.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:45.625 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:45.652 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:45.701 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:45.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:45.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:45.771 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 53 16 59673 59657 70.2978 43.375 0.015229 0.0142176 2023-09-23T23:04:45.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:45.835 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:45.882 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:45.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:45 smithi151 ceph-mon[159210]: pgmap v1304: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s 2023-09-23T23:04:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:45 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:45 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:04:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:45 smithi151 ceph-mon[159210]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T23:04:45.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:45 smithi151 ceph-mon[159210]: failed to encode map e419 with expected crc 2023-09-23T23:04:45.982 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:04:45.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:46.022 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:46.030 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:04:46.075 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[180979]: pgmap v1304: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s 2023-09-23T23:04:46.075 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:46.075 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:04:46.075 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[180979]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T23:04:46.075 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[180979]: failed to encode map e419 with expected crc 2023-09-23T23:04:46.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[187743]: pgmap v1304: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 44 MiB/s wr, 701 op/s 2023-09-23T23:04:46.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:04:46.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:04:46.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[187743]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T23:04:46.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:45 smithi052 ceph-mon[187743]: failed to encode map e419 with expected crc 2023-09-23T23:04:46.082 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:04:46.095 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:46.134 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:04:46.172 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:04:46.205 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:04:46.242 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:04:46.283 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:04:46.297 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.297 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.297 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.297 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.298 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.298 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.298 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.298 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:04:46.388 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:46.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:46.496 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:46.570 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:46.774 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 54 16 60393 60377 69.8266 45 0.00836154 0.0143028 2023-09-23T23:04:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3505874641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:47.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3505874641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:47.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:47.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:47.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3505874641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:47.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:47.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:47.310 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:47.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:47.420 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:47.446 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:47.493 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:47.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:47.599 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:47.627 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:47.673 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:47.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:47.776 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 55 16 61114 61098 69.3766 45.0625 0.0254392 0.0144083 2023-09-23T23:04:47.779 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:47.860 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:48.072 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:48.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:48.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:48.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[180979]: pgmap v1306: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T23:04:48.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:04:48.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:04:48.080 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:48.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[187743]: pgmap v1306: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T23:04:48.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:04:48.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:04:48.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:48.158 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T23:04:48.178 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:47 smithi151 ceph-mon[159210]: pgmap v1306: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T23:04:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:04:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:04:48.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:04:48.237 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:48.282 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:48.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:48.386 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:48.438 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:04:48.460 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:48.784 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 56 16 61848 61832 68.9442 45.875 0.00662394 0.0144966 2023-09-23T23:04:48.890 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:48.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:49.006 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:49.033 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:49.082 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:49.085 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1883890970' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:04:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:49.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:49.187 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:49.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1883890970' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:04:49.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1883890970' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:04:49.253 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:49.522 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:04:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T23:04:49.703 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:49.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:49.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 57 16 62603 62587 68.5566 47.1875 0.0309847 0.014579 2023-09-23T23:04:49.810 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:49.838 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:49.882 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:49.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:49.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:49.986 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:49.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:49 smithi052 ceph-mon[180979]: pgmap v1307: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T23:04:49.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:49.995 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:49 smithi052 ceph-mon[187743]: pgmap v1307: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T23:04:50.000 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:50.013 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:50.061 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:50.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:49 smithi151 ceph-mon[159210]: pgmap v1307: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T23:04:50.067 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:04:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:50.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:50.169 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:50.182 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:04:50.182 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:04:50.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:04:50.182 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:04:50.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.182 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114198 10.5 10:a53fd4ad:::benchmark_data_smithi052_461816_object17397:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.442820084, 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047289519, 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114198 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.183 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735120+0000", 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735128+0000", 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6639999999999998e-06 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735140+0000", 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1888999999999999e-05 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735151+0000", 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736424+0000", 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001272942 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736451+0000", 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7029000000000002e-05 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.187 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772313+0000", 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.035862128 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772461+0000", 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014821499999999999 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772545+0000", 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3183000000000006e-05 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779124+0000", 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065792929999999999 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782207+0000", 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030828169999999998 2023-09-23T23:04:50.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782247+0000", 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0198999999999999e-05 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782304+0000", 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7172000000000001e-05 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782412+0000", 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107539 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114209 10.5 10:a8019dd6:::benchmark_data_smithi052_461816_object17408:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.44162562700001, 2023-09-23T23:04:50.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.052215573, 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114209 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736316+0000", 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:50.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736321+0000", 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6640000000000003e-06 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736333+0000", 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1854e-05 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736343+0000", 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7399999999999999e-06 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736479+0000", 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013588000000000001 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736486+0000", 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3150000000000003e-06 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774671+0000", 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0381846910000001 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774856+0000", 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000185179 2023-09-23T23:04:50.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774979+0000", 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122843 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782160+0000", 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071812610000000004 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782451+0000", 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029058300000000002 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788381+0000", 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059298370000000003 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788422+0000", 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1409999999999998e-05 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788532+0000", 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110103 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.199 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114217 10.5 10:a151ffa4:::benchmark_data_smithi052_461816_object17416:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.441593146, 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068861074, 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.200 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114217 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736352+0000", 2023-09-23T23:04:50.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9399999999999998e-06 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736358+0000", 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736363+0000", 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3199999999999999e-06 2023-09-23T23:04:50.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737029+0000", 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000666032 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737038+0000", 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5490000000000006e-06 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779153+0000", 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0421153320000001 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779339+0000", 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000186016 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779477+0000", 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000137923 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782455+0000", 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029779519999999999 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790613+0000", 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081575689999999999 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805085+0000", 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014472021 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805126+0000", 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1619000000000003e-05 2023-09-23T23:04:50.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805210+0000", 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3728999999999997e-05 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114228 10.5 10:abf9b074:::benchmark_data_smithi052_461816_object17427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:04:50.208 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.44115710599999, 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068634885, 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114228 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736784+0000", 2023-09-23T23:04:50.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736797+0000", 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2558000000000001e-05 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736807+0000", 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7669999999999998e-06 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736815+0000", 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737073+0000", 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000257502 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737079+0000", 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5180000000000001e-06 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782471+0000", 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0453921399999999 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782629+0000", 2023-09-23T23:04:50.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015792200000000001 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782710+0000", 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0499000000000003e-05 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788710+0000", 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060006640000000002 2023-09-23T23:04:50.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796703+0000", 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079926860000000006 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805261+0000", 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085578570000000003 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805317+0000", 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5649000000000002e-05 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805420+0000", 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103467 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.216 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114233 10.5 10:adc2278f:::benchmark_data_smithi052_461816_object17432:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.44111184899998, 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068750884, 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.217 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114233 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:04:50.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736833+0000", 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0469999999999998e-06 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736840+0000", 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2130000000000002e-06 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736846+0000", 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1099999999999999e-06 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737097+0000", 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025043299999999999 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737103+0000", 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8150000000000002e-06 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.221 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785090+0000", 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047987419 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785226+0000", 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013601299999999999 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785376+0000", 2023-09-23T23:04:50.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000150407 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790542+0000", 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051658429999999998 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796752+0000", 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062099620000000003 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805468+0000", 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0087155189999999997 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805509+0000", 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1789000000000003e-05 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805581+0000", 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1771000000000001e-05 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114234 10.5 10:a1bb5c32:::benchmark_data_smithi052_461816_object17433:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.44109122499998, 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068915523, 2023-09-23T23:04:50.225 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114234 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:04:50.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736853+0000", 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3379999999999999e-06 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736859+0000", 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0499999999999997e-06 2023-09-23T23:04:50.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736870+0000", 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1021e-05 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737132+0000", 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00026153199999999998 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737138+0000", 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9839999999999998e-06 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786800+0000", 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.049662885 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786974+0000", 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017351000000000001 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.787107+0000", 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013272100000000001 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790552+0000", 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034448650000000001 2023-09-23T23:04:50.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797758+0000", 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072065200000000001 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805623+0000", 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0078654350000000005 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805667+0000", 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3665000000000003e-05 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805766+0000", 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9365999999999997e-05 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.233 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114235 10.5 10:a3d19bc6:::benchmark_data_smithi052_461816_object17434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.44106761099999, 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081414576, 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114235 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.234 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736874+0000", 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736877+0000", 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8949999999999998e-06 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736883+0000", 2023-09-23T23:04:50.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3459999999999999e-06 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736891+0000", 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8050000000000007e-06 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737166+0000", 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027465399999999999 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737172+0000", 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.207e-06 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788737+0000", 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0515644399999999 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788877+0000", 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014082 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788968+0000", 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0167999999999995e-05 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792267+0000", 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003299256 2023-09-23T23:04:50.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797797+0000", 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.005530274 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818146+0000", 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020349151999999999 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818185+0000", 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8575000000000003e-05 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818289+0000", 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010435899999999999 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114248 10.5 10:a5b9c0ca:::benchmark_data_smithi052_461816_object17447:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 404.44104645099998, 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081611235, 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114248 2023-09-23T23:04:50.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736895+0000", 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736898+0000", 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4250000000000001e-06 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736904+0000", 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736909+0000", 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2310000000000004e-06 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737193+0000", 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000283885 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737199+0000", 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9610000000000002e-06 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790684+0000", 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.053484358 2023-09-23T23:04:50.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790817+0000", 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000133693 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790895+0000", 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8218000000000002e-05 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792358+0000", 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001462758 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797833+0000", 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054744759999999998 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818343+0000", 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020510347000000002 2023-09-23T23:04:50.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818398+0000", 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4530999999999997e-05 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818507+0000", 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010940100000000001 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.249 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:116534 10.6 10:62c2756c:::benchmark_data_smithi052_461816_object19733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 363.29841140600001, 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18822530200000001, 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.250 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 116534 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879530+0000", 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879535+0000", 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879547+0000", 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1918e-05 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879555+0000", 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1459999999999996e-06 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879602+0000", 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6768999999999999e-05 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879810+0000", 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020844799999999999 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:04:50.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879951+0000", 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014103400000000001 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.881979+0000", 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020274709999999999 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.883452+0000", 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001472985 2023-09-23T23:04:50.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067651+0000", 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18419902199999999 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067709+0000", 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8180000000000002e-05 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067756+0000", 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7275999999999999e-05 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138140 10.8 10:10bb8653:::benchmark_data_smithi052_461816_object41339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 262.56520931799997, 2023-09-23T23:04:50.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.272336093, 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138140 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:04:50.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612731+0000", 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612741+0000", 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4800000000000007e-06 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612762+0000", 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1761000000000001e-05 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612772+0000", 2023-09-23T23:04:50.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6229999999999992e-06 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612837+0000", 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4789999999999995e-05 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613071+0000", 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023441400000000001 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613293+0000", 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022140800000000001 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.616187+0000", 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028945080000000001 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.618246+0000", 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020593949999999999 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883729+0000", 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265482459 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883802+0000", 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3008000000000003e-05 2023-09-23T23:04:50.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.885069+0000", 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012670579999999999 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138142 10.8 10:10a11927:::benchmark_data_smithi052_461816_object41341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:04:50.264 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 262.55111785299999, 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26049065999999998, 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138142 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.265 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626822+0000", 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:04:50.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626834+0000", 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626866+0000", 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1934999999999998e-05 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626881+0000", 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5491000000000001e-05 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626950+0000", 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8701000000000005e-05 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627110+0000", 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000160347 2023-09-23T23:04:50.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627220+0000", 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110009 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.629339+0000", 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021183280000000001 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.635341+0000", 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060026009999999998 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886154+0000", 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25081289200000001 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886210+0000", 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5321000000000001e-05 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.887315+0000", 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001105396 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:140723 10.6 10:68c22ebc:::benchmark_data_smithi052_461816_object43922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 250.962918906, 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.195061394, 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140723 2023-09-23T23:04:50.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.273 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215022+0000", 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215028+0000", 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215039+0000", 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0998e-05 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215044+0000", 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6380000000000001e-06 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215089+0000", 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4929000000000003e-05 2023-09-23T23:04:50.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215188+0000", 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8741999999999998e-05 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215268+0000", 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0049000000000006e-05 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217087+0000", 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018184539999999999 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217880+0000", 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079313599999999995 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410012+0000", 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19213219200000001 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410050+0000", 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8262000000000001e-05 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410085+0000", 2023-09-23T23:04:50.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4588000000000002e-05 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:150935 10.5 10:a2940dfc:::benchmark_data_smithi052_461816_object54134:head [delete] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 214.358273817, 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16991541099999999, 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.279 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 150935 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.280 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819668+0000", 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819671+0000", 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2229999999999999e-06 2023-09-23T23:04:50.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819678+0000", 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5080000000000002e-06 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.819683+0000", 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0799999999999996e-06 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.960363+0000", 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.140680312 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.960543+0000", 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018004800000000001 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.960725+0000", 2023-09-23T23:04:50.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018175299999999999 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.964365+0000", 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036400709999999999 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.964476+0000", 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000111432 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.965502+0000", 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010257280000000001 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.988206+0000", 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022703556999999999 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.285 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.988258+0000", 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2590000000000003e-05 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:01:15.989584+0000", 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013256590000000001 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:188923 10.d e403/354 10:ba1021e3:::benchmark_data_smithi052_461816_object92122:head v 403'11913, mlcod=344'7206)", 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 99.423500579999995, 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.205832343, 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754440+0000", 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:50.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754450+0000", 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3270000000000007e-06 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754462+0000", 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2313999999999999e-05 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754470+0000", 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2209999999999997e-06 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754510+0000", 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9437999999999997e-05 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.290 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754531+0000", 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1557999999999999e-05 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.960234+0000", 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20570290699999999 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.960274+0000", 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9780999999999997e-05 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.291 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:190119 10.5 10:afa20eef:::benchmark_data_smithi052_461816_object93318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e403)", 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:13.860114+0000", 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 96.317828449999993, 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17804316100000001, 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.292 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 190119 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860114+0000", 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860114+0000", 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860112+0000", 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860119+0000", 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6410000000000002e-06 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860132+0000", 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2799999999999999e-05 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860138+0000", 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.575e-06 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866187+0000", 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060495749999999997 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866370+0000", 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018212400000000001 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866544+0000", 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017491800000000001 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.296 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.033639+0000", 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16709457899999999 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.033676+0000", 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6943000000000002e-05 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.033709+0000", 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3056000000000003e-05 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.036843+0000", 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031338580000000002 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.036906+0000", 2023-09-23T23:04:50.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3314999999999999e-05 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.038157+0000", 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001250698 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:190121 10.4 10:29d183b6:::benchmark_data_smithi052_461816_object93320:head [delete] snapc 0=[] ondisk+write+known_if_redirected e403)", 2023-09-23T23:04:50.299 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:13.860334+0000", 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 96.317608221, 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17034539400000001, 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 190121 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860334+0000", 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860334+0000", 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860334+0000", 2023-09-23T23:04:50.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860336+0000", 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.43e-06 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860340+0000", 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1180000000000002e-06 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860343+0000", 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.253e-06 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860385+0000", 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1826999999999998e-05 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860479+0000", 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4007999999999998e-05 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,3,4", 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.860574+0000", 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4357999999999993e-05 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.863692+0000", 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031183970000000002 2023-09-23T23:04:50.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.869415+0000", 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057231770000000003 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.869643+0000", 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022767299999999999 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.030509+0000", 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.160865917 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.030557+0000", 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7809000000000003e-05 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.030679+0000", 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012279500000000001 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:190123 10.5 10:a45ecb25:::benchmark_data_smithi052_461816_object93322:head [delete] snapc 0=[] ondisk+write+known_if_redirected e403)", 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:13.866102+0000", 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 96.311840058000001, 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17939245500000001, 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.307 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 190123 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866102+0000", 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.308 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866102+0000", 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866101+0000", 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.309 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866108+0000", 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4410000000000004e-06 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866122+0000", 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3839000000000001e-05 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.866131+0000", 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5290000000000008e-06 2023-09-23T23:04:50.310 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.869417+0000", 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032860820000000001 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:13.869619+0000", 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020202100000000001 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.311 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.030598+0000", 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.16097892999999999 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.038214+0000", 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.007616902 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.312 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.038224+0000", 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5000000000000005e-06 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.038296+0000", 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2274999999999995e-05 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.044207+0000", 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059105520000000003 2023-09-23T23:04:50.313 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.044273+0000", 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5959999999999999e-05 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:14.045495+0000", 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012218629999999999 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.314 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:44802 11.6 11:6ab43c8b:::benchmark_data_smithi052_606809_object44801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 22.260016872000001, 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22081410400000001, 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:04:50.315 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 44802 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.316 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917924+0000", 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917950+0000", 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5999e-05 2023-09-23T23:04:50.317 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917964+0000", 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4412e-05 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917972+0000", 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2800000000000003e-06 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.318 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918102+0000", 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012992200000000001 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918229+0000", 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012657599999999999 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918357+0000", 2023-09-23T23:04:50.319 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012783599999999999 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.920436+0000", 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020790959999999999 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.921317+0000", 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088133199999999997 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.320 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138645+0000", 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21732747999999999 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138699+0000", 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4261000000000001e-05 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.321 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138739+0000", 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0448000000000003e-05 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:48636 11.6 11:6e1729a3:::benchmark_data_smithi052_606809_object48635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e418)", 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:32.274242+0000", 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.903699935999999, 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18169365500000001, 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.322 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48636 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274242+0000", 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.323 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274242+0000", 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274241+0000", 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.324 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274263+0000", 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2819999999999998e-05 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274285+0000", 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1384000000000001e-05 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.325 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274293+0000", 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5660000000000007e-06 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.276417+0000", 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021240949999999999 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.276566+0000", 2023-09-23T23:04:50.326 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000148715 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.276717+0000", 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015050999999999999 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.327 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283292+0000", 2023-09-23T23:04:50.328 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065753210000000003 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283301+0000", 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2660000000000007e-06 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.455847+0000", 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17254571799999999 2023-09-23T23:04:50.329 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.455898+0000", 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1140000000000002e-05 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.455936+0000", 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7761e-05 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.330 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:48637 11.6 11:6a40531f:::benchmark_data_smithi052_606809_object48636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e418)", 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.903431700999999, 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18496931799999999, 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:04:50.331 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48637 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:04:50.332 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274531+0000", 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1537000000000001e-05 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.333 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274548+0000", 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6458000000000001e-05 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274555+0000", 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0389999999999999e-06 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:04:50.334 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.278791+0000", 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042367860000000002 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.278955+0000", 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016336700000000001 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.279146+0000", 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019095900000000001 2023-09-23T23:04:50.335 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283306+0000", 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041598900000000003 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283525+0000", 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021912200000000001 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.336 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459382+0000", 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17585730099999999 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459433+0000", 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1341999999999997e-05 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459480+0000", 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6394999999999999e-05 2023-09-23T23:04:50.337 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.338 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.338 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.338 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:04:50.338 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:04:50.338 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:04:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:50.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:50.339 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:50.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:50.440 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:50.536 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:50.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 58 16 63354 63338 68.1841 46.9375 0.0156349 0.01466 2023-09-23T23:04:51.077 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:51.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:51.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:51.196 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:51.290 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:51.380 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:51 smithi052 ceph-mon[180979]: pgmap v1308: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T23:04:51.561 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:51.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:51.671 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:51.696 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:51 smithi151 ceph-mon[159210]: pgmap v1308: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T23:04:51.699 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:51.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:51 smithi052 ceph-mon[187743]: pgmap v1308: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 50 MiB/s wr, 798 op/s 2023-09-23T23:04:51.747 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:51.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:51.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 59 16 64128 64112 67.8486 48.375 0.00826818 0.0147336 2023-09-23T23:04:51.851 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:04:51.878 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:51.912 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:04:51.925 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:04:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:51.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:04:52.033 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:52.142 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:52.307 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:52.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:52.366 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:52.415 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:52.442 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:52.488 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:52.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:52.592 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:52.684 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:52.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T23:04:52.790424+0000 min lat: 0.00102195 max lat: 0.430607 avg lat: 0.0148018 2023-09-23T23:04:52.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:04:52.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 60 16 64914 64898 67.5367 49.125 0.0116529 0.0148018 2023-09-23T23:04:52.900 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:52.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:53.010 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:53.120 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:53.261 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:53.360 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:53.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:53.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:53.472 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:53.498 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:04:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:04:53] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.33.4" 2023-09-23T23:04:53.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:53.552 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:53.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:53.658 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:04:53.751 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:53.792 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 61 16 65690 65674 67.2246 48.5 0.00373267 0.0148684 2023-09-23T23:04:53.792 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:04:53.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:53 smithi052 ceph-mon[187743]: pgmap v1309: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T23:04:53.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/737905158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:53.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:53 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:53 smithi052 ceph-mon[180979]: pgmap v1309: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T23:04:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/737905158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:53.809 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:53 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:53 smithi151 ceph-mon[159210]: pgmap v1309: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T23:04:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/737905158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:53 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:04:54.272 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:04:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:54.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:04:54.380 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:54.465 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:54.627 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [7, 6] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:04:54.627 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:04:54.628 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T23:04:54.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T23:04:54.790 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 62 16 66517 66501 66.9741 51.6875 0.00290867 0.0149013 2023-09-23T23:04:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:04:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:04:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T23:04:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[187743]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: failed to encode map e420 with expected crc 2023-09-23T23:04:54.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:54 smithi052 ceph-mon[180979]: failed to encode map e420 with expected crc 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: failed to encode map e420 with expected crc 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: failed to encode map e420 with expected crc 2023-09-23T23:04:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: failed to encode map e420 with expected crc 2023-09-23T23:04:55.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:54 smithi151 ceph-mon[159210]: failed to encode map e420 with expected crc 2023-09-23T23:04:55.269 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:55.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:55.390 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:55.487 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:55.791 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 63 16 67219 67203 66.6077 43.875 0.0148164 0.015008 2023-09-23T23:04:55.831 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:55.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:55.939 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:55.967 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:56.018 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:56.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:56.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:55 smithi151 ceph-mon[159210]: pgmap v1311: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s 2023-09-23T23:04:56.103 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:55 smithi052 ceph-mon[180979]: pgmap v1311: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s 2023-09-23T23:04:56.103 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:55 smithi052 ceph-mon[187743]: pgmap v1311: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s 2023-09-23T23:04:56.123 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:04:56.238 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:56.250 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:04:56.313 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:04:56.369 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:04:56.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:56.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:04:56.481 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:56.503 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:04:56.503 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:04:56.503 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.504 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.505 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:04:56.527 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:56.799 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 64 16 67967 67951 66.2893 46.75 0.0134312 0.0150793 2023-09-23T23:04:56.854 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:56.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:56.966 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:56.993 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:57.023 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3109450801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:57.023 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3109450801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:57.047 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:57.050 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3109450801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:04:57.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:57.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:57.152 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:57.236 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:57.376 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:04:57.587 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:57.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:57.679 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:04:57.699 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:57.725 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:57.772 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:57.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:57.799 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 65 16 68736 68720 66.0091 48.0625 0.0250878 0.0151429 2023-09-23T23:04:57.878 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:04:57.905 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:57.947 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:57 smithi052 ceph-mon[180979]: pgmap v1312: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-09-23T23:04:57.947 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:57 smithi052 ceph-mon[187743]: pgmap v1312: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-09-23T23:04:57.952 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:04:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:57.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:04:58.061 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:04:58.158 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:57 smithi151 ceph-mon[159210]: pgmap v1312: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-09-23T23:04:58.366 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:04:58.366 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:04:58.425 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:04:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:58.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:04:58.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:04:58.567 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:58.613 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:04:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:58.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:04:58.718 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:58.799 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 66 16 69491 69475 65.7241 47.1875 0.0314101 0.0152088 2023-09-23T23:04:58.810 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:59.496 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:04:59.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:59.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:04:59.612 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:04:59.638 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:59.706 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:04:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:04:59.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:04:59.723 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:04:59.799 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 67 16 70269 70253 65.4688 48.625 0.0227264 0.0152688 2023-09-23T23:04:59.813 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:04:59.930 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:04:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:04:59 smithi151 ceph-mon[159210]: pgmap v1313: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-09-23T23:05:00.231 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:00.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:00.241 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:04:59 smithi052 ceph-mon[180979]: pgmap v1313: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-09-23T23:05:00.241 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:04:59 smithi052 ceph-mon[187743]: pgmap v1313: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 745 op/s 2023-09-23T23:05:00.341 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:00.368 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:00.380 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:00.413 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:00.418 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:05:00.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:00.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:00.520 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:00.546 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:00.590 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:00.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:00.696 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:00.722 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:00.768 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:00.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:00.800 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 68 16 71008 70992 65.1854 46.1875 0.0302704 0.0153331 2023-09-23T23:05:00.877 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:00.969 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:01.114 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:01.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:01.227 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:01.255 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:01.260 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:01.301 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:01.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:01.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:01.407 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:01.511 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:01.544 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:01 smithi052 ceph-mon[187743]: pgmap v1314: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T23:05:01.545 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:01 smithi052 ceph-mon[180979]: pgmap v1314: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T23:05:01.545 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:01 smithi052 conmon[180954]: 2023-09-23T23:05:01.541+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e420 definitely_dead 0 2023-09-23T23:05:01.643 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:01.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:01.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:01 smithi151 ceph-mon[159210]: pgmap v1314: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T23:05:01.747 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:01.773 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:01.799 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 69 16 71582 71566 64.7609 35.875 0.00266721 0.0154077 2023-09-23T23:05:01.813 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:01.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:01.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:01.920 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:01.946 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:01.988 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:01.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:02.094 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:02.162 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:02.461 INFO:teuthology.orchestra.run.smithi052.stderr:marked in osd.6. 2023-09-23T23:05:02.463 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3636958422' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:05:02.463 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3636958422' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:05:02.469 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:02.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:02.476 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:02.576 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:02.607 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:02.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3636958422' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T23:05:02.800 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 70 16 71944 71928 64.1594 22.625 0.0059996 0.0155044 2023-09-23T23:05:03.492 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:05:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:05:03] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.33.4" 2023-09-23T23:05:03.492 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3636958422' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[187743]: pgmap v1316: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3636958422' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T23:05:03.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:03 smithi052 ceph-mon[180979]: pgmap v1316: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s 2023-09-23T23:05:03.494 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:03.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:03.506 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T23:05:03.599 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:03.625 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:03.666 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:03.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:03.698 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3636958422' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T23:05:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T23:05:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: failed to encode map e421 with expected crc 2023-09-23T23:05:03.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:03 smithi151 ceph-mon[159210]: pgmap v1316: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s 2023-09-23T23:05:03.772 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:03.800 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 71 16 71944 71928 63.2566 0 - 0.0155044 2023-09-23T23:05:03.802 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:03.994 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:03.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:04.098 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:04.123 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:04.166 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:04.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:04.271 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:04.301 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:04.350 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:04.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:04.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:04.454 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:04.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.623 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.624 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T23:05:04.624 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.624 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.624 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[187743]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.624 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.625 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:04 smithi052 ceph-mon[180979]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.635 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T23:05:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:04 smithi151 ceph-mon[159210]: failed to encode map e422 with expected crc 2023-09-23T23:05:04.800 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 72 16 71951 71935 62.3848 0.21875 2.32232 0.0156981 2023-09-23T23:05:04.858 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:04.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:04.985 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:05.032 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:05.054 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:05:05.091 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:05:05.106 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:05.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:05.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:05.124 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:05:05.174 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:05:05.212 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:05:05.218 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:05.250 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:05.251 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:05:05.284 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:05:05.311 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:05.315 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:05:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:05.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.331 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:05:05.402 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:05:05.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:05.444 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:05.486 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:05.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:05.592 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:05.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T23:05:05.618 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: pgmap v1319: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 630 op/s 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3917861944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/34992043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:05.619 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:05.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/34992043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:05:05.620 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:05 smithi151 ceph-mon[159210]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T23:05:05.620 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:05.662 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:05.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:05.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: pgmap v1319: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 630 op/s 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3917861944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/34992043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/34992043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:05:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[180979]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: failed to encode map e423 with expected crc 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: pgmap v1319: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 630 op/s 2023-09-23T23:05:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3917861944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/34992043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/34992043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:05:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:05 smithi052 ceph-mon[187743]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T23:05:05.768 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:05.800 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 73 16 72371 72355 61.8902 26.25 0.00913991 0.0161513 2023-09-23T23:05:05.818 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:05.888 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:05.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:05.997 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:06.045 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:06.105 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:06.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:06.214 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:06.290 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:06.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[180979]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[180979]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[180979]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.548 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[180979]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T23:05:06.634 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:06.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:06.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:06.796 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:06.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:06 smithi151 ceph-mon[159210]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:06 smithi151 ceph-mon[159210]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:06 smithi151 ceph-mon[159210]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:06 smithi151 ceph-mon[159210]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T23:05:06.801 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 74 16 73066 73050 61.6398 43.4375 0.0141541 0.0162165 2023-09-23T23:05:06.869 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[187743]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.869 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[187743]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[187743]: failed to encode map e424 with expected crc 2023-09-23T23:05:06.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:06 smithi052 ceph-mon[187743]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T23:05:06.879 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:06.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:06.991 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:07.067 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:07.802 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 75 16 73713 73697 61.3573 40.4375 0.00619362 0.016293 2023-09-23T23:05:07.866 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:07.870 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[187743]: failed to encode map e424 with expected crc 2023-09-23T23:05:07.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[187743]: failed to encode map e424 with expected crc 2023-09-23T23:05:07.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[187743]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[187743]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[187743]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[187743]: pgmap v1322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 79 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 363 op/s; 31599/218127 objects degraded (14.487%); 8977/218127 objects misplaced (4.115%); 896 KiB/s, 2 keys/s, 15 objects/s recovering 2023-09-23T23:05:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:07.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:07.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[180979]: failed to encode map e424 with expected crc 2023-09-23T23:05:07.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[180979]: failed to encode map e424 with expected crc 2023-09-23T23:05:07.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[180979]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[180979]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[180979]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:07 smithi052 ceph-mon[180979]: pgmap v1322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 79 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 363 op/s; 31599/218127 objects degraded (14.487%); 8977/218127 objects misplaced (4.115%); 896 KiB/s, 2 keys/s, 15 objects/s recovering 2023-09-23T23:05:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:07 smithi151 ceph-mon[159210]: failed to encode map e424 with expected crc 2023-09-23T23:05:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:07 smithi151 ceph-mon[159210]: failed to encode map e424 with expected crc 2023-09-23T23:05:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:07 smithi151 ceph-mon[159210]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:07 smithi151 ceph-mon[159210]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:07 smithi151 ceph-mon[159210]: failed to encode map e425 with expected crc 2023-09-23T23:05:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:07 smithi151 ceph-mon[159210]: pgmap v1322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 79 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 363 op/s; 31599/218127 objects degraded (14.487%); 8977/218127 objects misplaced (4.115%); 896 KiB/s, 2 keys/s, 15 objects/s recovering 2023-09-23T23:05:07.973 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:08.040 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:08.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:08.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:08.281 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:08.292 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:05:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:05:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:05:08.293 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:05:08.293 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:05:08.364 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:08.507 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:05:08.507 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:05:08.507 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-09-23T23:05:08.507 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:05:08.624 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:08.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:08.733 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:08.801 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 76 16 74347 74331 61.0715 39.625 0.0125293 0.016367 2023-09-23T23:05:08.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:08 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 31599/218127 objects degraded (14.487%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:08.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:08 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 31599/218127 objects degraded (14.487%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:08.847 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:08.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:08 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 31599/218127 objects degraded (14.487%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:08.984 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:08.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:09.098 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:09.145 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:09.581 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:09.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:09.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:09.665 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:05:09.687 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:05:09.689 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:09.714 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:05:09.802 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 77 16 74908 74892 60.7339 35.0625 0.0230953 0.0164572 2023-09-23T23:05:09.809 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:09.869 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:09.948 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:09 smithi052 ceph-mon[187743]: pgmap v1323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 79 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 277 op/s; 31599/218127 objects degraded (14.487%); 8977/218127 objects misplaced (4.115%); 683 KiB/s, 1 keys/s, 11 objects/s recovering 2023-09-23T23:05:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:09 smithi151 ceph-mon[159210]: pgmap v1323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 79 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 277 op/s; 31599/218127 objects degraded (14.487%); 8977/218127 objects misplaced (4.115%); 683 KiB/s, 1 keys/s, 11 objects/s recovering 2023-09-23T23:05:09.950 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:09 smithi052 ceph-mon[180979]: pgmap v1323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 79 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 277 op/s; 31599/218127 objects degraded (14.487%); 8977/218127 objects misplaced (4.115%); 683 KiB/s, 1 keys/s, 11 objects/s recovering 2023-09-23T23:05:10.083 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:10.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:10.205 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:10.233 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:10.292 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:10.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:10.400 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:10.496 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:10.688 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:10.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:10.801 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:10.811 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 78 16 75418 75402 60.3625 31.875 0.04038 0.0165589 2023-09-23T23:05:10.894 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:11.034 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:11.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:11.147 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:11.190 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:11.259 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:11.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:11.367 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:11.393 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:11.433 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:11.438 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:11 smithi151 ceph-mon[159210]: pgmap v1324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 31599/220758 objects degraded (14.314%); 8977/220758 objects misplaced (4.066%); 574 KiB/s, 1 keys/s, 9 objects/s recovering 2023-09-23T23:05:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:11.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:11.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:11.568 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:11.609 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:11.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:11.715 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:11 smithi052 ceph-mon[180979]: pgmap v1324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 31599/220758 objects degraded (14.314%); 8977/220758 objects misplaced (4.066%); 574 KiB/s, 1 keys/s, 9 objects/s recovering 2023-09-23T23:05:11.747 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:11 smithi052 ceph-mon[187743]: pgmap v1324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 31599/220758 objects degraded (14.314%); 8977/220758 objects misplaced (4.066%); 574 KiB/s, 1 keys/s, 9 objects/s recovering 2023-09-23T23:05:11.790 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:11.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 79 16 75871 75855 59.9571 28.3125 0.0340983 0.0166716 2023-09-23T23:05:12.037 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:12.038 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:12.038 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:12.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:12.140 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:12.152 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:12.250 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:12.392 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:12.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:12.503 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:12.584 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:12.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:2023-09-23T23:05:12.804543+0000 min lat: 0.00102195 max lat: 2.44231 avg lat: 0.0167536 2023-09-23T23:05:12.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T23:05:12.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 80 16 76460 76444 59.668 36.8125 0.0224762 0.0167536 2023-09-23T23:05:13.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:12 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31599/226869 objects degraded (13.928%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:13.072 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:12 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31599/226869 objects degraded (13.928%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:12 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31599/226869 objects degraded (13.928%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:13.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:05:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:05:13] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.33.4" 2023-09-23T23:05:13.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 81 16 77101 77085 59.4261 40.0625 0.0310462 0.0168195 2023-09-23T23:05:13.892 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:05:13.897 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:05:14.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:13 smithi151 ceph-mon[159210]: pgmap v1325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 545 op/s; 31599/226869 objects degraded (13.928%); 8820/226869 objects misplaced (3.888%); 1.7 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:05:14.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:13 smithi052 ceph-mon[180979]: pgmap v1325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 545 op/s; 31599/226869 objects degraded (13.928%); 8820/226869 objects misplaced (3.888%); 1.7 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:05:14.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:13 smithi052 ceph-mon[187743]: pgmap v1325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 545 op/s; 31599/226869 objects degraded (13.928%); 8820/226869 objects misplaced (3.888%); 1.7 MiB/s, 1 keys/s, 27 objects/s recovering 2023-09-23T23:05:14.263 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:14.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:14.367 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:14.447 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:14.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 82 16 77724 77708 59.1764 38.9375 0.0142526 0.0168889 2023-09-23T23:05:14.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:14 smithi052 ceph-mon[180979]: from='client.46991 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:05:14.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:14 smithi052 ceph-mon[187743]: from='client.46991 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:05:15.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:14 smithi151 ceph-mon[159210]: from='client.46991 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:05:15.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 83 16 78362 78346 58.944 39.875 0.0068202 0.0169593 2023-09-23T23:05:15.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:15 smithi052 ceph-mon[187743]: pgmap v1326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 469 op/s; 31599/226869 objects degraded (13.928%); 8820/226869 objects misplaced (3.888%); 1.4 MiB/s, 1 keys/s, 23 objects/s recovering 2023-09-23T23:05:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:15 smithi052 ceph-mon[180979]: pgmap v1326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 469 op/s; 31599/226869 objects degraded (13.928%); 8820/226869 objects misplaced (3.888%); 1.4 MiB/s, 1 keys/s, 23 objects/s recovering 2023-09-23T23:05:16.117 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:16.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:15 smithi151 ceph-mon[159210]: pgmap v1326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 469 op/s; 31599/226869 objects degraded (13.928%); 8820/226869 objects misplaced (3.888%); 1.4 MiB/s, 1 keys/s, 23 objects/s recovering 2023-09-23T23:05:16.235 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:16.270 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:16.335 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:16.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:16.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:16.442 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:16.557 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:16.804 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 84 16 78942 78926 58.674 36.25 0.0138744 0.0170364 2023-09-23T23:05:17.135 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:17.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:17.243 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:17.276 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1325,"stamp":"2023-09-23T23:05:12.683711+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4955051936,"num_objects":75623,"num_object_clones":0,"num_object_copies":226869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31599,"num_objects_misplaced":8820,"num_objects_unfound":0,"num_objects_dirty":75623,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":75859,"num_write_kb":4840781,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":258,"num_bytes_recovered":14570300,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75793,"ondisk_log_size":75793,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19492796,"kb_used_data":14874436,"kb_used_omap":137,"kb_used_meta":4618166,"kb_avail":636723268,"statfs":{"total":671965249536,"available":652004626432,"internally_reserved":0,"allocated":15231422464,"data_stored":15221899823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":4729002168},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,5,11,5,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":286457856,"num_objects":4371,"num_object_clones":0,"num_object_copies":13113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31599,"num_objects_misplaced":8820,"num_objects_unfound":0,"num_objects_dirty":4371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4371,"num_write_kb":279744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":13964550,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.007531"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":437,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388620+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:05:05.388620+0000","last_peered":"2023-09-23T23:05:05.388620+0000","last_clean":"2023-09-23T23:05:05.388620+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:05:05.388620+0000","last_undegraded":"2023-09-23T23:05:05.388620+0000","last_fullsized":"2023-09-23T23:05:05.388620+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":96,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386967+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:05:05.386967+0000","last_peered":"2023-09-23T23:05:05.386967+0000","last_clean":"2023-09-23T23:05:05.386967+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:05:05.386967+0000","last_undegraded":"2023-09-23T23:05:05.386967+0000","last_fullsized":"2023-09-23T23:05:05.386967+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389370+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:05:05.389370+0000","last_peered":"2023-09-23T23:05:05.389370+0000","last_clean":"2023-09-23T23:05:05.389370+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:05:05.389370+0000","last_undegraded":"2023-09-23T23:05:05.389370+0000","last_fullsized":"2023-09-23T23:05:05.389370+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":423,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386036+0000","last_change":"2023-09-23T22:58:06.748407+0000","last_active":"2023-09-23T23:05:05.386036+0000","last_peered":"2023-09-23T23:05:05.386036+0000","last_clean":"2023-09-23T23:05:05.386036+0000","last_became_active":"2023-09-23T22:58:06.748107+0000","last_became_peered":"2023-09-23T22:58:06.748107+0000","last_unstale":"2023-09-23T23:05:05.386036+0000","last_undegraded":"2023-09-23T23:05:05.386036+0000","last_fullsized":"2023-09-23T23:05:05.386036+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:57.528235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":22,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.398841+0000","last_change":"2023-09-23T23:03:51.528123+0000","last_active":"2023-09-23T23:05:05.398841+0000","last_peered":"2023-09-23T23:05:05.398841+0000","last_clean":"2023-09-23T23:05:05.398841+0000","last_became_active":"2023-09-23T23:03:51.527770+0000","last_became_peered":"2023-09-23T23:03:51.527770+0000","last_unstale":"2023-09-23T23:05:05.398841+0000","last_undegraded":"2023-09-23T23:05:05.398841+0000","last_fullsized":"2023-09-23T23:05:05.398841+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:32.789048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.381045+0000","last_change":"2023-09-23T23:05:03.454677+0000","last_active":"2023-09-23T23:05:05.381045+0000","last_peered":"2023-09-23T23:05:05.381045+0000","last_clean":"2023-09-23T23:05:05.381045+0000","last_became_active":"2023-09-23T23:05:03.454290+0000","last_became_peered":"2023-09-23T23:05:03.454290+0000","last_unstale":"2023-09-23T23:05:05.381045+0000","last_undegraded":"2023-09-23T23:05:05.381045+0000","last_fullsized":"2023-09-23T23:05:05.381045+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":360,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380144+0000","last_change":"2023-09-23T22:58:06.766887+0000","last_active":"2023-09-23T23:05:05.380144+0000","last_peered":"2023-09-23T23:05:05.380144+0000","last_clean":"2023-09-23T23:05:05.380144+0000","last_became_active":"2023-09-23T22:58:06.766331+0000","last_became_peered":"2023-09-23T22:58:06.766331+0000","last_unstale":"2023-09-23T23:05:05.380144+0000","last_undegraded":"2023-09-23T23:05:05.380144+0000","last_fullsized":"2023-09-23T23:05:05.380144+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.395117+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:05:05.395117+0000","last_peered":"2023-09-23T23:05:05.395117+0000","last_clean":"2023-09-23T23:05:05.395117+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:05:05.395117+0000","last_undegraded":"2023-09-23T23:05:05.395117+0000","last_fullsized":"2023-09-23T23:05:05.395117+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":321,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.379910+0000","last_change":"2023-09-23T22:57:26.524373+0000","last_active":"2023-09-23T23:05:05.379910+0000","last_peered":"2023-09-23T23:05:05.379910+0000","last_clean":"2023-09-23T23:05:05.379910+0000","last_became_active":"2023-09-23T22:57:26.524076+0000","last_became_peered":"2023-09-23T22:57:26.524076+0000","last_unstale":"2023-09-23T23:05:05.379910+0000","last_undegraded":"2023-09-23T23:05:05.379910+0000","last_fullsized":"2023-09-23T23:05:05.379910+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389218+0000","last_change":"2023-09-23T23:05:03.447783+0000","last_active":"2023-09-23T23:05:05.389218+0000","last_peered":"2023-09-23T23:05:05.389218+0000","last_clean":"2023-09-23T23:05:05.389218+0000","last_became_active":"2023-09-23T23:05:03.447567+0000","last_became_peered":"2023-09-23T23:05:03.447567+0000","last_unstale":"2023-09-23T23:05:05.389218+0000","last_undegraded":"2023-09-23T23:05:05.389218+0000","last_fullsized":"2023-09-23T23:05:05.389218+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:05.175895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":131,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386887+0000","last_change":"2023-09-23T23:03:51.536661+0000","last_active":"2023-09-23T23:05:05.386887+0000","last_peered":"2023-09-23T23:05:05.386887+0000","last_clean":"2023-09-23T23:05:05.386887+0000","last_became_active":"2023-09-23T23:03:51.532360+0000","last_became_peered":"2023-09-23T23:03:51.532360+0000","last_unstale":"2023-09-23T23:05:05.386887+0000","last_undegraded":"2023-09-23T23:05:05.386887+0000","last_fullsized":"2023-09-23T23:05:05.386887+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:55.428059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380819+0000","last_change":"2023-09-23T23:05:03.446951+0000","last_active":"2023-09-23T23:05:05.380819+0000","last_peered":"2023-09-23T23:05:05.380819+0000","last_clean":"2023-09-23T23:05:05.380819+0000","last_became_active":"2023-09-23T23:05:03.445517+0000","last_became_peered":"2023-09-23T23:05:03.445517+0000","last_unstale":"2023-09-23T23:05:05.380819+0000","last_undegraded":"2023-09-23T23:05:05.380819+0000","last_fullsized":"2023-09-23T23:05:05.380819+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":515,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389485+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:05:05.389485+0000","last_peered":"2023-09-23T23:05:05.389485+0000","last_clean":"2023-09-23T23:05:05.389485+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:05:05.389485+0000","last_undegraded":"2023-09-23T23:05:05.389485+0000","last_fullsized":"2023-09-23T23:05:05.389485+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":408,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388797+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:05:05.388797+0000","last_peered":"2023-09-23T23:05:05.388797+0000","last_clean":"2023-09-23T23:05:05.388797+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:05:05.388797+0000","last_undegraded":"2023-09-23T23:05:05.388797+0000","last_fullsized":"2023-09-23T23:05:05.388797+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":340,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388899+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:05:05.388899+0000","last_peered":"2023-09-23T23:05:05.388899+0000","last_clean":"2023-09-23T23:05:05.388899+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:05:05.388899+0000","last_undegraded":"2023-09-23T23:05:05.388899+0000","last_fullsized":"2023-09-23T23:05:05.388899+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388438+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:05:05.388438+0000","last_peered":"2023-09-23T23:05:05.388438+0000","last_clean":"2023-09-23T23:05:05.388438+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:05:05.388438+0000","last_undegraded":"2023-09-23T23:05:05.388438+0000","last_fullsized":"2023-09-23T23:05:05.388438+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":416,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386110+0000","last_change":"2023-09-23T22:58:06.750639+0000","last_active":"2023-09-23T23:05:05.386110+0000","last_peered":"2023-09-23T23:05:05.386110+0000","last_clean":"2023-09-23T23:05:05.386110+0000","last_became_active":"2023-09-23T22:58:06.750329+0000","last_became_peered":"2023-09-23T22:58:06.750329+0000","last_unstale":"2023-09-23T23:05:05.386110+0000","last_undegraded":"2023-09-23T23:05:05.386110+0000","last_fullsized":"2023-09-23T23:05:05.386110+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.881149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.379337+0000","last_change":"2023-09-23T23:05:03.453683+0000","last_active":"2023-09-23T23:05:05.379337+0000","last_peered":"2023-09-23T23:05:05.379337+0000","last_clean":"2023-09-23T23:05:05.379337+0000","last_became_active":"2023-09-23T23:05:03.453332+0000","last_became_peered":"2023-09-23T23:05:03.453332+0000","last_unstale":"2023-09-23T23:05:05.379337+0000","last_undegraded":"2023-09-23T23:05:05.379337+0000","last_fullsized":"2023-09-23T23:05:05.379337+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":319,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388682+0000","last_change":"2023-09-23T23:05:03.450126+0000","last_active":"2023-09-23T23:05:05.388682+0000","last_peered":"2023-09-23T23:05:05.388682+0000","last_clean":"2023-09-23T23:05:05.388682+0000","last_became_active":"2023-09-23T23:05:03.449196+0000","last_became_peered":"2023-09-23T23:05:03.449196+0000","last_unstale":"2023-09-23T23:05:05.388682+0000","last_undegraded":"2023-09-23T23:05:05.388682+0000","last_fullsized":"2023-09-23T23:05:05.388682+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":440,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386128+0000","last_change":"2023-09-23T22:58:24.246265+0000","last_active":"2023-09-23T23:05:05.386128+0000","last_peered":"2023-09-23T23:05:05.386128+0000","last_clean":"2023-09-23T23:05:05.386128+0000","last_became_active":"2023-09-23T22:58:24.245847+0000","last_became_peered":"2023-09-23T22:58:24.245847+0000","last_unstale":"2023-09-23T23:05:05.386128+0000","last_undegraded":"2023-09-23T23:05:05.386128+0000","last_fullsized":"2023-09-23T23:05:05.386128+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:02.851904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":23,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389627+0000","last_change":"2023-09-23T23:05:03.802457+0000","last_active":"2023-09-23T23:05:05.389627+0000","last_peered":"2023-09-23T23:05:05.389627+0000","last_clean":"2023-09-23T23:05:05.389627+0000","last_became_active":"2023-09-23T23:05:03.454670+0000","last_became_peered":"2023-09-23T23:05:03.454670+0000","last_unstale":"2023-09-23T23:05:05.389627+0000","last_undegraded":"2023-09-23T23:05:05.389627+0000","last_fullsized":"2023-09-23T23:05:05.389627+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":120,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386293+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:05:05.386293+0000","last_peered":"2023-09-23T23:05:05.386293+0000","last_clean":"2023-09-23T23:05:05.386293+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:05:05.386293+0000","last_undegraded":"2023-09-23T23:05:05.386293+0000","last_fullsized":"2023-09-23T23:05:05.386293+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"425'4688","reported_seq":4711,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:10.358296+0000","last_change":"2023-09-23T23:03:43.131702+0000","last_active":"2023-09-23T23:05:10.358296+0000","last_peered":"2023-09-23T23:05:10.358296+0000","last_clean":"2023-09-23T23:05:10.358296+0000","last_became_active":"2023-09-23T23:03:43.131323+0000","last_became_peered":"2023-09-23T23:03:43.131323+0000","last_unstale":"2023-09-23T23:05:10.358296+0000","last_undegraded":"2023-09-23T23:05:10.358296+0000","last_fullsized":"2023-09-23T23:05:10.358296+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307232768,"num_objects":4688,"num_object_clones":0,"num_object_copies":14064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4688,"num_write_kb":300032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":309,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388780+0000","last_change":"2023-09-23T22:58:06.777384+0000","last_active":"2023-09-23T23:05:05.388780+0000","last_peered":"2023-09-23T23:05:05.388780+0000","last_clean":"2023-09-23T23:05:05.388780+0000","last_became_active":"2023-09-23T22:58:06.777004+0000","last_became_peered":"2023-09-23T22:58:06.777004+0000","last_unstale":"2023-09-23T23:05:05.388780+0000","last_undegraded":"2023-09-23T23:05:05.388780+0000","last_fullsized":"2023-09-23T23:05:05.388780+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":304,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380310+0000","last_change":"2023-09-23T23:05:03.450655+0000","last_active":"2023-09-23T23:05:05.380310+0000","last_peered":"2023-09-23T23:05:05.380310+0000","last_clean":"2023-09-23T23:05:05.380310+0000","last_became_active":"2023-09-23T23:05:03.449850+0000","last_became_peered":"2023-09-23T23:05:03.449850+0000","last_unstale":"2023-09-23T23:05:05.380310+0000","last_undegraded":"2023-09-23T23:05:05.380310+0000","last_fullsized":"2023-09-23T23:05:05.380310+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"425'4645","reported_seq":4668,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:10.358854+0000","last_change":"2023-09-23T23:03:43.132133+0000","last_active":"2023-09-23T23:05:10.358854+0000","last_peered":"2023-09-23T23:05:10.358854+0000","last_clean":"2023-09-23T23:05:10.358854+0000","last_became_active":"2023-09-23T23:03:43.131741+0000","last_became_peered":"2023-09-23T23:03:43.131741+0000","last_unstale":"2023-09-23T23:05:10.358854+0000","last_undegraded":"2023-09-23T23:05:10.358854+0000","last_fullsized":"2023-09-23T23:05:10.358854+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4645,"log_dups_size":0,"ondisk_log_size":4645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:36.498718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304414720,"num_objects":4645,"num_object_clones":0,"num_object_copies":13935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4645,"num_write_kb":297280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":26,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.400965+0000","last_change":"2023-09-23T23:03:51.533976+0000","last_active":"2023-09-23T23:05:05.400965+0000","last_peered":"2023-09-23T23:05:05.400965+0000","last_clean":"2023-09-23T23:05:05.400965+0000","last_became_active":"2023-09-23T23:03:51.528701+0000","last_became_peered":"2023-09-23T23:03:51.528701+0000","last_unstale":"2023-09-23T23:05:05.400965+0000","last_undegraded":"2023-09-23T23:05:05.400965+0000","last_fullsized":"2023-09-23T23:05:05.400965+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.921280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":432,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388617+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:05:05.388617+0000","last_peered":"2023-09-23T23:05:05.388617+0000","last_clean":"2023-09-23T23:05:05.388617+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:05:05.388617+0000","last_undegraded":"2023-09-23T23:05:05.388617+0000","last_fullsized":"2023-09-23T23:05:05.388617+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"425'4656","reported_seq":4687,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:10.345979+0000","last_change":"2023-09-23T23:05:04.829166+0000","last_active":"2023-09-23T23:05:10.345979+0000","last_peered":"2023-09-23T23:05:10.345979+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:05:04.822433+0000","last_became_peered":"2023-09-23T23:05:04.822433+0000","last_unstale":"2023-09-23T23:05:10.345979+0000","last_undegraded":"2023-09-23T23:05:04.452196+0000","last_fullsized":"2023-09-23T23:05:04.433561+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4656,"log_dups_size":0,"ondisk_log_size":4656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4656,"num_write_kb":297984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,3],"avail_no_missing":["5","0","3"],"object_location_counts":[{"shards":"0,3,5","objects":4480},{"shards":"3,5","objects":176}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":189,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380031+0000","last_change":"2023-09-23T23:05:03.504558+0000","last_active":"2023-09-23T23:05:05.380031+0000","last_peered":"2023-09-23T23:05:05.380031+0000","last_clean":"2023-09-23T23:05:05.380031+0000","last_became_active":"2023-09-23T23:05:03.453187+0000","last_became_peered":"2023-09-23T23:05:03.453187+0000","last_unstale":"2023-09-23T23:05:05.380031+0000","last_undegraded":"2023-09-23T23:05:05.380031+0000","last_fullsized":"2023-09-23T23:05:05.380031+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380672+0000","last_change":"2023-09-23T23:05:03.453364+0000","last_active":"2023-09-23T23:05:05.380672+0000","last_peered":"2023-09-23T23:05:05.380672+0000","last_clean":"2023-09-23T23:05:05.380672+0000","last_became_active":"2023-09-23T23:05:03.453021+0000","last_became_peered":"2023-09-23T23:05:03.453021+0000","last_unstale":"2023-09-23T23:05:05.380672+0000","last_undegraded":"2023-09-23T23:05:05.380672+0000","last_fullsized":"2023-09-23T23:05:05.380672+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"425'4798","reported_seq":4830,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:11.303211+0000","last_change":"2023-09-23T23:05:04.794249+0000","last_active":"2023-09-23T23:05:11.303211+0000","last_peered":"2023-09-23T23:05:11.303211+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:05:04.783302+0000","last_became_peered":"2023-09-23T23:05:04.783302+0000","last_unstale":"2023-09-23T23:05:11.303211+0000","last_undegraded":"2023-09-23T23:05:04.673306+0000","last_fullsized":"2023-09-23T23:05:04.432768+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4798,"log_dups_size":0,"ondisk_log_size":4798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:12.794503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314441728,"num_objects":4798,"num_object_clones":0,"num_object_copies":14394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4562,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4798,"num_write_kb":307072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":236},{"shards":"1,2,3","objects":4562}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":105,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388369+0000","last_change":"2023-09-23T23:05:03.450209+0000","last_active":"2023-09-23T23:05:05.388369+0000","last_peered":"2023-09-23T23:05:05.388369+0000","last_clean":"2023-09-23T23:05:05.388369+0000","last_became_active":"2023-09-23T23:05:03.449572+0000","last_became_peered":"2023-09-23T23:05:03.449572+0000","last_unstale":"2023-09-23T23:05:05.388369+0000","last_undegraded":"2023-09-23T23:05:05.388369+0000","last_fullsized":"2023-09-23T23:05:05.388369+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:15.428519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":326,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.383011+0000","last_change":"2023-09-23T22:58:06.749928+0000","last_active":"2023-09-23T23:05:05.383011+0000","last_peered":"2023-09-23T23:05:05.383011+0000","last_clean":"2023-09-23T23:05:05.383011+0000","last_became_active":"2023-09-23T22:58:06.749531+0000","last_became_peered":"2023-09-23T22:58:06.749531+0000","last_unstale":"2023-09-23T23:05:05.383011+0000","last_undegraded":"2023-09-23T23:05:05.383011+0000","last_fullsized":"2023-09-23T23:05:05.383011+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:34.034081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"425'4793","reported_seq":4830,"reported_epoch":425,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T23:05:11.258394+0000","last_change":"2023-09-23T23:05:04.792623+0000","last_active":"2023-09-23T23:05:11.258394+0000","last_peered":"2023-09-23T23:05:11.258394+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:05:04.742376+0000","last_became_peered":"2023-09-23T23:05:04.742376+0000","last_unstale":"2023-09-23T23:05:11.258394+0000","last_undegraded":"2023-09-23T23:05:11.258394+0000","last_fullsized":"2023-09-23T23:05:11.258394+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4793,"log_dups_size":0,"ondisk_log_size":4793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:27.469045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314114048,"num_objects":4793,"num_object_clones":0,"num_object_copies":14379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4516,"num_objects_unfound":0,"num_objects_dirty":4793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4793,"num_write_kb":306752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[2,1,5],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2,5","objects":277}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":115,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.385866+0000","last_change":"2023-09-23T22:58:06.746286+0000","last_active":"2023-09-23T23:05:05.385866+0000","last_peered":"2023-09-23T23:05:05.385866+0000","last_clean":"2023-09-23T23:05:05.385866+0000","last_became_active":"2023-09-23T22:58:06.745927+0000","last_became_peered":"2023-09-23T22:58:06.745927+0000","last_unstale":"2023-09-23T23:05:05.385866+0000","last_undegraded":"2023-09-23T23:05:05.385866+0000","last_fullsized":"2023-09-23T23:05:05.385866+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:50.765438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":427,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389390+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:05:05.389390+0000","last_peered":"2023-09-23T23:05:05.389390+0000","last_clean":"2023-09-23T23:05:05.389390+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:05:05.389390+0000","last_undegraded":"2023-09-23T23:05:05.389390+0000","last_fullsized":"2023-09-23T23:05:05.389390+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"425'4724","reported_seq":4748,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:11.310532+0000","last_change":"2023-09-23T23:03:42.889763+0000","last_active":"2023-09-23T23:05:11.310532+0000","last_peered":"2023-09-23T23:05:11.310532+0000","last_clean":"2023-09-23T23:05:11.310532+0000","last_became_active":"2023-09-23T23:03:42.889481+0000","last_became_peered":"2023-09-23T23:03:42.889481+0000","last_unstale":"2023-09-23T23:05:11.310532+0000","last_undegraded":"2023-09-23T23:05:11.310532+0000","last_fullsized":"2023-09-23T23:05:11.310532+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4724,"log_dups_size":0,"ondisk_log_size":4724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:57.841265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309592064,"num_objects":4724,"num_object_clones":0,"num_object_copies":14172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4724,"num_write_kb":302336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380598+0000","last_change":"2023-09-23T23:05:03.447155+0000","last_active":"2023-09-23T23:05:05.380598+0000","last_peered":"2023-09-23T23:05:05.380598+0000","last_clean":"2023-09-23T23:05:05.380598+0000","last_became_active":"2023-09-23T23:05:03.446361+0000","last_became_peered":"2023-09-23T23:05:03.446361+0000","last_unstale":"2023-09-23T23:05:05.380598+0000","last_undegraded":"2023-09-23T23:05:05.380598+0000","last_fullsized":"2023-09-23T23:05:05.380598+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":435,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389663+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:05:05.389663+0000","last_peered":"2023-09-23T23:05:05.389663+0000","last_clean":"2023-09-23T23:05:05.389663+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:05:05.389663+0000","last_undegraded":"2023-09-23T23:05:05.389663+0000","last_fullsized":"2023-09-23T23:05:05.389663+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"425'4726","reported_seq":4753,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:10.217464+0000","last_change":"2023-09-23T23:05:04.750351+0000","last_active":"2023-09-23T23:05:10.217464+0000","last_peered":"2023-09-23T23:05:10.217464+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:05:04.739837+0000","last_became_peered":"2023-09-23T23:05:04.739837+0000","last_unstale":"2023-09-23T23:05:10.217464+0000","last_undegraded":"2023-09-23T23:05:04.460015+0000","last_fullsized":"2023-09-23T23:05:04.429292+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4726,"log_dups_size":0,"ondisk_log_size":4726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:53:48.832829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309723136,"num_objects":4726,"num_object_clones":0,"num_object_copies":14178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4726,"num_write_kb":302464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":4559},{"shards":"0,3","objects":167}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":448,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388470+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:05:05.388470+0000","last_peered":"2023-09-23T23:05:05.388470+0000","last_clean":"2023-09-23T23:05:05.388470+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:05:05.388470+0000","last_undegraded":"2023-09-23T23:05:05.388470+0000","last_fullsized":"2023-09-23T23:05:05.388470+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":435,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389907+0000","last_change":"2023-09-23T23:05:03.450633+0000","last_active":"2023-09-23T23:05:05.389907+0000","last_peered":"2023-09-23T23:05:05.389907+0000","last_clean":"2023-09-23T23:05:05.389907+0000","last_became_active":"2023-09-23T23:05:03.450232+0000","last_became_peered":"2023-09-23T23:05:03.450232+0000","last_unstale":"2023-09-23T23:05:05.389907+0000","last_undegraded":"2023-09-23T23:05:05.389907+0000","last_fullsized":"2023-09-23T23:05:05.389907+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"425'4699","reported_seq":4731,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:11.327820+0000","last_change":"2023-09-23T23:05:04.855367+0000","last_active":"2023-09-23T23:05:11.327820+0000","last_peered":"2023-09-23T23:05:11.327820+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:04.842585+0000","last_became_peered":"2023-09-23T23:05:04.842585+0000","last_unstale":"2023-09-23T23:05:11.327820+0000","last_undegraded":"2023-09-23T23:05:04.506327+0000","last_fullsized":"2023-09-23T23:05:04.437731+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4699,"log_dups_size":0,"ondisk_log_size":4699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:45:12.973112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307953664,"num_objects":4699,"num_object_clones":0,"num_object_copies":14097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4699,"num_write_kb":300736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":4437},{"shards":"1,3","objects":262}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":96,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.397253+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:05:05.397253+0000","last_peered":"2023-09-23T23:05:05.397253+0000","last_clean":"2023-09-23T23:05:05.397253+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:05:05.397253+0000","last_undegraded":"2023-09-23T23:05:05.397253+0000","last_fullsized":"2023-09-23T23:05:05.397253+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":330,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:06.401638+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:05:06.401638+0000","last_peered":"2023-09-23T23:05:06.401638+0000","last_clean":"2023-09-23T23:05:06.401638+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:05:06.401638+0000","last_undegraded":"2023-09-23T23:05:06.401638+0000","last_fullsized":"2023-09-23T23:05:06.401638+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"425'4806","reported_seq":4841,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:11.331870+0000","last_change":"2023-09-23T23:05:04.841155+0000","last_active":"2023-09-23T23:05:11.331870+0000","last_peered":"2023-09-23T23:05:11.331870+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:05:11.331870+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4806,"log_dups_size":0,"ondisk_log_size":4806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314966016,"num_objects":4806,"num_object_clones":0,"num_object_copies":14418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4552,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4806,"num_write_kb":307584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":4552},{"shards":"2,4","objects":254}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":334,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.409263+0000","last_change":"2023-09-23T22:58:23.265626+0000","last_active":"2023-09-23T23:05:05.409263+0000","last_peered":"2023-09-23T23:05:05.409263+0000","last_clean":"2023-09-23T23:05:05.409263+0000","last_became_active":"2023-09-23T22:58:23.265285+0000","last_became_peered":"2023-09-23T22:58:23.265285+0000","last_unstale":"2023-09-23T23:05:05.409263+0000","last_undegraded":"2023-09-23T23:05:05.409263+0000","last_fullsized":"2023-09-23T23:05:05.409263+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:41.651285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":335,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.379254+0000","last_change":"2023-09-23T22:58:24.244520+0000","last_active":"2023-09-23T23:05:05.379254+0000","last_peered":"2023-09-23T23:05:05.379254+0000","last_clean":"2023-09-23T23:05:05.379254+0000","last_became_active":"2023-09-23T22:58:24.244237+0000","last_became_peered":"2023-09-23T22:58:24.244237+0000","last_unstale":"2023-09-23T23:05:05.379254+0000","last_undegraded":"2023-09-23T23:05:05.379254+0000","last_fullsized":"2023-09-23T23:05:05.379254+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":148,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.379390+0000","last_change":"2023-09-23T22:57:04.807822+0000","last_active":"2023-09-23T23:05:05.379390+0000","last_peered":"2023-09-23T23:05:05.379390+0000","last_clean":"2023-09-23T23:05:05.379390+0000","last_became_active":"2023-09-23T22:57:04.807537+0000","last_became_peered":"2023-09-23T22:57:04.807537+0000","last_unstale":"2023-09-23T23:05:05.379390+0000","last_undegraded":"2023-09-23T23:05:05.379390+0000","last_fullsized":"2023-09-23T23:05:05.379390+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"425'4720","reported_seq":4744,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:11.699510+0000","last_change":"2023-09-23T23:03:43.132707+0000","last_active":"2023-09-23T23:05:11.699510+0000","last_peered":"2023-09-23T23:05:11.699510+0000","last_clean":"2023-09-23T23:05:11.699510+0000","last_became_active":"2023-09-23T23:03:43.132372+0000","last_became_peered":"2023-09-23T23:03:43.132372+0000","last_unstale":"2023-09-23T23:05:11.699510+0000","last_undegraded":"2023-09-23T23:05:11.699510+0000","last_fullsized":"2023-09-23T23:05:11.699510+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4720,"log_dups_size":0,"ondisk_log_size":4720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:52:01.951582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4720,"num_write_kb":302080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380751+0000","last_change":"2023-09-23T23:05:03.453544+0000","last_active":"2023-09-23T23:05:05.380751+0000","last_peered":"2023-09-23T23:05:05.380751+0000","last_clean":"2023-09-23T23:05:05.380751+0000","last_became_active":"2023-09-23T23:05:03.453121+0000","last_became_peered":"2023-09-23T23:05:03.453121+0000","last_unstale":"2023-09-23T23:05:05.380751+0000","last_undegraded":"2023-09-23T23:05:05.380751+0000","last_fullsized":"2023-09-23T23:05:05.380751+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":454,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388987+0000","last_change":"2023-09-23T23:05:03.450518+0000","last_active":"2023-09-23T23:05:05.388987+0000","last_peered":"2023-09-23T23:05:05.388987+0000","last_clean":"2023-09-23T23:05:05.388987+0000","last_became_active":"2023-09-23T23:05:03.449416+0000","last_became_peered":"2023-09-23T23:05:03.449416+0000","last_unstale":"2023-09-23T23:05:05.388987+0000","last_undegraded":"2023-09-23T23:05:05.388987+0000","last_fullsized":"2023-09-23T23:05:05.388987+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":319,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.409178+0000","last_change":"2023-09-23T22:58:24.244480+0000","last_active":"2023-09-23T23:05:05.409178+0000","last_peered":"2023-09-23T23:05:05.409178+0000","last_clean":"2023-09-23T23:05:05.409178+0000","last_became_active":"2023-09-23T22:58:24.244159+0000","last_became_peered":"2023-09-23T22:58:24.244159+0000","last_unstale":"2023-09-23T23:05:05.409178+0000","last_undegraded":"2023-09-23T23:05:05.409178+0000","last_fullsized":"2023-09-23T23:05:05.409178+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:23.216035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"425'4779","reported_seq":4810,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:11.691664+0000","last_change":"2023-09-23T23:05:04.842583+0000","last_active":"2023-09-23T23:05:11.691664+0000","last_peered":"2023-09-23T23:05:11.691664+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:04.820928+0000","last_became_peered":"2023-09-23T23:05:04.820928+0000","last_unstale":"2023-09-23T23:05:11.691664+0000","last_undegraded":"2023-09-23T23:05:04.510581+0000","last_fullsized":"2023-09-23T23:05:04.439141+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4779,"log_dups_size":0,"ondisk_log_size":4779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:24.641416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313196544,"num_objects":4779,"num_object_clones":0,"num_object_copies":14337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4542,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4779,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":4542},{"shards":"3,5","objects":237}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":394,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.398075+0000","last_change":"2023-09-23T22:58:24.240789+0000","last_active":"2023-09-23T23:05:05.398075+0000","last_peered":"2023-09-23T23:05:05.398075+0000","last_clean":"2023-09-23T23:05:05.398075+0000","last_became_active":"2023-09-23T22:58:24.240095+0000","last_became_peered":"2023-09-23T22:58:24.240095+0000","last_unstale":"2023-09-23T23:05:05.398075+0000","last_undegraded":"2023-09-23T23:05:05.398075+0000","last_fullsized":"2023-09-23T23:05:05.398075+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:41.891448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":30,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.378970+0000","last_change":"2023-09-23T23:05:03.450848+0000","last_active":"2023-09-23T23:05:05.378970+0000","last_peered":"2023-09-23T23:05:05.378970+0000","last_clean":"2023-09-23T23:05:05.378970+0000","last_became_active":"2023-09-23T23:05:03.450288+0000","last_became_peered":"2023-09-23T23:05:03.450288+0000","last_unstale":"2023-09-23T23:05:05.378970+0000","last_undegraded":"2023-09-23T23:05:05.378970+0000","last_fullsized":"2023-09-23T23:05:05.378970+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":335,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380183+0000","last_change":"2023-09-23T22:58:06.772756+0000","last_active":"2023-09-23T23:05:05.380183+0000","last_peered":"2023-09-23T23:05:05.380183+0000","last_clean":"2023-09-23T23:05:05.380183+0000","last_became_active":"2023-09-23T22:58:06.767142+0000","last_became_peered":"2023-09-23T22:58:06.767142+0000","last_unstale":"2023-09-23T23:05:05.380183+0000","last_undegraded":"2023-09-23T23:05:05.380183+0000","last_fullsized":"2023-09-23T23:05:05.380183+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"425'4754","reported_seq":5181,"reported_epoch":425,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:11.718456+0000","last_change":"2023-09-23T23:05:04.825391+0000","last_active":"2023-09-23T23:05:11.718456+0000","last_peered":"2023-09-23T23:05:11.718456+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:04.725097+0000","last_became_peered":"2023-09-23T23:05:04.725097+0000","last_unstale":"2023-09-23T23:05:11.718456+0000","last_undegraded":"2023-09-23T23:05:04.547265+0000","last_fullsized":"2023-09-23T23:05:04.460221+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4754,"log_dups_size":0,"ondisk_log_size":4754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:00.694669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311558144,"num_objects":4754,"num_object_clones":0,"num_object_copies":14262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":4304,"num_objects_unfound":0,"num_objects_dirty":4754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4754,"num_write_kb":304256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":198,"num_bytes_recovered":12976128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":4304},{"shards":"3,5","objects":449}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":408,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.379851+0000","last_change":"2023-09-23T22:58:24.248405+0000","last_active":"2023-09-23T23:05:05.379851+0000","last_peered":"2023-09-23T23:05:05.379851+0000","last_clean":"2023-09-23T23:05:05.379851+0000","last_became_active":"2023-09-23T22:58:24.248031+0000","last_became_peered":"2023-09-23T22:58:24.248031+0000","last_unstale":"2023-09-23T23:05:05.379851+0000","last_undegraded":"2023-09-23T23:05:05.379851+0000","last_fullsized":"2023-09-23T23:05:05.379851+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":561,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:06.396832+0000","last_change":"2023-09-23T23:05:06.396832+0000","last_active":"2023-09-23T23:05:06.396832+0000","last_peered":"2023-09-23T23:05:06.396832+0000","last_clean":"2023-09-23T23:05:06.396832+0000","last_became_active":"2023-09-23T23:05:06.396448+0000","last_became_peered":"2023-09-23T23:05:06.396448+0000","last_unstale":"2023-09-23T23:05:06.396832+0000","last_undegraded":"2023-09-23T23:05:06.396832+0000","last_fullsized":"2023-09-23T23:05:06.396832+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:42.501436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":105,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389694+0000","last_change":"2023-09-23T23:05:03.448193+0000","last_active":"2023-09-23T23:05:05.389694+0000","last_peered":"2023-09-23T23:05:05.389694+0000","last_clean":"2023-09-23T23:05:05.389694+0000","last_became_active":"2023-09-23T23:05:03.447847+0000","last_became_peered":"2023-09-23T23:05:03.447847+0000","last_unstale":"2023-09-23T23:05:05.389694+0000","last_undegraded":"2023-09-23T23:05:05.389694+0000","last_fullsized":"2023-09-23T23:05:05.389694+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:42:14.731099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.381251+0000","last_change":"2023-09-23T23:05:03.450446+0000","last_active":"2023-09-23T23:05:05.381251+0000","last_peered":"2023-09-23T23:05:05.381251+0000","last_clean":"2023-09-23T23:05:05.381251+0000","last_became_active":"2023-09-23T23:05:03.450090+0000","last_became_peered":"2023-09-23T23:05:03.450090+0000","last_unstale":"2023-09-23T23:05:05.381251+0000","last_undegraded":"2023-09-23T23:05:05.381251+0000","last_fullsized":"2023-09-23T23:05:05.381251+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"425'4686","reported_seq":4721,"reported_epoch":425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:05:11.335283+0000","last_change":"2023-09-23T23:05:04.827791+0000","last_active":"2023-09-23T23:05:11.335283+0000","last_peered":"2023-09-23T23:05:11.335283+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:04.818427+0000","last_became_peered":"2023-09-23T23:05:04.818427+0000","last_unstale":"2023-09-23T23:05:11.335283+0000","last_undegraded":"2023-09-23T23:05:04.691344+0000","last_fullsized":"2023-09-23T23:05:04.438151+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4686,"log_dups_size":0,"ondisk_log_size":4686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:44.385276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307101696,"num_objects":4686,"num_object_clones":0,"num_object_copies":14058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":299904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,0],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":4466},{"shards":"0,3","objects":220}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":417,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.382131+0000","last_change":"2023-09-23T22:58:06.762467+0000","last_active":"2023-09-23T23:05:05.382131+0000","last_peered":"2023-09-23T23:05:05.382131+0000","last_clean":"2023-09-23T23:05:05.382131+0000","last_became_active":"2023-09-23T22:58:06.758832+0000","last_became_peered":"2023-09-23T22:58:06.758832+0000","last_unstale":"2023-09-23T23:05:05.382131+0000","last_undegraded":"2023-09-23T23:05:05.382131+0000","last_fullsized":"2023-09-23T23:05:05.382131+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:09.261404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":388,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.387043+0000","last_change":"2023-09-23T22:58:06.768731+0000","last_active":"2023-09-23T23:05:05.387043+0000","last_peered":"2023-09-23T23:05:05.387043+0000","last_clean":"2023-09-23T23:05:05.387043+0000","last_became_active":"2023-09-23T22:58:06.768458+0000","last_became_peered":"2023-09-23T22:58:06.768458+0000","last_unstale":"2023-09-23T23:05:05.387043+0000","last_undegraded":"2023-09-23T23:05:05.387043+0000","last_fullsized":"2023-09-23T23:05:05.387043+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:25.301136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":348,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.382922+0000","last_change":"2023-09-23T22:58:23.269237+0000","last_active":"2023-09-23T23:05:05.382922+0000","last_peered":"2023-09-23T23:05:05.382922+0000","last_clean":"2023-09-23T23:05:05.382922+0000","last_became_active":"2023-09-23T22:58:23.268933+0000","last_became_peered":"2023-09-23T22:58:23.268933+0000","last_unstale":"2023-09-23T23:05:05.382922+0000","last_undegraded":"2023-09-23T23:05:05.382922+0000","last_fullsized":"2023-09-23T23:05:05.382922+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:02.617390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"425'4742","reported_seq":4766,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:11.558376+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:05:11.558376+0000","last_peered":"2023-09-23T23:05:11.558376+0000","last_clean":"2023-09-23T23:05:11.558376+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:05:11.558376+0000","last_undegraded":"2023-09-23T23:05:11.558376+0000","last_fullsized":"2023-09-23T23:05:11.558376+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4742,"log_dups_size":0,"ondisk_log_size":4742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310771712,"num_objects":4742,"num_object_clones":0,"num_object_copies":14226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4742,"num_write_kb":303488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":363,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.382190+0000","last_change":"2023-09-23T22:58:06.749288+0000","last_active":"2023-09-23T23:05:05.382190+0000","last_peered":"2023-09-23T23:05:05.382190+0000","last_clean":"2023-09-23T23:05:05.382190+0000","last_became_active":"2023-09-23T22:58:06.749000+0000","last_became_peered":"2023-09-23T22:58:06.749000+0000","last_unstale":"2023-09-23T23:05:05.382190+0000","last_undegraded":"2023-09-23T23:05:05.382190+0000","last_fullsized":"2023-09-23T23:05:05.382190+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:39.493012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":271,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.409568+0000","last_change":"2023-09-23T22:58:23.259073+0000","last_active":"2023-09-23T23:05:05.409568+0000","last_peered":"2023-09-23T23:05:05.409568+0000","last_clean":"2023-09-23T23:05:05.409568+0000","last_became_active":"2023-09-23T22:58:23.258395+0000","last_became_peered":"2023-09-23T22:58:23.258395+0000","last_unstale":"2023-09-23T23:05:05.409568+0000","last_undegraded":"2023-09-23T23:05:05.409568+0000","last_fullsized":"2023-09-23T23:05:05.409568+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:06.399155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":238,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.379923+0000","last_change":"2023-09-23T22:58:06.766483+0000","last_active":"2023-09-23T23:05:05.379923+0000","last_peered":"2023-09-23T23:05:05.379923+0000","last_clean":"2023-09-23T23:05:05.379923+0000","last_became_active":"2023-09-23T22:58:06.766148+0000","last_became_peered":"2023-09-23T22:58:06.766148+0000","last_unstale":"2023-09-23T23:05:05.379923+0000","last_undegraded":"2023-09-23T23:05:05.379923+0000","last_fullsized":"2023-09-23T23:05:05.379923+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"425'4669","reported_seq":4693,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:11.282194+0000","last_change":"2023-09-23T23:03:43.132731+0000","last_active":"2023-09-23T23:05:11.282194+0000","last_peered":"2023-09-23T23:05:11.282194+0000","last_clean":"2023-09-23T23:05:11.282194+0000","last_became_active":"2023-09-23T23:03:43.132451+0000","last_became_peered":"2023-09-23T23:03:43.132451+0000","last_unstale":"2023-09-23T23:05:11.282194+0000","last_undegraded":"2023-09-23T23:05:11.282194+0000","last_fullsized":"2023-09-23T23:05:11.282194+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4669,"log_dups_size":0,"ondisk_log_size":4669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:51.837853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305987584,"num_objects":4669,"num_object_clones":0,"num_object_copies":14007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4669,"num_write_kb":298816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":120,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.386451+0000","last_change":"2023-09-23T22:58:07.735927+0000","last_active":"2023-09-23T23:05:05.386451+0000","last_peered":"2023-09-23T23:05:05.386451+0000","last_clean":"2023-09-23T23:05:05.386451+0000","last_became_active":"2023-09-23T22:58:07.735541+0000","last_became_peered":"2023-09-23T22:58:07.735541+0000","last_unstale":"2023-09-23T23:05:05.386451+0000","last_undegraded":"2023-09-23T23:05:05.386451+0000","last_fullsized":"2023-09-23T23:05:05.386451+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.780504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.381554+0000","last_change":"2023-09-23T23:05:03.447728+0000","last_active":"2023-09-23T23:05:05.381554+0000","last_peered":"2023-09-23T23:05:05.381554+0000","last_clean":"2023-09-23T23:05:05.381554+0000","last_became_active":"2023-09-23T23:05:03.446786+0000","last_became_peered":"2023-09-23T23:05:03.446786+0000","last_unstale":"2023-09-23T23:05:05.381554+0000","last_undegraded":"2023-09-23T23:05:05.381554+0000","last_fullsized":"2023-09-23T23:05:05.381554+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":111,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.395193+0000","last_change":"2023-09-23T22:58:06.746930+0000","last_active":"2023-09-23T23:05:05.395193+0000","last_peered":"2023-09-23T23:05:05.395193+0000","last_clean":"2023-09-23T23:05:05.395193+0000","last_became_active":"2023-09-23T22:58:06.746152+0000","last_became_peered":"2023-09-23T22:58:06.746152+0000","last_unstale":"2023-09-23T23:05:05.395193+0000","last_undegraded":"2023-09-23T23:05:05.395193+0000","last_fullsized":"2023-09-23T23:05:05.395193+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:12:45.332681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"425'4714","reported_seq":4737,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T23:05:10.297827+0000","last_change":"2023-09-23T23:03:43.131930+0000","last_active":"2023-09-23T23:05:10.297827+0000","last_peered":"2023-09-23T23:05:10.297827+0000","last_clean":"2023-09-23T23:05:10.297827+0000","last_became_active":"2023-09-23T23:03:43.131552+0000","last_became_peered":"2023-09-23T23:03:43.131552+0000","last_unstale":"2023-09-23T23:05:10.297827+0000","last_undegraded":"2023-09-23T23:05:10.297827+0000","last_fullsized":"2023-09-23T23:05:10.297827+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4714,"log_dups_size":0,"ondisk_log_size":4714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4714,"num_write_kb":301696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.380971+0000","last_change":"2023-09-23T23:05:03.448207+0000","last_active":"2023-09-23T23:05:05.380971+0000","last_peered":"2023-09-23T23:05:05.380971+0000","last_clean":"2023-09-23T23:05:05.380971+0000","last_became_active":"2023-09-23T23:05:03.447838+0000","last_became_peered":"2023-09-23T23:05:03.447838+0000","last_unstale":"2023-09-23T23:05:05.380971+0000","last_undegraded":"2023-09-23T23:05:05.380971+0000","last_fullsized":"2023-09-23T23:05:05.380971+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":330,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.382858+0000","last_change":"2023-09-23T22:58:06.778652+0000","last_active":"2023-09-23T23:05:05.382858+0000","last_peered":"2023-09-23T23:05:05.382858+0000","last_clean":"2023-09-23T23:05:05.382858+0000","last_became_active":"2023-09-23T22:58:06.778290+0000","last_became_peered":"2023-09-23T22:58:06.778290+0000","last_unstale":"2023-09-23T23:05:05.382858+0000","last_undegraded":"2023-09-23T23:05:05.382858+0000","last_fullsized":"2023-09-23T23:05:05.382858+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:14.683598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":408,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388525+0000","last_change":"2023-09-23T23:05:03.451106+0000","last_active":"2023-09-23T23:05:05.388525+0000","last_peered":"2023-09-23T23:05:05.388525+0000","last_clean":"2023-09-23T23:05:05.388525+0000","last_became_active":"2023-09-23T23:05:03.449760+0000","last_became_peered":"2023-09-23T23:05:03.449760+0000","last_unstale":"2023-09-23T23:05:05.388525+0000","last_undegraded":"2023-09-23T23:05:05.388525+0000","last_fullsized":"2023-09-23T23:05:05.388525+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":373,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.383107+0000","last_change":"2023-09-23T23:03:51.529572+0000","last_active":"2023-09-23T23:05:05.383107+0000","last_peered":"2023-09-23T23:05:05.383107+0000","last_clean":"2023-09-23T23:05:05.383107+0000","last_became_active":"2023-09-23T23:03:51.529215+0000","last_became_peered":"2023-09-23T23:03:51.529215+0000","last_unstale":"2023-09-23T23:05:05.383107+0000","last_undegraded":"2023-09-23T23:05:05.383107+0000","last_fullsized":"2023-09-23T23:05:05.383107+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:33.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":174,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.409639+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:05:05.409639+0000","last_peered":"2023-09-23T23:05:05.409639+0000","last_clean":"2023-09-23T23:05:05.409639+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:05:05.409639+0000","last_undegraded":"2023-09-23T23:05:05.409639+0000","last_fullsized":"2023-09-23T23:05:05.409639+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":234,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389309+0000","last_change":"2023-09-23T22:45:51.513443+0000","last_active":"2023-09-23T23:05:05.389309+0000","last_peered":"2023-09-23T23:05:05.389309+0000","last_clean":"2023-09-23T23:05:05.389309+0000","last_became_active":"2023-09-23T22:45:51.513045+0000","last_became_peered":"2023-09-23T22:45:51.513045+0000","last_unstale":"2023-09-23T23:05:05.389309+0000","last_undegraded":"2023-09-23T23:05:05.389309+0000","last_fullsized":"2023-09-23T23:05:05.389309+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":448,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389063+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:05:05.389063+0000","last_peered":"2023-09-23T23:05:05.389063+0000","last_clean":"2023-09-23T23:05:05.389063+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:05:05.389063+0000","last_undegraded":"2023-09-23T23:05:05.389063+0000","last_fullsized":"2023-09-23T23:05:05.389063+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":261,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389715+0000","last_change":"2023-09-23T23:05:03.846235+0000","last_active":"2023-09-23T23:05:05.389715+0000","last_peered":"2023-09-23T23:05:05.389715+0000","last_clean":"2023-09-23T23:05:05.389715+0000","last_became_active":"2023-09-23T23:05:03.451815+0000","last_became_peered":"2023-09-23T23:05:03.451815+0000","last_unstale":"2023-09-23T23:05:05.389715+0000","last_undegraded":"2023-09-23T23:05:05.389715+0000","last_fullsized":"2023-09-23T23:05:05.389715+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":335,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388709+0000","last_change":"2023-09-23T22:58:06.758141+0000","last_active":"2023-09-23T23:05:05.388709+0000","last_peered":"2023-09-23T23:05:05.388709+0000","last_clean":"2023-09-23T23:05:05.388709+0000","last_became_active":"2023-09-23T22:58:06.757856+0000","last_became_peered":"2023-09-23T22:58:06.757856+0000","last_unstale":"2023-09-23T23:05:05.388709+0000","last_undegraded":"2023-09-23T23:05:05.388709+0000","last_fullsized":"2023-09-23T23:05:05.388709+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.381481+0000","last_change":"2023-09-23T23:05:03.454542+0000","last_active":"2023-09-23T23:05:05.381481+0000","last_peered":"2023-09-23T23:05:05.381481+0000","last_clean":"2023-09-23T23:05:05.381481+0000","last_became_active":"2023-09-23T23:05:03.454113+0000","last_became_peered":"2023-09-23T23:05:03.454113+0000","last_unstale":"2023-09-23T23:05:05.381481+0000","last_undegraded":"2023-09-23T23:05:05.381481+0000","last_fullsized":"2023-09-23T23:05:05.381481+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":123,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.389488+0000","last_change":"2023-09-23T22:58:24.243556+0000","last_active":"2023-09-23T23:05:05.389488+0000","last_peered":"2023-09-23T23:05:05.389488+0000","last_clean":"2023-09-23T23:05:05.389488+0000","last_became_active":"2023-09-23T22:58:24.243233+0000","last_became_peered":"2023-09-23T22:58:24.243233+0000","last_unstale":"2023-09-23T23:05:05.389488+0000","last_undegraded":"2023-09-23T23:05:05.389488+0000","last_fullsized":"2023-09-23T23:05:05.389488+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:07.909426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":369,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.382843+0000","last_change":"2023-09-23T22:58:23.302195+0000","last_active":"2023-09-23T23:05:05.382843+0000","last_peered":"2023-09-23T23:05:05.382843+0000","last_clean":"2023-09-23T23:05:05.382843+0000","last_became_active":"2023-09-23T22:58:23.301677+0000","last_became_peered":"2023-09-23T22:58:23.301677+0000","last_unstale":"2023-09-23T23:05:05.382843+0000","last_undegraded":"2023-09-23T23:05:05.382843+0000","last_fullsized":"2023-09-23T23:05:05.382843+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:32.529363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":419,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T23:05:05.388464+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:05:05.388464+0000","last_peered":"2023-09-23T23:05:05.388464+0000","last_clean":"2023-09-23T23:05:05.388464+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:05:05.388464+0000","last_undegraded":"2023-09-23T23:05:05.388464+0000","last_fullsized":"2023-09-23T23:05:05.388464+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4954456064,"num_objects":75599,"num_object_clones":0,"num_object_copies":226797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31599,"num_objects_misplaced":8820,"num_objects_unfound":0,"num_objects_dirty":75599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":75599,"num_write_kb":4838336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":204,"num_bytes_recovered":13369344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14890827776,"data_stored":14890827776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75599,"ondisk_log_size":75599,"up":48,"acting":40,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":20220,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430666,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626891,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1383384,"kb_used_data":480656,"kb_used_omap":0,"kb_used_meta":902720,"kb_avail":92361768,"statfs":{"total":95995035648,"available":94578450432,"internally_reserved":0,"allocated":492191744,"data_stored":491311692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924385280},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":6.4199999999999999,"15min":6.2539999999999996},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":6.6500000000000004,"5min":115.32899999999999,"15min":115.32899999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.77200000000000002,"5min":6.4610000000000003,"15min":6.3200000000000003},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":6.7039999999999997,"5min":115.348,"15min":115.348},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:04:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":2.1709999999999998,"15min":3.5880000000000001},"min":{"1min":0.27900000000000003,"5min":0.26000000000000001,"15min":0.23100000000000001},"max":{"1min":1.956,"5min":115.497,"15min":254.62100000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":2.0369999999999999,"15min":3.9340000000000002},"min":{"1min":0.34399999999999997,"5min":0.28999999999999998,"15min":0.24199999999999999},"max":{"1min":0.73199999999999998,"5min":115.43600000000001,"15min":255.20099999999999},"last":2.774}]},{"osd":2,"last update":"Sat Sep 23 23:04:36 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":2.3119999999999998,"15min":1.431},"min":{"1min":0.39500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.8550000000000004,"5min":115.45699999999999,"15min":115.45699999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.5800000000000001,"15min":1.641},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.5529999999999999,"5min":115.53100000000001,"15min":115.53100000000001},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:04:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":2.468,"15min":2.3849999999999998},"min":{"1min":0.35799999999999998,"5min":0.34300000000000003,"15min":0.26800000000000002},"max":{"1min":5.2409999999999997,"5min":115.369,"15min":115.369},"last":2.149},{"interface":"front","average":{"1min":0.88700000000000001,"5min":2.3199999999999998,"15min":2.3820000000000001},"min":{"1min":0.35299999999999998,"5min":0.33500000000000002,"15min":0.309},"max":{"1min":3.492,"5min":115.616,"15min":115.616},"last":1.222}]},{"osd":4,"last update":"Sat Sep 23 23:04:09 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":2.2559999999999998,"15min":1.819},"min":{"1min":0.28599999999999998,"5min":0.26200000000000001,"15min":0.23899999999999999},"max":{"1min":12.353,"5min":119.804,"15min":119.804},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.0960000000000001,"5min":2.5289999999999999,"15min":1.857},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":12.388,"5min":122.31999999999999,"15min":122.31999999999999},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:04:09 2023","interfaces":[{"interface":"back","average":{"1min":4.516,"5min":4.9459999999999997,"15min":2.5099999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":44.781999999999996,"5min":186.27600000000001,"15min":186.27600000000001},"last":0.439},{"interface":"front","average":{"1min":4.6719999999999997,"5min":5.1790000000000003,"15min":2.5539999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":44.601999999999997,"5min":191.28700000000001,"15min":191.28700000000001},"last":0.46600000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":2.036,"15min":1.4590000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":1.7070000000000001,"5min":115.61,"15min":115.61},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":2.3889999999999998,"15min":1.7050000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":0.77300000000000002,"5min":115.578,"15min":115.578},"last":0.44700000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253204,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4407324,"kb_used_data":3596692,"kb_used_omap":9,"kb_used_meta":810614,"kb_avail":89337828,"statfs":{"total":95995035648,"available":91481935872,"internally_reserved":0,"allocated":3683012608,"data_stored":3681382098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":830069737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:14 2023","interfaces":[{"interface":"back","average":{"1min":20.364999999999998,"5min":5.7789999999999999,"15min":2.161},"min":{"1min":0.224,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":279.30500000000001,"5min":279.30500000000001,"15min":279.30500000000001},"last":2.9910000000000001},{"interface":"front","average":{"1min":20.523,"5min":6.0369999999999999,"15min":2.5830000000000002},"min":{"1min":0.26800000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":279.39800000000002,"5min":279.39800000000002,"15min":279.39800000000002},"last":2.407}]},{"osd":2,"last update":"Sat Sep 23 23:04:31 2023","interfaces":[{"interface":"back","average":{"1min":17.202999999999999,"5min":3.8980000000000001,"15min":1.651},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":161.61699999999999,"5min":161.61699999999999,"15min":161.61699999999999},"last":11.631},{"interface":"front","average":{"1min":16.652999999999999,"5min":3.871,"15min":1.774},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":162.61699999999999,"5min":162.61699999999999,"15min":162.61699999999999},"last":6.6840000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:04:48 2023","interfaces":[{"interface":"back","average":{"1min":8.0649999999999995,"5min":2.1579999999999999,"15min":1.591},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":59.424999999999997,"5min":59.424999999999997,"15min":59.424999999999997},"last":3.0579999999999998},{"interface":"front","average":{"1min":7.4500000000000002,"5min":2,"15min":1.6739999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":55.174999999999997,"5min":55.174999999999997,"15min":55.174999999999997},"last":12.569000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:04:48 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.8160000000000001,"15min":2.1840000000000002},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.9620000000000002,"5min":67.738,"15min":151.036},"last":2.3090000000000002},{"interface":"front","average":{"1min":1.744,"5min":2.2240000000000002,"15min":2.5870000000000002},"min":{"1min":0.40600000000000003,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":5.6340000000000003,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.434}]},{"osd":5,"last update":"Sat Sep 23 23:04:48 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.9750000000000001,"15min":2.294},"min":{"1min":0.32800000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":8.0190000000000001,"5min":68.259,"15min":116.098},"last":2.948},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.8420000000000001,"15min":2.2050000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":3.931,"5min":70.057000000000002,"15min":116.08499999999999},"last":2.3519999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:04:48 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":2.27,"15min":2.1760000000000002},"min":{"1min":0.38500000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":5.6710000000000003,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.609,"5min":1.9350000000000001,"15min":1.8160000000000001},"min":{"1min":0.36299999999999999,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":8.0690000000000008,"5min":67.366,"15min":67.366},"last":3.0249999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:04:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86099999999999999,"15min":1.2909999999999999},"min":{"1min":0.40999999999999998,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":4.032,"5min":4.1669999999999998,"15min":9.9329999999999998},"last":2.3799999999999999},{"interface":"front","average":{"1min":1.829,"5min":1.2250000000000001,"15min":1.772},"min":{"1min":0.372,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":5.6970000000000001,"5min":5.6970000000000001,"15min":20.902999999999999},"last":0.46600000000000003}]}]},{"osd":0,"up_from":353,"seq":1516123455574,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3211868,"kb_used_data":2954260,"kb_used_omap":37,"kb_used_meta":257562,"kb_avail":90533284,"statfs":{"total":95995035648,"available":92706082816,"internally_reserved":0,"allocated":3025162240,"data_stored":3023353278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":263743650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.83699999999999997,"15min":0.63900000000000001},"min":{"1min":0.216,"5min":0.214,"15min":0.214},"max":{"1min":2.7570000000000001,"5min":8.234,"15min":8.234},"last":6.0519999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.86899999999999999,"15min":0.73699999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.867,"5min":6.4459999999999997,"15min":6.4459999999999997},"last":0.78200000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:04:36 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":1.4319999999999999,"15min":0.997},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":34.234000000000002,"5min":34.234000000000002,"15min":34.234000000000002},"last":6.0800000000000001},{"interface":"front","average":{"1min":2.5619999999999998,"5min":2.141,"15min":1.157},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":34.284999999999997,"5min":74.698999999999998,"15min":74.698999999999998},"last":0.51000000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":6.7220000000000004,"5min":2.25,"15min":1.052},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":74.744,"5min":74.744,"15min":74.744},"last":5.9850000000000003},{"interface":"front","average":{"1min":6.8479999999999999,"5min":2.125,"15min":1.3140000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":74.777000000000001,"5min":74.777000000000001,"15min":74.777000000000001},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":4.7409999999999997,"5min":4.0750000000000002,"15min":1.931},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":74.805000000000007,"5min":183.148,"15min":183.148},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":3.4119999999999999,"15min":1.583},"min":{"1min":0.38,"5min":0.31,"15min":0.31},"max":{"1min":1.8320000000000001,"5min":183.39500000000001,"15min":183.39500000000001},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.0640000000000001,"15min":0.80700000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":2.2530000000000001,"5min":8.5950000000000006,"15min":8.5950000000000006},"last":4.3330000000000002},{"interface":"front","average":{"1min":4.774,"5min":1.7809999999999999,"15min":1.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":74.858000000000004,"5min":74.858000000000004,"15min":74.858000000000004},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.0549999999999999,"15min":0.90800000000000003},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.893,"5min":5.7309999999999999,"15min":5.7309999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.0029999999999999,"15min":0.81699999999999995},"min":{"1min":0.46800000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.7149999999999999,"5min":6.375,"15min":6.375},"last":4.3780000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.0209999999999999,"15min":1.2090000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.917,"5min":5.4779999999999998,"15min":5.4779999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.0680000000000001,"15min":1.208},"min":{"1min":0.48799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.79,"5min":6.343,"15min":6.343},"last":4.4029999999999996}]}]},{"osd":2,"up_from":359,"seq":1541893259347,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2640420,"kb_used_data":2414748,"kb_used_omap":83,"kb_used_meta":225580,"kb_avail":91104732,"statfs":{"total":95995035648,"available":93291245568,"internally_reserved":0,"allocated":2472701952,"data_stored":2470949951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":230994373},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,5,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":2.5350000000000001,"15min":1.081},"min":{"1min":0.20999999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.8370000000000002,"5min":74.241,"15min":74.241},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.47,"5min":2.7480000000000002,"15min":1.3879999999999999},"min":{"1min":0.25600000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.3079999999999998,"5min":74.430000000000007,"15min":74.430000000000007},"last":6.1230000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":0.89400000000000002,"15min":0.74199999999999999},"min":{"1min":0.251,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.4569999999999999,"5min":5.9710000000000001,"15min":5.9710000000000001},"last":9.7989999999999995},{"interface":"front","average":{"1min":1.677,"5min":1.1679999999999999,"15min":1.159},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7270000000000003,"5min":11.397,"15min":11.397},"last":12.574999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":7.125,"5min":2.105,"15min":1.177},"min":{"1min":0.27400000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":110.31699999999999,"5min":110.31699999999999,"15min":110.31699999999999},"last":5.8559999999999999},{"interface":"front","average":{"1min":7.5739999999999998,"5min":2.2229999999999999,"15min":1.2849999999999999},"min":{"1min":0.28000000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":110.56999999999999,"5min":110.56999999999999,"15min":110.56999999999999},"last":4.1630000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":1.3109999999999999,"15min":0.81299999999999994},"min":{"1min":0.34499999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":33.476999999999997,"5min":33.476999999999997,"15min":33.476999999999997},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.7000000000000002,"5min":1.3440000000000001,"15min":0.996},"min":{"1min":0.39800000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":33.497999999999998,"5min":33.497999999999998,"15min":33.497999999999998},"last":5.9180000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":9.8460000000000001,"5min":2.7709999999999999,"15min":1.5600000000000001},"min":{"1min":0.38600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":139.405,"5min":139.405,"15min":139.405},"last":5.8959999999999999},{"interface":"front","average":{"1min":9.5690000000000008,"5min":2.7130000000000001,"15min":1.3580000000000001},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":139.42099999999999,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.2290000000000001,"15min":1.056},"min":{"1min":0.42199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.1420000000000003,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":5.9420000000000002},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1,"15min":0.73799999999999999},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.8100000000000001,"5min":6.9939999999999998,"15min":6.9939999999999998},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 23:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.141,"15min":1.0409999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.7839999999999998,"5min":7.0499999999999998,"15min":7.0499999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.117,"5min":1.032,"15min":1.1299999999999999},"min":{"1min":0.40899999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.4929999999999999,"5min":6.8520000000000003,"15min":6.8520000000000003},"last":5.9649999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926151,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3327728,"kb_used_data":2709032,"kb_used_omap":7,"kb_used_meta":618680,"kb_avail":90417424,"statfs":{"total":95995035648,"available":92587442176,"internally_reserved":0,"allocated":2774048768,"data_stored":2772339070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":633528936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.5419999999999998,"15min":1.0600000000000001},"min":{"1min":0.25600000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.7130000000000001,"5min":122.289,"15min":122.289},"last":4.1929999999999996},{"interface":"front","average":{"1min":0.53100000000000003,"5min":2.04,"15min":0.95099999999999996},"min":{"1min":0.218,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":1.6839999999999999,"5min":122.041,"15min":122.041},"last":1.024}]},{"osd":1,"last update":"Sat Sep 23 23:05:02 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":1.6080000000000001,"15min":2.1739999999999999},"min":{"1min":0.23699999999999999,"5min":0.22,"15min":0.187},"max":{"1min":10.617000000000001,"5min":15.025,"15min":123.866},"last":3.4780000000000002},{"interface":"front","average":{"1min":3.8300000000000001,"5min":1.6870000000000001,"15min":2.1789999999999998},"min":{"1min":0.34899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.6899999999999995,"5min":9.718,"15min":122.265},"last":4.4050000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":0.78200000000000003,"15min":0.73999999999999999},"min":{"1min":0.309,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":5.056,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":3.4580000000000002},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.4239999999999999,"15min":0.96999999999999997},"min":{"1min":0.29299999999999998,"5min":0.215,"15min":0.215},"max":{"1min":9.9789999999999992,"5min":9.9789999999999992,"15min":9.9789999999999992},"last":2.952}]},{"osd":4,"last update":"Sat Sep 23 23:05:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.8420000000000001,"15min":1.8819999999999999},"min":{"1min":0.33300000000000002,"5min":0.30199999999999999,"15min":0.28699999999999998},"max":{"1min":10.327,"5min":64.406999999999996,"15min":87.046999999999997},"last":1.754},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.9099999999999999,"15min":1.875},"min":{"1min":0.309,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.7560000000000002,"5min":64.468000000000004,"15min":87.066000000000003},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:05:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":1.4139999999999999,"15min":1.405},"min":{"1min":0.34899999999999998,"5min":0.33000000000000002,"15min":0.28499999999999998},"max":{"1min":8.7089999999999996,"5min":8.7089999999999996,"15min":11.803000000000001},"last":4.3550000000000004},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.2410000000000001,"15min":1.5800000000000001},"min":{"1min":0.40300000000000002,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":4.6589999999999998,"5min":10.246,"15min":11.951000000000001},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 23:05:02 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":1.5049999999999999,"15min":1.702},"min":{"1min":0.41299999999999998,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":8.7929999999999993,"5min":8.7929999999999993,"15min":14.097},"last":0.435},{"interface":"front","average":{"1min":1.764,"5min":1.228,"15min":1.7},"min":{"1min":0.436,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":4.6769999999999996,"5min":5.79,"15min":12.887},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:05:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.1779999999999999,"15min":1.5129999999999999},"min":{"1min":0.58799999999999997,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":4.6920000000000002,"5min":5.2409999999999997,"15min":11.694000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.542,"15min":1.5149999999999999},"min":{"1min":0.45000000000000001,"5min":0.40300000000000002,"15min":0.33200000000000002},"max":{"1min":10.278,"5min":10.278,"15min":10.278},"last":1.7110000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019348,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2120980,"kb_used_data":1209420,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":91624172,"statfs":{"total":95995035648,"available":93823152128,"internally_reserved":0,"allocated":1238446080,"data_stored":1237569532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":4.1989999999999998,"15min":1.714},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.93400000000000005,"5min":156.375,"15min":156.375},"last":2.294},{"interface":"front","average":{"1min":0.67900000000000005,"5min":4.351,"15min":1.847},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.5509999999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":2.9420000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:05:09 2023","interfaces":[{"interface":"back","average":{"1min":11.801,"5min":3.419,"15min":3.9969999999999999},"min":{"1min":0.32800000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":95.591999999999999,"5min":95.591999999999999,"15min":132.304},"last":1.8720000000000001},{"interface":"front","average":{"1min":11.077999999999999,"5min":3.1019999999999999,"15min":3.891},"min":{"1min":0.255,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":97.929000000000002,"5min":97.929000000000002,"15min":132.453},"last":2.3119999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:04:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":1.26,"15min":1.1000000000000001},"min":{"1min":0.33300000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.1619999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":6.0449999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.202,"15min":0.879},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.605,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":2.1579999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:05:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6859999999999999,"5min":1.754,"15min":2.4620000000000002},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":60.274000000000001,"5min":60.274000000000001,"15min":164.453},"last":2.3380000000000001},{"interface":"front","average":{"1min":5.5430000000000001,"5min":2.1890000000000001,"15min":2.581},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":60.677999999999997,"5min":60.677999999999997,"15min":167.05199999999999},"last":6.5970000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:04:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.95199999999999996,"15min":1.698},"min":{"1min":0.22,"5min":0.22,"15min":0.214},"max":{"1min":0.872,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.86799999999999999,"15min":1.778},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.73199999999999998,"5min":8.7810000000000006,"15min":151.905},"last":2.0699999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.04,"15min":1.226},"min":{"1min":0.316,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.55300000000000005,"5min":8.9610000000000003,"15min":16.943000000000001},"last":2.1110000000000002},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.878,"15min":1.157},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":4.0460000000000003,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.222,"15min":1.252},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.28399999999999997},"max":{"1min":0.90900000000000003,"5min":11.045999999999999,"15min":12.84},"last":1.7170000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.96599999999999997,"15min":1.1140000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":0.71399999999999997,"5min":10.227,"15min":10.587},"last":2.262}]}]},{"osd":5,"up_from":36,"seq":154618823122,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2401092,"kb_used_data":1509628,"kb_used_omap":0,"kb_used_meta":891456,"kb_avail":91344060,"statfs":{"total":95995035648,"available":93536317440,"internally_reserved":0,"allocated":1545859072,"data_stored":1544994202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912850944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.8859999999999999,"15min":1.032},"min":{"1min":0.26900000000000002,"5min":0.221,"15min":0.221},"max":{"1min":7.7469999999999999,"5min":42.155000000000001,"15min":42.155000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.96,"15min":1.3740000000000001},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.7000000000000002,"5min":43.378999999999998,"15min":43.378999999999998},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:04:46 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":0.89800000000000002,"15min":2.0089999999999999},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":9.9009999999999998,"5min":9.9009999999999998,"15min":110.36499999999999},"last":2.5600000000000001},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.167,"15min":2.4060000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25600000000000001},"max":{"1min":5.1360000000000001,"5min":7.1219999999999999,"15min":110.89},"last":0.93400000000000005}]},{"osd":2,"last update":"Sat Sep 23 23:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.024,"15min":0.875},"min":{"1min":0.29999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.5549999999999999,"5min":6.601,"15min":6.601},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.94099999999999995,"15min":0.97699999999999998},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":1.9770000000000001,"5min":4.9530000000000003,"15min":4.9530000000000003},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:04:46 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":1.5489999999999999,"15min":2.5659999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":27.890000000000001,"5min":27.890000000000001,"15min":98.641000000000005},"last":1.272},{"interface":"front","average":{"1min":2.794,"5min":1.4359999999999999,"15min":2.6120000000000001},"min":{"1min":0.39700000000000002,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":27.806000000000001,"5min":27.806000000000001,"15min":98.906000000000006},"last":11.065}]},{"osd":4,"last update":"Sat Sep 23 23:04:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.98699999999999999,"15min":1.552},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23300000000000001},"max":{"1min":1.748,"5min":7.0380000000000003,"15min":115.06699999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.90500000000000003,"15min":1.496},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.253},"max":{"1min":1.6040000000000001,"5min":9.5890000000000004,"15min":114.908},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:04:46 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.117,"15min":1.2130000000000001},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":1.7769999999999999,"5min":7.4630000000000001,"15min":11.121},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.81399999999999995,"15min":1.0549999999999999},"min":{"1min":0.33200000000000002,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.6519999999999999,"5min":5.4909999999999997,"15min":12.137},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.94599999999999995,"15min":1.079},"min":{"1min":0.375,"5min":0.34399999999999997,"15min":0.29299999999999998},"max":{"1min":1.048,"5min":9.5980000000000008,"15min":12.23},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.2350000000000001,"15min":1.2030000000000001},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.31},"max":{"1min":1.2689999999999999,"5min":7.6719999999999997,"15min":11.112},"last":0.63300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3027763200,"data_stored":3027763200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3698982912,"data_stored":3698982912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2476408832,"data_stored":2476408832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2777808896,"data_stored":2777808896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1236795392,"data_stored":1236795392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1543045120,"data_stored":1543045120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":130023424,"data_stored":130023424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:05:17.283 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:05:17.283 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-09-23T23:05:17.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:17.546 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:17.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:17.649 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:17.793 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:17.810 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 85 16 79471 79455 58.3694 33.0625 0.0366284 0.0171225 2023-09-23T23:05:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:17 smithi052 ceph-mon[180979]: pgmap v1327: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 36 MiB/s wr, 578 op/s; 31599/232938 objects degraded (13.565%); 8820/232938 objects misplaced (3.786%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:05:17.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:17 smithi052 ceph-mon[187743]: pgmap v1327: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 36 MiB/s wr, 578 op/s; 31599/232938 objects degraded (13.565%); 8820/232938 objects misplaced (3.786%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:05:18.107 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:05:18.139 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:18.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:18.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:23232 11.2 11:4d24e738:::benchmark_data_smithi052_606809_object23231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:07.112594+0000", 2023-09-23T23:05:18.166 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 71.044166669000006, 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22801025999999999, 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 23232 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.167 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112594+0000", 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112594+0000", 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112593+0000", 2023-09-23T23:05:18.168 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112617+0000", 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.357e-05 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112625+0000", 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8650000000000001e-06 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.169 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112631+0000", 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8579999999999996e-06 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112677+0000", 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6675999999999998e-05 2023-09-23T23:05:18.170 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112796+0000", 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000118863 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.112924+0000", 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012802199999999999 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.171 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.115431+0000", 2023-09-23T23:05:18.172 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025071519999999999 2023-09-23T23:05:18.172 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.172 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.172 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.172 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.117200+0000", 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017691180000000001 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.340530+0000", 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.223330046 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.340571+0000", 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0191000000000002e-05 2023-09-23T23:05:18.173 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:07.340604+0000", 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3515e-05 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:41359 11.3 e417/410 11:cd5108ed:::benchmark_data_smithi052_606809_object41358:head v 417'2595, mlcod=417'2590)", 2023-09-23T23:05:18.174 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.158364+0000", 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 53.998396202000002, 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23886327299999999, 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158364+0000", 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.175 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158364+0000", 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158363+0000", 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158407+0000", 2023-09-23T23:05:18.176 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3223999999999998e-05 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158421+0000", 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4093e-05 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158429+0000", 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9969999999999998e-06 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.177 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158493+0000", 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3806000000000005e-05 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158516+0000", 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3459999999999999e-05 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.178 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.397172+0000", 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23865600100000001 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.397228+0000", 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5553000000000002e-05 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.179 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:41357 11.f 11:f99afbff:::benchmark_data_smithi052_606809_object41356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.159108+0000", 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 53.997652862000002, 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23755769099999999, 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 41357 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.180 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159108+0000", 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159108+0000", 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.181 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159106+0000", 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159133+0000", 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6750999999999998e-05 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159145+0000", 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2359e-05 2023-09-23T23:05:18.182 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159153+0000", 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8269999999999998e-06 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159204+0000", 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1050000000000001e-05 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.183 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159329+0000", 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012519399999999999 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159505+0000", 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017603399999999999 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.184 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.283664+0000", 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12415902500000001 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395574+0000", 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.111909323 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.396576+0000", 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001002537 2023-09-23T23:05:18.185 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.396629+0000", 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2593999999999998e-05 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.396665+0000", 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6476000000000001e-05 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.186 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:41361 11.2 11:47bb5080:::benchmark_data_smithi052_606809_object41360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 53.997589130000001, 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23663515700000001, 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.187 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 41361 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.188 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159194+0000", 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3439999999999999e-05 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.189 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159202+0000", 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2790000000000001e-06 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159207+0000", 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4609999999999999e-06 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.190 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160385+0000", 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00117815 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160518+0000", 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013272699999999999 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160643+0000", 2023-09-23T23:05:18.191 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000125251 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.285496+0000", 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.124852347 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395674+0000", 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11017845 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.192 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395719+0000", 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4589000000000003e-05 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395761+0000", 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2354000000000001e-05 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.193 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395807+0000", 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5643000000000003e-05 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:41365 11.3 e417/410 11:c59232f5:::benchmark_data_smithi052_606809_object41364:head v 417'2596, mlcod=417'2590)", 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.160978+0000", 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 53.995782871000003, 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.236275239, 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.194 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160978+0000", 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160978+0000", 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.195 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160977+0000", 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.161011+0000", 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4246999999999999e-05 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.196 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.283561+0000", 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122549825 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.283572+0000", 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0465000000000001e-05 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.285799+0000", 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0022272500000000001 2023-09-23T23:05:18.197 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.285820+0000", 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1509e-05 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.397233+0000", 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.111412681 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.198 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.397253+0000", 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9899999999999999e-05 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:44791 11.c 11:3dd520ea:::benchmark_data_smithi052_606809_object44790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:27.913708+0000", 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 50.243052347999999, 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22476216299999999, 2023-09-23T23:05:18.199 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 44791 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913708+0000", 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.200 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913708+0000", 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913707+0000", 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.201 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913735+0000", 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8411999999999998e-05 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913748+0000", 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2313999999999999e-05 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.202 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913759+0000", 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1163e-05 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.914081+0000", 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00032271300000000003 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.914203+0000", 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000121643 2023-09-23T23:05:18.203 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.056512+0000", 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14230867799999999 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.060879+0000", 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043670879999999999 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.204 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.061153+0000", 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000273725 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138388+0000", 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.077235411000000004 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.205 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138430+0000", 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2098000000000002e-05 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138470+0000", 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0228999999999997e-05 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.206 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:44792 11.c 11:389e2b86:::benchmark_data_smithi052_606809_object44791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:27.913779+0000", 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 50.242981030000003, 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22482278, 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 44792 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.207 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913779+0000", 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913779+0000", 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.208 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913779+0000", 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913801+0000", 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2476999999999998e-05 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913810+0000", 2023-09-23T23:05:18.209 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3170000000000002e-06 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913816+0000", 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5250000000000002e-06 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.058992+0000", 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.14517571000000001 2023-09-23T23:05:18.210 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.059118+0000", 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000126193 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.059248+0000", 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000129529 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.211 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.063094+0000", 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0038464599999999999 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138352+0000", 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.075257827999999999 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138518+0000", 2023-09-23T23:05:18.212 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016565800000000001 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138561+0000", 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3038999999999999e-05 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138602+0000", 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1641999999999999e-05 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.213 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:44794 11.4 11:2a6192c4:::benchmark_data_smithi052_606809_object44793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:27.913873+0000", 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 50.242887236000001, 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.225052061, 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.214 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 44794 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913873+0000", 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913873+0000", 2023-09-23T23:05:18.215 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913873+0000", 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913893+0000", 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.022e-05 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.216 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913900+0000", 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0029999999999997e-06 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.913906+0000", 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9619999999999998e-06 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.217 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.060905+0000", 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.146999201 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.061024+0000", 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011916700000000001 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.061167+0000", 2023-09-23T23:05:18.218 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014237899999999999 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138361+0000", 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.077193922999999998 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138644+0000", 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000283176 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.219 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138844+0000", 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000200051 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138890+0000", 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5884999999999999e-05 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.220 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138925+0000", 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5612999999999999e-05 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66493 11.4 11:2854aa07:::benchmark_data_smithi052_606809_object66492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.667304+0000", 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.489456968999999, 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248592391, 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.221 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66493 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667304+0000", 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.222 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667304+0000", 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667303+0000", 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.223 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667322+0000", 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9454999999999999e-05 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667329+0000", 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0049999999999998e-06 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667335+0000", 2023-09-23T23:05:18.224 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6500000000000001e-06 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667382+0000", 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7197000000000002e-05 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667501+0000", 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119177 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.225 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667635+0000", 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013359600000000001 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.675585+0000", 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079498629999999997 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.226 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682693+0000", 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071082100000000002 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915822+0000", 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23312934099999999 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915860+0000", 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7058000000000003e-05 2023-09-23T23:05:18.227 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915896+0000", 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6412000000000003e-05 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66496 11.d 11:bbe513fe:::benchmark_data_smithi052_606809_object66495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.676916+0000", 2023-09-23T23:05:18.228 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.479844515, 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239258154, 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66496 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.229 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676916+0000", 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676916+0000", 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676915+0000", 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.230 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676940+0000", 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5650999999999999e-05 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676951+0000", 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0159e-05 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.231 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676958+0000", 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7260000000000002e-06 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677258+0000", 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029930899999999999 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677369+0000", 2023-09-23T23:05:18.232 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011102700000000001 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677498+0000", 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012934200000000001 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682271+0000", 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004772844 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.233 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682278+0000", 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5519999999999999e-06 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916127+0000", 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23384854799999999 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.234 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916153+0000", 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5812e-05 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916174+0000", 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1461e-05 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.235 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66498 11.f 11:f0c9ffe3:::benchmark_data_smithi052_606809_object66497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.676974+0000", 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.479786869000002, 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23892223100000001, 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66498 2023-09-23T23:05:18.236 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676974+0000", 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676974+0000", 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.237 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676973+0000", 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677006+0000", 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2792e-05 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.238 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677014+0000", 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5860000000000005e-06 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677021+0000", 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5930000000000002e-06 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.678110+0000", 2023-09-23T23:05:18.239 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001088756 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.678230+0000", 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012026799999999999 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.678360+0000", 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013007399999999999 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.240 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.681128+0000", 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027677370000000001 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915745+0000", 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.234617042 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.241 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915810+0000", 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5512999999999994e-05 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915856+0000", 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5179999999999998e-05 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915896+0000", 2023-09-23T23:05:18.242 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0346000000000003e-05 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66499 11.d 11:baa8f49a:::benchmark_data_smithi052_606809_object66498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.47972605, 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239213228, 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.243 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66499 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.244 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677069+0000", 2023-09-23T23:05:18.245 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4826999999999997e-05 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677077+0000", 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7479999999999999e-06 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677083+0000", 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9499999999999998e-06 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.246 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.679753+0000", 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026696469999999998 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.679881+0000", 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000128369 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.247 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680012+0000", 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000130987 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682407+0000", 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023948989999999998 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.248 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682482+0000", 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4179999999999998e-05 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916204+0000", 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23372264200000001 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916229+0000", 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5060000000000001e-05 2023-09-23T23:05:18.249 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916248+0000", 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8491e-05 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66501 11.c 11:338ca9b7:::benchmark_data_smithi052_606809_object66500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.250 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.677089+0000", 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.479671623000002, 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.240935287, 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66501 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.251 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677089+0000", 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677089+0000", 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.252 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677088+0000", 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677111+0000", 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2957999999999999e-05 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677118+0000", 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5710000000000003e-06 2023-09-23T23:05:18.253 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677123+0000", 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1699999999999996e-06 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680249+0000", 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031262730000000002 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.254 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680354+0000", 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0001044 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680473+0000", 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011943 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.684728+0000", 2023-09-23T23:05:18.255 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004254982 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915325+0000", 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.230596933 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917951+0000", 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026257350000000001 2023-09-23T23:05:18.256 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917989+0000", 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7696e-05 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918024+0000", 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5638e-05 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.257 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66503 11.2 11:43ad7ec3:::benchmark_data_smithi052_606809_object66502:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.682102+0000", 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.474658589000001, 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.235809987, 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.258 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66503 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682102+0000", 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682102+0000", 2023-09-23T23:05:18.259 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682101+0000", 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682129+0000", 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8011999999999999e-05 2023-09-23T23:05:18.260 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682141+0000", 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2864000000000001e-05 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682149+0000", 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6799999999999993e-06 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.261 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682204+0000", 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4954000000000003e-05 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682327+0000", 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012244299999999999 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:05:18.262 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682467+0000", 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000140192 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917780+0000", 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23531313000000001 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917819+0000", 2023-09-23T23:05:18.263 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9368e-05 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917852+0000", 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2512000000000003e-05 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917890+0000", 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8127000000000003e-05 2023-09-23T23:05:18.264 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917912+0000", 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2084000000000001e-05 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66504 11.f 11:f9bc871c:::benchmark_data_smithi052_606809_object66503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.682167+0000", 2023-09-23T23:05:18.265 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.474593534, 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23627304499999999, 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66504 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.266 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682167+0000", 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682167+0000", 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682166+0000", 2023-09-23T23:05:18.267 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682188+0000", 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1736999999999999e-05 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682198+0000", 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4399999999999994e-06 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.268 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682205+0000", 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0230000000000004e-06 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682261+0000", 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5943999999999998e-05 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.269 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682389+0000", 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012866 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682506+0000", 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000116469 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915330+0000", 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.232824326 2023-09-23T23:05:18.270 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915776+0000", 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00044591800000000001 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918359+0000", 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025832989999999998 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.271 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918404+0000", 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5092999999999997e-05 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918440+0000", 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5748999999999999e-05 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.272 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66505 11.2 11:4c8f9ba6:::benchmark_data_smithi052_606809_object66504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.685302+0000", 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.471458506000001, 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23625360300000001, 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66505 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.273 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685302+0000", 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685302+0000", 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.274 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685301+0000", 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685323+0000", 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1742e-05 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685334+0000", 2023-09-23T23:05:18.275 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1735e-05 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685342+0000", 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8089999999999993e-06 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915388+0000", 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23004556500000001 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.276 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915544+0000", 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015627800000000001 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915667+0000", 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012271800000000001 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.277 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918228+0000", 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0025608979999999998 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.921314+0000", 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003086367 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.921464+0000", 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014956400000000001 2023-09-23T23:05:18.278 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.921518+0000", 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4125000000000001e-05 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.921556+0000", 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7970999999999999e-05 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.279 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:66507 11.3 e420/410 11:cfd5fc4c:::benchmark_data_smithi052_606809_object66506:head v 420'4190, mlcod=420'4188)", 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.686002+0000", 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.470758845999999, 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.232201463, 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.280 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.686002+0000", 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.686002+0000", 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.686001+0000", 2023-09-23T23:05:18.281 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.686033+0000", 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1816000000000002e-05 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915317+0000", 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22928442199999999 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.282 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915340+0000", 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2565000000000002e-05 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915388+0000", 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8683999999999997e-05 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.283 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915420+0000", 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1257999999999998e-05 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918165+0000", 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027452209999999999 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918203+0000", 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8220000000000003e-05 2023-09-23T23:05:18.284 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:66511 11.1 e420/410 11:808fe917:::benchmark_data_smithi052_606809_object66510:head v 420'4177, mlcod=420'4175)", 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.689148+0000", 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 23.467612297999999, 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23099603899999999, 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.285 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.689148+0000", 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.689148+0000", 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.286 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.689147+0000", 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.689171+0000", 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4023999999999999e-05 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915320+0000", 2023-09-23T23:05:18.287 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22614892 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915339+0000", 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9023999999999999e-05 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915432+0000", 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2417999999999995e-05 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.288 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915454+0000", 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2727000000000001e-05 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.920094+0000", 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046391619999999996 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.289 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.920144+0000", 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0606999999999998e-05 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72288 11.2 11:44178e5e:::benchmark_data_smithi052_606809_object72287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.394288+0000", 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 12.762472314, 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.355785346, 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.290 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72288 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394288+0000", 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.291 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394288+0000", 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394287+0000", 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.292 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394315+0000", 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8002999999999999e-05 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394324+0000", 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7780000000000002e-06 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394331+0000", 2023-09-23T23:05:18.293 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9120000000000001e-06 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394497+0000", 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000165167 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394798+0000", 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00030159799999999998 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.294 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394917+0000", 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011914800000000001 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.403946+0000", 2023-09-23T23:05:18.295 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090287289999999992 2023-09-23T23:05:18.295 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:18.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:17 smithi151 ceph-mon[159210]: pgmap v1327: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 36 MiB/s wr, 578 op/s; 31599/232938 objects degraded (13.565%); 8820/232938 objects misplaced (3.786%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T23:05:18.297 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:05:18.297 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.297 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.297 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.297 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.404128+0000", 2023-09-23T23:05:18.297 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018199700000000001 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.749969+0000", 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34584069200000001 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.750027+0000", 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8128999999999999e-05 2023-09-23T23:05:18.298 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.750074+0000", 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6765999999999999e-05 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75762 11.2 11:402b2427:::benchmark_data_smithi052_606809_object75761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.316624+0000", 2023-09-23T23:05:18.299 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 6.8401369169999997, 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.37235110799999999, 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75762 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.300 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316624+0000", 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316624+0000", 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:05:18.301 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316623+0000", 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316646+0000", 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2690000000000001e-05 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316652+0000", 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4999999999999996e-06 2023-09-23T23:05:18.302 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316657+0000", 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0499999999999999e-06 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316746+0000", 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8312000000000004e-05 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.303 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316851+0000", 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105086 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316971+0000", 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119861 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.304 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.321893+0000", 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049227719999999997 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.321963+0000", 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9827999999999996e-05 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688891+0000", 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36692815499999998 2023-09-23T23:05:18.305 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688940+0000", 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8717999999999997e-05 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688975+0000", 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4561999999999999e-05 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.306 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.307 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:05:18.307 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:05:18.307 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:05:18.307 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:05:18.307 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:05:18.325 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:18.327 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:05:18.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:18.385 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:05:18.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:18.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:18.419 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:05:18.469 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:05:18.492 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:18.511 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:05:18.521 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:18.594 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:18.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:18.701 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:18.786 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:18.810 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 86 16 80000 79984 58.0753 33.0625 0.00547858 0.0172086 2023-09-23T23:05:19.119 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:05:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3123422792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2643957346' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:18 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:19.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3123422792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:19.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2643957346' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:19.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:18 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3123422792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2643957346' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:18 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:19.811 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 87 16 80449 80433 57.7296 28.0625 0.0207568 0.0173157 2023-09-23T23:05:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: pgmap v1328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 31 MiB/s wr, 492 op/s; 31599/235866 objects degraded (13.397%); 8655/235866 objects misplaced (3.669%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:05:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31599/235866 objects degraded (13.397%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:05:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T23:05:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: failed to encode map e426 with expected crc 2023-09-23T23:05:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: failed to encode map e426 with expected crc 2023-09-23T23:05:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:20.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: pgmap v1328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 31 MiB/s wr, 492 op/s; 31599/235866 objects degraded (13.397%); 8655/235866 objects misplaced (3.669%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31599/235866 objects degraded (13.397%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: failed to encode map e426 with expected crc 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: failed to encode map e426 with expected crc 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:20.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:20.249 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: pgmap v1328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 31 MiB/s wr, 492 op/s; 31599/235866 objects degraded (13.397%); 8655/235866 objects misplaced (3.669%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31599/235866 objects degraded (13.397%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: failed to encode map e426 with expected crc 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: failed to encode map e426 with expected crc 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:20.250 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:20.335 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:20.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:20.441 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:20.534 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:20.534 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:05:20.811 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 88 16 80956 80940 57.4338 31.6875 0.131607 0.0174021 2023-09-23T23:05:21.263 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:21.336 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:21.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:21.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:21 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:21.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:21 smithi151 ceph-mon[159210]: failed to encode map e426 with expected crc 2023-09-23T23:05:21.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:21 smithi151 ceph-mon[159210]: pgmap v1330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 585 op/s; 31599/238326 objects degraded (13.259%); 8655/238326 objects misplaced (3.632%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:05:21.462 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:21.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:21 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:21.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:21 smithi052 ceph-mon[187743]: failed to encode map e426 with expected crc 2023-09-23T23:05:21.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:21 smithi052 ceph-mon[187743]: pgmap v1330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 585 op/s; 31599/238326 objects degraded (13.259%); 8655/238326 objects misplaced (3.632%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:05:21.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:21 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:21.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:21 smithi052 ceph-mon[180979]: failed to encode map e426 with expected crc 2023-09-23T23:05:21.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:21 smithi052 ceph-mon[180979]: pgmap v1330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 585 op/s; 31599/238326 objects degraded (13.259%); 8655/238326 objects misplaced (3.632%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T23:05:21.550 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:21.716 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:21.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:21.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:21.813 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 89 16 81561 81545 57.2128 37.8125 0.013944 0.0174713 2023-09-23T23:05:21.827 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:21.876 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:21.951 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:21.952 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:21.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:22.057 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:22.090 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:22.151 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:22.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:22.259 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:22.291 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:22.363 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:22.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:22.475 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:22.518 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:22.584 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:22.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:22.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:22.699 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:22.725 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:22.766 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:22.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:22.813 INFO:tasks.radosbench.radosbench.0.smithi052.stdout: 90 16 82151 82135 56.987 36.875 0.0037933 0.0175141 2023-09-23T23:05:22.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:22.897 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:22.938 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:22.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:23.043 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:23.077 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:23.276 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:23.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:23.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:23.318 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:05:23.360 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:05:23.380 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:23.446 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:23.517 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:23.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:23.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:23.623 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:23.637 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:05:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:05:23] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.33.4" 2023-09-23T23:05:23.659 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:23.813 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total time run: 90.2242 2023-09-23T23:05:23.813 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Total writes made: 82151 2023-09-23T23:05:23.813 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Write size: 65536 2023-09-23T23:05:23.813 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Object size: 65536 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Bandwidth (MB/sec): 56.9076 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Bandwidth: 25.1097 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max bandwidth (MB/sec): 139.312 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min bandwidth (MB/sec): 0 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average IOPS: 910 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev IOPS: 401.768 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max IOPS: 2229 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min IOPS: 0 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Average Latency(s): 0.0175666 2023-09-23T23:05:23.814 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Stddev Latency(s): 0.0394045 2023-09-23T23:05:23.815 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Max latency(s): 2.44231 2023-09-23T23:05:23.815 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Min latency(s): 0.00102195 2023-09-23T23:05:23.823 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T23:05:23.953 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:23.964 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:23 smithi052 ceph-mon[180979]: pgmap v1331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 31599/243792 objects degraded (12.961%); 8487/243792 objects misplaced (3.481%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:23.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:23.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:23.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:23 smithi052 ceph-mon[187743]: pgmap v1331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 31599/243792 objects degraded (12.961%); 8487/243792 objects misplaced (3.481%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:24.065 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:24.101 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:23 smithi151 ceph-mon[159210]: pgmap v1331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 31599/243792 objects degraded (12.961%); 8487/243792 objects misplaced (3.481%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:24.105 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:24.186 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:24.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:24.301 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:24.345 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:24.418 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:24.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:24.530 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:24.563 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:24.680 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:24.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:24.801 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:24.847 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:24.924 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:24.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:25.037 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:25.100 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:25.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:25.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:25.309 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:25.327 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:25.345 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:25.448 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:25.823 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:25.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:25.927 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:25.993 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:05:25.994 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:05:26.006 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:25 smithi052 ceph-mon[180979]: pgmap v1332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 31599/243792 objects degraded (12.961%); 8487/243792 objects misplaced (3.481%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:26.006 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31599/243792 objects degraded (12.961%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:26.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:25 smithi052 ceph-mon[187743]: pgmap v1332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 31599/243792 objects degraded (12.961%); 8487/243792 objects misplaced (3.481%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:26.007 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31599/243792 objects degraded (12.961%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:26.060 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:25 smithi151 ceph-mon[159210]: pgmap v1332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 31599/243792 objects degraded (12.961%); 8487/243792 objects misplaced (3.481%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31599/243792 objects degraded (12.961%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:26.811 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 11 pg_num to 26 2023-09-23T23:05:27.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3186597802' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:05:27.060 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:26 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:05:27.060 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3186597802' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:05:27.060 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:26 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:05:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3186597802' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:05:27.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:26 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T23:05:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[180979]: pgmap v1333: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 523 op/s; 31599/243189 objects degraded (12.994%); 8487/243189 objects misplaced (3.490%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[180979]: failed to encode map e427 with expected crc 2023-09-23T23:05:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:05:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[180979]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T23:05:27.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[180979]: failed to encode map e427 with expected crc 2023-09-23T23:05:27.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[187743]: pgmap v1333: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 523 op/s; 31599/243189 objects degraded (12.994%); 8487/243189 objects misplaced (3.490%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:27.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[187743]: failed to encode map e427 with expected crc 2023-09-23T23:05:27.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:05:27.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[187743]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T23:05:27.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:27 smithi052 ceph-mon[187743]: failed to encode map e427 with expected crc 2023-09-23T23:05:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:27 smithi151 ceph-mon[159210]: pgmap v1333: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 523 op/s; 31599/243189 objects degraded (12.994%); 8487/243189 objects misplaced (3.490%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T23:05:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:27 smithi151 ceph-mon[159210]: failed to encode map e427 with expected crc 2023-09-23T23:05:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T23:05:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:27 smithi151 ceph-mon[159210]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T23:05:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:27 smithi151 ceph-mon[159210]: failed to encode map e427 with expected crc 2023-09-23T23:05:28.460 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:28.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:28.572 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:28.602 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:28.682 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:28.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:28.788 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:28.957 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:28 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:05:29.026 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:29.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:28 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:05:29.195 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:28 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T23:05:29.810 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:29.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:29.915 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:29.973 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:30.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:05:30.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[187743]: pgmap v1335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 104 B/s rd, 23 MiB/s wr, 512 op/s; 31599/242079 objects degraded (13.053%); 8251/242079 objects misplaced (3.408%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:05:30.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[187743]: failed to encode map e428 with expected crc 2023-09-23T23:05:30.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:05:30.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[187743]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T23:05:30.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[187743]: failed to encode map e428 with expected crc 2023-09-23T23:05:30.077 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[180979]: pgmap v1335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 104 B/s rd, 23 MiB/s wr, 512 op/s; 31599/242079 objects degraded (13.053%); 8251/242079 objects misplaced (3.408%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:05:30.078 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[180979]: failed to encode map e428 with expected crc 2023-09-23T23:05:30.078 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:05:30.078 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[180979]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T23:05:30.078 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:29 smithi052 ceph-mon[180979]: failed to encode map e428 with expected crc 2023-09-23T23:05:30.089 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:30.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:30.196 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:29 smithi151 ceph-mon[159210]: pgmap v1335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 104 B/s rd, 23 MiB/s wr, 512 op/s; 31599/242079 objects degraded (13.053%); 8251/242079 objects misplaced (3.408%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T23:05:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:29 smithi151 ceph-mon[159210]: failed to encode map e428 with expected crc 2023-09-23T23:05:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:29 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T23:05:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:29 smithi151 ceph-mon[159210]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T23:05:30.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:29 smithi151 ceph-mon[159210]: failed to encode map e428 with expected crc 2023-09-23T23:05:30.270 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:30.312 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:30.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:30.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:30.417 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:30.444 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:30.485 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:30.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:30.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:30.591 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:30.624 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:05:30.630 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:30.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:05:30.674 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:05:30.697 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:05:30.716 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:05:30.735 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:05:30.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:05:30.785 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:30.786 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:05:30.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.789 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.789 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:05:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:30.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:30.825 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:30.891 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.942 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 31599/242079 objects degraded (13.053%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1045102198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: failed to encode map e429 with expected crc 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 31599/242079 objects degraded (13.053%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:30.943 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:30 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1045102198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:30.946 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: failed to encode map e429 with expected crc 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: failed to encode map e429 with expected crc 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: failed to encode map e429 with expected crc 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: failed to encode map e429 with expected crc 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: failed to encode map e429 with expected crc 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 31599/242079 objects degraded (13.053%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:31.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:30 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1045102198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:31.831 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: pgmap v1338: 99 pgs: 1 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 4 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 9.5 MiB/s wr, 538 op/s; 20206/239571 objects degraded (8.434%); 3735/239571 objects misplaced (1.559%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2104257593' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:05:32.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:31 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:05:32.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: pgmap v1338: 99 pgs: 1 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 4 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 9.5 MiB/s wr, 538 op/s; 20206/239571 objects degraded (8.434%); 3735/239571 objects misplaced (1.559%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:05:32.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2104257593' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:05:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: pgmap v1338: 99 pgs: 1 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 4 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 170 B/s rd, 9.5 MiB/s wr, 538 op/s; 20206/239571 objects degraded (8.434%); 3735/239571 objects misplaced (1.559%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: failed to encode map e430 with expected crc 2023-09-23T23:05:32.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2104257593' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:05:32.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:31 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:05:32.458 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:32.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:32.565 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:32.599 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:33.173 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:33.173 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[187743]: failed to encode map e431 with expected crc 2023-09-23T23:05:33.173 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[187743]: failed to encode map e431 with expected crc 2023-09-23T23:05:33.173 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:05:33.173 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[187743]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T23:05:33.174 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:33.174 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[180979]: failed to encode map e431 with expected crc 2023-09-23T23:05:33.174 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[180979]: failed to encode map e431 with expected crc 2023-09-23T23:05:33.174 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:05:33.174 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:32 smithi052 ceph-mon[180979]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T23:05:33.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:32 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:05:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:32 smithi151 ceph-mon[159210]: failed to encode map e431 with expected crc 2023-09-23T23:05:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:32 smithi151 ceph-mon[159210]: failed to encode map e431 with expected crc 2023-09-23T23:05:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:32 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:05:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:32 smithi151 ceph-mon[159210]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T23:05:33.375 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:33.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:33.468 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:05:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:05:33] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.33.4" 2023-09-23T23:05:33.479 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:33.513 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:33.563 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:33.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:33.667 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:33.694 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:33.739 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:33.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:33.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:33.844 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:33.874 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:33.928 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:33.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:33.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:33.948 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:33 smithi151 ceph-mon[159210]: pgmap v1341: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 6 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 19426/237072 objects degraded (8.194%); 4233/237072 objects misplaced (1.786%); 4.3 MiB/s, 71 objects/s recovering 2023-09-23T23:05:34.034 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:34.062 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:34.118 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:34.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:34.224 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:34.230 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3, 6] out_osds: [7] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:05:34.230 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:05:34.230 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 4, 5, 3, 6] 2023-09-23T23:05:34.230 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-23T23:05:34.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:33 smithi052 ceph-mon[187743]: pgmap v1341: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 6 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 19426/237072 objects degraded (8.194%); 4233/237072 objects misplaced (1.786%); 4.3 MiB/s, 71 objects/s recovering 2023-09-23T23:05:34.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:33 smithi052 ceph-mon[180979]: pgmap v1341: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 6 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 19426/237072 objects degraded (8.194%); 4233/237072 objects misplaced (1.786%); 4.3 MiB/s, 71 objects/s recovering 2023-09-23T23:05:34.253 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:34.299 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:34.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:34.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:34.406 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:34.434 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:34.506 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:34.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:34.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:34.644 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:34.899 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:34.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:34.985 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:35.010 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:35.046 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:35.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:35.125 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:35.159 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:35.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:35.191 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:05:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:35.271 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:35.309 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:35.436 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:05:35.436 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:05:35.436 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:05:35.436 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:05:35.436 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:05:35.456 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:35.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:35.542 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:05:35.562 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:35.636 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:35 smithi151 ceph-mon[159210]: pgmap v1342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 6 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 19426/237072 objects degraded (8.194%); 4233/237072 objects misplaced (1.786%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:05:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 19426/237072 objects degraded (8.194%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:35 smithi052 ceph-mon[187743]: pgmap v1342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 6 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 19426/237072 objects degraded (8.194%); 4233/237072 objects misplaced (1.786%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:05:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 19426/237072 objects degraded (8.194%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:36.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:35 smithi052 ceph-mon[180979]: pgmap v1342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 6 active+recovery_wait+degraded, 3 creating+peering, 2 active+recovery_wait, 4 peering, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 283 op/s; 19426/237072 objects degraded (8.194%); 4233/237072 objects misplaced (1.786%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T23:05:36.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 19426/237072 objects degraded (8.194%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:36.913 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:36.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:37.019 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:37.068 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:37.130 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:37.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:37.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:37.239 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:37.291 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:37.382 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:37.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:37.491 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:37.538 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:37.621 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:37.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:37.664 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:05:37.733 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:37.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:37.855 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:37.857 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:37.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:37.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:37.881 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:37.967 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:37.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:37 smithi052 ceph-mon[180979]: pgmap v1343: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 13 active+recovery_wait+degraded, 2 active+recovery_wait, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 35197/235659 objects degraded (14.936%); 3895/235659 objects misplaced (1.653%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T23:05:37.969 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:37 smithi052 ceph-mon[187743]: pgmap v1343: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 13 active+recovery_wait+degraded, 2 active+recovery_wait, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 35197/235659 objects degraded (14.936%); 3895/235659 objects misplaced (1.653%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T23:05:38.001 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:38.055 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:38.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:38.062 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:37 smithi151 ceph-mon[159210]: pgmap v1343: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded, 13 active+recovery_wait+degraded, 2 active+recovery_wait, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 35197/235659 objects degraded (14.936%); 3895/235659 objects misplaced (1.653%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T23:05:38.163 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:38.201 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:38.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:38.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:38.391 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:38.438 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:38.468 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:38.851 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:38 smithi052 conmon[180954]: 2023-09-23T23:05:38.553+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e431 definitely_dead 0 2023-09-23T23:05:38.852 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/225514294' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T23:05:38.852 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:38 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T23:05:38.852 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:38 smithi052 ceph-mon[180979]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T23:05:38.878 INFO:teuthology.orchestra.run.smithi052.stderr:marked out osd.0. 2023-09-23T23:05:39.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/225514294' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T23:05:39.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:38 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T23:05:39.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:38 smithi151 ceph-mon[159210]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T23:05:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/225514294' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T23:05:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:38 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T23:05:39.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:38 smithi052 ceph-mon[187743]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: pgmap v1344: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+degraded, 2 active+recovery_wait, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 35007/234690 objects degraded (14.916%); 3895/234690 objects misplaced (1.660%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: osdmap e432: 8 total, 8 up, 6 in 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:39 smithi151 ceph-mon[159210]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: pgmap v1344: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+degraded, 2 active+recovery_wait, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 35007/234690 objects degraded (14.916%); 3895/234690 objects misplaced (1.660%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: osdmap e432: 8 total, 8 up, 6 in 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[180979]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: pgmap v1344: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+degraded, 2 active+recovery_wait, 81 active+clean; 4.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 35007/234690 objects degraded (14.916%); 3895/234690 objects misplaced (1.660%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:05:40.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T23:05:40.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: osdmap e432: 8 total, 8 up, 6 in 2023-09-23T23:05:40.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:39 smithi052 ceph-mon[187743]: failed to encode map e432 with expected crc 2023-09-23T23:05:40.817 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:40.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:40.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:40.921 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:40.963 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:41.048 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:41.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:41.155 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e432 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: osdmap e433: 8 total, 8 up, 6 in 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 35007/234690 objects degraded (14.916%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e432 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: osdmap e433: 8 total, 8 up, 6 in 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 35007/234690 objects degraded (14.916%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e432 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: osdmap e433: 8 total, 8 up, 6 in 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 35007/234690 objects degraded (14.916%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:41.291 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:42.008 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:42.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:42.112 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: failed to encode map e433 with expected crc 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: pgmap v1347: 99 pgs: 1 active+recovering, 4 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 79 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 28061/233583 objects degraded (12.013%); 6049/233583 objects misplaced (2.590%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: osdmap e434: 8 total, 8 up, 6 in 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[187743]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.150 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: failed to encode map e433 with expected crc 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: pgmap v1347: 99 pgs: 1 active+recovering, 4 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 79 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 28061/233583 objects degraded (12.013%); 6049/233583 objects misplaced (2.590%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: osdmap e434: 8 total, 8 up, 6 in 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.151 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:41 smithi052 ceph-mon[180979]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.152 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: failed to encode map e433 with expected crc 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: pgmap v1347: 99 pgs: 1 active+recovering, 4 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+degraded, 2 active+recovery_wait, 79 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 28061/233583 objects degraded (12.013%); 6049/233583 objects misplaced (2.590%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: osdmap e434: 8 total, 8 up, 6 in 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:41 smithi151 ceph-mon[159210]: failed to encode map e434 with expected crc 2023-09-23T23:05:42.308 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:42.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:42.412 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:42.439 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:42.490 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:42.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:42.597 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:42.623 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:42.671 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:42.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:42.779 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:42.812 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:42.897 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:05:42.922 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:05:42.924 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:42.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:42.948 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:05:42.968 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:05:42.988 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:05:43.008 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:05:43.027 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:05:43.029 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:43.051 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:05:43.052 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.052 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.052 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.053 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.053 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.053 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.053 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.053 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:05:43.056 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:43.077 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:42 smithi052 ceph-mon[187743]: failed to encode map e434 with expected crc 2023-09-23T23:05:43.078 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:42 smithi052 ceph-mon[180979]: failed to encode map e434 with expected crc 2023-09-23T23:05:43.101 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:43.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:43.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:42 smithi151 ceph-mon[159210]: failed to encode map e434 with expected crc 2023-09-23T23:05:43.208 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:43.240 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:43.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:05:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:05:43] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-09-23T23:05:43.895 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:05:44.042 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:44.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:44.147 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:44.174 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[187743]: pgmap v1349: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 4 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 50732/232266 objects degraded (21.842%); 3991/232266 objects misplaced (1.718%); 11 MiB/s, 174 objects/s recovering 2023-09-23T23:05:44.174 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1112848156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:44.174 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/179334486' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:44.174 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:44.175 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[180979]: pgmap v1349: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 4 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 50732/232266 objects degraded (21.842%); 3991/232266 objects misplaced (1.718%); 11 MiB/s, 174 objects/s recovering 2023-09-23T23:05:44.175 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1112848156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:44.175 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/179334486' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:44.175 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:43 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:44.180 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:43 smithi151 ceph-mon[159210]: pgmap v1349: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 4 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 50732/232266 objects degraded (21.842%); 3991/232266 objects misplaced (1.718%); 11 MiB/s, 174 objects/s recovering 2023-09-23T23:05:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1112848156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/179334486' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:43 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:05:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:05:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:05:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[180979]: osdmap e435: 8 total, 8 up, 6 in 2023-09-23T23:05:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[180979]: failed to encode map e435 with expected crc 2023-09-23T23:05:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[180979]: failed to encode map e435 with expected crc 2023-09-23T23:05:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:05:44.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:05:44.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[187743]: osdmap e435: 8 total, 8 up, 6 in 2023-09-23T23:05:44.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[187743]: failed to encode map e435 with expected crc 2023-09-23T23:05:44.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:44 smithi052 ceph-mon[187743]: failed to encode map e435 with expected crc 2023-09-23T23:05:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:44 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:05:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:05:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:44 smithi151 ceph-mon[159210]: osdmap e435: 8 total, 8 up, 6 in 2023-09-23T23:05:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:44 smithi151 ceph-mon[159210]: failed to encode map e435 with expected crc 2023-09-23T23:05:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:44 smithi151 ceph-mon[159210]: failed to encode map e435 with expected crc 2023-09-23T23:05:45.994 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:46.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:46.101 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:46.163 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:45 smithi151 ceph-mon[159210]: pgmap v1351: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 4 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 50732/232266 objects degraded (21.842%); 3991/232266 objects misplaced (1.718%); 9.0 MiB/s, 147 objects/s recovering 2023-09-23T23:05:46.164 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 50732/232266 objects degraded (21.842%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:46.164 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:46.208 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:45 smithi052 ceph-mon[187743]: pgmap v1351: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 4 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 50732/232266 objects degraded (21.842%); 3991/232266 objects misplaced (1.718%); 9.0 MiB/s, 147 objects/s recovering 2023-09-23T23:05:46.208 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 50732/232266 objects degraded (21.842%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:46.209 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:45 smithi052 ceph-mon[180979]: pgmap v1351: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 4 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 50732/232266 objects degraded (21.842%); 3991/232266 objects misplaced (1.718%); 9.0 MiB/s, 147 objects/s recovering 2023-09-23T23:05:46.209 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 50732/232266 objects degraded (21.842%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:46.257 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:46.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:46.363 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:46.402 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:46.454 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:05:46.455 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:05:46.463 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:46.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:46.567 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:05:46.613 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:46.681 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:05:46.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:46.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:05:46.787 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:46.820 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:46.879 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:46.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:46.985 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:47.046 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:47.141 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:47.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:47.203 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 5, 3, 6] out_osds: [7, 0] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:05:47.203 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:05:47.204 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [1, 2, 4, 5, 3, 6] 2023-09-23T23:05:47.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T23:05:47.255 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:47.296 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:47.371 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:47.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:47.380 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:05:47.477 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:47.525 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:47.595 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:47.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:47.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:47.702 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:47.705 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:47.769 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:47.826 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:05:47.862 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:47.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:47.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:47.982 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:47.994 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:05:48.051 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:47 smithi151 ceph-mon[159210]: pgmap v1352: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 12 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 58909/230937 objects degraded (25.509%); 6140/230937 objects misplaced (2.659%); 9.0 MiB/s, 146 objects/s recovering 2023-09-23T23:05:48.051 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:47 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:48.054 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:48.122 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:47 smithi052 ceph-mon[180979]: pgmap v1352: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 12 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 58909/230937 objects degraded (25.509%); 6140/230937 objects misplaced (2.659%); 9.0 MiB/s, 146 objects/s recovering 2023-09-23T23:05:48.123 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:47 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:48.123 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:47 smithi052 ceph-mon[187743]: pgmap v1352: 99 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering, 12 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 58909/230937 objects degraded (25.509%); 6140/230937 objects misplaced (2.659%); 9.0 MiB/s, 146 objects/s recovering 2023-09-23T23:05:48.123 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:47 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:05:48.126 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:48.154 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:48.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:48.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:48.325 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:48.387 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:48.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:48.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:48.499 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:48.619 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:48.714 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:48.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:48.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:48.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:48.871 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:49.290 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:49.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:49.398 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:49.500 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:49.633 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:49.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:49.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.640 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:49 smithi151 ceph-mon[159210]: pgmap v1353: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 58785/230064 objects degraded (25.552%); 5384/230064 objects misplaced (2.340%); 13 MiB/s, 204 objects/s recovering 2023-09-23T23:05:49.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:49.649 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:49.741 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:05:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[187743]: pgmap v1353: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 58785/230064 objects degraded (25.552%); 5384/230064 objects misplaced (2.340%); 13 MiB/s, 204 objects/s recovering 2023-09-23T23:05:49.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:49.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:49 smithi052 ceph-mon[180979]: pgmap v1353: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 58785/230064 objects degraded (25.552%); 5384/230064 objects misplaced (2.340%); 13 MiB/s, 204 objects/s recovering 2023-09-23T23:05:49.792 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:49.915 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:05:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:49.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:05:50.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:05:50.112 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:50.232 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:05:50.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:50.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:05:50.346 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:50.414 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:50.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:50.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:50.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 58785/230064 objects degraded (25.552%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:50.548 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:50.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:50.657 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:50.692 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 58785/230064 objects degraded (25.552%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:05:50.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 58785/230064 objects degraded (25.552%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:51.101 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:51.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:51.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:51.205 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:51.242 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:51.625 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:51.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:51.729 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:51.779 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:51.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:51 smithi151 ceph-mon[159210]: pgmap v1354: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 179 op/s; 58510/228288 objects degraded (25.630%); 5368/228288 objects misplaced (2.351%); 12 MiB/s, 195 objects/s recovering 2023-09-23T23:05:51.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:51 smithi052 ceph-mon[180979]: pgmap v1354: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 179 op/s; 58510/228288 objects degraded (25.630%); 5368/228288 objects misplaced (2.351%); 12 MiB/s, 195 objects/s recovering 2023-09-23T23:05:51.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:51 smithi052 conmon[180954]: 2023-09-23T23:05:51.612+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e435 definitely_dead 0 2023-09-23T23:05:51.996 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:51 smithi052 ceph-mon[187743]: pgmap v1354: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 179 op/s; 58510/228288 objects degraded (25.630%); 5368/228288 objects misplaced (2.351%); 12 MiB/s, 195 objects/s recovering 2023-09-23T23:05:52.040 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:52.131 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:52.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:52.241 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:05:52.291 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:52.456 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:05:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:52.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:05:52.565 INFO:teuthology.orchestra.run.smithi052.stderr:marked out osd.5. 2023-09-23T23:05:52.567 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:52.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1152072357' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T23:05:52.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:52 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T23:05:52.589 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:52 smithi052 ceph-mon[180979]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T23:05:52.589 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1152072357' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T23:05:52.589 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:52 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T23:05:52.589 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:52 smithi052 ceph-mon[187743]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T23:05:52.619 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1152072357' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T23:05:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:52 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T23:05:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:52 smithi151 ceph-mon[159210]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T23:05:53.732 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:05:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:05:53] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.33.4" 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: osdmap e436: 8 total, 8 up, 5 in 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: pgmap v1356: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 258 op/s; 58308/225423 objects degraded (25.866%); 4852/225423 objects misplaced (2.152%); 14 MiB/s, 229 objects/s recovering 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:53.733 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: osdmap e436: 8 total, 8 up, 5 in 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: pgmap v1356: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 258 op/s; 58308/225423 objects degraded (25.866%); 4852/225423 objects misplaced (2.152%); 14 MiB/s, 229 objects/s recovering 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:53.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T23:05:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: osdmap e436: 8 total, 8 up, 5 in 2023-09-23T23:05:53.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: failed to encode map e436 with expected crc 2023-09-23T23:05:53.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: pgmap v1356: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 258 op/s; 58308/225423 objects degraded (25.866%); 4852/225423 objects misplaced (2.152%); 14 MiB/s, 229 objects/s recovering 2023-09-23T23:05:53.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:53.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:53.983 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:53.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:54.011 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:05:54.011 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:05:54.088 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:54.119 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: osdmap e437: 8 total, 8 up, 5 in 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:54.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:54 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:54.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: osdmap e437: 8 total, 8 up, 5 in 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: osdmap e437: 8 total, 8 up, 5 in 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: failed to encode map e437 with expected crc 2023-09-23T23:05:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:54.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:54 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:55.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: osdmap e438: 8 total, 8 up, 5 in 2023-09-23T23:05:55.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: pgmap v1359: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 257 op/s; 58308/225423 objects degraded (25.866%); 4852/225423 objects misplaced (2.152%); 10 MiB/s, 167 objects/s recovering 2023-09-23T23:05:55.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 58308/225423 objects degraded (25.866%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:55.782 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:55.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.784 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: osdmap e438: 8 total, 8 up, 5 in 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: pgmap v1359: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 257 op/s; 58308/225423 objects degraded (25.866%); 4852/225423 objects misplaced (2.152%); 10 MiB/s, 167 objects/s recovering 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 58308/225423 objects degraded (25.866%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: osdmap e438: 8 total, 8 up, 5 in 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: failed to encode map e438 with expected crc 2023-09-23T23:05:55.785 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: pgmap v1359: 99 pgs: 2 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+recovery_wait, 73 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 257 op/s; 58308/225423 objects degraded (25.866%); 4852/225423 objects misplaced (2.152%); 10 MiB/s, 167 objects/s recovering 2023-09-23T23:05:55.786 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 58308/225423 objects degraded (25.866%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T23:05:55.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:55.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:55.887 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:55.926 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:56.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:56.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:56.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:56.264 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:05:56.296 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:56.524 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:05:56.544 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:05:56.563 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:05:56.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:05:56.607 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:05:56.629 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:05:56.653 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:05:56.676 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:05:56.677 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.677 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.677 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.677 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.678 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.678 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.678 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.678 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:05:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:56 smithi151 ceph-mon[159210]: failed to encode map e438 with expected crc 2023-09-23T23:05:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/192201258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:56.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:56 smithi052 ceph-mon[187743]: failed to encode map e438 with expected crc 2023-09-23T23:05:56.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/192201258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:56 smithi052 ceph-mon[180979]: failed to encode map e438 with expected crc 2023-09-23T23:05:56.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/192201258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:05:57.953 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:57 smithi151 ceph-mon[159210]: pgmap v1360: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 218 op/s; 56435/224358 objects degraded (25.154%); 9266/224358 objects misplaced (4.130%); 13 MiB/s, 210 objects/s recovering 2023-09-23T23:05:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:57 smithi052 ceph-mon[187743]: pgmap v1360: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 218 op/s; 56435/224358 objects degraded (25.154%); 9266/224358 objects misplaced (4.130%); 13 MiB/s, 210 objects/s recovering 2023-09-23T23:05:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:57 smithi052 ceph-mon[180979]: pgmap v1360: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 218 op/s; 56435/224358 objects degraded (25.154%); 9266/224358 objects misplaced (4.130%); 13 MiB/s, 210 objects/s recovering 2023-09-23T23:05:58.451 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:05:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:58.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:05:58.555 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:58.587 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:58.682 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:05:58.742 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:58.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:58.852 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:05:58.884 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:58.949 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:05:58.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:58.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:05:59.053 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:59.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:59.182 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:59.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:59.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:59.255 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:05:59.287 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:05:59.326 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:59.392 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:05:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:59.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:05:59.497 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:59.529 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:59.633 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:05:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:05:59.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:05:59.739 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:05:59.768 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:05:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[187743]: pgmap v1361: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 72 op/s; 62915/224082 objects degraded (28.077%); 8701/224082 objects misplaced (3.883%); 10 MiB/s, 161 objects/s recovering 2023-09-23T23:05:59.780 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1061971185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:59.781 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:59.781 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:59.781 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:59.781 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[180979]: pgmap v1361: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 72 op/s; 62915/224082 objects degraded (28.077%); 8701/224082 objects misplaced (3.883%); 10 MiB/s, 161 objects/s recovering 2023-09-23T23:05:59.782 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1061971185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:59.782 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:59.782 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:59.782 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:05:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:59.782 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:05:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:59 smithi151 ceph-mon[159210]: pgmap v1361: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 72 op/s; 62915/224082 objects degraded (28.077%); 8701/224082 objects misplaced (3.883%); 10 MiB/s, 161 objects/s recovering 2023-09-23T23:05:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1061971185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:59 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:05:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:05:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:05:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:00.501 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:06:00.501 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:06:00.502 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:06:00.502 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-09-23T23:06:00.502 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-09-23T23:06:00.641 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:06:00.698 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:00.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:00.802 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:00.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:00.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: failed to encode map e439 with expected crc 2023-09-23T23:06:00.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: failed to encode map e439 with expected crc 2023-09-23T23:06:00.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:06:00.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: osdmap e439: 8 total, 8 up, 5 in 2023-09-23T23:06:00.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: failed to encode map e439 with expected crc 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 62915/224082 objects degraded (28.077%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: failed to encode map e439 with expected crc 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: failed to encode map e439 with expected crc 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: osdmap e439: 8 total, 8 up, 5 in 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: failed to encode map e439 with expected crc 2023-09-23T23:06:00.830 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 62915/224082 objects degraded (28.077%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T23:06:00.831 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: failed to encode map e439 with expected crc 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: failed to encode map e439 with expected crc 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: osdmap e439: 8 total, 8 up, 5 in 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: failed to encode map e439 with expected crc 2023-09-23T23:06:01.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 62915/224082 objects degraded (28.077%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T23:06:01.593 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:01.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:01.697 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:01.728 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:01.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:01 smithi052 ceph-mon[187743]: pgmap v1363: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 63 op/s; 62915/224052 objects degraded (28.081%); 8584/224052 objects misplaced (3.831%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T23:06:01.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:01 smithi052 ceph-mon[180979]: pgmap v1363: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 63 op/s; 62915/224052 objects degraded (28.081%); 8584/224052 objects misplaced (3.831%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T23:06:02.071 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:02.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:02.178 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:02.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:01 smithi151 ceph-mon[159210]: pgmap v1363: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 2 remapped+peering, 2 activating+undersized+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 63 op/s; 62915/224052 objects degraded (28.081%); 8584/224052 objects misplaced (3.831%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T23:06:02.212 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:02.252 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:02.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:02.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:02.396 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:02.436 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:02.520 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:02.714 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:02.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:02.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:02.818 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:02.866 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:03.189 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:03.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:03.298 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:03.345 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:03.426 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:06:03.426 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:06:03.427 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:06:03.427 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:06:03.427 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:06:03.647 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:06:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:06:03] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.33.4" 2023-09-23T23:06:03.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:03 smithi052 ceph-mon[187743]: pgmap v1364: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 67 op/s; 69906/223773 objects degraded (31.240%); 7726/223773 objects misplaced (3.453%); 15 MiB/s, 245 objects/s recovering 2023-09-23T23:06:03.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:03 smithi052 ceph-mon[180979]: pgmap v1364: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 67 op/s; 69906/223773 objects degraded (31.240%); 7726/223773 objects misplaced (3.453%); 15 MiB/s, 245 objects/s recovering 2023-09-23T23:06:04.190 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:06:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:03 smithi151 ceph-mon[159210]: pgmap v1364: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 67 op/s; 69906/223773 objects degraded (31.240%); 7726/223773 objects misplaced (3.453%); 15 MiB/s, 245 objects/s recovering 2023-09-23T23:06:05.472 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:05.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:05.576 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:05.633 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[187743]: pgmap v1365: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 54 op/s; 69906/223773 objects degraded (31.240%); 7726/223773 objects misplaced (3.453%); 12 MiB/s, 199 objects/s recovering 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 69906/223773 objects degraded (31.240%), 28 pgs degraded (PG_DEGRADED) 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[180979]: pgmap v1365: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 54 op/s; 69906/223773 objects degraded (31.240%); 7726/223773 objects misplaced (3.453%); 12 MiB/s, 199 objects/s recovering 2023-09-23T23:06:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 69906/223773 objects degraded (31.240%), 28 pgs degraded (PG_DEGRADED) 2023-09-23T23:06:05.783 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:05.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:05.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:05.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:05 smithi151 ceph-mon[159210]: pgmap v1365: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 54 op/s; 69906/223773 objects degraded (31.240%); 7726/223773 objects misplaced (3.453%); 12 MiB/s, 199 objects/s recovering 2023-09-23T23:06:05.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 69906/223773 objects degraded (31.240%), 28 pgs degraded (PG_DEGRADED) 2023-09-23T23:06:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:05.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:05.899 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:05.961 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:06.045 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:06.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:06.151 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:06.181 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:06.261 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:06.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:06.353 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:06:06.373 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:06.404 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:06.499 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:06.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:06.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:06.573 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:06.611 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:06.655 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:06.840 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:06.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:06.946 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:06.990 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:07.116 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:07.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:07.209 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:07.212 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:07.230 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:07.245 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:07.285 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:07.322 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:07.327 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:07.352 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:07.411 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:07.454 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:07.460 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:07.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:07.481 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:07.482 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.482 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.482 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.482 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.483 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.483 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.483 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.483 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:06:07.574 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:07.611 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.6 primary-affinity to 0 (802) 2023-09-23T23:06:07.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.638 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: pgmap v1366: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 80 op/s; 69595/221946 objects degraded (31.357%); 6841/221946 objects misplaced (3.082%); 16 MiB/s, 255 objects/s recovering 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3283054071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1561186279' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: pgmap v1366: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 80 op/s; 69595/221946 objects degraded (31.357%); 6841/221946 objects misplaced (3.082%); 16 MiB/s, 255 objects/s recovering 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3283054071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:07.639 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1561186279' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T23:06:07.640 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:06:07.640 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:06:07.640 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: pgmap v1366: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 80 op/s; 69595/221946 objects degraded (31.357%); 6841/221946 objects misplaced (3.082%); 16 MiB/s, 255 objects/s recovering 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3283054071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1561186279' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:06:07.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:07.659 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:07.716 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:07.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:07.824 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:07.851 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:07.896 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:07.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:08.003 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:08.035 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:08.535 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:08.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:08.639 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:08.669 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:08.814 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1561186279' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-23T23:06:08.814 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[187743]: osdmap e440: 8 total, 8 up, 5 in 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[187743]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[187743]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[187743]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[187743]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1561186279' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[180979]: osdmap e440: 8 total, 8 up, 5 in 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[180979]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[180979]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.815 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[180979]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.816 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:08 smithi052 ceph-mon[180979]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1561186279' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-09-23T23:06:08.962 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:08 smithi151 ceph-mon[159210]: osdmap e440: 8 total, 8 up, 5 in 2023-09-23T23:06:08.962 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:08 smithi151 ceph-mon[159210]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.962 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:08 smithi151 ceph-mon[159210]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.962 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:08 smithi151 ceph-mon[159210]: failed to encode map e440 with expected crc 2023-09-23T23:06:08.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:08 smithi151 ceph-mon[159210]: failed to encode map e440 with expected crc 2023-09-23T23:06:09.678 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:10.445 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.445 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.445 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: osdmap e441: 8 total, 8 up, 5 in 2023-09-23T23:06:10.446 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: pgmap v1369: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 125 op/s; 69420/221028 objects degraded (31.408%); 6841/221028 objects misplaced (3.095%); 16 MiB/s, 263 objects/s recovering 2023-09-23T23:06:10.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:09 smithi151 ceph-mon[159210]: failed to encode map e440 with expected crc 2023-09-23T23:06:10.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:10.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:10.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: osdmap e441: 8 total, 8 up, 5 in 2023-09-23T23:06:10.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.450 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: pgmap v1369: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 125 op/s; 69420/221028 objects degraded (31.408%); 6841/221028 objects misplaced (3.095%); 16 MiB/s, 263 objects/s recovering 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[187743]: failed to encode map e440 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.451 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: osdmap e441: 8 total, 8 up, 5 in 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e441 with expected crc 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: pgmap v1369: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 71 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 125 op/s; 69420/221028 objects degraded (31.408%); 6841/221028 objects misplaced (3.095%); 16 MiB/s, 263 objects/s recovering 2023-09-23T23:06:10.452 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:09 smithi052 ceph-mon[180979]: failed to encode map e440 with expected crc 2023-09-23T23:06:10.551 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:10.584 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:10.646 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:10.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:10.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: osdmap e442: 8 total, 8 up, 5 in 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 69420/221028 objects degraded (31.408%), 28 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: osdmap e442: 8 total, 8 up, 5 in 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 69420/221028 objects degraded (31.408%), 28 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:10.751 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:10.787 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: osdmap e442: 8 total, 8 up, 5 in 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: failed to encode map e442 with expected crc 2023-09-23T23:06:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 69420/221028 objects degraded (31.408%), 28 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:11.909 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:11 smithi052 ceph-mon[187743]: pgmap v1371: 99 pgs: 1 activating, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 70 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 161 op/s; 69420/220866 objects degraded (31.431%); 6725/220866 objects misplaced (3.045%); 14 MiB/s, 222 objects/s recovering 2023-09-23T23:06:11.910 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:11 smithi052 ceph-mon[180979]: pgmap v1371: 99 pgs: 1 activating, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 70 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 161 op/s; 69420/220866 objects degraded (31.431%); 6725/220866 objects misplaced (3.045%); 14 MiB/s, 222 objects/s recovering 2023-09-23T23:06:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:11 smithi151 ceph-mon[159210]: pgmap v1371: 99 pgs: 1 activating, 2 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 70 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 161 op/s; 69420/220866 objects degraded (31.431%); 6725/220866 objects misplaced (3.045%); 14 MiB/s, 222 objects/s recovering 2023-09-23T23:06:11.963 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:11.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:12.068 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:12.101 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:12.194 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:12.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:12.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:12.298 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:12.330 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:12.454 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:06:12.456 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:12.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:12.562 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:12.597 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:12.653 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:12.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:12.658 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:06:12.658 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:06:12.757 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:12.760 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:06:12.801 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:12.980 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:12.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:13.084 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:13.111 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3192134021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:13.111 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:12 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:13.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3192134021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:13.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:12 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:13.113 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3192134021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:12 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:13.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:06:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:06:13] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.33.4" 2023-09-23T23:06:14.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: pgmap v1372: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 activating, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 72 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 91 op/s; 64289/220293 objects degraded (29.183%); 6254/220293 objects misplaced (2.839%); 9.5 MiB/s, 192 objects/s recovering 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: osdmap e443: 8 total, 8 up, 5 in 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[180979]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: pgmap v1372: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 activating, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 72 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 91 op/s; 64289/220293 objects degraded (29.183%); 6254/220293 objects misplaced (2.839%); 9.5 MiB/s, 192 objects/s recovering 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: osdmap e443: 8 total, 8 up, 5 in 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:13 smithi052 ceph-mon[187743]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: pgmap v1372: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 activating, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 72 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 91 op/s; 64289/220293 objects degraded (29.183%); 6254/220293 objects misplaced (2.839%); 9.5 MiB/s, 192 objects/s recovering 2023-09-23T23:06:14.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:14.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:14.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: osdmap e443: 8 total, 8 up, 5 in 2023-09-23T23:06:14.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T23:06:14.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:13 smithi151 ceph-mon[159210]: failed to encode map e443 with expected crc 2023-09-23T23:06:14.607 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:14.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:14.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:14.738 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:14.790 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:14.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:14.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:14.900 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:14.927 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:14.961 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:06:14.973 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:14.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:14.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:15.051 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:15.081 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:15.114 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:15.178 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:15.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:15.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:15.281 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:15.312 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:15.362 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:15.416 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:15.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:15.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:15.486 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:06:15.486 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:06:15.486 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T23:06:15.487 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:06:15.520 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:15.549 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:15.605 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:15.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:15.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:15.742 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:15.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:15 smithi151 ceph-mon[159210]: pgmap v1374: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 activating, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 72 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 64289/220293 objects degraded (29.183%); 6254/220293 objects misplaced (2.839%); 8.1 MiB/s, 170 objects/s recovering 2023-09-23T23:06:15.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 64289/220293 objects degraded (29.183%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:15.945 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:15.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:15.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:15.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:15 smithi052 ceph-mon[187743]: pgmap v1374: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 activating, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 72 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 64289/220293 objects degraded (29.183%); 6254/220293 objects misplaced (2.839%); 8.1 MiB/s, 170 objects/s recovering 2023-09-23T23:06:15.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 64289/220293 objects degraded (29.183%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:15 smithi052 ceph-mon[180979]: pgmap v1374: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 activating, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 72 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 64289/220293 objects degraded (29.183%); 6254/220293 objects misplaced (2.839%); 8.1 MiB/s, 170 objects/s recovering 2023-09-23T23:06:15.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 64289/220293 objects degraded (29.183%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:16.050 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:16.079 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:16.662 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:16.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:16.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:16.767 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:16.808 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:16.870 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:16.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:16.949 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:16.977 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:17.009 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:06:17.013 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:17.061 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:17.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:17.164 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:17.198 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:17.274 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:17.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:17.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:17.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:17.430 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:17.503 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:17.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:17.547 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:17.570 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:17.593 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:17.608 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:17.622 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:17.646 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:17.647 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:17.670 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:17.691 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:17.715 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:17.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:17.728 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:17.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.730 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.731 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:06:17.822 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:17.860 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:17.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:17 smithi151 ceph-mon[159210]: pgmap v1375: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 73 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 42 op/s; 66315/220005 objects degraded (30.142%); 4194/220005 objects misplaced (1.906%); 7.8 MiB/s, 154 objects/s recovering 2023-09-23T23:06:17.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1079596279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:17.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:17 smithi052 ceph-mon[187743]: pgmap v1375: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 73 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 42 op/s; 66315/220005 objects degraded (30.142%); 4194/220005 objects misplaced (1.906%); 7.8 MiB/s, 154 objects/s recovering 2023-09-23T23:06:17.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1079596279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:17 smithi052 ceph-mon[180979]: pgmap v1375: 99 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 2 activating+undersized+remapped, 1 activating+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 73 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 42 op/s; 66315/220005 objects degraded (30.142%); 4194/220005 objects misplaced (1.906%); 7.8 MiB/s, 154 objects/s recovering 2023-09-23T23:06:17.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1079596279' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:18.220 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:18.220 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPushReply(11.e 443/433 [PushReplyOp(11:783e9fa1:::benchmark_data_smithi052_606809_object16385:head)])", 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:18.217055+0000", 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 0.0029647620000000001, 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029920020000000001, 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:18.221 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217055+0000", 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217055+0000", 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217053+0000", 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0195e-05 2023-09-23T23:06:18.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217063+0000", 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0290999999999999e-05 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217083+0000", 2023-09-23T23:06:18.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3190000000000001e-06 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217092+0000", 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6191000000000004e-05 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:18.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:18.217159+0000", 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103916 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: ], 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 1 2023-09-23T23:06:18.226 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:18.372 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:18.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:18.476 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:18.531 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:18.608 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:18.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:18.712 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:18.754 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:18.811 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:18 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T23:06:18.813 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:18.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:18.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:18.920 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:18.965 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:19.153 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:06:19.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:18 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T23:06:19.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:18 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T23:06:19.328 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:19.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:19.406 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:19.433 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:19.470 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:19.593 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:19.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:19.698 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:19.759 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:19.772 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:19.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:19.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:19.945 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:19.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:19 smithi052 ceph-mon[180979]: pgmap v1376: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 52 op/s; 68439/219600 objects degraded (31.165%); 5607/219600 objects misplaced (2.553%); 12 MiB/s, 225 objects/s recovering 2023-09-23T23:06:19.979 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:19.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:19 smithi052 ceph-mon[187743]: pgmap v1376: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 52 op/s; 68439/219600 objects degraded (31.165%); 5607/219600 objects misplaced (2.553%); 12 MiB/s, 225 objects/s recovering 2023-09-23T23:06:19.979 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:19.983 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:20.035 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:19 smithi151 ceph-mon[159210]: pgmap v1376: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 52 op/s; 68439/219600 objects degraded (31.165%); 5607/219600 objects misplaced (2.553%); 12 MiB/s, 225 objects/s recovering 2023-09-23T23:06:20.035 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:20.060 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:20.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:20.165 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:20.198 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:20.337 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:06:20.338 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:06:20.354 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:20.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:20.458 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:20.490 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:20.544 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:20.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:20.647 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:20.683 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:20.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 68439/219600 objects degraded (31.165%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:20.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 68439/219600 objects degraded (31.165%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:21.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 68439/219600 objects degraded (31.165%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:21.328 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:06:21.691 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:21.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:21.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:21.794 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:21.820 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:21 smithi151 ceph-mon[159210]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:06:21.821 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:21 smithi151 ceph-mon[159210]: pgmap v1377: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 51 op/s; 68439/219309 objects degraded (31.207%); 5494/219309 objects misplaced (2.505%); 11 MiB/s, 205 objects/s recovering 2023-09-23T23:06:21.822 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:21.822 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:06:21.822 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:06:21.822 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:06:21.822 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:21.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:21 smithi052 ceph-mon[187743]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:06:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:21 smithi052 ceph-mon[187743]: pgmap v1377: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 51 op/s; 68439/219309 objects degraded (31.207%); 5494/219309 objects misplaced (2.505%); 11 MiB/s, 205 objects/s recovering 2023-09-23T23:06:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:21 smithi052 ceph-mon[180979]: from='client.36987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:06:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:21 smithi052 ceph-mon[180979]: pgmap v1377: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 51 op/s; 68439/219309 objects degraded (31.207%); 5494/219309 objects misplaced (2.505%); 11 MiB/s, 205 objects/s recovering 2023-09-23T23:06:22.176 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1376,"stamp":"2023-09-23T23:06:18.712240+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4796192696,"num_objects":73200,"num_object_clones":0,"num_object_copies":219600,"num_objects_missing_on_primary":10421,"num_objects_missing":10421,"num_objects_degraded":68439,"num_objects_misplaced":5607,"num_objects_unfound":0,"num_objects_dirty":73200,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":91388,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9936,"num_bytes_recovered":629821888,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142305,"ondisk_log_size":142305,"up":297,"acting":276,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":14033996,"kb_used_data":10436964,"kb_used_omap":99,"kb_used_meta":3596892,"kb_avail":454691764,"statfs":{"total":479975178240,"available":465604366336,"internally_reserved":0,"allocated":10687451136,"data_stored":10680161532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3683217430},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,3],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-31195136,"num_objects":-476,"num_object_clones":0,"num_object_copies":-1428,"num_objects_missing_on_primary":-4316,"num_objects_missing":-4316,"num_objects_degraded":-981,"num_objects_misplaced":-1234,"num_objects_unfound":0,"num_objects_dirty":-476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":476,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2061,"num_bytes_recovered":118750688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.128121"},"pg_stats":[{"pgid":"11.12","version":"443'5520","reported_seq":5583,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.459711+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:06:16.459711+0000","last_peered":"2023-09-23T23:06:16.459711+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:06:16.459711+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149651456,"num_objects":2284,"num_object_clones":0,"num_object_copies":6852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2857,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":2315},{"shards":"2,3","objects":-173}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":455,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592207+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:06:09.592207+0000","last_peered":"2023-09-23T23:06:09.592207+0000","last_clean":"2023-09-23T23:06:09.592207+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:06:09.592207+0000","last_undegraded":"2023-09-23T23:06:09.592207+0000","last_fullsized":"2023-09-23T23:06:09.592207+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":114,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.594047+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:06:09.594047+0000","last_peered":"2023-09-23T23:06:09.594047+0000","last_clean":"2023-09-23T23:06:09.594047+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:06:09.594047+0000","last_undegraded":"2023-09-23T23:06:09.594047+0000","last_fullsized":"2023-09-23T23:06:09.594047+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"443'5483","reported_seq":5665,"reported_epoch":443,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:16.937253+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:06:16.937253+0000","last_peered":"2023-09-23T23:06:16.937253+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:06:16.937253+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:06:16.937253+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5483,"log_dups_size":0,"ondisk_log_size":5483,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150765568,"num_objects":2301,"num_object_clones":0,"num_object_copies":6903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2587,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2830,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":3604480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":1974},{"shards":"0,3,6","objects":270},{"shards":"2,3,6","objects":57}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":113,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595506+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:06:09.595506+0000","last_peered":"2023-09-23T23:06:09.595506+0000","last_clean":"2023-09-23T23:06:09.595506+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:06:09.595506+0000","last_undegraded":"2023-09-23T23:06:09.595506+0000","last_fullsized":"2023-09-23T23:06:09.595506+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":460,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.600347+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:06:09.600347+0000","last_peered":"2023-09-23T23:06:09.600347+0000","last_clean":"2023-09-23T23:06:09.600347+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:06:09.600347+0000","last_undegraded":"2023-09-23T23:06:09.600347+0000","last_fullsized":"2023-09-23T23:06:09.600347+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"443'5497","reported_seq":9522,"reported_epoch":443,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:16.778938+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:06:16.778938+0000","last_peered":"2023-09-23T23:06:16.778938+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:06:16.778938+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:06:16.778938+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148504576,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2841,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":163},{"shards":"1,3,6","objects":2103}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.588160+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:06:09.588160+0000","last_peered":"2023-09-23T23:06:09.588160+0000","last_clean":"2023-09-23T23:06:09.588160+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:06:09.588160+0000","last_undegraded":"2023-09-23T23:06:09.588160+0000","last_fullsized":"2023-09-23T23:06:09.588160+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":39,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:11.952798+0000","last_change":"2023-09-23T23:06:08.605727+0000","last_active":"2023-09-23T23:06:11.952798+0000","last_peered":"2023-09-23T23:06:11.952798+0000","last_clean":"2023-09-23T23:06:11.952798+0000","last_became_active":"2023-09-23T23:06:08.605421+0000","last_became_peered":"2023-09-23T23:06:08.605421+0000","last_unstale":"2023-09-23T23:06:11.952798+0000","last_undegraded":"2023-09-23T23:06:11.952798+0000","last_fullsized":"2023-09-23T23:06:11.952798+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:10.283307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":386,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.606406+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:06:09.606406+0000","last_peered":"2023-09-23T23:06:09.606406+0000","last_clean":"2023-09-23T23:06:09.606406+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:06:09.606406+0000","last_undegraded":"2023-09-23T23:06:09.606406+0000","last_fullsized":"2023-09-23T23:06:09.606406+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":132,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595440+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:06:09.595440+0000","last_peered":"2023-09-23T23:06:09.595440+0000","last_clean":"2023-09-23T23:06:09.595440+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:06:09.595440+0000","last_undegraded":"2023-09-23T23:06:09.595440+0000","last_fullsized":"2023-09-23T23:06:09.595440+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":248,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.600213+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:06:09.600213+0000","last_peered":"2023-09-23T23:06:09.600213+0000","last_clean":"2023-09-23T23:06:09.600213+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:06:09.600213+0000","last_undegraded":"2023-09-23T23:06:09.600213+0000","last_fullsized":"2023-09-23T23:06:09.600213+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":146,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595372+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:06:09.595372+0000","last_peered":"2023-09-23T23:06:09.595372+0000","last_clean":"2023-09-23T23:06:09.595372+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:06:09.595372+0000","last_undegraded":"2023-09-23T23:06:09.595372+0000","last_fullsized":"2023-09-23T23:06:09.595372+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":179,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593979+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:06:09.593979+0000","last_peered":"2023-09-23T23:06:09.593979+0000","last_clean":"2023-09-23T23:06:09.593979+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:06:09.593979+0000","last_undegraded":"2023-09-23T23:06:09.593979+0000","last_fullsized":"2023-09-23T23:06:09.593979+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":179,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:11.625572+0000","last_change":"2023-09-23T23:06:08.609112+0000","last_active":"2023-09-23T23:06:11.625572+0000","last_peered":"2023-09-23T23:06:11.625572+0000","last_clean":"2023-09-23T23:06:11.625572+0000","last_became_active":"2023-09-23T23:06:08.608811+0000","last_became_peered":"2023-09-23T23:06:08.608811+0000","last_unstale":"2023-09-23T23:06:11.625572+0000","last_undegraded":"2023-09-23T23:06:11.625572+0000","last_fullsized":"2023-09-23T23:06:11.625572+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:01.339818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":533,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593515+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:06:09.593515+0000","last_peered":"2023-09-23T23:06:09.593515+0000","last_clean":"2023-09-23T23:06:09.593515+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:06:09.593515+0000","last_undegraded":"2023-09-23T23:06:09.593515+0000","last_fullsized":"2023-09-23T23:06:09.593515+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":426,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592733+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:06:09.592733+0000","last_peered":"2023-09-23T23:06:09.592733+0000","last_clean":"2023-09-23T23:06:09.592733+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:06:09.592733+0000","last_undegraded":"2023-09-23T23:06:09.592733+0000","last_fullsized":"2023-09-23T23:06:09.592733+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":358,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592826+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:06:09.592826+0000","last_peered":"2023-09-23T23:06:09.592826+0000","last_clean":"2023-09-23T23:06:09.592826+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:06:09.592826+0000","last_undegraded":"2023-09-23T23:06:09.592826+0000","last_fullsized":"2023-09-23T23:06:09.592826+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":125,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.607356+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:06:09.607356+0000","last_peered":"2023-09-23T23:06:09.607356+0000","last_clean":"2023-09-23T23:06:09.607356+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:06:09.607356+0000","last_undegraded":"2023-09-23T23:06:09.607356+0000","last_fullsized":"2023-09-23T23:06:09.607356+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":443,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.597398+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:06:09.597398+0000","last_peered":"2023-09-23T23:06:09.597398+0000","last_clean":"2023-09-23T23:06:09.597398+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:06:09.597398+0000","last_undegraded":"2023-09-23T23:06:09.597398+0000","last_fullsized":"2023-09-23T23:06:09.597398+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":340,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593877+0000","last_change":"2023-09-23T23:05:53.598631+0000","last_active":"2023-09-23T23:06:09.593877+0000","last_peered":"2023-09-23T23:06:09.593877+0000","last_clean":"2023-09-23T23:06:09.593877+0000","last_became_active":"2023-09-23T23:05:53.598050+0000","last_became_peered":"2023-09-23T23:05:53.598050+0000","last_unstale":"2023-09-23T23:06:09.593877+0000","last_undegraded":"2023-09-23T23:06:09.593877+0000","last_fullsized":"2023-09-23T23:06:09.593877+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:24.263110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"443'5473","reported_seq":7378,"reported_epoch":443,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T23:06:16.937277+0000","last_change":"2023-09-23T23:06:12.301951+0000","last_active":"2023-09-23T23:06:16.937277+0000","last_peered":"2023-09-23T23:06:16.937277+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.040640+0000","last_became_peered":"2023-09-23T23:05:44.040640+0000","last_unstale":"2023-09-23T23:06:16.937277+0000","last_undegraded":"2023-09-23T23:06:16.937277+0000","last_fullsized":"2023-09-23T23:05:40.871164+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148799488,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1413,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2824,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":918,"num_bytes_recovered":60063744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":1106},{"shards":"0,3,6","objects":307},{"shards":"3,6","objects":858}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":346,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591965+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:06:09.591965+0000","last_peered":"2023-09-23T23:06:09.591965+0000","last_clean":"2023-09-23T23:06:09.591965+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:06:09.591965+0000","last_undegraded":"2023-09-23T23:06:09.591965+0000","last_fullsized":"2023-09-23T23:06:09.591965+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":467,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.600416+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:06:09.600416+0000","last_peered":"2023-09-23T23:06:09.600416+0000","last_clean":"2023-09-23T23:06:09.600416+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:06:09.600416+0000","last_undegraded":"2023-09-23T23:06:09.600416+0000","last_fullsized":"2023-09-23T23:06:09.600416+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"443'5566","reported_seq":6053,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.661228+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:06:16.661228+0000","last_peered":"2023-09-23T23:06:16.661228+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:06:16.661228+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5566,"log_dups_size":0,"ondisk_log_size":5566,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150634496,"num_objects":2299,"num_object_clones":0,"num_object_copies":6897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2876,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":188,"num_bytes_recovered":12124160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1955},{"shards":"3,5,6","objects":317},{"shards":"3,6","objects":27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":56,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:11.951162+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:06:11.951162+0000","last_peered":"2023-09-23T23:06:11.951162+0000","last_clean":"2023-09-23T23:06:11.951162+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:06:11.951162+0000","last_undegraded":"2023-09-23T23:06:11.951162+0000","last_fullsized":"2023-09-23T23:06:11.951162+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":138,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.606497+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:06:09.606497+0000","last_peered":"2023-09-23T23:06:09.606497+0000","last_clean":"2023-09-23T23:06:09.606497+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:06:09.606497+0000","last_undegraded":"2023-09-23T23:06:09.606497+0000","last_fullsized":"2023-09-23T23:06:09.606497+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"443'5463","reported_seq":5531,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:14.167335+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:06:14.167335+0000","last_peered":"2023-09-23T23:06:14.167335+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:06:14.167335+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5463,"log_dups_size":0,"ondisk_log_size":5463,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149651456,"num_objects":2283,"num_object_clones":0,"num_object_copies":6849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2812,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-82},{"shards":"1,4,5","objects":2325}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":353,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593839+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:06:09.593839+0000","last_peered":"2023-09-23T23:06:09.593839+0000","last_clean":"2023-09-23T23:06:09.593839+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:06:09.593839+0000","last_undegraded":"2023-09-23T23:06:09.593839+0000","last_fullsized":"2023-09-23T23:06:09.593839+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":309,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591408+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:06:09.591408+0000","last_peered":"2023-09-23T23:06:09.591408+0000","last_clean":"2023-09-23T23:06:09.591408+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:06:09.591408+0000","last_undegraded":"2023-09-23T23:06:09.591408+0000","last_fullsized":"2023-09-23T23:06:09.591408+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"443'5451","reported_seq":10427,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:14.190293+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:06:14.190293+0000","last_peered":"2023-09-23T23:06:14.190293+0000","last_clean":"2023-09-23T23:06:14.190293+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:06:14.190293+0000","last_undegraded":"2023-09-23T23:06:14.190293+0000","last_fullsized":"2023-09-23T23:06:14.190293+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147030016,"num_objects":2243,"num_object_clones":0,"num_object_copies":6729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2819,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":71,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.596392+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:06:09.596392+0000","last_peered":"2023-09-23T23:06:09.596392+0000","last_clean":"2023-09-23T23:06:09.596392+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:06:09.596392+0000","last_undegraded":"2023-09-23T23:06:09.596392+0000","last_fullsized":"2023-09-23T23:06:09.596392+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":450,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591878+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:06:09.591878+0000","last_peered":"2023-09-23T23:06:09.591878+0000","last_clean":"2023-09-23T23:06:09.591878+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:06:09.591878+0000","last_undegraded":"2023-09-23T23:06:09.591878+0000","last_fullsized":"2023-09-23T23:06:09.591878+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"443'5471","reported_seq":5948,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:17.303368+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:06:17.303368+0000","last_peered":"2023-09-23T23:06:17.303368+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:06:17.303368+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5471,"log_dups_size":0,"ondisk_log_size":5471,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148340736,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":2091,"num_objects_missing":2091,"num_objects_degraded":4623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2822,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1996},{"shards":"0,3,5,6","objects":5},{"shards":"1,6","objects":-46},{"shards":"3,5,6","objects":308}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":100,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.280419+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:06:12.280419+0000","last_peered":"2023-09-23T23:06:12.280419+0000","last_clean":"2023-09-23T23:06:12.280419+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:06:12.280419+0000","last_undegraded":"2023-09-23T23:06:12.280419+0000","last_fullsized":"2023-09-23T23:06:12.280419+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":39,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:11.952594+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:06:11.952594+0000","last_peered":"2023-09-23T23:06:11.952594+0000","last_clean":"2023-09-23T23:06:11.952594+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:06:11.952594+0000","last_undegraded":"2023-09-23T23:06:11.952594+0000","last_fullsized":"2023-09-23T23:06:11.952594+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"443'5613","reported_seq":6020,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:14.365367+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:06:14.365367+0000","last_peered":"2023-09-23T23:06:14.365367+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:06:14.365367+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152403968,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2897,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":281},{"shards":"1,2,3","objects":2034}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.607288+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:06:09.607288+0000","last_peered":"2023-09-23T23:06:09.607288+0000","last_clean":"2023-09-23T23:06:09.607288+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:06:09.607288+0000","last_undegraded":"2023-09-23T23:06:09.607288+0000","last_fullsized":"2023-09-23T23:06:09.607288+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":265,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592659+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:06:09.592659+0000","last_peered":"2023-09-23T23:06:09.592659+0000","last_clean":"2023-09-23T23:06:09.592659+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:06:09.592659+0000","last_undegraded":"2023-09-23T23:06:09.592659+0000","last_fullsized":"2023-09-23T23:06:09.592659+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"443'5539","reported_seq":5966,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.919428+0000","last_change":"2023-09-23T23:06:13.792666+0000","last_active":"2023-09-23T23:06:16.919428+0000","last_peered":"2023-09-23T23:06:16.919428+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:06:13.017017+0000","last_became_peered":"2023-09-23T23:06:13.017017+0000","last_unstale":"2023-09-23T23:06:16.919428+0000","last_undegraded":"2023-09-23T23:06:10.970550+0000","last_fullsized":"2023-09-23T23:06:09.601294+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153878528,"num_objects":2348,"num_object_clones":0,"num_object_copies":7044,"num_objects_missing_on_primary":2036,"num_objects_missing":2036,"num_objects_degraded":2157,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2856,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[6,1],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":1963},{"shards":"1,6","objects":385}],"blocked_by":[],"up_primary":2,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":146,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.604544+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:06:09.604544+0000","last_peered":"2023-09-23T23:06:09.604544+0000","last_clean":"2023-09-23T23:06:09.604544+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:06:09.604544+0000","last_undegraded":"2023-09-23T23:06:09.604544+0000","last_fullsized":"2023-09-23T23:06:09.604544+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":445,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593410+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:06:09.593410+0000","last_peered":"2023-09-23T23:06:09.593410+0000","last_clean":"2023-09-23T23:06:09.593410+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:06:09.593410+0000","last_undegraded":"2023-09-23T23:06:09.593410+0000","last_fullsized":"2023-09-23T23:06:09.593410+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"443'5511","reported_seq":5575,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.479127+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:06:16.479127+0000","last_peered":"2023-09-23T23:06:16.479127+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:06:16.479127+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5511,"log_dups_size":0,"ondisk_log_size":5511,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150241280,"num_objects":2292,"num_object_clones":0,"num_object_copies":6876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2847,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":2261},{"shards":"2,3","objects":-103}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":216,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592403+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:06:09.592403+0000","last_peered":"2023-09-23T23:06:09.592403+0000","last_clean":"2023-09-23T23:06:09.592403+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:06:09.592403+0000","last_undegraded":"2023-09-23T23:06:09.592403+0000","last_fullsized":"2023-09-23T23:06:09.592403+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":453,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591950+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:06:09.591950+0000","last_peered":"2023-09-23T23:06:09.591950+0000","last_clean":"2023-09-23T23:06:09.591950+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:06:09.591950+0000","last_undegraded":"2023-09-23T23:06:09.591950+0000","last_fullsized":"2023-09-23T23:06:09.591950+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"443'5476","reported_seq":5885,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:17.276619+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:06:17.276619+0000","last_peered":"2023-09-23T23:06:17.276619+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:06:17.276619+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151224320,"num_objects":2307,"num_object_clones":0,"num_object_copies":6921,"num_objects_missing_on_primary":2149,"num_objects_missing":2149,"num_objects_degraded":2580,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2822,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":10944512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,2,3","objects":2062},{"shards":"0,3,6","objects":300},{"shards":"2,6","objects":-55}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":466,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591607+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:06:09.591607+0000","last_peered":"2023-09-23T23:06:09.591607+0000","last_clean":"2023-09-23T23:06:09.591607+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:06:09.591607+0000","last_undegraded":"2023-09-23T23:06:09.591607+0000","last_fullsized":"2023-09-23T23:06:09.591607+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":453,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593714+0000","last_change":"2023-09-23T23:05:03.450633+0000","last_active":"2023-09-23T23:06:09.593714+0000","last_peered":"2023-09-23T23:06:09.593714+0000","last_clean":"2023-09-23T23:06:09.593714+0000","last_became_active":"2023-09-23T23:05:03.450232+0000","last_became_peered":"2023-09-23T23:05:03.450232+0000","last_unstale":"2023-09-23T23:06:09.593714+0000","last_undegraded":"2023-09-23T23:06:09.593714+0000","last_fullsized":"2023-09-23T23:06:09.593714+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"443'5499","reported_seq":5936,"reported_epoch":443,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:16.788487+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:06:16.788487+0000","last_peered":"2023-09-23T23:06:16.788487+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:06:16.788487+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:06:16.788487+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2843,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":187,"num_bytes_recovered":12255232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1900},{"shards":"1,3,6","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":115,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.598154+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:06:09.598154+0000","last_peered":"2023-09-23T23:06:09.598154+0000","last_clean":"2023-09-23T23:06:09.598154+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:06:09.598154+0000","last_undegraded":"2023-09-23T23:06:09.598154+0000","last_fullsized":"2023-09-23T23:06:09.598154+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":348,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.596535+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:06:09.596535+0000","last_peered":"2023-09-23T23:06:09.596535+0000","last_clean":"2023-09-23T23:06:09.596535+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:06:09.596535+0000","last_undegraded":"2023-09-23T23:06:09.596535+0000","last_fullsized":"2023-09-23T23:06:09.596535+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"443'5799","reported_seq":7235,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.591614+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:06:16.591614+0000","last_peered":"2023-09-23T23:06:16.591614+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:06:16.591614+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135640,"num_objects":4657,"num_object_clones":0,"num_object_copies":13971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4657,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5799,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":3659},{"shards":"2,4","objects":791}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":361,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595163+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:06:09.595163+0000","last_peered":"2023-09-23T23:06:09.595163+0000","last_clean":"2023-09-23T23:06:09.595163+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:06:09.595163+0000","last_undegraded":"2023-09-23T23:06:09.595163+0000","last_fullsized":"2023-09-23T23:06:09.595163+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":280,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593502+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:06:09.593502+0000","last_peered":"2023-09-23T23:06:09.593502+0000","last_clean":"2023-09-23T23:06:09.593502+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:06:09.593502+0000","last_undegraded":"2023-09-23T23:06:09.593502+0000","last_fullsized":"2023-09-23T23:06:09.593502+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592118+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:06:09.592118+0000","last_peered":"2023-09-23T23:06:09.592118+0000","last_clean":"2023-09-23T23:06:09.592118+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:06:09.592118+0000","last_undegraded":"2023-09-23T23:06:09.592118+0000","last_fullsized":"2023-09-23T23:06:09.592118+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"443'5722","reported_seq":5808,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.754418+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:06:16.754418+0000","last_peered":"2023-09-23T23:06:16.754418+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:06:16.754418+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"415'1167","ondisk_log_start":"415'1167","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298450944,"num_objects":4554,"num_object_clones":0,"num_object_copies":13662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5722,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":4554},{"shards":"1,3","objects":-271}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":39,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:11.952628+0000","last_change":"2023-09-23T23:06:08.607076+0000","last_active":"2023-09-23T23:06:11.952628+0000","last_peered":"2023-09-23T23:06:11.952628+0000","last_clean":"2023-09-23T23:06:11.952628+0000","last_became_active":"2023-09-23T23:06:08.606649+0000","last_became_peered":"2023-09-23T23:06:08.606649+0000","last_unstale":"2023-09-23T23:06:11.952628+0000","last_undegraded":"2023-09-23T23:06:11.952628+0000","last_fullsized":"2023-09-23T23:06:11.952628+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:47.677948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":481,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592982+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:06:09.592982+0000","last_peered":"2023-09-23T23:06:09.592982+0000","last_clean":"2023-09-23T23:06:09.592982+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:06:09.592982+0000","last_undegraded":"2023-09-23T23:06:09.592982+0000","last_fullsized":"2023-09-23T23:06:09.592982+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":346,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595092+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:06:09.595092+0000","last_peered":"2023-09-23T23:06:09.595092+0000","last_clean":"2023-09-23T23:06:09.595092+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:06:09.595092+0000","last_undegraded":"2023-09-23T23:06:09.595092+0000","last_fullsized":"2023-09-23T23:06:09.595092+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"443'5582","reported_seq":6096,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.597467+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:06:16.597467+0000","last_peered":"2023-09-23T23:06:16.597467+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:06:16.597467+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5582,"log_dups_size":0,"ondisk_log_size":5582,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149585920,"num_objects":2282,"num_object_clones":0,"num_object_copies":6846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4674,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2891,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":202,"num_bytes_recovered":13172736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1994},{"shards":"3,5,6","objects":315},{"shards":"3,6","objects":-27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":421,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.588660+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:06:09.588660+0000","last_peered":"2023-09-23T23:06:09.588660+0000","last_clean":"2023-09-23T23:06:09.588660+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:06:09.588660+0000","last_undegraded":"2023-09-23T23:06:09.588660+0000","last_fullsized":"2023-09-23T23:06:09.588660+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":274,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.588728+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:06:09.588728+0000","last_peered":"2023-09-23T23:06:09.588728+0000","last_clean":"2023-09-23T23:06:09.588728+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:06:09.588728+0000","last_undegraded":"2023-09-23T23:06:09.588728+0000","last_fullsized":"2023-09-23T23:06:09.588728+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":361,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.606326+0000","last_change":"2023-09-23T23:05:53.573682+0000","last_active":"2023-09-23T23:06:09.606326+0000","last_peered":"2023-09-23T23:06:09.606326+0000","last_clean":"2023-09-23T23:06:09.606326+0000","last_became_active":"2023-09-23T23:05:53.573311+0000","last_became_peered":"2023-09-23T23:05:53.573311+0000","last_unstale":"2023-09-23T23:06:09.606326+0000","last_undegraded":"2023-09-23T23:06:09.606326+0000","last_fullsized":"2023-09-23T23:06:09.606326+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:26.957590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"443'5684","reported_seq":7590,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.771448+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:06:16.771448+0000","last_peered":"2023-09-23T23:06:16.771448+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:06:16.771448+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"415'1134","ondisk_log_start":"415'1134","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298844160,"num_objects":4560,"num_object_clones":0,"num_object_copies":13680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4658,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5684,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":922,"num_bytes_recovered":60424192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":3580},{"shards":"3,5","objects":1},{"shards":"3,5,6","objects":619},{"shards":"3,6","objects":360}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":353,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.589215+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:06:09.589215+0000","last_peered":"2023-09-23T23:06:09.589215+0000","last_clean":"2023-09-23T23:06:09.589215+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:06:09.589215+0000","last_undegraded":"2023-09-23T23:06:09.589215+0000","last_fullsized":"2023-09-23T23:06:09.589215+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":554,"reported_epoch":442,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:11.625481+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:06:11.625481+0000","last_peered":"2023-09-23T23:06:11.625481+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:06:11.625481+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:06:11.625481+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":133,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591389+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:06:09.591389+0000","last_peered":"2023-09-23T23:06:09.591389+0000","last_clean":"2023-09-23T23:06:09.591389+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:06:09.591389+0000","last_undegraded":"2023-09-23T23:06:09.591389+0000","last_fullsized":"2023-09-23T23:06:09.591389+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":343,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.173520+0000","last_change":"2023-09-23T23:06:12.173156+0000","last_active":"2023-09-23T23:06:12.173520+0000","last_peered":"2023-09-23T23:06:12.173520+0000","last_clean":"2023-09-23T23:06:12.173520+0000","last_became_active":"2023-09-23T23:06:12.172747+0000","last_became_peered":"2023-09-23T23:06:12.172747+0000","last_unstale":"2023-09-23T23:06:12.173520+0000","last_undegraded":"2023-09-23T23:06:12.173520+0000","last_fullsized":"2023-09-23T23:06:12.173520+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"443'5511","reported_seq":5705,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.730353+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:06:16.730353+0000","last_peered":"2023-09-23T23:06:16.730353+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:06:16.730353+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5511,"log_dups_size":0,"ondisk_log_size":5511,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146309120,"num_objects":2232,"num_object_clones":0,"num_object_copies":6696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2861,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4030464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":2027},{"shards":"0,3,6","objects":321},{"shards":"1,3","objects":-266}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":366,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591389+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:06:09.591389+0000","last_peered":"2023-09-23T23:06:09.591389+0000","last_clean":"2023-09-23T23:06:09.591389+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:06:09.591389+0000","last_undegraded":"2023-09-23T23:06:09.591389+0000","last_fullsized":"2023-09-23T23:06:09.591389+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":337,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591524+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:06:09.591524+0000","last_peered":"2023-09-23T23:06:09.591524+0000","last_clean":"2023-09-23T23:06:09.591524+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:06:09.591524+0000","last_undegraded":"2023-09-23T23:06:09.591524+0000","last_fullsized":"2023-09-23T23:06:09.591524+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":302,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.596282+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:06:09.596282+0000","last_peered":"2023-09-23T23:06:09.596282+0000","last_clean":"2023-09-23T23:06:09.596282+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:06:09.596282+0000","last_undegraded":"2023-09-23T23:06:09.596282+0000","last_fullsized":"2023-09-23T23:06:09.596282+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"443'5657","reported_seq":5700,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:16.549532+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:06:16.549532+0000","last_peered":"2023-09-23T23:06:16.549532+0000","last_clean":"2023-09-23T23:06:16.549532+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:06:16.549532+0000","last_undegraded":"2023-09-23T23:06:16.549532+0000","last_fullsized":"2023-09-23T23:06:16.549532+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5657,"log_dups_size":0,"ondisk_log_size":5657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299827200,"num_objects":4575,"num_object_clones":0,"num_object_copies":13725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5657,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":302,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591882+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:06:09.591882+0000","last_peered":"2023-09-23T23:06:09.591882+0000","last_clean":"2023-09-23T23:06:09.591882+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:06:09.591882+0000","last_undegraded":"2023-09-23T23:06:09.591882+0000","last_fullsized":"2023-09-23T23:06:09.591882+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":298,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595528+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:06:09.595528+0000","last_peered":"2023-09-23T23:06:09.595528+0000","last_clean":"2023-09-23T23:06:09.595528+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:06:09.595528+0000","last_undegraded":"2023-09-23T23:06:09.595528+0000","last_fullsized":"2023-09-23T23:06:09.595528+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":264,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.588871+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:06:09.588871+0000","last_peered":"2023-09-23T23:06:09.588871+0000","last_clean":"2023-09-23T23:06:09.588871+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:06:09.588871+0000","last_undegraded":"2023-09-23T23:06:09.588871+0000","last_fullsized":"2023-09-23T23:06:09.588871+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"443'5608","reported_seq":5659,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.502616+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:06:16.502616+0000","last_peered":"2023-09-23T23:06:16.502616+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:06:16.502616+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296353792,"num_objects":4522,"num_object_clones":0,"num_object_copies":13566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5608,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":4522},{"shards":"1,2","objects":-247}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":165,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.606653+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:06:09.606653+0000","last_peered":"2023-09-23T23:06:09.606653+0000","last_clean":"2023-09-23T23:06:09.606653+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:06:09.606653+0000","last_undegraded":"2023-09-23T23:06:09.606653+0000","last_fullsized":"2023-09-23T23:06:09.606653+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":141,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593634+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:06:09.593634+0000","last_peered":"2023-09-23T23:06:09.593634+0000","last_clean":"2023-09-23T23:06:09.593634+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:06:09.593634+0000","last_undegraded":"2023-09-23T23:06:09.593634+0000","last_fullsized":"2023-09-23T23:06:09.593634+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":148,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595520+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:06:09.595520+0000","last_peered":"2023-09-23T23:06:09.595520+0000","last_clean":"2023-09-23T23:06:09.595520+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:06:09.595520+0000","last_undegraded":"2023-09-23T23:06:09.595520+0000","last_fullsized":"2023-09-23T23:06:09.595520+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"443'5716","reported_seq":6772,"reported_epoch":443,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:15.377278+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:06:15.377278+0000","last_peered":"2023-09-23T23:06:15.377278+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:06:15.377278+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302514176,"num_objects":4616,"num_object_clones":0,"num_object_copies":13848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":4194,"num_objects_unfound":0,"num_objects_dirty":4616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5716,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":4194},{"shards":"1,4","objects":238}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":390,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.596415+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:06:09.596415+0000","last_peered":"2023-09-23T23:06:09.596415+0000","last_clean":"2023-09-23T23:06:09.596415+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:06:09.596415+0000","last_undegraded":"2023-09-23T23:06:09.596415+0000","last_fullsized":"2023-09-23T23:06:09.596415+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":64,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593803+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:06:09.593803+0000","last_peered":"2023-09-23T23:06:09.593803+0000","last_clean":"2023-09-23T23:06:09.593803+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:06:09.593803+0000","last_undegraded":"2023-09-23T23:06:09.593803+0000","last_fullsized":"2023-09-23T23:06:09.593803+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":435,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591741+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:06:09.591741+0000","last_peered":"2023-09-23T23:06:09.591741+0000","last_clean":"2023-09-23T23:06:09.591741+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:06:09.591741+0000","last_undegraded":"2023-09-23T23:06:09.591741+0000","last_fullsized":"2023-09-23T23:06:09.591741+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"443'5562","reported_seq":6034,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:17.216206+0000","last_change":"2023-09-23T23:06:13.782895+0000","last_active":"2023-09-23T23:06:17.216206+0000","last_peered":"2023-09-23T23:06:17.216206+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:06:13.017993+0000","last_became_peered":"2023-09-23T23:06:13.017993+0000","last_unstale":"2023-09-23T23:06:17.216206+0000","last_undegraded":"2023-09-23T23:06:12.255966+0000","last_fullsized":"2023-09-23T23:06:10.977105+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152371200,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":2111,"num_objects_missing":2111,"num_objects_degraded":2252,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2879,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":196,"num_bytes_recovered":12812288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[6,1],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":2022},{"shards":"1,6","objects":303}],"blocked_by":[],"up_primary":2,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":16,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592508+0000","last_change":"2023-09-23T23:05:53.573701+0000","last_active":"2023-09-23T23:06:09.592508+0000","last_peered":"2023-09-23T23:06:09.592508+0000","last_clean":"2023-09-23T23:06:09.592508+0000","last_became_active":"2023-09-23T23:05:53.573268+0000","last_became_peered":"2023-09-23T23:05:53.573268+0000","last_unstale":"2023-09-23T23:06:09.592508+0000","last_undegraded":"2023-09-23T23:06:09.592508+0000","last_fullsized":"2023-09-23T23:06:09.592508+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":193,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595602+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:06:09.595602+0000","last_peered":"2023-09-23T23:06:09.595602+0000","last_clean":"2023-09-23T23:06:09.595602+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:06:09.595602+0000","last_undegraded":"2023-09-23T23:06:09.595602+0000","last_fullsized":"2023-09-23T23:06:09.595602+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"443'5599","reported_seq":5991,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:14.450525+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:06:14.450525+0000","last_peered":"2023-09-23T23:06:14.450525+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:06:14.450525+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5599,"log_dups_size":0,"ondisk_log_size":5599,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153321472,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2884,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":365},{"shards":"1,2,3","objects":1970}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":265,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593320+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:06:09.593320+0000","last_peered":"2023-09-23T23:06:09.593320+0000","last_clean":"2023-09-23T23:06:09.593320+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:06:09.593320+0000","last_undegraded":"2023-09-23T23:06:09.593320+0000","last_fullsized":"2023-09-23T23:06:09.593320+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":466,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.593058+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:06:09.593058+0000","last_peered":"2023-09-23T23:06:09.593058+0000","last_clean":"2023-09-23T23:06:09.593058+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:06:09.593058+0000","last_undegraded":"2023-09-23T23:06:09.593058+0000","last_fullsized":"2023-09-23T23:06:09.593058+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"443'5469","reported_seq":5949,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:17.315817+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:06:17.315817+0000","last_peered":"2023-09-23T23:06:17.315817+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:06:17.315817+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148471808,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":2032,"num_objects_missing":2032,"num_objects_degraded":4479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11206656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1916},{"shards":"0,3,5,6","objects":4},{"shards":"3,5,6","objects":298},{"shards":"3,6","objects":48}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":292,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592067+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:06:09.592067+0000","last_peered":"2023-09-23T23:06:09.592067+0000","last_clean":"2023-09-23T23:06:09.592067+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:06:09.592067+0000","last_undegraded":"2023-09-23T23:06:09.592067+0000","last_fullsized":"2023-09-23T23:06:09.592067+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":362,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.592325+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:06:09.592325+0000","last_peered":"2023-09-23T23:06:09.592325+0000","last_clean":"2023-09-23T23:06:09.592325+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:06:09.592325+0000","last_undegraded":"2023-09-23T23:06:09.592325+0000","last_fullsized":"2023-09-23T23:06:09.592325+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"443'5456","reported_seq":6437,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:16.517968+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:06:16.517968+0000","last_peered":"2023-09-23T23:06:16.517968+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:06:16.517968+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146702336,"num_objects":2239,"num_object_clones":0,"num_object_copies":6717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2825,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":1866},{"shards":"1,2","objects":252}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":17,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.600138+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:06:09.600138+0000","last_peered":"2023-09-23T23:06:09.600138+0000","last_clean":"2023-09-23T23:06:09.600138+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:06:09.600138+0000","last_undegraded":"2023-09-23T23:06:09.600138+0000","last_fullsized":"2023-09-23T23:06:09.600138+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":159,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.595618+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:06:09.595618+0000","last_peered":"2023-09-23T23:06:09.595618+0000","last_clean":"2023-09-23T23:06:09.595618+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:06:09.595618+0000","last_undegraded":"2023-09-23T23:06:09.595618+0000","last_fullsized":"2023-09-23T23:06:09.595618+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"443'5485","reported_seq":5543,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:13.885003+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:06:13.885003+0000","last_peered":"2023-09-23T23:06:13.885003+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:06:13.885003+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5485,"log_dups_size":0,"ondisk_log_size":5485,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148209664,"num_objects":2262,"num_object_clones":0,"num_object_copies":6786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2835,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":6},{"shards":"1,4,5","objects":2220}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":210,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.596624+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:06:09.596624+0000","last_peered":"2023-09-23T23:06:09.596624+0000","last_clean":"2023-09-23T23:06:09.596624+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:06:09.596624+0000","last_undegraded":"2023-09-23T23:06:09.596624+0000","last_fullsized":"2023-09-23T23:06:09.596624+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":437,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T23:06:09.591490+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:06:09.591490+0000","last_peered":"2023-09-23T23:06:09.591490+0000","last_clean":"2023-09-23T23:06:09.591490+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:06:09.591490+0000","last_undegraded":"2023-09-23T23:06:09.591490+0000","last_fullsized":"2023-09-23T23:06:09.591490+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":4795596824,"num_objects":73176,"num_object_clones":0,"num_object_copies":219528,"num_objects_missing_on_primary":10419,"num_objects_missing":10419,"num_objects_degraded":68437,"num_objects_misplaced":5607,"num_objects_unfound":0,"num_objects_dirty":73176,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":91128,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9866,"num_bytes_recovered":629800960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15245520896,"data_stored":15245508680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142111,"ondisk_log_size":142111,"up":78,"acting":57,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430679,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626904,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1482604,"kb_used_data":588068,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92262548,"statfs":{"total":95995035648,"available":94476849152,"internally_reserved":0,"allocated":602181632,"data_stored":601236588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,3],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:05:15 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":6.1130000000000004,"15min":5.9800000000000004},"min":{"1min":0.38400000000000001,"5min":0.249,"15min":0.249},"max":{"1min":9.75,"5min":115.32899999999999,"15min":115.32899999999999},"last":1.401},{"interface":"front","average":{"1min":1.3979999999999999,"5min":5.9420000000000002,"15min":5.9960000000000004},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":5.673,"5min":115.348,"15min":115.348},"last":0.94299999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":3.8370000000000002,"15min":3.9540000000000002},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.23100000000000001},"max":{"1min":5.3819999999999997,"5min":149.947,"15min":254.62100000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.992,"5min":4.0800000000000001,"15min":4.3520000000000003},"min":{"1min":0.35899999999999999,"5min":0.28999999999999998,"15min":0.24199999999999999},"max":{"1min":10.922000000000001,"5min":150.22499999999999,"15min":255.20099999999999},"last":1.4379999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:05:40 2023","interfaces":[{"interface":"back","average":{"1min":16.077000000000002,"5min":5.2679999999999998,"15min":2.4369999999999998},"min":{"1min":0.376,"5min":0.307,"15min":0.307},"max":{"1min":138.43299999999999,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":15.375,"5min":5.4400000000000004,"15min":2.5880000000000001},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":138.45599999999999,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.88400000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":7.1200000000000001,"5min":5.2779999999999996,"15min":3.2010000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":70.521000000000001,"5min":142.24700000000001,"15min":142.24700000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":7.9740000000000002,"5min":5.3710000000000004,"15min":3.2370000000000001},"min":{"1min":0.45600000000000002,"5min":0.34899999999999998,"15min":0.309},"max":{"1min":70.497,"5min":143.86699999999999,"15min":143.86699999999999},"last":1.4590000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":2.1920000000000002,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.26200000000000001,"15min":0.23899999999999999},"max":{"1min":12.407999999999999,"5min":119.804,"15min":119.804},"last":1.4770000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.0030000000000001,"15min":1.861},"min":{"1min":0.39500000000000002,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":3.6699999999999999,"5min":122.31999999999999,"15min":122.31999999999999},"last":1.1000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":4.7850000000000001,"15min":2.5049999999999999},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":10.946,"5min":186.27600000000001,"15min":186.27600000000001},"last":1.49},{"interface":"front","average":{"1min":0.69699999999999995,"5min":4.6920000000000002,"15min":2.5289999999999999},"min":{"1min":0.39500000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":1.768,"5min":191.28700000000001,"15min":191.28700000000001},"last":0.92600000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:05:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.917,"15min":1.458},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":2.4660000000000002,"5min":115.61,"15min":115.61},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":2.246,"15min":1.6930000000000001},"min":{"1min":0.39800000000000002,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":0.749,"5min":115.578,"15min":115.578},"last":0.90100000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253217,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4377752,"kb_used_data":3478672,"kb_used_omap":9,"kb_used_meta":899062,"kb_avail":89367400,"statfs":{"total":95995035648,"available":91512217600,"internally_reserved":0,"allocated":3562160128,"data_stored":3560432397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":920640489},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:05:15 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":6.5289999999999999,"15min":2.415},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":59.475000000000001,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":4.6449999999999996,"5min":6.8819999999999997,"15min":2.835},"min":{"1min":0.25600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":59.393999999999998,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:05:31 2023","interfaces":[{"interface":"back","average":{"1min":4.1920000000000002,"5min":4.6550000000000002,"15min":1.895},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":62.436,"5min":161.61699999999999,"15min":161.61699999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":6.2800000000000002,"5min":5.0220000000000002,"15min":2.1440000000000001},"min":{"1min":0.32200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":62.756999999999998,"5min":162.61699999999999,"15min":162.61699999999999},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0529999999999999,"5min":2.4700000000000002,"15min":1.665},"min":{"1min":0.309,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":8.5559999999999992,"5min":59.424999999999997,"15min":59.424999999999997},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.1259999999999999,"5min":2.3279999999999998,"15min":1.681},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":12.569000000000001,"5min":55.174999999999997,"15min":55.174999999999997},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.7669999999999999,"15min":2.1480000000000001},"min":{"1min":0.36199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3609999999999998,"5min":67.738,"15min":151.036},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.548,"5min":2.218,"15min":2.4630000000000001},"min":{"1min":0.39400000000000002,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":5.5419999999999998,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":2.1320000000000001,"15min":2.3399999999999999},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":6.04,"5min":68.259,"15min":116.098},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.5489999999999999,"5min":2.004,"15min":2.2189999999999999},"min":{"1min":0.41499999999999998,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":5.6970000000000001,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":2.2959999999999998,"15min":2.0619999999999998},"min":{"1min":0.35699999999999998,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":5.4989999999999997,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.1299999999999999,"15min":1.8640000000000001},"min":{"1min":0.41499999999999998,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":5.8099999999999996,"5min":67.366,"15min":67.366},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.93799999999999994,"15min":1.276},"min":{"1min":0.48499999999999999,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":4.3849999999999998,"5min":4.3849999999999998,"15min":9.9329999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.1309999999999998,"5min":1.4219999999999999,"15min":1.677},"min":{"1min":0.438,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":5.8109999999999999,"5min":5.8109999999999999,"15min":20.902999999999999},"last":0.54800000000000004}]}]},{"osd":0,"up_from":0,"seq":1516123455587,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259360,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2611240,"kb_used_data":2357664,"kb_used_omap":83,"kb_used_meta":253484,"kb_avail":91133912,"statfs":{"total":95995035648,"available":93321125888,"internally_reserved":0,"allocated":2414247936,"data_stored":2412391137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":259568069},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":2.657,"15min":1.147},"min":{"1min":0.221,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.199999999999999,"5min":74.241,"15min":74.241},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.9810000000000001,"5min":3.0470000000000002,"15min":1.4730000000000001},"min":{"1min":0.29299999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.6790000000000003,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":1.6679999999999999,"15min":0.98299999999999998},"min":{"1min":0.246,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":18.396999999999998,"5min":18.396999999999998,"15min":18.396999999999998},"last":0.309},{"interface":"front","average":{"1min":3.081,"5min":1.601,"15min":1.2869999999999999},"min":{"1min":0.26400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.744999999999999,"5min":12.744999999999999,"15min":12.744999999999999},"last":0.25900000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":2.637,"15min":1.361},"min":{"1min":0.30599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":17.786999999999999,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":3.008,"5min":2.6150000000000002,"15min":1.431},"min":{"1min":0.32600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":14.863,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.29999999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.28,"15min":0.86199999999999999},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.4829999999999997,"5min":33.476999999999997,"15min":33.476999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.5409999999999999,"15min":1.0620000000000001},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.75,"5min":33.497999999999998,"15min":33.497999999999998},"last":0.36899999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":2.9329999999999998,"15min":1.6259999999999999},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.8019999999999996,"5min":139.405,"15min":139.405},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.5189999999999999,"5min":2.7959999999999998,"15min":1.4139999999999999},"min":{"1min":0.377,"5min":0.371,"15min":0.36299999999999999},"max":{"1min":10.109999999999999,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":1.3560000000000001,"15min":1.1240000000000001},"min":{"1min":0.379,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.9009999999999998,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.0640000000000001,"15min":0.81000000000000005},"min":{"1min":0.42399999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.233000000000001,"5min":10.233000000000001,"15min":10.233000000000001},"last":3.552}]},{"osd":7,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.165,"15min":1.0920000000000001},"min":{"1min":0.49399999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.164,"5min":10.164,"15min":10.164},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.1910000000000001,"15min":1.1719999999999999},"min":{"1min":0.435,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.7830000000000004,"5min":7.7830000000000004,"15min":7.7830000000000004},"last":0.47699999999999998}]}]},{"osd":3,"up_from":94,"seq":403726926164,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3301044,"kb_used_data":2662764,"kb_used_omap":7,"kb_used_meta":638264,"kb_avail":90444108,"statfs":{"total":95995035648,"available":92614766592,"internally_reserved":0,"allocated":2726670336,"data_stored":2724855872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":653582952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:05:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3900000000000001,"5min":3.1349999999999998,"15min":1.264},"min":{"1min":0.505,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.7119999999999997,"5min":122.289,"15min":122.289},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.9790000000000001,"5min":2.3519999999999999,"15min":1.056},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.17799999999999999},"max":{"1min":5.8529999999999998,"5min":122.041,"15min":122.041},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":4.2759999999999998,"5min":2.2429999999999999,"15min":2.2130000000000001},"min":{"1min":0.30299999999999999,"5min":0.22,"15min":0.187},"max":{"1min":23.969000000000001,"5min":23.969000000000001,"15min":123.866},"last":0.435},{"interface":"front","average":{"1min":2.9609999999999999,"5min":2.0880000000000001,"15min":2.1989999999999998},"min":{"1min":0.27600000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":16.986000000000001,"5min":16.986000000000001,"15min":122.265},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:05:37 2023","interfaces":[{"interface":"back","average":{"1min":4.1260000000000003,"5min":1.504,"15min":0.96699999999999997},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":18.588000000000001,"5min":18.588000000000001,"15min":18.588000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":4.1539999999999999,"5min":2.0819999999999999,"15min":1.1970000000000001},"min":{"1min":0.251,"5min":0.215,"15min":0.215},"max":{"1min":10.353999999999999,"5min":10.353999999999999,"15min":10.353999999999999},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.661,"15min":1.8959999999999999},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":9.1099999999999994,"5min":10.327,"15min":87.046999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.6549999999999998,"5min":1.6399999999999999,"15min":1.8779999999999999},"min":{"1min":0.37,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":16.931000000000001,"5min":16.931000000000001,"15min":87.066000000000003},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5369999999999999,"5min":1.6839999999999999,"15min":1.4019999999999999},"min":{"1min":0.35899999999999999,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":16.959,"5min":16.959,"15min":16.959},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.2769999999999999,"15min":1.488},"min":{"1min":0.34399999999999997,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":8.5820000000000007,"5min":8.5820000000000007,"15min":10.246},"last":0.502}]},{"osd":6,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7410000000000001,"5min":1.7090000000000001,"15min":1.704},"min":{"1min":0.435,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.2330000000000001,"15min":1.625},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":8.1999999999999993,"5min":8.1999999999999993,"15min":12.887},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.242,"15min":1.4079999999999999},"min":{"1min":0.36899999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":8.1620000000000008,"5min":8.1620000000000008,"15min":8.8160000000000007},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.556,"5min":1.7709999999999999,"15min":1.4950000000000001},"min":{"1min":0.41199999999999998,"5min":0.40300000000000002,"15min":0.33200000000000002},"max":{"1min":9.1880000000000006,"5min":10.278,"15min":10.278},"last":0.59199999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019361,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2261356,"kb_used_data":1349796,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":91483796,"statfs":{"total":95995035648,"available":93679407104,"internally_reserved":0,"allocated":1382191104,"data_stored":1381245538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:05:18 2023","interfaces":[{"interface":"back","average":{"1min":6.7160000000000002,"5min":5.2140000000000004,"15min":2.1299999999999999},"min":{"1min":0.25,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":99.132999999999996,"5min":156.375,"15min":156.375},"last":0.41199999999999998},{"interface":"front","average":{"1min":6.8470000000000004,"5min":5.5439999999999996,"15min":2.2639999999999998},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":101.062,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.53900000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":3.464,"15min":3.871},"min":{"1min":0.34799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.9279999999999999,"5min":95.591999999999999,"15min":132.304},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.284,"5min":3.1720000000000002,"15min":3.8759999999999999},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.5629999999999997,"5min":97.929000000000002,"15min":132.453},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:05:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9960000000000004,"5min":2.157,"15min":1.4319999999999999},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":56.531999999999996,"5min":56.531999999999996,"15min":56.531999999999996},"last":1.2490000000000001},{"interface":"front","average":{"1min":5.8680000000000003,"5min":2.1379999999999999,"15min":1.222},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":56.204000000000001,"5min":56.204000000000001,"15min":56.204000000000001},"last":2.2240000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:06:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":1.9770000000000001,"15min":2.4239999999999999},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.972,"5min":60.274000000000001,"15min":164.453},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.0070000000000001,"5min":2.234,"15min":2.5289999999999999},"min":{"1min":0.36599999999999999,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":10.888999999999999,"5min":60.677999999999997,"15min":167.05199999999999},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 23:05:36 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.84399999999999997,"15min":1.655},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.214},"max":{"1min":4.0039999999999996,"5min":8.3439999999999994,"15min":148.61600000000001},"last":6.7080000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.70899999999999996,"15min":1.738},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":2.0699999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0069999999999999,"15min":1.2310000000000001},"min":{"1min":0.32500000000000001,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":5.3209999999999997,"5min":8.9610000000000003,"15min":16.943000000000001},"last":1.917},{"interface":"front","average":{"1min":1.149,"5min":0.92100000000000004,"15min":1.1359999999999999},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":4.5220000000000002,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:05:27 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.0840000000000001,"15min":1.2549999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":6.726,"5min":7.1050000000000004,"15min":12.84},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.81100000000000005,"15min":1.0920000000000001},"min":{"1min":0.33800000000000002,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":3.6539999999999999,"5min":3.6539999999999999,"15min":10.587},"last":0.51600000000000001}]}]},{"osd":5,"up_from":0,"seq":154618823135,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3074031616,"data_stored":3074031616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3549233152,"data_stored":3549233152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2406813696,"data_stored":2406809624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2716860416,"data_stored":2716860416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1377898496,"data_stored":1377894424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":588648448,"data_stored":588644376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:06:22.183 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-09-23T23:06:22.222 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:22.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:22.327 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:22.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:22.474 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:22.779 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:22.790 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:06:22.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:22.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:22.893 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:22.945 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:22.956 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:23.080 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:23.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:23.216 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:23.283 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:23.363 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:23.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:23.452 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:23.473 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:23.527 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:23.615 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:23.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:23.721 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:06:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:06:23] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.33.4" 2023-09-23T23:06:23.761 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:23.850 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:06:23.884 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:23.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:23.988 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:24.020 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:23 smithi052 ceph-mon[180979]: pgmap v1378: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 68018/216789 objects degraded (31.375%); 5063/216789 objects misplaced (2.335%); 12 MiB/s, 184 objects/s recovering 2023-09-23T23:06:24.020 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1553510189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:24.020 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:24.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:23 smithi052 ceph-mon[187743]: pgmap v1378: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 68018/216789 objects degraded (31.375%); 5063/216789 objects misplaced (2.335%); 12 MiB/s, 184 objects/s recovering 2023-09-23T23:06:24.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1553510189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:24.022 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:24.024 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:24.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:23 smithi151 ceph-mon[159210]: pgmap v1378: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 68018/216789 objects degraded (31.375%); 5063/216789 objects misplaced (2.335%); 12 MiB/s, 184 objects/s recovering 2023-09-23T23:06:24.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1553510189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:24.153 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:24.348 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:24.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:24.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:24.453 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:24.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:24.886 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:24.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:24.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:24.990 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:25.026 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:25.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:24 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:25.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:06:25.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:24 smithi151 ceph-mon[159210]: osdmap e444: 8 total, 8 up, 5 in 2023-09-23T23:06:25.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:24 smithi151 ceph-mon[159210]: failed to encode map e444 with expected crc 2023-09-23T23:06:25.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:24 smithi151 ceph-mon[159210]: failed to encode map e444 with expected crc 2023-09-23T23:06:25.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:25.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:25.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:25.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:06:25.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[180979]: osdmap e444: 8 total, 8 up, 5 in 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[180979]: failed to encode map e444 with expected crc 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[180979]: failed to encode map e444 with expected crc 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[187743]: osdmap e444: 8 total, 8 up, 5 in 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[187743]: failed to encode map e444 with expected crc 2023-09-23T23:06:25.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:24 smithi052 ceph-mon[187743]: failed to encode map e444 with expected crc 2023-09-23T23:06:25.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:25.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:25.494 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:25.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:25.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:25.517 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:25.539 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:25.560 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:25.570 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:25.570 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:06:25.570 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114197 10.6 10:6cbf7a9f:::benchmark_data_smithi052_461816_object17396:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.638868+0000", 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 531.92749168600005, 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0609476529999999, 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:25.571 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114197 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638868+0000", 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638868+0000", 2023-09-23T23:06:25.572 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638867+0000", 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6000000000000003e-06 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638873+0000", 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2157999999999999e-05 2023-09-23T23:06:25.573 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638885+0000", 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6440000000000003e-06 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638892+0000", 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4397000000000001e-05 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.574 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638946+0000", 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4919999999999993e-06 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638955+0000", 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0465119249999999 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.575 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.685467+0000", 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024042859999999998 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.687871+0000", 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000155284 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.688026+0000", 2023-09-23T23:06:25.576 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.009768288 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.697795+0000", 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018617110000000001 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699656+0000", 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4583999999999998e-05 2023-09-23T23:06:25.577 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699711+0000", 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000104646 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699816+0000", 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0609476529999999 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.578 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114199 10.6 10:63a78c60:::benchmark_data_smithi052_461816_object17398:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.638909+0000", 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 531.92745026, 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0610776770000001, 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:25.579 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114199 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638909+0000", 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638909+0000", 2023-09-23T23:06:25.580 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638909+0000", 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.937e-06 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638912+0000", 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9689999999999999e-06 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.581 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638918+0000", 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.126e-06 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.638922+0000", 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045948000000000001 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.582 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.639381+0000", 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6140000000000002e-06 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.639386+0000", 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.05181372 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691200+0000", 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000126706 2023-09-23T23:06:25.583 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691326+0000", 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000119423 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691446+0000", 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0063587169999999998 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.584 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.697805+0000", 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020814200000000001 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699886+0000", 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3377999999999997e-05 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699929+0000", 2023-09-23T23:06:25.585 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7663e-05 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.699987+0000", 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0610776770000001 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.586 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114201 10.6 10:63900912:::benchmark_data_smithi052_461816_object17400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.646195+0000", 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 531.92016425199995, 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.053975222, 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114201 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.587 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646195+0000", 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646195+0000", 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.588 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646194+0000", 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8939999999999993e-06 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646202+0000", 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4620000000000002e-06 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646211+0000", 2023-09-23T23:06:25.589 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8049999999999997e-06 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646216+0000", 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1202999999999997e-05 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646267+0000", 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0254999999999999e-05 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.590 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.646277+0000", 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.048210514 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.694488+0000", 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000117799 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.591 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.694606+0000", 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012992200000000001 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 5", 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.694736+0000", 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031115409999999998 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.697847+0000", 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021845380000000002 2023-09-23T23:06:25.592 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.700032+0000", 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7054e-05 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.700079+0000", 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1739999999999999e-05 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.593 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.700171+0000", 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.053975222 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114204 10.d 10:b3b88576:::benchmark_data_smithi052_461816_object17403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e345)", 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.664220+0000", 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 531.90213919300004, 2023-09-23T23:06:25.594 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0153031850000001, 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114204 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664220+0000", 2023-09-23T23:06:25.595 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664220+0000", 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664219+0000", 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2110000000000001e-06 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.596 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664226+0000", 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4001e-05 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664240+0000", 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6459999999999999e-06 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.597 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664249+0000", 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3594000000000002e-05 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664302+0000", 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1223000000000001e-05 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.664313+0000", 2023-09-23T23:06:25.598 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0007254290000001 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.665039+0000", 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014699799999999999 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.665186+0000", 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018358900000000001 2023-09-23T23:06:25.599 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.665369+0000", 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033414080000000001 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.668711+0000", 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004222188 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.600 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.672933+0000", 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0063869929999999997 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.679320+0000", 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4951999999999997e-05 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.679385+0000", 2023-09-23T23:06:25.601 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013865500000000001 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.679524+0000", 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0153031850000001 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.602 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:17355 11.e 11:7857a061:::benchmark_data_smithi052_606809_object17354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 142.958539836, 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15692616000000001, 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 17355 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.603 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.604 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607819+0000", 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3510999999999995e-05 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607892+0000", 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0942e-05 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607903+0000", 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8921999999999999e-05 2023-09-23T23:06:25.605 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607922+0000", 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4350000000000001e-05 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607956+0000", 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6907000000000001e-05 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.606 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608033+0000", 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8525999999999999e-05 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608122+0000", 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00055712700000000003 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.607 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608679+0000", 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049612479999999997 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.613640+0000", 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15104679400000001 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764687+0000", 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5262999999999998e-05 2023-09-23T23:06:25.608 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764722+0000", 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3643999999999999e-05 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764746+0000", 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15692616000000001 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.609 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:17367 11.e 11:71b796f5:::benchmark_data_smithi052_606809_object17366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:02.615149+0000", 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 142.951210782, 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.152643889, 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.610 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 17367 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615149+0000", 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615149+0000", 2023-09-23T23:06:25.611 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615148+0000", 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1823999999999999e-05 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615180+0000", 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4660000000000005e-06 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.612 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615189+0000", 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8840000000000002e-06 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615194+0000", 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8225e-05 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.613 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615242+0000", 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4358999999999997e-05 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615317+0000", 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9893000000000001e-05 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:06:25.614 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615407+0000", 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00063447099999999995 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.616041+0000", 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00090008299999999998 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.616941+0000", 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15079795700000001 2023-09-23T23:06:25.615 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.767739+0000", 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1355e-05 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.767770+0000", 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2345e-05 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.616 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.767793+0000", 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.152643889 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:21315 11.e 11:7fea961b:::benchmark_data_smithi052_606809_object21314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.667901+0000", 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 139.89845860099999, 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13239036300000001, 2023-09-23T23:06:25.617 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 21315 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667901+0000", 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.618 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667901+0000", 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667899+0000", 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4780999999999999e-05 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.619 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667994+0000", 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9510000000000001e-06 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668004+0000", 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0350000000000001e-06 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.620 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668009+0000", 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3237e-05 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668052+0000", 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5195e-05 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668138+0000", 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100846 2023-09-23T23:06:25.621 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668238+0000", 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083182799999999995 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.669070+0000", 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13107804100000001 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.622 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800148+0000", 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5852000000000001e-05 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800194+0000", 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2657000000000001e-05 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.623 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800247+0000", 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4549000000000003e-05 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800291+0000", 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13239036300000001 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.624 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:21320 11.c e415/410 11:3aa1cab5:::benchmark_data_smithi052_606809_object21319:head v 415'1301, mlcod=415'1298)", 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 139.895228352, 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12977512799999999, 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.625 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1396000000000001e-05 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.626 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671172+0000", 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2270000000000001e-06 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671178+0000", 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1e-06 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671181+0000", 2023-09-23T23:06:25.627 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00070157599999999998 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671882+0000", 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1986e-05 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671894+0000", 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.128977016 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.628 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800871+0000", 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5169999999999997e-05 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800906+0000", 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12977512799999999 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.629 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:21321 11.c e415/410 11:30848661:::benchmark_data_smithi052_606809_object21320:head v 415'1302, mlcod=415'1298)", 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.671193+0000", 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 139.895166983, 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13194534299999999, 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.630 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671193+0000", 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671193+0000", 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671192+0000", 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2735000000000003e-05 2023-09-23T23:06:25.631 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671235+0000", 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3810000000000004e-06 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671241+0000", 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0230000000000001e-06 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.632 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671245+0000", 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010633789999999999 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672308+0000", 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3658e-05 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.633 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672322+0000", 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13077090799999999 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.803092+0000", 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5525000000000002e-05 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.803138+0000", 2023-09-23T23:06:25.634 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13194534299999999 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop_reply(client.46892.0:21315 11.e e415/410 ondisk, result = 0)", 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.672760+0000", 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 139.893599526, 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12739600200000001, 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T23:06:25.635 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672760+0000", 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672760+0000", 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.636 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672759+0000", 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1360000000000002e-06 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672765+0000", 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12722345900000001 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.799989+0000", 2023-09-23T23:06:25.637 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9403e-05 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800018+0000", 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010932 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800128+0000", 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1924e-05 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.638 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800140+0000", 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6569999999999999e-05 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800156+0000", 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12739600200000001 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.639 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:61498 11.e 11:74d9cd66:::benchmark_data_smithi052_606809_object61497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e419)", 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:48.235116+0000", 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 97.331243650999994, 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.141686063, 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61498 2023-09-23T23:06:25.640 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235116+0000", 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235116+0000", 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.641 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235115+0000", 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7705999999999999e-05 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235172+0000", 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2746e-05 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.642 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235185+0000", 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2739999999999998e-06 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235190+0000", 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9527000000000003e-05 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235230+0000", 2023-09-23T23:06:25.643 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9593999999999995e-05 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235330+0000", 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116106 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235446+0000", 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001788544 2023-09-23T23:06:25.644 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.237234+0000", 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00053666399999999996 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.237771+0000", 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13892012300000001 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.645 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.376691+0000", 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1877000000000004e-05 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.376753+0000", 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9186e-05 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.646 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.376802+0000", 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.141686063 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:74473 11.e 11:7aba8dc9:::benchmark_data_smithi052_606809_object74472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:08.997352+0000", 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 76.569007689000003, 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.127978069, 2023-09-23T23:06:25.647 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 74473 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997352+0000", 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.648 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997352+0000", 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997349+0000", 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1848999999999996e-05 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.649 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997421+0000", 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1778999999999999e-05 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997443+0000", 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1619999999999997e-06 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.650 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997452+0000", 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6990000000000002e-05 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997509+0000", 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000128899 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997638+0000", 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012929900000000001 2023-09-23T23:06:25.651 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997767+0000", 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003043263 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.000811+0000", 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001496559 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.652 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.002307+0000", 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12291558800000001 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.125223+0000", 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1513000000000005e-05 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.653 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.125284+0000", 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5707999999999997e-05 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.125330+0000", 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.127978069 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.654 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:86860 11.e 11:75ab6736:::benchmark_data_smithi052_606809_object4706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 46.683050535, 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3618323549999998, 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 86860 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.655 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.656 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883307+0000", 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3599999999999996e-06 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883316+0000", 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7278e-05 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883333+0000", 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2119999999999992e-06 2023-09-23T23:06:25.657 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883342+0000", 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011255879999999999 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.894598+0000", 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2694000000000001e-05 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.658 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.894611+0000", 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99447565000000004 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889087+0000", 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2336e-05 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889099+0000", 2023-09-23T23:06:25.659 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3388404170000001 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.227939+0000", 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017411099999999999 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.228113+0000", 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012214100000000001 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.660 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.228236+0000", 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.01031818 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238554+0000", 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9231000000000002e-05 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.661 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238623+0000", 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039840279999999997 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.242607+0000", 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5633999999999997e-05 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.242653+0000", 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024888699999999998 2023-09-23T23:06:25.662 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.245141+0000", 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3618323549999998 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:86874 11.e 11:77be3fa7:::benchmark_data_smithi052_606809_object4720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:06:25.663 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 46.543459970999997, 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.23339197, 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 86874 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.664 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999943 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.665 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022894+0000", 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5319e-05 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022909+0000", 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3163000000000002e-05 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022932+0000", 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0350000000000007e-06 2023-09-23T23:06:25.666 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022941+0000", 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3570000000000002e-05 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.023015+0000", 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4547e-05 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.667 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.023030+0000", 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86609229700000001 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889122+0000", 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8590000000000005e-06 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:25.668 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889130+0000", 2023-09-23T23:06:25.669 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3413657240000001 2023-09-23T23:06:25.669 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.669 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.669 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.670 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230495+0000", 2023-09-23T23:06:25.670 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000186393 2023-09-23T23:06:25.670 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.670 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230682+0000", 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196875 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230879+0000", 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076860059999999996 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.671 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238565+0000", 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8522000000000004e-05 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238653+0000", 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013606922 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.252260+0000", 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8609000000000006e-05 2023-09-23T23:06:25.672 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.252339+0000", 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039528990000000002 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.256292+0000", 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.23339197 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.673 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:91816 11.e 11:720de07a:::benchmark_data_smithi052_606809_object9662:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:20.536407+0000", 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 5.029952421, 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030330213000000002, 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.674 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 91816 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.536407+0000", 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.675 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.536407+0000", 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.536406+0000", 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4790000000000004e-06 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.536411+0000", 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4849999999999998e-06 2023-09-23T23:06:25.676 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.536421+0000", 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4530000000000004e-06 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.536425+0000", 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012797675999999999 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.677 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.549223+0000", 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000144766 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.549368+0000", 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000137782 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.678 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.549506+0000", 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032062839999999998 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.552712+0000", 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00056374599999999995 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.553276+0000", 2023-09-23T23:06:25.679 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011080311000000001 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.564356+0000", 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4030999999999999e-05 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.564400+0000", 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023374419999999999 2023-09-23T23:06:25.680 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:20.566737+0000", 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030330213000000002 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:92039 11.e 11:70dd89d0:::benchmark_data_smithi052_606809_object9885:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-09-23T23:06:25.681 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:21.534692+0000", 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.0316679400000002, 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029925634999999999, 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 92039 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.682 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.534692+0000", 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.534692+0000", 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.683 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.534689+0000", 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9330000000000007e-06 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.534698+0000", 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1382e-05 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.534709+0000", 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1389999999999998e-06 2023-09-23T23:06:25.684 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.534717+0000", 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010613248 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.545330+0000", 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015805300000000001 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.685 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.545488+0000", 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013112099999999999 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.545619+0000", 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012137847 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.686 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.557757+0000", 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00365403 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.561411+0000", 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3010000000000003e-05 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.561454+0000", 2023-09-23T23:06:25.687 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4121000000000005e-05 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.561518+0000", 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030992709999999998 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.564617+0000", 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029925634999999999 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.688 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:92040 11.e 11:7d386cdb:::benchmark_data_smithi052_606809_object9886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:21.538283+0000", 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.0280768389999997, 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.036402640999999999, 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.689 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 92040 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.538283+0000", 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.690 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.538283+0000", 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.538281+0000", 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0668e-05 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.538292+0000", 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8335999999999999e-05 2023-09-23T23:06:25.691 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.538310+0000", 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2169999999999994e-06 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.538318+0000", 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016293625999999999 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.692 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.554612+0000", 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015630400000000001 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.554768+0000", 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000181893 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.693 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.554950+0000", 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0097427360000000001 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.564693+0000", 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3127999999999997e-05 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.564746+0000", 2023-09-23T23:06:25.694 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062885770000000001 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.571034+0000", 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0997000000000001e-05 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.571075+0000", 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036101169999999999 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.695 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.574685+0000", 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.036402640999999999 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:92041 11.e 11:734452d9:::benchmark_data_smithi052_606809_object9887:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:21.540284+0000", 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.0260752889999996, 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.049368038000000003, 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.696 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 92041 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.540284+0000", 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.697 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.540284+0000", 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999938 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.540278+0000", 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8187999999999999e-05 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.698 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.540296+0000", 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1463000000000001e-05 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.540318+0000", 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0217999999999999e-05 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.540328+0000", 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024446720000000002 2023-09-23T23:06:25.699 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.564775+0000", 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000182535 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.564957+0000", 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020432600000000001 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.700 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.565162+0000", 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0096097769999999999 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.574771+0000", 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8582e-05 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.574860+0000", 2023-09-23T23:06:25.701 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010633949 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.585494+0000", 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3854000000000002e-05 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.702 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.585558+0000", 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040945030000000002 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.589652+0000", 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.049368038000000003 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.703 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:92046 11.e 11:73be7c2f:::benchmark_data_smithi052_606809_object9892:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:21.563147+0000", 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.0032121629999997, 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030653478000000001, 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:25.704 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 92046 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.563147+0000", 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.563147+0000", 2023-09-23T23:06:25.705 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.563145+0000", 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3889999999999994e-06 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.563154+0000", 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3393e-05 2023-09-23T23:06:25.706 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.563168+0000", 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5340000000000002e-06 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.563175+0000", 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052204950000000003 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.707 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.568396+0000", 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015881700000000001 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.568555+0000", 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012347399999999999 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.568678+0000", 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0061685999999999998 2023-09-23T23:06:25.708 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.574847+0000", 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4632000000000001e-05 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.574871+0000", 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014850722 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.709 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.589722+0000", 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7951e-05 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.589790+0000", 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040108790000000002 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:21.593801+0000", 2023-09-23T23:06:25.710 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030653478000000001 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:92423 11.b e443/433 11:dc0fd95e:::benchmark_data_smithi052_606809_object10269:head v 443'5804, mlcod=414'297)", 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:23.434648+0000", 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 2.1317113480000001, 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034527215, 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:25.711 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434648+0000", 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434648+0000", 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.712 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434648+0000", 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8100000000000003e-06 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434654+0000", 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7039999999999999e-06 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:25.713 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434659+0000", 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8360000000000002e-06 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434663+0000", 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9129999999999998e-05 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434722+0000", 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9703999999999999e-05 2023-09-23T23:06:25.714 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.434752+0000", 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034386422999999999 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.469139+0000", 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.6913999999999999e-05 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:25.715 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:23.469175+0000", 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.034527215 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:25.716 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:25.716 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:25.717 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:25.718 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:25.718 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:25.718 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:25.718 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:25.718 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.718 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.719 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:06:25.752 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:25 smithi151 ceph-mon[159210]: pgmap v1380: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 68018/216789 objects degraded (31.375%); 5063/216789 objects misplaced (2.335%); 12 MiB/s, 184 objects/s recovering 2023-09-23T23:06:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 68018/216789 objects degraded (31.375%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:25.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1596761610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:25 smithi052 ceph-mon[180979]: pgmap v1380: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 68018/216789 objects degraded (31.375%); 5063/216789 objects misplaced (2.335%); 12 MiB/s, 184 objects/s recovering 2023-09-23T23:06:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 68018/216789 objects degraded (31.375%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:25.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1596761610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:25 smithi052 ceph-mon[187743]: pgmap v1380: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 68018/216789 objects degraded (31.375%); 5063/216789 objects misplaced (2.335%); 12 MiB/s, 184 objects/s recovering 2023-09-23T23:06:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 68018/216789 objects degraded (31.375%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:25.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1596761610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:26.108 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:26.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:26.212 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:26.258 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:26.522 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:26.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:26.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:26.543 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:06:26.637 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:26.679 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:26.760 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:26.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:26.866 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:26.907 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:27.017 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:27.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:27.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:27.121 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:27.132 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:27.160 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:27.211 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:27.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:27.317 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:27.377 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:27.474 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:27.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:27.549 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T23:06:27.587 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:27.621 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:27.797 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:27.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:27.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:27.907 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:27.939 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:27 smithi151 ceph-mon[159210]: pgmap v1381: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 155 op/s; 68018/215334 objects degraded (31.587%); 4948/215334 objects misplaced (2.298%); 11 MiB/s, 176 objects/s recovering 2023-09-23T23:06:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3345623815' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:06:27.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:06:27.995 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:27.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:28.099 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:28.124 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:27 smithi052 ceph-mon[187743]: pgmap v1381: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 155 op/s; 68018/215334 objects degraded (31.587%); 4948/215334 objects misplaced (2.298%); 11 MiB/s, 176 objects/s recovering 2023-09-23T23:06:28.124 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3345623815' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:06:28.124 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:06:28.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:27 smithi052 ceph-mon[180979]: pgmap v1381: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 155 op/s; 68018/215334 objects degraded (31.587%); 4948/215334 objects misplaced (2.298%); 11 MiB/s, 176 objects/s recovering 2023-09-23T23:06:28.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3345623815' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:06:28.125 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T23:06:28.127 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:28.172 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:28.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:28.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:28.276 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:28.321 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:28.393 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:28.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:28.498 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:28.539 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:06:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T23:06:28.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:28.764 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:28.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:28.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:28.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:06:28.871 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:28.911 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:28.995 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:29.062 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:29.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:29.168 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:29.205 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:29.314 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:06:29.315 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:06:29.327 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:29.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:29.432 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:29.486 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:29.529 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:29.541 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:29.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:29.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:29.646 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:29.685 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:29.752 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:29.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:29.856 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:29.886 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:29.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:29 smithi151 ceph-mon[159210]: pgmap v1382: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 190 op/s; 67620/213876 objects degraded (31.616%); 4521/213876 objects misplaced (2.114%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:06:30.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:29 smithi052 ceph-mon[180979]: pgmap v1382: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 190 op/s; 67620/213876 objects degraded (31.616%); 4521/213876 objects misplaced (2.114%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:06:30.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:29 smithi052 ceph-mon[187743]: pgmap v1382: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 190 op/s; 67620/213876 objects degraded (31.616%); 4521/213876 objects misplaced (2.114%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:06:30.335 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:06:30.336 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:30.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:30.440 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:30.480 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:30.581 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:30.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:30.706 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:30.761 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:30.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 67620/213876 objects degraded (31.616%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:30.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 67620/213876 objects degraded (31.616%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:30.924 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:30.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:30.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:30.941 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:30.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 67620/213876 objects degraded (31.616%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:31.029 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:31.070 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:31.781 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:31.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:31.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:31.835 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:31 smithi151 ceph-mon[159210]: pgmap v1383: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 67620/213498 objects degraded (31.672%); 4408/213498 objects misplaced (2.065%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:06:31.891 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:31.933 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:31.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:31 smithi052 ceph-mon[180979]: pgmap v1383: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 67620/213498 objects degraded (31.672%); 4408/213498 objects misplaced (2.065%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:06:31.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:31 smithi052 ceph-mon[187743]: pgmap v1383: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 67620/213498 objects degraded (31.672%); 4408/213498 objects misplaced (2.065%); 10 MiB/s, 165 objects/s recovering 2023-09-23T23:06:32.325 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:32.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:32.365 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:32.386 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:32.405 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:32.425 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:32.435 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:32.445 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:32.463 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:32.483 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:32.483 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:32.506 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.509 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:06:32.574 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:32.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:32.678 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:32.712 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:32.768 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:32.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:32.831 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:32.832 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:06:32.832 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:06:32.832 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:06:32.832 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: osdmap e445: 8 total, 8 up, 5 in 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2026948261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:32.848 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2436547009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: osdmap e445: 8 total, 8 up, 5 in 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: failed to encode map e445 with expected crc 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2026948261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:32.849 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2436547009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:32.871 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:06:32.874 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:32.915 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:32.971 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:32.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:33.075 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:33.109 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: osdmap e445: 8 total, 8 up, 5 in 2023-09-23T23:06:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: failed to encode map e445 with expected crc 2023-09-23T23:06:33.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: failed to encode map e445 with expected crc 2023-09-23T23:06:33.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: failed to encode map e445 with expected crc 2023-09-23T23:06:33.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: failed to encode map e445 with expected crc 2023-09-23T23:06:33.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2026948261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:33.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2436547009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:33.315 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:33.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:33.381 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:06:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:06:33] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.33.4" 2023-09-23T23:06:33.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:33.464 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:33.623 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:06:33.623 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:06:33.624 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T23:06:33.624 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:06:33.816 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:33.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:33.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:33.855 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:06:33.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: pgmap v1385: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 221 op/s; 67180/210882 objects degraded (31.857%); 3853/210882 objects misplaced (1.827%); 13 MiB/s, 218 objects/s recovering 2023-09-23T23:06:33.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:06:33.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: failed to encode map e446 with expected crc 2023-09-23T23:06:33.915 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: failed to encode map e446 with expected crc 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2026948261' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: osdmap e446: 8 total, 8 up, 5 in 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[187743]: failed to encode map e446 with expected crc 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: pgmap v1385: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 221 op/s; 67180/210882 objects degraded (31.857%); 3853/210882 objects misplaced (1.827%); 13 MiB/s, 218 objects/s recovering 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: failed to encode map e446 with expected crc 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: failed to encode map e446 with expected crc 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2026948261' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:33.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: osdmap e446: 8 total, 8 up, 5 in 2023-09-23T23:06:33.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:33 smithi052 ceph-mon[180979]: failed to encode map e446 with expected crc 2023-09-23T23:06:33.936 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:34.017 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:34.109 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:34.130 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:34.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:34.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:34.142 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: pgmap v1385: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 221 op/s; 67180/210882 objects degraded (31.857%); 3853/210882 objects misplaced (1.827%); 13 MiB/s, 218 objects/s recovering 2023-09-23T23:06:34.142 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:06:34.142 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: failed to encode map e446 with expected crc 2023-09-23T23:06:34.142 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: failed to encode map e446 with expected crc 2023-09-23T23:06:34.142 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2026948261' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:34.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: osdmap e446: 8 total, 8 up, 5 in 2023-09-23T23:06:34.143 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:33 smithi151 ceph-mon[159210]: failed to encode map e446 with expected crc 2023-09-23T23:06:34.242 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:34.365 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:34.443 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:34.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:34.551 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:34.599 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:34.611 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:34.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:34.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:34.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:34.838 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:34.877 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:34.961 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:34.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:34 smithi052 ceph-mon[187743]: failed to encode map e446 with expected crc 2023-09-23T23:06:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:34 smithi052 ceph-mon[180979]: failed to encode map e446 with expected crc 2023-09-23T23:06:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:35.068 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:35.115 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:34 smithi151 ceph-mon[159210]: failed to encode map e446 with expected crc 2023-09-23T23:06:35.115 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:35.115 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:35.122 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:35.252 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:35.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:35.359 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:35.398 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:06:35.403 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:35.490 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:35.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:35.594 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:35.636 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:35.788 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:35.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:35.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:35.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:35.928 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:35 smithi151 ceph-mon[159210]: pgmap v1387: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 185 op/s; 67180/210882 objects degraded (31.857%); 3853/210882 objects misplaced (1.827%); 13 MiB/s, 228 objects/s recovering 2023-09-23T23:06:35.928 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 67180/210882 objects degraded (31.857%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:35.931 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:35.944 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:06:35.954 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:35.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:35 smithi052 ceph-mon[180979]: pgmap v1387: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 185 op/s; 67180/210882 objects degraded (31.857%); 3853/210882 objects misplaced (1.827%); 13 MiB/s, 228 objects/s recovering 2023-09-23T23:06:35.991 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 67180/210882 objects degraded (31.857%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:35.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:35 smithi052 ceph-mon[187743]: pgmap v1387: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 185 op/s; 67180/210882 objects degraded (31.857%); 3853/210882 objects misplaced (1.827%); 13 MiB/s, 228 objects/s recovering 2023-09-23T23:06:35.991 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 67180/210882 objects degraded (31.857%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:36.187 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:06:36.189 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:06:36.198 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:36.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:36.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:36.317 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:36.364 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:36.723 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:36.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97333 10.7 10:e37b5792:::benchmark_data_smithi052_461816_object532:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.854177+0000", 2023-09-23T23:06:36.771 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 572.91326134099995, 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94848450200000001, 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97333 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854177+0000", 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854177+0000", 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854175+0000", 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8010000000000004e-06 2023-09-23T23:06:36.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854183+0000", 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5661000000000001e-05 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854199+0000", 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.114e-06 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854206+0000", 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.6054000000000001e-05 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854272+0000", 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3147e-05 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.854285+0000", 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92368130199999998 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.777966+0000", 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013291099999999999 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.778099+0000", 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014655700000000001 2023-09-23T23:06:36.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.778246+0000", 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012212475 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.790458+0000", 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0099334079999999995 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.777 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800392+0000", 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7689999999999998e-05 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800449+0000", 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8202000000000003e-05 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.800508+0000", 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021539710000000002 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.802662+0000", 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.94848450200000001 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97340 10.9 10:92897f12:::benchmark_data_smithi052_461816_object539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.860204+0000", 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 572.90723431000004, 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93248454700000005, 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97340 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.780 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860204+0000", 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860204+0000", 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860202+0000", 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.465e-06 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860211+0000", 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.808e-05 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860229+0000", 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1040000000000007e-06 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860238+0000", 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0139000000000002e-05 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860288+0000", 2023-09-23T23:06:36.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0019999999999995e-06 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.860297+0000", 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90190687999999997 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.762204+0000", 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002049875 2023-09-23T23:06:36.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.764253+0000", 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0403999999999997e-05 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.764344+0000", 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0083045500000000008 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.785 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.772648+0000", 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019917872 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792566+0000", 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5818000000000001e-05 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792612+0000", 2023-09-23T23:06:36.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6701999999999998e-05 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792689+0000", 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.93248454700000005 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.787 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97348 10.9 10:9297810a:::benchmark_data_smithi052_461816_object547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.871378+0000", 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 572.89606074599999, 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92148298399999995, 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97348 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871378+0000", 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871378+0000", 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871376+0000", 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8349999999999999e-06 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871386+0000", 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5957999999999999e-05 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871401+0000", 2023-09-23T23:06:36.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9549999999999992e-06 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871409+0000", 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000133244 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871543+0000", 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0589e-05 2023-09-23T23:06:36.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.871553+0000", 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.89562549300000005 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767179+0000", 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000122841 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767302+0000", 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013168399999999999 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.767433+0000", 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0078544730000000007 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.775288+0000", 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017446303999999999 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792734+0000", 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5383999999999998e-05 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792779+0000", 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1309000000000004e-05 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.792861+0000", 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.92148298399999995 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.795 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97368 10.9 10:977fa808:::benchmark_data_smithi052_461816_object567:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.885379+0000", 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 572.88205964999997, 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.908468681, 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97368 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.796 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885379+0000", 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885379+0000", 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885377+0000", 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1136e-05 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885388+0000", 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1736e-05 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885409+0000", 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9409999999999996e-06 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.885418+0000", 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00073186099999999999 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.886150+0000", 2023-09-23T23:06:36.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9389999999999992e-06 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.886160+0000", 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88392771000000003 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770088+0000", 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010628699999999999 2023-09-23T23:06:36.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770194+0000", 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100766 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4", 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.770295+0000", 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076737589999999996 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.777969+0000", 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015755661000000001 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793724+0000", 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0615000000000003e-05 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793765+0000", 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2626999999999993e-05 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.793848+0000", 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.908468681 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.803 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:97377 10.7 10:e47c8c4a:::benchmark_data_smithi052_461816_object576:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:03.898849+0000", 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 572.86858908900001, 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90806386800000005, 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 97377 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.804 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898849+0000", 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898849+0000", 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898847+0000", 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0633e-05 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898858+0000", 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8763e-05 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898877+0000", 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0770000000000008e-06 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898885+0000", 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2481e-05 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898947+0000", 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9250000000000007e-06 2023-09-23T23:06:36.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:03.898957+0000", 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.88245452400000002 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781412+0000", 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013464799999999999 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781547+0000", 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017924 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.781726+0000", 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.016204418000000002 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.797930+0000", 2023-09-23T23:06:36.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00481702 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.802747+0000", 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001920106 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.804667+0000", 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8881000000000001e-05 2023-09-23T23:06:36.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.804726+0000", 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021871030000000001 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:04.806913+0000", 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.90806386800000005 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.811 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114190 10.a 10:552a2e4c:::benchmark_data_smithi052_461816_object17389:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637641+0000", 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 543.12979729100005, 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.055871596, 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.812 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114190 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637641+0000", 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637641+0000", 2023-09-23T23:06:36.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637640+0000", 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7300000000000002e-06 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637646+0000", 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1294999999999999e-05 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637657+0000", 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6370000000000004e-06 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637663+0000", 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.3239000000000003e-05 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637716+0000", 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0421000000000001e-05 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637726+0000", 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0540947350000001 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691821+0000", 2023-09-23T23:06:36.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00169184 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.693513+0000", 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.055871596 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114194 10.8 10.1b16bda8 (undecoded) ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:06:36.817 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 543.12976138099998, 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058402421, 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114194 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637677+0000", 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9859999999999999e-06 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637680+0000", 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.246e-06 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637686+0000", 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0130000000000002e-06 2023-09-23T23:06:36.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637689+0000", 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6356000000000001e-05 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637735+0000", 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1199999999999996e-06 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637742+0000", 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058305386 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.696048+0000", 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1797000000000001e-05 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.822 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.696080+0000", 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.058402421 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114198 10.5 10.b52bfca5 (undecoded) ondisk+retry+write+known_if_redirected e345)", 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:57:33.637701+0000", 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 543.12973782100005, 2023-09-23T23:06:36.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0533679819999999, 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114198 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637701+0000", 2023-09-23T23:06:36.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637701+0000", 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637700+0000", 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6850000000000001e-06 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637703+0000", 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4110000000000004e-06 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637708+0000", 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.89e-06 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637712+0000", 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0080000000000002e-05 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637762+0000", 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0999999999999993e-06 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:33.637771+0000", 2023-09-23T23:06:36.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0532533829999999 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691025+0000", 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3807000000000003e-05 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:57:34.691069+0000", 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0533679819999999 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.828 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114194 10.8 10:15bd68d8:::benchmark_data_smithi052_461816_object17393:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e352)", 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733139+0000", 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.034299589, 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.051601287, 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.829 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114194 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733139+0000", 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733139+0000", 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999919 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733131+0000", 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6708999999999999e-05 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733148+0000", 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.889e-05 2023-09-23T23:06:36.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733167+0000", 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9740000000000008e-06 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733176+0000", 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00030421400000000003 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733480+0000", 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0818999999999999e-05 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733491+0000", 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.047172094 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.780663+0000", 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0040775170000000001 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.784740+0000", 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.051601287 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.834 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114211 10.8 10:14a263e5:::benchmark_data_smithi052_461816_object17410:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.034244376, 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083107408, 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114211 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.835 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733194+0000", 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9090000000000002e-06 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733197+0000", 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0261000000000001e-05 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733208+0000", 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6239999999999999e-06 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733213+0000", 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045623199999999999 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733670+0000", 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0780000000000001e-06 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733676+0000", 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.05109607 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.784772+0000", 2023-09-23T23:06:36.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00058614799999999999 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.785358+0000", 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015455199999999999 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.785512+0000", 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017875975999999998 2023-09-23T23:06:36.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803388+0000", 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012748966 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816137+0000", 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0511999999999999e-05 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816198+0000", 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103699 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816302+0000", 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083107408 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.842 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114213 10.8 10:18638f4e:::benchmark_data_smithi052_461816_object17412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733219+0000", 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.03421955699997, 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0833211110000001, 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.843 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114213 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733219+0000", 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733219+0000", 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733218+0000", 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.574e-06 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733222+0000", 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6910000000000003e-06 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733231+0000", 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2459999999999997e-06 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733235+0000", 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000902928 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734138+0000", 2023-09-23T23:06:36.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3409999999999997e-06 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734145+0000", 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.054099326 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788245+0000", 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000139839 2023-09-23T23:06:36.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788384+0000", 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013002399999999999 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.788514+0000", 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014879720000000001 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803394+0000", 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012988685999999999 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816383+0000", 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0869999999999999e-05 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.849 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816434+0000", 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000106334 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816540+0000", 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0833211110000001 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114214 10.8 10:12e4921e:::benchmark_data_smithi052_461816_object17413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733241+0000", 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.034197861, 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083505803, 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.851 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114214 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733241+0000", 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733241+0000", 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733240+0000", 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9979999999999999e-06 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733243+0000", 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3750000000000003e-06 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733252+0000", 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4340000000000002e-06 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733256+0000", 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001189924 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734446+0000", 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.5579999999999997e-06 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.734452+0000", 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0568490829999999 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791302+0000", 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124091 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791426+0000", 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019142000000000001 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.791617+0000", 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011782971999999999 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803400+0000", 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013200309 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816600+0000", 2023-09-23T23:06:36.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7296999999999998e-05 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816648+0000", 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8789000000000006e-05 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816746+0000", 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083505803 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.858 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114223 10.8 10:1be79ee9:::benchmark_data_smithi052_461816_object17422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.03417818100002, 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0836756359999999, 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.859 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114223 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733260+0000", 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9550000000000001e-06 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733263+0000", 2023-09-23T23:06:36.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9780000000000005e-06 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733271+0000", 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4569999999999998e-06 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733275+0000", 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001796401 2023-09-23T23:06:36.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735072+0000", 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7419999999999997e-06 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735079+0000", 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0592689369999999 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794348+0000", 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129245 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794478+0000", 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016843500000000001 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.794646+0000", 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021672981000000001 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816319+0000", 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00047669500000000002 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816796+0000", 2023-09-23T23:06:36.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5318000000000002e-05 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816841+0000", 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4890999999999997e-05 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816936+0000", 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0836756359999999 2023-09-23T23:06:36.866 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114229 10.8 10:11c38b0a:::benchmark_data_smithi052_461816_object17428:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733280+0000", 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.03415868799999, 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083835967, 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.867 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114229 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733280+0000", 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733280+0000", 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733279+0000", 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.3809999999999999e-06 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733283+0000", 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.2919999999999996e-06 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733291+0000", 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8060000000000002e-06 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733296+0000", 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022410709999999999 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735537+0000", 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7049999999999994e-06 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735547+0000", 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0619144519999999 2023-09-23T23:06:36.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797461+0000", 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000124339 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797585+0000", 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013691500000000001 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.797722+0000", 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018607162999999999 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816330+0000", 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00064679199999999998 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816976+0000", 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5336000000000001e-05 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817022+0000", 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4147000000000005e-05 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817116+0000", 2023-09-23T23:06:36.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.083835967 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114239 10.8 10:1ab2d4f2:::benchmark_data_smithi052_461816_object17438:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.034138359, 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0839988570000001, 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.875 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114239 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733300+0000", 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.101e-06 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733303+0000", 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9850000000000003e-06 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733309+0000", 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0110000000000002e-06 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733313+0000", 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023753369999999999 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735688+0000", 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5669999999999995e-06 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.735697+0000", 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.064783783 2023-09-23T23:06:36.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800481+0000", 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011374699999999999 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800594+0000", 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012472500000000001 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.800719+0000", 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015619889 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.816339+0000", 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00081903599999999998 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817158+0000", 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5899999999999998e-05 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817204+0000", 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.5204000000000005e-05 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817299+0000", 2023-09-23T23:06:36.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0839988570000001 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:114244 10.8 10:16611384:::benchmark_data_smithi052_461816_object17443:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:05.733317+0000", 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 511.03412184799998, 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0845525629999999, 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.883 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 114244 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733317+0000", 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733317+0000", 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733316+0000", 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.531e-06 2023-09-23T23:06:36.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733319+0000", 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6490000000000008e-06 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733326+0000", 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0069999999999999e-06 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.733330+0000", 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031344089999999999 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.736465+0000", 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1509999999999998e-06 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:06:36.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:05.736472+0000", 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0669546030000001 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803427+0000", 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000114894 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:36.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803542+0000", 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000129417 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0", 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.803671+0000", 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013664033000000001 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817335+0000", 2023-09-23T23:06:36.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037598800000000001 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817711+0000", 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1017999999999998e-05 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817772+0000", 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7250999999999994e-05 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:06.817869+0000", 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0845525629999999 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPull(11.15 e445/440 cost 1049576)", 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:32.504189+0000", 2023-09-23T23:06:36.891 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 4.263249794, 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026280380000000001, 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504189+0000", 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504189+0000", 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504187+0000", 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.9281000000000001e-05 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504207+0000", 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3118000000000001e-05 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504230+0000", 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4120000000000001e-06 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504238+0000", 2023-09-23T23:06:36.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7599000000000003e-05 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.504286+0000", 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0025310340000000001 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:32.506817+0000", 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026280380000000001 2023-09-23T23:06:36.895 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPull(11.3 e446/441 cost 1049576)", 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:35.411952+0000", 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 1.3554865729999999, 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005384425, 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.411952+0000", 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.411952+0000", 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.411951+0000", 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.0752000000000001e-05 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.411971+0000", 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3300000000000001e-05 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.411995+0000", 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3420000000000008e-06 2023-09-23T23:06:36.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.412003+0000", 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0977999999999997e-05 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.412074+0000", 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0052624639999999997 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.417336+0000", 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005384425 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPull(11.13 e446/441 cost 1049576)", 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:35.414687+0000", 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 1.3527517840000001, 2023-09-23T23:06:36.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050571269999999998, 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.414687+0000", 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.414687+0000", 2023-09-23T23:06:36.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.414685+0000", 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.215e-05 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.414707+0000", 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2507999999999999e-05 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.414730+0000", 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.4479999999999994e-06 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.414738+0000", 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002351213 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.417089+0000", 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026544229999999999 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.419744+0000", 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050571269999999998 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.904 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPull(11.15 e446/440 cost 1049576)", 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:35.528620+0000", 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 1.2388184499999999, 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030810270000000001, 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528620+0000", 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:36.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528620+0000", 2023-09-23T23:06:36.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:36.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.906 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:36.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:36.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528618+0000", 2023-09-23T23:06:36.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1812999999999999e-05 2023-09-23T23:06:36.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528639+0000", 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3332e-05 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528663+0000", 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2459999999999992e-06 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:36.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528672+0000", 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1938999999999999e-05 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.528724+0000", 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029770980000000001 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:35.531701+0000", 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030810270000000001 2023-09-23T23:06:36.909 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.910 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.910 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.910 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:36.910 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:36.910 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:36.982 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:37.046 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:37.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:37.152 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:37.184 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:37.228 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:37.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:37.332 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:37.373 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:37.655 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1387,"stamp":"2023-09-23T23:06:34.718131+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4605745080,"num_objects":70294,"num_object_clones":0,"num_object_copies":210882,"num_objects_missing_on_primary":9911,"num_objects_missing":9911,"num_objects_degraded":67180,"num_objects_misplaced":3853,"num_objects_unfound":0,"num_objects_dirty":70294,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":94294,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12710,"num_bytes_recovered":802116032,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":145011,"ondisk_log_size":145011,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":13717092,"kb_used_data":10118396,"kb_used_omap":99,"kb_used_meta":3598556,"kb_avail":455008668,"statfs":{"total":479975178240,"available":465928876032,"internally_reserved":0,"allocated":10361237504,"data_stored":10353917857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3684921366},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-97255424,"num_objects":-1484,"num_object_clones":0,"num_object_copies":-4452,"num_objects_missing_on_primary":-346,"num_objects_missing":-346,"num_objects_degraded":-838,"num_objects_misplaced":-1095,"num_objects_unfound":0,"num_objects_dirty":-1484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1484,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1830,"num_bytes_recovered":110428160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002945"},"pg_stats":[{"pgid":"11.12","version":"444'5609","reported_seq":5673,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.425507+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:06:31.425507+0000","last_peered":"2023-09-23T23:06:31.425507+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:06:31.425507+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143818752,"num_objects":2195,"num_object_clones":0,"num_object_copies":6585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2946,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":2226},{"shards":"2,3","objects":-262}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":456,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.765157+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:06:12.765157+0000","last_peered":"2023-09-23T23:06:12.765157+0000","last_clean":"2023-09-23T23:06:12.765157+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:06:12.765157+0000","last_undegraded":"2023-09-23T23:06:12.765157+0000","last_fullsized":"2023-09-23T23:06:12.765157+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":116,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.849661+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:06:23.849661+0000","last_peered":"2023-09-23T23:06:23.849661+0000","last_clean":"2023-09-23T23:06:23.849661+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:06:23.849661+0000","last_undegraded":"2023-09-23T23:06:23.849661+0000","last_fullsized":"2023-09-23T23:06:23.849661+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"444'5581","reported_seq":5959,"reported_epoch":444,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:31.705254+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:06:31.705254+0000","last_peered":"2023-09-23T23:06:31.705254+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:06:31.705254+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:06:31.705254+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5581,"log_dups_size":0,"ondisk_log_size":5581,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144343040,"num_objects":2203,"num_object_clones":0,"num_object_copies":6609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2490,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2928,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":9961472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":1877},{"shards":"0,3,6","objects":270},{"shards":"2,3,6","objects":56}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":115,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.841508+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:06:23.841508+0000","last_peered":"2023-09-23T23:06:23.841508+0000","last_clean":"2023-09-23T23:06:23.841508+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:06:23.841508+0000","last_undegraded":"2023-09-23T23:06:23.841508+0000","last_fullsized":"2023-09-23T23:06:23.841508+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":462,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846390+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:06:23.846390+0000","last_peered":"2023-09-23T23:06:23.846390+0000","last_clean":"2023-09-23T23:06:23.846390+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:06:23.846390+0000","last_undegraded":"2023-09-23T23:06:23.846390+0000","last_fullsized":"2023-09-23T23:06:23.846390+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"444'5597","reported_seq":9624,"reported_epoch":444,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:31.845396+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:06:31.845396+0000","last_peered":"2023-09-23T23:06:31.845396+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:06:31.845396+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:06:31.845396+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5597,"log_dups_size":0,"ondisk_log_size":5597,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141950976,"num_objects":2166,"num_object_clones":0,"num_object_copies":6498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2941,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":163},{"shards":"1,3,6","objects":2003}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":23,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.837085+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:06:23.837085+0000","last_peered":"2023-09-23T23:06:23.837085+0000","last_clean":"2023-09-23T23:06:23.837085+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:06:23.837085+0000","last_undegraded":"2023-09-23T23:06:23.837085+0000","last_fullsized":"2023-09-23T23:06:23.837085+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":41,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846747+0000","last_change":"2023-09-23T23:06:08.605727+0000","last_active":"2023-09-23T23:06:23.846747+0000","last_peered":"2023-09-23T23:06:23.846747+0000","last_clean":"2023-09-23T23:06:23.846747+0000","last_became_active":"2023-09-23T23:06:08.605421+0000","last_became_peered":"2023-09-23T23:06:08.605421+0000","last_unstale":"2023-09-23T23:06:23.846747+0000","last_undegraded":"2023-09-23T23:06:23.846747+0000","last_fullsized":"2023-09-23T23:06:23.846747+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:10.283307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":388,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.847345+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:06:23.847345+0000","last_peered":"2023-09-23T23:06:23.847345+0000","last_clean":"2023-09-23T23:06:23.847345+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:06:23.847345+0000","last_undegraded":"2023-09-23T23:06:23.847345+0000","last_fullsized":"2023-09-23T23:06:23.847345+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":134,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.841372+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:06:23.841372+0000","last_peered":"2023-09-23T23:06:23.841372+0000","last_clean":"2023-09-23T23:06:23.841372+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:06:23.841372+0000","last_undegraded":"2023-09-23T23:06:23.841372+0000","last_fullsized":"2023-09-23T23:06:23.841372+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":250,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846270+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:06:23.846270+0000","last_peered":"2023-09-23T23:06:23.846270+0000","last_clean":"2023-09-23T23:06:23.846270+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:06:23.846270+0000","last_undegraded":"2023-09-23T23:06:23.846270+0000","last_fullsized":"2023-09-23T23:06:23.846270+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":148,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.838358+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:06:23.838358+0000","last_peered":"2023-09-23T23:06:23.838358+0000","last_clean":"2023-09-23T23:06:23.838358+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:06:23.838358+0000","last_undegraded":"2023-09-23T23:06:23.838358+0000","last_fullsized":"2023-09-23T23:06:23.838358+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":181,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846870+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:06:23.846870+0000","last_peered":"2023-09-23T23:06:23.846870+0000","last_clean":"2023-09-23T23:06:23.846870+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:06:23.846870+0000","last_undegraded":"2023-09-23T23:06:23.846870+0000","last_fullsized":"2023-09-23T23:06:23.846870+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":180,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767394+0000","last_change":"2023-09-23T23:06:08.609112+0000","last_active":"2023-09-23T23:06:12.767394+0000","last_peered":"2023-09-23T23:06:12.767394+0000","last_clean":"2023-09-23T23:06:12.767394+0000","last_became_active":"2023-09-23T23:06:08.608811+0000","last_became_peered":"2023-09-23T23:06:08.608811+0000","last_unstale":"2023-09-23T23:06:12.767394+0000","last_undegraded":"2023-09-23T23:06:12.767394+0000","last_fullsized":"2023-09-23T23:06:12.767394+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:01.339818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":534,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760002+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:06:12.760002+0000","last_peered":"2023-09-23T23:06:12.760002+0000","last_clean":"2023-09-23T23:06:12.760002+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:06:12.760002+0000","last_undegraded":"2023-09-23T23:06:12.760002+0000","last_fullsized":"2023-09-23T23:06:12.760002+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":427,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759993+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:06:12.759993+0000","last_peered":"2023-09-23T23:06:12.759993+0000","last_clean":"2023-09-23T23:06:12.759993+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:06:12.759993+0000","last_undegraded":"2023-09-23T23:06:12.759993+0000","last_fullsized":"2023-09-23T23:06:12.759993+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":359,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760071+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:06:12.760071+0000","last_peered":"2023-09-23T23:06:12.760071+0000","last_clean":"2023-09-23T23:06:12.760071+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:06:12.760071+0000","last_undegraded":"2023-09-23T23:06:12.760071+0000","last_fullsized":"2023-09-23T23:06:12.760071+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.850091+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:06:23.850091+0000","last_peered":"2023-09-23T23:06:23.850091+0000","last_clean":"2023-09-23T23:06:23.850091+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:06:23.850091+0000","last_undegraded":"2023-09-23T23:06:23.850091+0000","last_fullsized":"2023-09-23T23:06:23.850091+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":445,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.845211+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:06:23.845211+0000","last_peered":"2023-09-23T23:06:23.845211+0000","last_clean":"2023-09-23T23:06:23.845211+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:06:23.845211+0000","last_undegraded":"2023-09-23T23:06:23.845211+0000","last_fullsized":"2023-09-23T23:06:23.845211+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":342,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846781+0000","last_change":"2023-09-23T23:05:53.598631+0000","last_active":"2023-09-23T23:06:23.846781+0000","last_peered":"2023-09-23T23:06:23.846781+0000","last_clean":"2023-09-23T23:06:23.846781+0000","last_became_active":"2023-09-23T23:05:53.598050+0000","last_became_peered":"2023-09-23T23:05:53.598050+0000","last_unstale":"2023-09-23T23:06:23.846781+0000","last_undegraded":"2023-09-23T23:06:23.846781+0000","last_fullsized":"2023-09-23T23:06:23.846781+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:24.263110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"444'5552","reported_seq":10683,"reported_epoch":445,"state":"peering","last_fresh":"2023-09-23T23:06:31.851692+0000","last_change":"2023-09-23T23:06:31.851692+0000","last_active":"2023-09-23T23:06:31.851594+0000","last_peered":"2023-09-23T23:06:31.820303+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.040640+0000","last_became_peered":"2023-09-23T23:05:44.040640+0000","last_unstale":"2023-09-23T23:06:31.851692+0000","last_undegraded":"2023-09-23T23:06:31.851692+0000","last_fullsized":"2023-09-23T23:06:31.851692+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143622144,"num_objects":2192,"num_object_clones":0,"num_object_copies":6576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2903,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2476,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[6],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.761456+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:06:12.761456+0000","last_peered":"2023-09-23T23:06:12.761456+0000","last_clean":"2023-09-23T23:06:12.761456+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:06:12.761456+0000","last_undegraded":"2023-09-23T23:06:12.761456+0000","last_fullsized":"2023-09-23T23:06:12.761456+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":469,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846435+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:06:23.846435+0000","last_peered":"2023-09-23T23:06:23.846435+0000","last_clean":"2023-09-23T23:06:23.846435+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:06:23.846435+0000","last_undegraded":"2023-09-23T23:06:23.846435+0000","last_fullsized":"2023-09-23T23:06:23.846435+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"444'5651","reported_seq":6395,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.827460+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:06:31.827460+0000","last_peered":"2023-09-23T23:06:31.827460+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:06:31.827460+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145063936,"num_objects":2214,"num_object_clones":0,"num_object_copies":6642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4389,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2961,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":273,"num_bytes_recovered":17694720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1870},{"shards":"3,5,6","objects":317},{"shards":"3,6","objects":27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":58,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.837560+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:06:23.837560+0000","last_peered":"2023-09-23T23:06:23.837560+0000","last_clean":"2023-09-23T23:06:23.837560+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:06:23.837560+0000","last_undegraded":"2023-09-23T23:06:23.837560+0000","last_fullsized":"2023-09-23T23:06:23.837560+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":140,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.847423+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:06:23.847423+0000","last_peered":"2023-09-23T23:06:23.847423+0000","last_clean":"2023-09-23T23:06:23.847423+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:06:23.847423+0000","last_undegraded":"2023-09-23T23:06:23.847423+0000","last_fullsized":"2023-09-23T23:06:23.847423+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"444'5547","reported_seq":5616,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:30.245084+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:06:30.245084+0000","last_peered":"2023-09-23T23:06:30.245084+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:06:30.245084+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5547,"log_dups_size":0,"ondisk_log_size":5547,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144146432,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2896,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-166},{"shards":"1,4,5","objects":2241}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":354,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760014+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:06:12.760014+0000","last_peered":"2023-09-23T23:06:12.760014+0000","last_clean":"2023-09-23T23:06:12.760014+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:06:12.760014+0000","last_undegraded":"2023-09-23T23:06:12.760014+0000","last_fullsized":"2023-09-23T23:06:12.760014+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":310,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759586+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:06:12.759586+0000","last_peered":"2023-09-23T23:06:12.759586+0000","last_clean":"2023-09-23T23:06:12.759586+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:06:12.759586+0000","last_undegraded":"2023-09-23T23:06:12.759586+0000","last_fullsized":"2023-09-23T23:06:12.759586+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"444'5536","reported_seq":10513,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:31.538312+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:06:31.538312+0000","last_peered":"2023-09-23T23:06:31.538312+0000","last_clean":"2023-09-23T23:06:31.538312+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:06:31.538312+0000","last_undegraded":"2023-09-23T23:06:31.538312+0000","last_fullsized":"2023-09-23T23:06:31.538312+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5536,"log_dups_size":0,"ondisk_log_size":5536,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141459456,"num_objects":2158,"num_object_clones":0,"num_object_copies":6474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2904,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":72,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.774033+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:06:12.774033+0000","last_peered":"2023-09-23T23:06:12.774033+0000","last_clean":"2023-09-23T23:06:12.774033+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:06:12.774033+0000","last_undegraded":"2023-09-23T23:06:12.774033+0000","last_fullsized":"2023-09-23T23:06:12.774033+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":451,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.761369+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:06:12.761369+0000","last_peered":"2023-09-23T23:06:12.761369+0000","last_clean":"2023-09-23T23:06:12.761369+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:06:12.761369+0000","last_undegraded":"2023-09-23T23:06:12.761369+0000","last_fullsized":"2023-09-23T23:06:12.761369+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"445'5581","reported_seq":6384,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:32.301731+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:06:32.301731+0000","last_peered":"2023-09-23T23:06:32.301731+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:06:32.301731+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5581,"log_dups_size":0,"ondisk_log_size":5581,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141131776,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":1986,"num_objects_missing":1986,"num_objects_degraded":4409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2932,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":284,"num_bytes_recovered":18612224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1891},{"shards":"0,3,5,6","objects":1},{"shards":"1,6","objects":-47},{"shards":"3,5,6","objects":308}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":102,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846695+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:06:23.846695+0000","last_peered":"2023-09-23T23:06:23.846695+0000","last_clean":"2023-09-23T23:06:23.846695+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:06:23.846695+0000","last_undegraded":"2023-09-23T23:06:23.846695+0000","last_fullsized":"2023-09-23T23:06:23.846695+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":41,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.837453+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:06:23.837453+0000","last_peered":"2023-09-23T23:06:23.837453+0000","last_clean":"2023-09-23T23:06:23.837453+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:06:23.837453+0000","last_undegraded":"2023-09-23T23:06:23.837453+0000","last_fullsized":"2023-09-23T23:06:23.837453+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"444'5710","reported_seq":6118,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.548442+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:06:31.548442+0000","last_peered":"2023-09-23T23:06:31.548442+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:06:31.548442+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5710,"log_dups_size":0,"ondisk_log_size":5710,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146046976,"num_objects":2228,"num_object_clones":0,"num_object_copies":6684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2994,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":184},{"shards":"1,2,3","objects":1937}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":135,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:26.956209+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:06:26.956209+0000","last_peered":"2023-09-23T23:06:26.956209+0000","last_clean":"2023-09-23T23:06:26.956209+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:06:26.956209+0000","last_undegraded":"2023-09-23T23:06:26.956209+0000","last_fullsized":"2023-09-23T23:06:26.956209+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":266,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759930+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:06:12.759930+0000","last_peered":"2023-09-23T23:06:12.759930+0000","last_clean":"2023-09-23T23:06:12.759930+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:06:12.759930+0000","last_undegraded":"2023-09-23T23:06:12.759930+0000","last_fullsized":"2023-09-23T23:06:12.759930+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"445'5639","reported_seq":6270,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:32.416305+0000","last_change":"2023-09-23T23:06:13.792666+0000","last_active":"2023-09-23T23:06:32.416305+0000","last_peered":"2023-09-23T23:06:32.416305+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:06:13.017017+0000","last_became_peered":"2023-09-23T23:06:13.017017+0000","last_unstale":"2023-09-23T23:06:32.416305+0000","last_undegraded":"2023-09-23T23:06:10.970550+0000","last_fullsized":"2023-09-23T23:06:09.601294+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147324928,"num_objects":2248,"num_object_clones":0,"num_object_copies":6744,"num_objects_missing_on_primary":1935,"num_objects_missing":1935,"num_objects_degraded":2056,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2956,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":275,"num_bytes_recovered":17924096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[6,1],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":1862},{"shards":"1,6","objects":386}],"blocked_by":[],"up_primary":2,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":149,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:26.956183+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:06:26.956183+0000","last_peered":"2023-09-23T23:06:26.956183+0000","last_clean":"2023-09-23T23:06:26.956183+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:06:26.956183+0000","last_undegraded":"2023-09-23T23:06:26.956183+0000","last_fullsized":"2023-09-23T23:06:26.956183+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":446,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759907+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:06:12.759907+0000","last_peered":"2023-09-23T23:06:12.759907+0000","last_clean":"2023-09-23T23:06:12.759907+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:06:12.759907+0000","last_undegraded":"2023-09-23T23:06:12.759907+0000","last_fullsized":"2023-09-23T23:06:12.759907+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"444'5599","reported_seq":5664,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.255559+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:06:31.255559+0000","last_peered":"2023-09-23T23:06:31.255559+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:06:31.255559+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5599,"log_dups_size":0,"ondisk_log_size":5599,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144474112,"num_objects":2204,"num_object_clones":0,"num_object_copies":6612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2935,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":2173},{"shards":"2,3","objects":-191}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.765320+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:06:12.765320+0000","last_peered":"2023-09-23T23:06:12.765320+0000","last_clean":"2023-09-23T23:06:12.765320+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:06:12.765320+0000","last_undegraded":"2023-09-23T23:06:12.765320+0000","last_fullsized":"2023-09-23T23:06:12.765320+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":454,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760381+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:06:12.760381+0000","last_peered":"2023-09-23T23:06:12.760381+0000","last_clean":"2023-09-23T23:06:12.760381+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:06:12.760381+0000","last_undegraded":"2023-09-23T23:06:12.760381+0000","last_fullsized":"2023-09-23T23:06:12.760381+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"445'5580","reported_seq":6200,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:32.396181+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:06:32.396181+0000","last_peered":"2023-09-23T23:06:32.396181+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:06:32.396181+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144408576,"num_objects":2203,"num_object_clones":0,"num_object_copies":6609,"num_objects_missing_on_primary":2044,"num_objects_missing":2044,"num_objects_degraded":2475,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":273,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1957},{"shards":"0,2,3,6","objects":1},{"shards":"0,3,6","objects":300},{"shards":"2,6","objects":-55}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":467,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.761135+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:06:12.761135+0000","last_peered":"2023-09-23T23:06:12.761135+0000","last_clean":"2023-09-23T23:06:12.761135+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:06:12.761135+0000","last_undegraded":"2023-09-23T23:06:12.761135+0000","last_fullsized":"2023-09-23T23:06:12.761135+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":454,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760568+0000","last_change":"2023-09-23T23:05:03.450633+0000","last_active":"2023-09-23T23:06:12.760568+0000","last_peered":"2023-09-23T23:06:12.760568+0000","last_clean":"2023-09-23T23:06:12.760568+0000","last_became_active":"2023-09-23T23:05:03.450232+0000","last_became_peered":"2023-09-23T23:05:03.450232+0000","last_unstale":"2023-09-23T23:06:12.760568+0000","last_undegraded":"2023-09-23T23:06:12.760568+0000","last_fullsized":"2023-09-23T23:06:12.760568+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"444'5589","reported_seq":6208,"reported_epoch":444,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:31.791235+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:06:31.791235+0000","last_peered":"2023-09-23T23:06:31.791235+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:06:31.791235+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:06:31.791235+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142475264,"num_objects":2174,"num_object_clones":0,"num_object_copies":6522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1900,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2933,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1810},{"shards":"1,3,6","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":117,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.843566+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:06:23.843566+0000","last_peered":"2023-09-23T23:06:23.843566+0000","last_clean":"2023-09-23T23:06:23.843566+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:06:23.843566+0000","last_undegraded":"2023-09-23T23:06:23.843566+0000","last_fullsized":"2023-09-23T23:06:23.843566+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":349,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767794+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:06:12.767794+0000","last_peered":"2023-09-23T23:06:12.767794+0000","last_clean":"2023-09-23T23:06:12.767794+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:06:12.767794+0000","last_undegraded":"2023-09-23T23:06:12.767794+0000","last_fullsized":"2023-09-23T23:06:12.767794+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"444'5979","reported_seq":7416,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.274255+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:06:31.274255+0000","last_peered":"2023-09-23T23:06:31.274255+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:06:31.274255+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293339160,"num_objects":4477,"num_object_clones":0,"num_object_copies":13431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5979,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":3659},{"shards":"2,4","objects":611}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":362,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.770083+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:06:12.770083+0000","last_peered":"2023-09-23T23:06:12.770083+0000","last_clean":"2023-09-23T23:06:12.770083+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:06:12.770083+0000","last_undegraded":"2023-09-23T23:06:12.770083+0000","last_fullsized":"2023-09-23T23:06:12.770083+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":282,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846573+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:06:23.846573+0000","last_peered":"2023-09-23T23:06:23.846573+0000","last_clean":"2023-09-23T23:06:23.846573+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:06:23.846573+0000","last_undegraded":"2023-09-23T23:06:23.846573+0000","last_fullsized":"2023-09-23T23:06:23.846573+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":185,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.765063+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:06:12.765063+0000","last_peered":"2023-09-23T23:06:12.765063+0000","last_clean":"2023-09-23T23:06:12.765063+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:06:12.765063+0000","last_undegraded":"2023-09-23T23:06:12.765063+0000","last_fullsized":"2023-09-23T23:06:12.765063+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"445'5907","reported_seq":5995,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.860424+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:06:31.860424+0000","last_peered":"2023-09-23T23:06:31.860424+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:06:31.860424+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"415'1267","ondisk_log_start":"415'1267","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4640,"log_dups_size":0,"ondisk_log_size":4640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286326784,"num_objects":4369,"num_object_clones":0,"num_object_copies":13107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5907,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":4369},{"shards":"1,3","objects":-456}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":41,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.845114+0000","last_change":"2023-09-23T23:06:08.607076+0000","last_active":"2023-09-23T23:06:23.845114+0000","last_peered":"2023-09-23T23:06:23.845114+0000","last_clean":"2023-09-23T23:06:23.845114+0000","last_became_active":"2023-09-23T23:06:08.606649+0000","last_became_peered":"2023-09-23T23:06:08.606649+0000","last_unstale":"2023-09-23T23:06:23.845114+0000","last_undegraded":"2023-09-23T23:06:23.845114+0000","last_fullsized":"2023-09-23T23:06:23.845114+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:47.677948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":482,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760208+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:06:12.760208+0000","last_peered":"2023-09-23T23:06:12.760208+0000","last_clean":"2023-09-23T23:06:12.760208+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:06:12.760208+0000","last_undegraded":"2023-09-23T23:06:12.760208+0000","last_fullsized":"2023-09-23T23:06:12.760208+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":347,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.769955+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:06:12.769955+0000","last_peered":"2023-09-23T23:06:12.769955+0000","last_clean":"2023-09-23T23:06:12.769955+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:06:12.769955+0000","last_undegraded":"2023-09-23T23:06:12.769955+0000","last_fullsized":"2023-09-23T23:06:12.769955+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"445'5673","reported_seq":6462,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.920063+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:06:31.920063+0000","last_peered":"2023-09-23T23:06:31.920063+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:06:31.920063+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143622144,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4492,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2982,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":293,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1903},{"shards":"3,5,6","objects":315},{"shards":"3,6","objects":-27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":422,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767206+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:06:12.767206+0000","last_peered":"2023-09-23T23:06:12.767206+0000","last_clean":"2023-09-23T23:06:12.767206+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:06:12.767206+0000","last_undegraded":"2023-09-23T23:06:12.767206+0000","last_fullsized":"2023-09-23T23:06:12.767206+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":275,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767068+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:06:12.767068+0000","last_peered":"2023-09-23T23:06:12.767068+0000","last_clean":"2023-09-23T23:06:12.767068+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:06:12.767068+0000","last_undegraded":"2023-09-23T23:06:12.767068+0000","last_fullsized":"2023-09-23T23:06:12.767068+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":363,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846893+0000","last_change":"2023-09-23T23:05:53.573682+0000","last_active":"2023-09-23T23:06:23.846893+0000","last_peered":"2023-09-23T23:06:23.846893+0000","last_clean":"2023-09-23T23:06:23.846893+0000","last_became_active":"2023-09-23T23:05:53.573311+0000","last_became_peered":"2023-09-23T23:05:53.573311+0000","last_unstale":"2023-09-23T23:06:23.846893+0000","last_undegraded":"2023-09-23T23:06:23.846893+0000","last_fullsized":"2023-09-23T23:06:23.846893+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:26.957590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"445'5871","reported_seq":7779,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.910037+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:06:31.910037+0000","last_peered":"2023-09-23T23:06:31.910037+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:06:31.910037+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"415'1234","ondisk_log_start":"415'1234","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4637,"log_dups_size":0,"ondisk_log_size":4637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286588928,"num_objects":4373,"num_object_clones":0,"num_object_copies":13119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4658,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5871,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":922,"num_bytes_recovered":60424192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":3580},{"shards":"3,5","objects":1},{"shards":"3,5,6","objects":619},{"shards":"3,6","objects":173}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":354,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767627+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:06:12.767627+0000","last_peered":"2023-09-23T23:06:12.767627+0000","last_clean":"2023-09-23T23:06:12.767627+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:06:12.767627+0000","last_undegraded":"2023-09-23T23:06:12.767627+0000","last_fullsized":"2023-09-23T23:06:12.767627+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":555,"reported_epoch":443,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:06:12.767276+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:06:12.767276+0000","last_peered":"2023-09-23T23:06:12.767276+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:06:12.767276+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:06:12.767276+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.837543+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:06:23.837543+0000","last_peered":"2023-09-23T23:06:23.837543+0000","last_clean":"2023-09-23T23:06:23.837543+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:06:23.837543+0000","last_undegraded":"2023-09-23T23:06:23.837543+0000","last_fullsized":"2023-09-23T23:06:23.837543+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":344,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760554+0000","last_change":"2023-09-23T23:06:12.173156+0000","last_active":"2023-09-23T23:06:12.760554+0000","last_peered":"2023-09-23T23:06:12.760554+0000","last_clean":"2023-09-23T23:06:12.760554+0000","last_became_active":"2023-09-23T23:06:12.172747+0000","last_became_peered":"2023-09-23T23:06:12.172747+0000","last_unstale":"2023-09-23T23:06:12.760554+0000","last_undegraded":"2023-09-23T23:06:12.760554+0000","last_fullsized":"2023-09-23T23:06:12.760554+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"444'5590","reported_seq":5786,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.394095+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:06:31.394095+0000","last_peered":"2023-09-23T23:06:31.394095+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:06:31.394095+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5590,"log_dups_size":0,"ondisk_log_size":5590,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141131776,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2940,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4030464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":1948},{"shards":"0,3,6","objects":321},{"shards":"1,3","objects":-345}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":368,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.840191+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:06:23.840191+0000","last_peered":"2023-09-23T23:06:23.840191+0000","last_clean":"2023-09-23T23:06:23.840191+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:06:23.840191+0000","last_undegraded":"2023-09-23T23:06:23.840191+0000","last_fullsized":"2023-09-23T23:06:23.840191+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":338,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759698+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:06:12.759698+0000","last_peered":"2023-09-23T23:06:12.759698+0000","last_clean":"2023-09-23T23:06:12.759698+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:06:12.759698+0000","last_undegraded":"2023-09-23T23:06:12.759698+0000","last_fullsized":"2023-09-23T23:06:12.759698+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":303,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.771300+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:06:12.771300+0000","last_peered":"2023-09-23T23:06:12.771300+0000","last_clean":"2023-09-23T23:06:12.771300+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:06:12.771300+0000","last_undegraded":"2023-09-23T23:06:12.771300+0000","last_fullsized":"2023-09-23T23:06:12.771300+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"444'5825","reported_seq":5869,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:31.397969+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:06:31.397969+0000","last_peered":"2023-09-23T23:06:31.397969+0000","last_clean":"2023-09-23T23:06:31.397969+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:06:31.397969+0000","last_undegraded":"2023-09-23T23:06:31.397969+0000","last_fullsized":"2023-09-23T23:06:31.397969+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5825,"log_dups_size":0,"ondisk_log_size":5825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288817152,"num_objects":4407,"num_object_clones":0,"num_object_copies":13221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5825,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":303,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760306+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:06:12.760306+0000","last_peered":"2023-09-23T23:06:12.760306+0000","last_clean":"2023-09-23T23:06:12.760306+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:06:12.760306+0000","last_undegraded":"2023-09-23T23:06:12.760306+0000","last_fullsized":"2023-09-23T23:06:12.760306+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":299,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.770268+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:06:12.770268+0000","last_peered":"2023-09-23T23:06:12.770268+0000","last_clean":"2023-09-23T23:06:12.770268+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:06:12.770268+0000","last_undegraded":"2023-09-23T23:06:12.770268+0000","last_fullsized":"2023-09-23T23:06:12.770268+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":265,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767167+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:06:12.767167+0000","last_peered":"2023-09-23T23:06:12.767167+0000","last_clean":"2023-09-23T23:06:12.767167+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:06:12.767167+0000","last_undegraded":"2023-09-23T23:06:12.767167+0000","last_fullsized":"2023-09-23T23:06:12.767167+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"444'5791","reported_seq":5843,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.274554+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:06:31.274554+0000","last_peered":"2023-09-23T23:06:31.274554+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:06:31.274554+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284360704,"num_objects":4339,"num_object_clones":0,"num_object_copies":13017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5791,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":4339},{"shards":"1,2","objects":-430}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":167,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.847558+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:06:23.847558+0000","last_peered":"2023-09-23T23:06:23.847558+0000","last_clean":"2023-09-23T23:06:23.847558+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:06:23.847558+0000","last_undegraded":"2023-09-23T23:06:23.847558+0000","last_fullsized":"2023-09-23T23:06:23.847558+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":142,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760505+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:06:12.760505+0000","last_peered":"2023-09-23T23:06:12.760505+0000","last_clean":"2023-09-23T23:06:12.760505+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:06:12.760505+0000","last_undegraded":"2023-09-23T23:06:12.760505+0000","last_fullsized":"2023-09-23T23:06:12.760505+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":150,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.841449+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:06:23.841449+0000","last_peered":"2023-09-23T23:06:23.841449+0000","last_clean":"2023-09-23T23:06:23.841449+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:06:23.841449+0000","last_undegraded":"2023-09-23T23:06:23.841449+0000","last_fullsized":"2023-09-23T23:06:23.841449+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"444'5875","reported_seq":7614,"reported_epoch":444,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:30.382173+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:06:30.382173+0000","last_peered":"2023-09-23T23:06:30.382173+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:06:30.382173+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5875,"log_dups_size":0,"ondisk_log_size":5875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":3853,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5875,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":3853},{"shards":"1,4","objects":420}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":391,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767607+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:06:12.767607+0000","last_peered":"2023-09-23T23:06:12.767607+0000","last_clean":"2023-09-23T23:06:12.767607+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:06:12.767607+0000","last_undegraded":"2023-09-23T23:06:12.767607+0000","last_fullsized":"2023-09-23T23:06:12.767607+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":65,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760653+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:06:12.760653+0000","last_peered":"2023-09-23T23:06:12.760653+0000","last_clean":"2023-09-23T23:06:12.760653+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:06:12.760653+0000","last_undegraded":"2023-09-23T23:06:12.760653+0000","last_fullsized":"2023-09-23T23:06:12.760653+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":436,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.761227+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:06:12.761227+0000","last_peered":"2023-09-23T23:06:12.761227+0000","last_clean":"2023-09-23T23:06:12.761227+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:06:12.761227+0000","last_undegraded":"2023-09-23T23:06:12.761227+0000","last_fullsized":"2023-09-23T23:06:12.761227+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"445'5659","reported_seq":6327,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:32.338584+0000","last_change":"2023-09-23T23:06:13.782895+0000","last_active":"2023-09-23T23:06:32.338584+0000","last_peered":"2023-09-23T23:06:32.338584+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:06:13.017993+0000","last_became_peered":"2023-09-23T23:06:13.017993+0000","last_unstale":"2023-09-23T23:06:32.338584+0000","last_undegraded":"2023-09-23T23:06:12.255966+0000","last_fullsized":"2023-09-23T23:06:10.977105+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5659,"log_dups_size":0,"ondisk_log_size":5659,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146014208,"num_objects":2228,"num_object_clones":0,"num_object_copies":6684,"num_objects_missing_on_primary":2014,"num_objects_missing":2014,"num_objects_degraded":2155,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2976,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":293,"num_bytes_recovered":19169280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[6,1],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":1925},{"shards":"1,6","objects":303}],"blocked_by":[],"up_primary":2,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":17,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759807+0000","last_change":"2023-09-23T23:05:53.573701+0000","last_active":"2023-09-23T23:06:12.759807+0000","last_peered":"2023-09-23T23:06:12.759807+0000","last_clean":"2023-09-23T23:06:12.759807+0000","last_became_active":"2023-09-23T23:05:53.573268+0000","last_became_peered":"2023-09-23T23:05:53.573268+0000","last_unstale":"2023-09-23T23:06:12.759807+0000","last_undegraded":"2023-09-23T23:06:12.759807+0000","last_fullsized":"2023-09-23T23:06:12.759807+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":194,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.770268+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:06:12.770268+0000","last_peered":"2023-09-23T23:06:12.770268+0000","last_clean":"2023-09-23T23:06:12.770268+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:06:12.770268+0000","last_undegraded":"2023-09-23T23:06:12.770268+0000","last_fullsized":"2023-09-23T23:06:12.770268+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"444'5694","reported_seq":6087,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.254655+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:06:31.254655+0000","last_peered":"2023-09-23T23:06:31.254655+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:06:31.254655+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147095552,"num_objects":2245,"num_object_clones":0,"num_object_copies":6735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2979,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":270},{"shards":"1,2,3","objects":1875}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":266,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759821+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:06:12.759821+0000","last_peered":"2023-09-23T23:06:12.759821+0000","last_clean":"2023-09-23T23:06:12.759821+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:06:12.759821+0000","last_undegraded":"2023-09-23T23:06:12.759821+0000","last_fullsized":"2023-09-23T23:06:12.759821+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":467,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760278+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:06:12.760278+0000","last_peered":"2023-09-23T23:06:12.760278+0000","last_clean":"2023-09-23T23:06:12.760278+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:06:12.760278+0000","last_undegraded":"2023-09-23T23:06:12.760278+0000","last_fullsized":"2023-09-23T23:06:12.760278+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"445'5570","reported_seq":6356,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:32.399541+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:06:32.399541+0000","last_peered":"2023-09-23T23:06:32.399541+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:06:32.399541+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5570,"log_dups_size":0,"ondisk_log_size":5570,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141852672,"num_objects":2165,"num_object_clones":0,"num_object_copies":6495,"num_objects_missing_on_primary":1930,"num_objects_missing":1930,"num_objects_degraded":4276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2922,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":273,"num_bytes_recovered":17825792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1814},{"shards":"0,3,5,6","objects":5},{"shards":"3,5,6","objects":298},{"shards":"3,6","objects":48}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":293,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.760472+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:06:12.760472+0000","last_peered":"2023-09-23T23:06:12.760472+0000","last_clean":"2023-09-23T23:06:12.760472+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:06:12.760472+0000","last_undegraded":"2023-09-23T23:06:12.760472+0000","last_fullsized":"2023-09-23T23:06:12.760472+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":363,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.765238+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:06:12.765238+0000","last_peered":"2023-09-23T23:06:12.765238+0000","last_clean":"2023-09-23T23:06:12.765238+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:06:12.765238+0000","last_undegraded":"2023-09-23T23:06:12.765238+0000","last_fullsized":"2023-09-23T23:06:12.765238+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"444'5541","reported_seq":6524,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:31.100216+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:06:31.100216+0000","last_peered":"2023-09-23T23:06:31.100216+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:06:31.100216+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141131776,"num_objects":2154,"num_object_clones":0,"num_object_copies":6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2910,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":1866},{"shards":"1,2","objects":167}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":19,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.846219+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:06:23.846219+0000","last_peered":"2023-09-23T23:06:23.846219+0000","last_clean":"2023-09-23T23:06:23.846219+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:06:23.846219+0000","last_undegraded":"2023-09-23T23:06:23.846219+0000","last_fullsized":"2023-09-23T23:06:23.846219+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":161,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T23:06:23.841594+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:06:23.841594+0000","last_peered":"2023-09-23T23:06:23.841594+0000","last_clean":"2023-09-23T23:06:23.841594+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:06:23.841594+0000","last_undegraded":"2023-09-23T23:06:23.841594+0000","last_fullsized":"2023-09-23T23:06:23.841594+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"444'5572","reported_seq":5631,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:06:30.352397+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:06:30.352397+0000","last_peered":"2023-09-23T23:06:30.352397+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:06:30.352397+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5572,"log_dups_size":0,"ondisk_log_size":5572,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142508032,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2922,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-81},{"shards":"1,4,5","objects":2133}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":211,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.767909+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:06:12.767909+0000","last_peered":"2023-09-23T23:06:12.767909+0000","last_clean":"2023-09-23T23:06:12.767909+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:06:12.767909+0000","last_undegraded":"2023-09-23T23:06:12.767909+0000","last_fullsized":"2023-09-23T23:06:12.767909+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":438,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T23:06:12.759645+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:06:12.759645+0000","last_peered":"2023-09-23T23:06:12.759645+0000","last_clean":"2023-09-23T23:06:12.759645+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:06:12.759645+0000","last_undegraded":"2023-09-23T23:06:12.759645+0000","last_fullsized":"2023-09-23T23:06:12.759645+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":4605149208,"num_objects":70270,"num_object_clones":0,"num_object_copies":210810,"num_objects_missing_on_primary":9909,"num_objects_missing":9909,"num_objects_degraded":67178,"num_objects_misplaced":3853,"num_objects_unfound":0,"num_objects_dirty":70270,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":94034,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12640,"num_bytes_recovered":802095104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14899949568,"data_stored":14899937352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":144817,"ondisk_log_size":144817,"up":78,"acting":58,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430682,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626907,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1509060,"kb_used_data":614524,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92236092,"statfs":{"total":95995035648,"available":94449758208,"internally_reserved":0,"allocated":629272576,"data_stored":628319995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":4.7830000000000004,"15min":5.6699999999999999},"min":{"1min":0.27100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":10.853,"5min":115.32899999999999,"15min":115.32899999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.038,"5min":4.6440000000000001,"15min":5.649},"min":{"1min":0.27400000000000002,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":7.5949999999999998,"5min":115.348,"15min":115.348},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":3.8370000000000002,"15min":3.9540000000000002},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.23100000000000001},"max":{"1min":5.3819999999999997,"5min":149.947,"15min":254.62100000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.992,"5min":4.0800000000000001,"15min":4.3520000000000003},"min":{"1min":0.35899999999999999,"5min":0.28999999999999998,"15min":0.24199999999999999},"max":{"1min":10.922000000000001,"5min":150.22499999999999,"15min":255.20099999999999},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:05:40 2023","interfaces":[{"interface":"back","average":{"1min":16.077000000000002,"5min":5.2679999999999998,"15min":2.4369999999999998},"min":{"1min":0.376,"5min":0.307,"15min":0.307},"max":{"1min":138.43299999999999,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":15.375,"5min":5.4400000000000004,"15min":2.5880000000000001},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":138.45599999999999,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":7.1200000000000001,"5min":5.2779999999999996,"15min":3.2010000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":70.521000000000001,"5min":142.24700000000001,"15min":142.24700000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":7.9740000000000002,"5min":5.3710000000000004,"15min":3.2370000000000001},"min":{"1min":0.45600000000000002,"5min":0.34899999999999998,"15min":0.309},"max":{"1min":70.497,"5min":143.86699999999999,"15min":143.86699999999999},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":2.1920000000000002,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.26200000000000001,"15min":0.23899999999999999},"max":{"1min":12.407999999999999,"5min":119.804,"15min":119.804},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.0030000000000001,"15min":1.861},"min":{"1min":0.39500000000000002,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":3.6699999999999999,"5min":122.31999999999999,"15min":122.31999999999999},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 23:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":4.7850000000000001,"15min":2.5049999999999999},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":10.946,"5min":186.27600000000001,"15min":186.27600000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.69699999999999995,"5min":4.6920000000000002,"15min":2.5289999999999999},"min":{"1min":0.39500000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":1.768,"5min":191.28700000000001,"15min":191.28700000000001},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 23:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":2.0409999999999999,"15min":1.498},"min":{"1min":0.41799999999999998,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":8.5299999999999994,"5min":115.61,"15min":115.61},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.0710000000000002,"15min":1.7050000000000001},"min":{"1min":0.38300000000000001,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":5.968,"5min":115.578,"15min":115.578},"last":0.77600000000000002}]}]},{"osd":1,"up_from":84,"seq":360777253220,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4253556,"kb_used_data":3358572,"kb_used_omap":9,"kb_used_meta":894966,"kb_avail":89491596,"statfs":{"total":95995035648,"available":91639394304,"internally_reserved":0,"allocated":3439177728,"data_stored":3437438753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":916446185},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:06:19 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":5.9859999999999998,"15min":2.532},"min":{"1min":0.20899999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":27.271999999999998,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.546,"5min":6.1900000000000004,"15min":2.8809999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":24.844000000000001,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.28000000000000003}]},{"osd":2,"last update":"Sat Sep 23 23:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":4.8010000000000002,"15min":1.9390000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":14.445,"5min":161.61699999999999,"15min":161.61699999999999},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.1359999999999999,"5min":5.1470000000000002,"15min":2.1720000000000002},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":14.558999999999999,"5min":162.61699999999999,"15min":162.61699999999999},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0529999999999999,"5min":2.4700000000000002,"15min":1.665},"min":{"1min":0.309,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":8.5559999999999992,"5min":59.424999999999997,"15min":59.424999999999997},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.1259999999999999,"5min":2.3279999999999998,"15min":1.681},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":12.569000000000001,"5min":55.174999999999997,"15min":55.174999999999997},"last":0.48299999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.7669999999999999,"15min":2.1480000000000001},"min":{"1min":0.36199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3609999999999998,"5min":67.738,"15min":151.036},"last":0.371},{"interface":"front","average":{"1min":1.548,"5min":2.218,"15min":2.4630000000000001},"min":{"1min":0.39400000000000002,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":5.5419999999999998,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":2.1320000000000001,"15min":2.3399999999999999},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":6.04,"5min":68.259,"15min":116.098},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.5489999999999999,"5min":2.004,"15min":2.2189999999999999},"min":{"1min":0.41499999999999998,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":5.6970000000000001,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.59399999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":2.2959999999999998,"15min":2.0619999999999998},"min":{"1min":0.35699999999999998,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":5.4989999999999997,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.1299999999999999,"15min":1.8640000000000001},"min":{"1min":0.41499999999999998,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":5.8099999999999996,"5min":67.366,"15min":67.366},"last":1.548}]},{"osd":7,"last update":"Sat Sep 23 23:05:51 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.93799999999999994,"15min":1.276},"min":{"1min":0.48499999999999999,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":4.3849999999999998,"5min":4.3849999999999998,"15min":9.9329999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.1309999999999998,"5min":1.4219999999999999,"15min":1.677},"min":{"1min":0.438,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":5.8109999999999999,"5min":5.8109999999999999,"15min":20.902999999999999},"last":0.54500000000000004}]}]},{"osd":0,"up_from":0,"seq":1516123455590,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259363,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527604,"kb_used_data":2265836,"kb_used_omap":83,"kb_used_meta":261676,"kb_avail":91217548,"statfs":{"total":95995035648,"available":93406769152,"internally_reserved":0,"allocated":2320216064,"data_stored":2318355511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":267956677},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":2.657,"15min":1.147},"min":{"1min":0.221,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.199999999999999,"5min":74.241,"15min":74.241},"last":0.246},{"interface":"front","average":{"1min":1.9810000000000001,"5min":3.0470000000000002,"15min":1.4730000000000001},"min":{"1min":0.29299999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.6790000000000003,"5min":74.430000000000007,"15min":74.430000000000007},"last":0.30499999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":1.6679999999999999,"15min":0.98299999999999998},"min":{"1min":0.246,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":18.396999999999998,"5min":18.396999999999998,"15min":18.396999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.081,"5min":1.601,"15min":1.2869999999999999},"min":{"1min":0.26400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.744999999999999,"5min":12.744999999999999,"15min":12.744999999999999},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":2.637,"15min":1.361},"min":{"1min":0.30599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":17.786999999999999,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.008,"5min":2.6150000000000002,"15min":1.431},"min":{"1min":0.32600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":14.863,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.28,"15min":0.86199999999999999},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.4829999999999997,"5min":33.476999999999997,"15min":33.476999999999997},"last":1.141},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.5409999999999999,"15min":1.0620000000000001},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.75,"5min":33.497999999999998,"15min":33.497999999999998},"last":1.224}]},{"osd":5,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":2.9329999999999998,"15min":1.6259999999999999},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.8019999999999996,"5min":139.405,"15min":139.405},"last":2.2269999999999999},{"interface":"front","average":{"1min":1.5189999999999999,"5min":2.7959999999999998,"15min":1.4139999999999999},"min":{"1min":0.377,"5min":0.371,"15min":0.36299999999999999},"max":{"1min":10.109999999999999,"5min":139.42099999999999,"15min":139.42099999999999},"last":6.8600000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":1.3560000000000001,"15min":1.1240000000000001},"min":{"1min":0.379,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.9009999999999998,"5min":9.9359999999999999,"15min":9.9359999999999999},"last":3.3130000000000002},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.0640000000000001,"15min":0.81000000000000005},"min":{"1min":0.42399999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.233000000000001,"5min":10.233000000000001,"15min":10.233000000000001},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.165,"15min":1.0920000000000001},"min":{"1min":0.49399999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.164,"5min":10.164,"15min":10.164},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.1910000000000001,"15min":1.1719999999999999},"min":{"1min":0.435,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.7830000000000004,"5min":7.7830000000000004,"15min":7.7830000000000004},"last":4.1609999999999996}]}]},{"osd":3,"up_from":94,"seq":403726926167,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3227008,"kb_used_data":2578872,"kb_used_omap":7,"kb_used_meta":648120,"kb_avail":90518144,"statfs":{"total":95995035648,"available":92690579456,"internally_reserved":0,"allocated":2640764928,"data_stored":2638946710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":663675496},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:06:18 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":2.1970000000000001,"15min":1.3600000000000001},"min":{"1min":0.20399999999999999,"5min":0.191,"15min":0.17299999999999999},"max":{"1min":17.015000000000001,"5min":37.249000000000002,"15min":122.289},"last":0.311},{"interface":"front","average":{"1min":1.196,"5min":1.359,"15min":1.1080000000000001},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.17799999999999999},"max":{"1min":11.382999999999999,"5min":35.991,"15min":122.041},"last":0.29699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":4.2759999999999998,"5min":2.2429999999999999,"15min":2.2130000000000001},"min":{"1min":0.30299999999999999,"5min":0.22,"15min":0.187},"max":{"1min":23.969000000000001,"5min":23.969000000000001,"15min":123.866},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.9609999999999999,"5min":2.0880000000000001,"15min":2.1989999999999998},"min":{"1min":0.27600000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":16.986000000000001,"5min":16.986000000000001,"15min":122.265},"last":0.39500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:05:37 2023","interfaces":[{"interface":"back","average":{"1min":4.1260000000000003,"5min":1.504,"15min":0.96699999999999997},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":18.588000000000001,"5min":18.588000000000001,"15min":18.588000000000001},"last":0.377},{"interface":"front","average":{"1min":4.1539999999999999,"5min":2.0819999999999999,"15min":1.1970000000000001},"min":{"1min":0.251,"5min":0.215,"15min":0.215},"max":{"1min":10.353999999999999,"5min":10.353999999999999,"15min":10.353999999999999},"last":0.28000000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.661,"15min":1.8959999999999999},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":9.1099999999999994,"5min":10.327,"15min":87.046999999999997},"last":3.0390000000000001},{"interface":"front","average":{"1min":2.6549999999999998,"5min":1.6399999999999999,"15min":1.8779999999999999},"min":{"1min":0.37,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":16.931000000000001,"5min":16.931000000000001,"15min":87.066000000000003},"last":0.36099999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5369999999999999,"5min":1.6839999999999999,"15min":1.4019999999999999},"min":{"1min":0.35899999999999999,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":16.959,"5min":16.959,"15min":16.959},"last":10.422000000000001},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.2769999999999999,"15min":1.488},"min":{"1min":0.34399999999999997,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":8.5820000000000007,"5min":8.5820000000000007,"15min":10.246},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7410000000000001,"5min":1.7090000000000001,"15min":1.704},"min":{"1min":0.435,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":3.3969999999999998},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.2330000000000001,"15min":1.625},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.32400000000000001},"max":{"1min":8.1999999999999993,"5min":8.1999999999999993,"15min":12.887},"last":3.4089999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:06:04 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.242,"15min":1.4079999999999999},"min":{"1min":0.36899999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":8.1620000000000008,"5min":8.1620000000000008,"15min":8.8160000000000007},"last":3.1909999999999998},{"interface":"front","average":{"1min":2.556,"5min":1.7709999999999999,"15min":1.4950000000000001},"min":{"1min":0.41199999999999998,"5min":0.40300000000000002,"15min":0.33200000000000002},"max":{"1min":9.1880000000000006,"5min":10.278,"15min":10.278},"last":3.1760000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019364,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2199864,"kb_used_data":1300592,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":91545288,"statfs":{"total":95995035648,"available":93742374912,"internally_reserved":0,"allocated":1331806208,"data_stored":1330856888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:06:21 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":5.1660000000000004,"15min":2.145},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.4750000000000001,"5min":156.375,"15min":156.375},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.85799999999999998,"5min":5.5359999999999996,"15min":2.282},"min":{"1min":0.315,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.5169999999999999,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.623}]},{"osd":1,"last update":"Sat Sep 23 23:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":3.464,"15min":3.871},"min":{"1min":0.34799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.9279999999999999,"5min":95.591999999999999,"15min":132.304},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.284,"5min":3.1720000000000002,"15min":3.8759999999999999},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.5629999999999997,"5min":97.929000000000002,"15min":132.453},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:05:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9960000000000004,"5min":2.157,"15min":1.4319999999999999},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":56.531999999999996,"5min":56.531999999999996,"15min":56.531999999999996},"last":0.629},{"interface":"front","average":{"1min":5.8680000000000003,"5min":2.1379999999999999,"15min":1.222},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":56.204000000000001,"5min":56.204000000000001,"15min":56.204000000000001},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:06:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":1.9770000000000001,"15min":2.4239999999999999},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.972,"5min":60.274000000000001,"15min":164.453},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.0070000000000001,"5min":2.234,"15min":2.5289999999999999},"min":{"1min":0.36599999999999999,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":10.888999999999999,"5min":60.677999999999997,"15min":167.05199999999999},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:05:36 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.84399999999999997,"15min":1.655},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.214},"max":{"1min":4.0039999999999996,"5min":8.3439999999999994,"15min":148.61600000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.70899999999999996,"15min":1.738},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":2.0699999999999998,"5min":8.7810000000000006,"15min":151.905},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0069999999999999,"15min":1.2310000000000001},"min":{"1min":0.32500000000000001,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":5.3209999999999997,"5min":8.9610000000000003,"15min":16.943000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.149,"5min":0.92100000000000004,"15min":1.1359999999999999},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":4.5220000000000002,"5min":7.7409999999999997,"15min":14.875999999999999},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:05:27 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.0840000000000001,"15min":1.2549999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":6.726,"5min":7.1050000000000004,"15min":12.84},"last":1.3520000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.81100000000000005,"15min":1.0920000000000001},"min":{"1min":0.33800000000000002,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":3.6539999999999999,"5min":3.6539999999999999,"15min":10.587},"last":0.44}]}]},{"osd":5,"up_from":0,"seq":154618823138,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3044540416,"data_stored":3044540416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3433234432,"data_stored":3433234432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2311131136,"data_stored":2311127064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2630615040,"data_stored":2630615040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1324158976,"data_stored":1324154904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":624234496,"data_stored":624230424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:06:37.662 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T23:06:37.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-09-23T23:06:37.718 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:37.729 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:06:37.740 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:37.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:37.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:37.846 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:37.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:37 smithi052 ceph-mon[180979]: from='client.47117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:06:37.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:37 smithi052 ceph-mon[180979]: pgmap v1388: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 194 op/s; 67180/209202 objects degraded (32.113%); 3739/209202 objects misplaced (1.787%); 8.5 MiB/s, 154 objects/s recovering 2023-09-23T23:06:37.887 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:37.965 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:37.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:38.069 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:38.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:37 smithi151 ceph-mon[159210]: from='client.47117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:06:38.116 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:37 smithi151 ceph-mon[159210]: pgmap v1388: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 194 op/s; 67180/209202 objects degraded (32.113%); 3739/209202 objects misplaced (1.787%); 8.5 MiB/s, 154 objects/s recovering 2023-09-23T23:06:38.120 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:38.211 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:37 smithi052 ceph-mon[187743]: from='client.47117 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:06:38.212 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:37 smithi052 ceph-mon[187743]: pgmap v1388: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 194 op/s; 67180/209202 objects degraded (32.113%); 3739/209202 objects misplaced (1.787%); 8.5 MiB/s, 154 objects/s recovering 2023-09-23T23:06:38.213 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:38.214 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:38.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:38.240 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:38.263 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:38.285 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:38.292 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:38.321 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:38.327 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:38.358 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:38.379 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:38.387 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:38.418 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:38.420 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.421 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:06:38.443 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:38.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:38.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:38.548 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:38.576 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:38.622 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:38.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:38.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:38.725 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:38.798 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:38.859 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:38 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1241633950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:38.946 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:38.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:38.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:39.052 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:39.087 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:38 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1241633950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:39.103 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:39.173 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:39.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:39.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:39.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:38 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1241633950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:39.280 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:39.326 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:39.655 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:39.716 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:39.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:39.822 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:39.855 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:39.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:39 smithi151 ceph-mon[159210]: pgmap v1389: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 246 op/s; 66740/207588 objects degraded (32.150%); 3739/207588 objects misplaced (1.801%); 10 MiB/s, 178 objects/s recovering 2023-09-23T23:06:39.973 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:39.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:40.079 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:40.111 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:39 smithi052 ceph-mon[187743]: pgmap v1389: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 246 op/s; 66740/207588 objects degraded (32.150%); 3739/207588 objects misplaced (1.801%); 10 MiB/s, 178 objects/s recovering 2023-09-23T23:06:40.111 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:39 smithi052 ceph-mon[180979]: pgmap v1389: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 246 op/s; 66740/207588 objects degraded (32.150%); 3739/207588 objects misplaced (1.801%); 10 MiB/s, 178 objects/s recovering 2023-09-23T23:06:40.122 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:40.185 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:40.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:40.269 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:06:40.270 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:06:40.289 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:40.325 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:40.468 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:40.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:40.576 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:40.603 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:40.647 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:40.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:40.750 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:40.788 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:40.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3766849520' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:06:40.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:40 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:06:40.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 66740/207588 objects degraded (32.150%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:40.891 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:06:41.034 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:41.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3766849520' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:06:41.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:40 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:06:41.037 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 66740/207588 objects degraded (32.150%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:41.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3766849520' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:06:41.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:40 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:06:41.037 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 66740/207588 objects degraded (32.150%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T23:06:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:41.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:41.139 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:41.156 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:06:41.201 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:41.289 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:41.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:41.404 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:41.481 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:41.604 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:41.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:41.649 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:06:41.714 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:41.756 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:41.832 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:41.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:41.867 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:41.938 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 198 op/s; 66740/207180 objects degraded (32.214%); 3627/207180 objects misplaced (1.751%); 9.8 MiB/s, 173 objects/s recovering 2023-09-23T23:06:41.938 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:41.938 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: osdmap e447: 8 total, 8 up, 5 in 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[187743]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 198 op/s; 66740/207180 objects degraded (32.214%); 3627/207180 objects misplaced (1.751%); 9.8 MiB/s, 173 objects/s recovering 2023-09-23T23:06:41.939 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:41.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:06:41.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: osdmap e447: 8 total, 8 up, 5 in 2023-09-23T23:06:41.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.940 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:41 smithi052 ceph-mon[180979]: failed to encode map e447 with expected crc 2023-09-23T23:06:41.955 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:41.993 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:42.066 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:42.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 198 op/s; 66740/207180 objects degraded (32.214%); 3627/207180 objects misplaced (1.751%); 9.8 MiB/s, 173 objects/s recovering 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: failed to encode map e447 with expected crc 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: osdmap e447: 8 total, 8 up, 5 in 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: failed to encode map e447 with expected crc 2023-09-23T23:06:42.117 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: failed to encode map e447 with expected crc 2023-09-23T23:06:42.118 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:41 smithi151 ceph-mon[159210]: failed to encode map e447 with expected crc 2023-09-23T23:06:42.171 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:42.171 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:42.211 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:42.332 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:42.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:42.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:42.437 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:42.448 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T23:06:42.476 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:42 smithi151 ceph-mon[159210]: failed to encode map e447 with expected crc 2023-09-23T23:06:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1389392182' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:06:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:42 smithi052 ceph-mon[180979]: failed to encode map e447 with expected crc 2023-09-23T23:06:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1389392182' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:06:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:42 smithi052 ceph-mon[187743]: failed to encode map e447 with expected crc 2023-09-23T23:06:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1389392182' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T23:06:43.439 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:43.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:43.542 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:43.581 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:43.680 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:06:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:06:43] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.33.4" 2023-09-23T23:06:43.699 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:43.703 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T23:06:43.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:43.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:43.806 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:43.860 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:43.941 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:43 smithi052 ceph-mon[187743]: pgmap v1392: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 222 op/s; 66257/204291 objects degraded (32.433%); 3627/204291 objects misplaced (1.775%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:06:43.941 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:43 smithi052 ceph-mon[180979]: pgmap v1392: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 222 op/s; 66257/204291 objects degraded (32.433%); 3627/204291 objects misplaced (1.775%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:06:43.972 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:06:43.981 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:43.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:43.990 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:43 smithi151 ceph-mon[159210]: pgmap v1392: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 222 op/s; 66257/204291 objects degraded (32.433%); 3627/204291 objects misplaced (1.775%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T23:06:44.047 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:06:44.047 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:06:44.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:44.152 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:44.305 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:44.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:44.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:44.408 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:44.441 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:44.463 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:44.478 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:44.486 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:44.496 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:44.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:44.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:44.512 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:44.536 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:44.560 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:44.584 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:44.602 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:44.606 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:44.638 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:44.641 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.641 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.642 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:06:44.647 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:44.702 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:44.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:44.807 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:44.858 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:44.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4164265379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4164265379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:45.057 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:06:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4164265379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:45.670 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:45.884 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:45.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:45.957 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:45 smithi052 ceph-mon[187743]: pgmap v1393: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 219 op/s; 66257/204291 objects degraded (32.433%); 3627/204291 objects misplaced (1.775%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T23:06:45.958 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 66257/204291 objects degraded (32.433%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:45.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:45 smithi052 ceph-mon[180979]: pgmap v1393: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 219 op/s; 66257/204291 objects degraded (32.433%); 3627/204291 objects misplaced (1.775%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T23:06:45.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 66257/204291 objects degraded (32.433%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:45.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:46.028 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:46.183 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:45 smithi151 ceph-mon[159210]: pgmap v1393: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 219 op/s; 66257/204291 objects degraded (32.433%); 3627/204291 objects misplaced (1.775%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T23:06:46.183 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 66257/204291 objects degraded (32.433%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:46.217 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:46.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:46.324 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:46.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:46.461 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:46.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:46.565 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:46.599 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:46.651 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:46.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:46.756 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:46.790 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:46.870 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:46.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:46.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:46.977 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:47.004 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:47.049 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:47.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:47.153 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:47.183 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:47.230 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:47.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:47.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:47.334 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:47.389 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:47.461 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:47.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:47.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:47.567 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:47.596 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:47.641 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:47.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:47.744 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:47.752 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:47.752 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:06:47.752 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116332 10.b 10:dc130431:::benchmark_data_smithi052_461816_object19531:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 482.14398167399997, 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001, 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116332 2023-09-23T23:06:47.753 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604201+0000", 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:47.754 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604199+0000", 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1632e-05 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604211+0000", 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7592999999999999e-05 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.755 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604228+0000", 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0749999999999998e-06 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.604236+0000", 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.223173936 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.827410+0000", 2023-09-23T23:06:47.756 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015809969 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.843220+0000", 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013161054 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.856381+0000", 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.035988506000000003 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.757 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.892370+0000", 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.027739362 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.920109+0000", 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020550299999999998 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.758 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922164+0000", 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.28725175800000002 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.209416+0000", 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0092034360000000006 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.218619+0000", 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030490359000000002 2023-09-23T23:06:47.759 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.249110+0000", 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6449085590000001 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.45953.0:116347 10.b 10:db51aae1:::benchmark_data_smithi052_461816_object19546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:06:47.760 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 482.09365178899998, 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001, 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.45953", 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 116347 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:06:47.761 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654531+0000", 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654529+0000", 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.5270000000000001e-06 2023-09-23T23:06:47.762 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654537+0000", 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5132e-05 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654552+0000", 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5509999999999999e-06 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.763 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:45.654557+0000", 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2683612470000001 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.922919+0000", 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0088446870000000004 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.764 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.931763+0000", 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011420990000000001 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.943184+0000", 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040857233999999999 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.984042+0000", 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022559659999999999 2023-09-23T23:06:47.765 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.986298+0000", 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002071696 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:46.988369+0000", 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.26508044400000003 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.766 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.253450+0000", 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055181509999999998 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.258968+0000", 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017719586999999998 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T22:58:47.276687+0000", 2023-09-23T23:06:47.767 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6221564320000001 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85301 11.4 11:20006e33:::benchmark_data_smithi052_606809_object3147:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.923239+0000", 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 76.824943435999998, 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2027856080000001, 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.768 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85301 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923239+0000", 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.769 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923239+0000", 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923238+0000", 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8920000000000003e-06 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.770 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923245+0000", 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1908e-05 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923257+0000", 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.443e-06 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923263+0000", 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8831311619999997 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.771 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.806394+0000", 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037811699999999999 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.810175+0000", 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017708903000000002 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.827884+0000", 2023-09-23T23:06:47.772 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029209840000000002 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.830805+0000", 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25806233699999997 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.088868+0000", 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030479840000000001 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.773 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.091916+0000", 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068492040000000002 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.098765+0000", 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062520520000000001 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.774 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.105017+0000", 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011661407 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.116678+0000", 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001103129 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.117781+0000", 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015897769999999999 2023-09-23T23:06:47.775 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.119371+0000", 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066538880000000002 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.126025+0000", 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2027856080000001 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.776 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85304 11.4 11:25d25a9c:::benchmark_data_smithi052_606809_object3150:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 76.824913378000005, 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2751230859999998, 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.777 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85304 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:06:47.778 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4080000000000001e-06 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923271+0000", 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1699999999999996e-06 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.779 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923277+0000", 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4489999999999999e-06 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923280+0000", 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8877396659999999 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.811020+0000", 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038907109999999998 2023-09-23T23:06:47.780 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.814910+0000", 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030186995000000001 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.845097+0000", 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031564060000000001 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.781 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.848254+0000", 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.297196077 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.145450+0000", 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004836478 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.782 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.150286+0000", 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0097873660000000005 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.160074+0000", 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062974859999999997 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.166371+0000", 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00091250000000000001 2023-09-23T23:06:47.783 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.167284+0000", 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024166043000000002 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.191450+0000", 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014562430000000001 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.784 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.192906+0000", 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005486505 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.198393+0000", 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2751230859999998 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.785 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85312 11.1 11:80e4190c:::benchmark_data_smithi052_606809_object3158:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.947654+0000", 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 76.800528420999996, 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7308309500000001, 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85312 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.786 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947654+0000", 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947654+0000", 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.787 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947652+0000", 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1599e-05 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947664+0000", 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7906000000000001e-05 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947682+0000", 2023-09-23T23:06:47.788 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1664999999999999e-05 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947693+0000", 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.513713555 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.461407+0000", 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003630253 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.789 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.465037+0000", 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015287227 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.480324+0000", 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003515991 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.790 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.483840+0000", 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16787706099999999 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.651717+0000", 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035492700000000002 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.655267+0000", 2023-09-23T23:06:47.791 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077437690000000002 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.663010+0000", 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066746690000000003 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.669685+0000", 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093855499999999999 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.792 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.670624+0000", 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00061731399999999999 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.671241+0000", 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015721909999999999 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.793 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.672813+0000", 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056723030000000001 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.678485+0000", 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7308309500000001 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.794 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85313 11.11 11:88ebcce4:::benchmark_data_smithi052_606809_object3159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.949419+0000", 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 76.798764239999997, 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7675347939999999, 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85313 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.795 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949419+0000", 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949419+0000", 2023-09-23T23:06:47.796 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949416+0000", 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3740000000000004e-06 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949426+0000", 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4061e-05 2023-09-23T23:06:47.797 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949440+0000", 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0450000000000001e-06 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949448+0000", 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5184956220000001 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.798 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.467943+0000", 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043664400000000001 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.472310+0000", 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023586055000000002 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.495896+0000", 2023-09-23T23:06:47.799 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043012090000000003 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.500197+0000", 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19441752200000001 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.694615+0000", 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023650899999999998 2023-09-23T23:06:47.800 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.696980+0000", 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050274810000000003 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.702007+0000", 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046063249999999997 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.801 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.706613+0000", 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001047886 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.707661+0000", 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00031287699999999998 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.707974+0000", 2023-09-23T23:06:47.802 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020656289999999998 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.710040+0000", 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069135209999999997 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.716953+0000", 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7675347939999999 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.803 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85369 11.4 11:24b22a8c:::benchmark_data_smithi052_606809_object3215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.515955+0000", 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 75.232227612000003, 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.953101872, 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.804 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85369 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515955+0000", 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515955+0000", 2023-09-23T23:06:47.805 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515953+0000", 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1839999999999999e-06 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515961+0000", 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.428e-05 2023-09-23T23:06:47.806 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515976+0000", 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.148e-06 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515983+0000", 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.333705964 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.807 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.849689+0000", 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038120300000000001 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.853501+0000", 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021648292999999999 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.875149+0000", 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045914110000000001 2023-09-23T23:06:47.808 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.879741+0000", 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30051507500000002 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.180256+0000", 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026806270000000001 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.809 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.182936+0000", 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048667820000000001 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.187803+0000", 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011472008000000001 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.810 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.199275+0000", 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010707579 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.209983+0000", 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24838902900000001 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.458372+0000", 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020854889999999998 2023-09-23T23:06:47.811 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.460457+0000", 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0085999939999999997 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.469057+0000", 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.953101872 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.812 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85403 11.4 11:26ba6595:::benchmark_data_smithi052_606809_object3249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.632315+0000", 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 75.115867667000003, 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.84570751, 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85403 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.813 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632315+0000", 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632315+0000", 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.814 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632314+0000", 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2310000000000004e-06 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632319+0000", 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2821e-05 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.815 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632332+0000", 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4320000000000004e-06 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632338+0000", 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.221519861 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.853858+0000", 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044097279999999999 2023-09-23T23:06:47.816 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.858268+0000", 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.038489075999999997 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.896757+0000", 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035420310000000002 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.817 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.900299+0000", 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31082930399999997 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.211128+0000", 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038503690000000002 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.214979+0000", 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075619830000000004 2023-09-23T23:06:47.818 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.222541+0000", 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069424860000000003 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.229483+0000", 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0078532020000000001 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.819 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.237336+0000", 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23273159299999999 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.470068+0000", 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014063319999999999 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.471474+0000", 2023-09-23T23:06:47.820 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006548262 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.478023+0000", 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.84570751 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85405 11.14 11:2a8d78d3:::benchmark_data_smithi052_606809_object3251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:06:47.821 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.634161+0000", 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 75.114021905000001, 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.832578955, 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85405 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.822 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634161+0000", 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634161+0000", 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.823 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634160+0000", 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0370000000000002e-06 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634165+0000", 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.52e-06 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634173+0000", 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7840000000000001e-06 2023-09-23T23:06:47.824 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634177+0000", 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.263104448 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.897281+0000", 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039633519999999998 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.825 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.901244+0000", 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015875415 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.917120+0000", 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003554771 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.826 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.920674+0000", 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29039737500000001 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.211072+0000", 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039948229999999998 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.827 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.215067+0000", 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0079402069999999995 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.223007+0000", 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069677610000000003 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.229975+0000", 2023-09-23T23:06:47.828 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081174709999999994 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.238092+0000", 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22017930799999999 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.458271+0000", 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015077059999999999 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.829 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.459779+0000", 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069607460000000003 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.466740+0000", 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.832578955 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.830 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85433 11.4 11:2052dad0:::benchmark_data_smithi052_606809_object3279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.737596+0000", 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 75.010587174999998, 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7512567720000001, 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.831 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85433 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737596+0000", 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737596+0000", 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:47.832 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737594+0000", 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8739999999999995e-06 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737602+0000", 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2435999999999999e-05 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.833 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737614+0000", 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.677e-06 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737620+0000", 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1630549939999999 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.834 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.900675+0000", 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031074190000000002 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.903782+0000", 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013217001000000001 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.916999+0000", 2023-09-23T23:06:47.835 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036349080000000001 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.920634+0000", 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31714880899999998 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.237783+0000", 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002564555 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.836 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.240348+0000", 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050599310000000002 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.245408+0000", 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023625132 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.269033+0000", 2023-09-23T23:06:47.837 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037696600000000001 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.269410+0000", 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.209935662 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.479345+0000", 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021585160000000001 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.838 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.481504+0000", 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073485110000000003 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.488852+0000", 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7512567720000001 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.839 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85464 11.14 11:289df81e:::benchmark_data_smithi052_606809_object3310:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.835884+0000", 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 74.912298660999994, 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6773388709999999, 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.840 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85464 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835884+0000", 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835884+0000", 2023-09-23T23:06:47.841 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835882+0000", 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.679e-06 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835892+0000", 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1365999999999999e-05 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.842 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835903+0000", 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2380000000000002e-06 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835909+0000", 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0858108289999999 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.843 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.921720+0000", 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043400929999999997 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.926060+0000", 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023379476999999999 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.949440+0000", 2023-09-23T23:06:47.844 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031490149999999998 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.952589+0000", 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.32618674399999997 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.278776+0000", 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033116529999999999 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.845 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.282087+0000", 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0059453020000000004 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.288033+0000", 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044534190000000001 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.292486+0000", 2023-09-23T23:06:47.846 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00032079800000000001 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.292807+0000", 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20849826399999999 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.501305+0000", 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002349056 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.847 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.503654+0000", 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095689150000000008 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.513223+0000", 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6773388709999999 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.848 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89654 11.15 11:afe848ca:::benchmark_data_smithi052_606809_object7500:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:58.974110+0000", 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.774072519999997, 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2612470760000001, 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89654 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.849 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974110+0000", 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974110+0000", 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.850 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974109+0000", 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5589999999999996e-06 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974115+0000", 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9350000000000006e-06 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974125+0000", 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0460000000000002e-06 2023-09-23T23:06:47.851 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974131+0000", 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7024555349999999 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.676586+0000", 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004985817 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.852 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.681572+0000", 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.036358366000000003 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.717930+0000", 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046866579999999998 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.722617+0000", 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.481659956 2023-09-23T23:06:47.853 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.204277+0000", 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037195980000000002 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.207997+0000", 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075747210000000004 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.854 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.215571+0000", 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066158290000000002 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.222187+0000", 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012446479999999999 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.223432+0000", 2023-09-23T23:06:47.855 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045506700000000003 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.223887+0000", 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020853339999999999 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.225972+0000", 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0093852569999999993 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.856 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.235357+0000", 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2612470760000001 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:89665 11.d e438/422 11:b4df92a6:::benchmark_data_smithi052_606809_object7511:head v 438'5708, mlcod=415'892)", 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.012900+0000", 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.735282755, 2023-09-23T23:06:47.857 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5605278060000001, 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012900+0000", 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012900+0000", 2023-09-23T23:06:47.858 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012899+0000", 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0849000000000001e-05 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012909+0000", 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1338999999999999e-05 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.859 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012921+0000", 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4239999999999998e-06 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012927+0000", 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.542952552 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.555880+0000", 2023-09-23T23:06:47.860 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5443999999999999e-05 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.555905+0000", 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017479251000000001 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573384+0000", 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3417999999999997e-05 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.861 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573428+0000", 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5605278060000001 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89672 11.5 11:a3f2d77a:::benchmark_data_smithi052_606809_object7518:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.061551+0000", 2023-09-23T23:06:47.862 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.686631847999998, 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0987285240000002, 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89672 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061551+0000", 2023-09-23T23:06:47.863 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061551+0000", 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061549+0000", 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8250000000000005e-06 2023-09-23T23:06:47.864 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061557+0000", 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2894000000000001e-05 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061570+0000", 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2576e-05 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.865 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061583+0000", 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6154805059999999 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.677063+0000", 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0047144320000000002 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.681778+0000", 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031613074999999997 2023-09-23T23:06:47.866 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.713391+0000", 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039974870000000001 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.717388+0000", 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.416516314 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.867 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.133904+0000", 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024337619999999999 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.136338+0000", 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0059120209999999999 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.142250+0000", 2023-09-23T23:06:47.868 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049678300000000003 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.147218+0000", 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001389562 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.148608+0000", 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039206070000000004 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.869 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.152528+0000", 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015177439999999999 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.154046+0000", 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062334870000000002 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.870 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.160280+0000", 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0987285240000002 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:89701 11.d e438/422 11:b8619b1e:::benchmark_data_smithi052_606809_object7547:head v 438'5709, mlcod=415'892)", 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.226962+0000", 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.521220474000003, 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3464912619999998, 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.871 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226962+0000", 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226962+0000", 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.872 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226961+0000", 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1880000000000008e-06 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226971+0000", 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9430000000000006e-06 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226980+0000", 2023-09-23T23:06:47.873 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9900000000000002e-06 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226986+0000", 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3334172569999998 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.560403+0000", 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7268999999999999e-05 2023-09-23T23:06:47.874 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.560430+0000", 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013000843 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573431+0000", 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2730000000000001e-05 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.875 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573454+0000", 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3464912619999998 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89719 11.5 11:a1dc5e4d:::benchmark_data_smithi052_606809_object7565:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.354009+0000", 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.394174237000001, 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9292337449999999, 2023-09-23T23:06:47.876 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89719 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354009+0000", 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.877 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354009+0000", 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354007+0000", 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3188e-05 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.878 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354020+0000", 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2751e-05 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354033+0000", 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0747000000000001e-05 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354043+0000", 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3637451550000002 2023-09-23T23:06:47.879 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.717788+0000", 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032941609999999999 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.721083+0000", 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.033079843999999997 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.880 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.754162+0000", 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002939999 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.757102+0000", 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.47205112399999999 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.229154+0000", 2023-09-23T23:06:47.881 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029339689999999998 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.232088+0000", 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072230259999999996 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.239311+0000", 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012806079 2023-09-23T23:06:47.882 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.252117+0000", 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001104449 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.253221+0000", 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020816254999999999 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.883 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.274037+0000", 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002526169 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.276564+0000", 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066788419999999999 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.283242+0000", 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9292337449999999 2023-09-23T23:06:47.884 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:89722 11.d e438/422 11:bfa4ed94:::benchmark_data_smithi052_606809_object7568:head v 438'5710, mlcod=415'892)", 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.370512+0000", 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.377670696000003, 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.202966194, 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.885 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370512+0000", 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370512+0000", 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.886 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370511+0000", 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.453e-05 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370526+0000", 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7280999999999999e-05 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370543+0000", 2023-09-23T23:06:47.887 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9209999999999998e-06 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370552+0000", 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.193936415 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.564488+0000", 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2923000000000001e-05 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.888 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.564511+0000", 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089455330000000003 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573457+0000", 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1611999999999999e-05 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.889 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573478+0000", 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.202966194 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89724 11.15 11:a935a078:::benchmark_data_smithi052_606809_object7570:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.376891+0000", 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.371291937999999, 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9619000199999999, 2023-09-23T23:06:47.890 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89724 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376891+0000", 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.891 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376891+0000", 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376889+0000", 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3060000000000003e-06 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.892 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376898+0000", 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3233e-05 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376911+0000", 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3850000000000002e-06 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376920+0000", 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.347168988 2023-09-23T23:06:47.893 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.724089+0000", 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035878080000000001 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.727677+0000", 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021852636000000002 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.894 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.749529+0000", 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041318489999999999 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.753661+0000", 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.54881714199999998 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.302478+0000", 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043085609999999998 2023-09-23T23:06:47.895 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.306787+0000", 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0085451180000000009 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.315332+0000", 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0083029590000000004 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.896 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.323635+0000", 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019104199999999999 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.325545+0000", 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00072450300000000004 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.326270+0000", 2023-09-23T23:06:47.897 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036638439999999999 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.329934+0000", 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0088572140000000004 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.338791+0000", 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9619000199999999 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.898 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89726 11.15 11:a840aef0:::benchmark_data_smithi052_606809_object7572:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.395826+0000", 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.352357208000001, 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1034079110000001, 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.899 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89726 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395826+0000", 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395826+0000", 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T23:06:47.900 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395823+0000", 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0997e-05 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395834+0000", 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8947999999999999e-05 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.901 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395853+0000", 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3639999999999999e-06 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395861+0000", 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3322495170000002 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.728111+0000", 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041596289999999998 2023-09-23T23:06:47.902 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.732270+0000", 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.063996094000000003 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.796266+0000", 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053571390000000003 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.903 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.801624+0000", 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.65385371299999995 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.455477+0000", 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034231679999999999 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.458900+0000", 2023-09-23T23:06:47.904 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081103259999999993 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.467011+0000", 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019588700000000001 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.486599+0000", 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034858699999999997 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.905 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.486948+0000", 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001419462 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.488368+0000", 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018342149999999999 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.906 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.490202+0000", 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0090318140000000009 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.499234+0000", 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1034079110000001 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89728 11.5 11:a12ca5da:::benchmark_data_smithi052_606809_object7574:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:06:47.907 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.431675+0000", 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 48.316507578, 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0659816869999998, 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89728 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:06:47.908 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431675+0000", 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431675+0000", 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431673+0000", 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9670000000000001e-06 2023-09-23T23:06:47.909 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431681+0000", 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6407000000000001e-05 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431698+0000", 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7449999999999995e-06 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.910 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431705+0000", 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3256917989999999 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.757397+0000", 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002954422 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.760352+0000", 2023-09-23T23:06:47.911 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031998626000000002 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.792350+0000", 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030543110000000001 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.795404+0000", 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.66251963899999999 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.912 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.457924+0000", 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034388040000000002 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.461363+0000", 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076716960000000004 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:06:47.913 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.469035+0000", 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018468185000000002 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.487503+0000", 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00090221400000000003 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.488405+0000", 2023-09-23T23:06:47.914 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00091449000000000001 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.489319+0000", 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014892130000000001 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.490809+0000", 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068482480000000004 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:06:47.915 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.497657+0000", 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0659816869999998 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:06:47.916 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:47.917 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:06:47.917 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:47.958 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:06:47.959 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:47.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:48.064 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:48.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:47 smithi151 ceph-mon[159210]: pgmap v1394: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 223 op/s; 66257/202485 objects degraded (32.722%); 3516/202485 objects misplaced (1.736%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:06:48.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3276197748' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:48.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:48.097 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:48.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:47 smithi052 ceph-mon[187743]: pgmap v1394: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 223 op/s; 66257/202485 objects degraded (32.722%); 3516/202485 objects misplaced (1.736%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:06:48.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3276197748' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:48.138 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:48.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:47 smithi052 ceph-mon[180979]: pgmap v1394: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 223 op/s; 66257/202485 objects degraded (32.722%); 3516/202485 objects misplaced (1.736%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T23:06:48.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3276197748' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:48.139 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:06:48.155 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:48.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:48.259 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:48.296 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:48.554 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:48.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:48.626 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:06:48.659 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:48.687 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:48.705 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:06:48.705 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:06:48.705 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T23:06:48.705 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T23:06:48.819 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:48.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:48.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:48.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:48 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:48.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:48 smithi151 ceph-mon[159210]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:48 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:48.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:48 smithi151 ceph-mon[159210]: osdmap e448: 8 total, 8 up, 5 in 2023-09-23T23:06:48.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:48 smithi151 ceph-mon[159210]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.888 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:48 smithi151 ceph-mon[159210]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.915 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:48.931 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:48.990 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[180979]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[180979]: osdmap e448: 8 total, 8 up, 5 in 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[180979]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[180979]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[187743]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[187743]: osdmap e448: 8 total, 8 up, 5 in 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[187743]: failed to encode map e448 with expected crc 2023-09-23T23:06:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:48 smithi052 ceph-mon[187743]: failed to encode map e448 with expected crc 2023-09-23T23:06:49.103 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:49.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:49.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:49.231 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:49.251 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:49.266 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:49.526 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:49.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:49.631 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:49.659 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:49.704 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:49.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:49.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:49.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:49.847 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:49.956 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:49.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:49.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:50.066 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:50.092 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:50.138 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:50.139 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:50.140 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:49 smithi151 ceph-mon[159210]: failed to encode map e448 with expected crc 2023-09-23T23:06:50.140 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:49 smithi151 ceph-mon[159210]: pgmap v1396: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 272 op/s; 65732/200643 objects degraded (32.761%); 3516/200643 objects misplaced (1.752%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T23:06:50.140 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:50.140 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:50.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:50.192 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[187743]: failed to encode map e448 with expected crc 2023-09-23T23:06:50.192 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[187743]: pgmap v1396: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 272 op/s; 65732/200643 objects degraded (32.761%); 3516/200643 objects misplaced (1.752%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T23:06:50.192 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:50.192 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:50.192 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[180979]: failed to encode map e448 with expected crc 2023-09-23T23:06:50.192 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[180979]: pgmap v1396: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 272 op/s; 65732/200643 objects degraded (32.761%); 3516/200643 objects misplaced (1.752%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T23:06:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:06:50.193 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:06:50.242 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:50.279 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:50.341 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:50.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:50.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:50.478 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:50.786 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:50.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:50.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:50.816 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:06:50.890 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:50.905 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 65732/200643 objects degraded (32.761%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:50.958 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:51.037 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:51.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:51.145 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:51.159 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:51.179 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:51.203 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:51.213 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:51.230 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:51.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 65732/200643 objects degraded (32.761%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:51.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 65732/200643 objects degraded (32.761%), 24 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:51.255 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:51.289 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:51.314 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:51.317 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:51.318 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:06:51.318 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T23:06:51.318 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:51.334 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:51.335 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.336 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:06:51.342 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:51.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:51.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:51.446 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:51.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:51.614 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:51.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:51.720 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:51.754 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:51.814 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:51.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:51.900 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:51 smithi151 ceph-mon[159210]: pgmap v1397: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 65732/200166 objects degraded (32.839%); 3404/200166 objects misplaced (1.701%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:51.901 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1229153862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:51.918 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:51.945 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:52.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:51 smithi052 ceph-mon[187743]: pgmap v1397: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 65732/200166 objects degraded (32.839%); 3404/200166 objects misplaced (1.701%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:52.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1229153862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:52.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:51 smithi052 ceph-mon[180979]: pgmap v1397: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 65732/200166 objects degraded (32.839%); 3404/200166 objects misplaced (1.701%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:52.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1229153862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:52.351 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:06:52.466 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:52.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:52.570 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:52.628 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:52.700 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:52.709 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:52.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:52.811 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:52.851 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:52.907 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:52.965 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:52.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:moved 0 / 297 (0%) 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:avg 59.4 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 8.54634 -> 8.54634 (expected baseline 6.89348) 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.2 with 50 -> 50 pgs (0.841751 -> 0.841751 * mean) 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.1 with 73 -> 73 pgs (1.22896 -> 1.22896 * mean) 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 0.05 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T23:06:53.031 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0280 2023-09-23T23:06:53.032 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.2802 2023-09-23T23:06:53.033 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T23:06:53.070 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:53.104 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:53.374 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:53.379 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:06:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:06:53] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T23:06:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:53.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:53.481 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:53.520 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:53.640 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:53.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:53.745 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:53.779 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:53.826 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:53.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:53.929 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:06:53.984 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:54.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:53 smithi151 ceph-mon[159210]: pgmap v1398: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 226 op/s; 65240/197508 objects degraded (33.032%); 3404/197508 objects misplaced (1.723%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:54.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:53 smithi052 ceph-mon[187743]: pgmap v1398: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 226 op/s; 65240/197508 objects degraded (33.032%); 3404/197508 objects misplaced (1.723%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:54.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:53 smithi052 ceph-mon[180979]: pgmap v1398: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 226 op/s; 65240/197508 objects degraded (33.032%); 3404/197508 objects misplaced (1.723%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:54.310 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:06:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:54.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:06:54.398 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T23:06:54.398 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T23:06:54.418 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:54.474 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:54.547 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:54.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:54.561 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:06:54.650 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:54.747 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:54.852 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:06:54.852 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:06:54.920 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:54.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:55.024 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:06:55.061 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:55.113 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:06:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:55.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:06:55.134 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:55.171 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:06:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:54 smithi151 ceph-mon[159210]: from='client.47159 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:06:55.217 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:55.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:54 smithi052 ceph-mon[187743]: from='client.47159 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:06:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:54 smithi052 ceph-mon[180979]: from='client.47159 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:06:55.245 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:55.295 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:55.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:55.399 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:06:55.441 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:55.747 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:06:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:55.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:06:55.814 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:06:55.853 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:55.887 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:55.948 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:06:55.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:55 smithi052 ceph-mon[187743]: pgmap v1399: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 226 op/s; 65240/197508 objects degraded (33.032%); 3404/197508 objects misplaced (1.723%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:55.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1205574763' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:55.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 65240/197508 objects degraded (33.032%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:55.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:55 smithi052 ceph-mon[180979]: pgmap v1399: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 226 op/s; 65240/197508 objects degraded (33.032%); 3404/197508 objects misplaced (1.723%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:55.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1205574763' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:55.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 65240/197508 objects degraded (33.032%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:56.194 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:56.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:56.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:55 smithi151 ceph-mon[159210]: pgmap v1399: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 226 op/s; 65240/197508 objects degraded (33.032%); 3404/197508 objects misplaced (1.723%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T23:06:56.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1205574763' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:06:56.203 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 65240/197508 objects degraded (33.032%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:06:56.298 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:56.364 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:56.424 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:06:56.519 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:56.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:56.631 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:06:56.670 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:57.133 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:56 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:57.133 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:56 smithi151 ceph-mon[159210]: failed to encode map e449 with expected crc 2023-09-23T23:06:57.134 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1205574763' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:06:57.134 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:56 smithi151 ceph-mon[159210]: osdmap e449: 8 total, 8 up, 5 in 2023-09-23T23:06:57.134 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:56 smithi151 ceph-mon[159210]: failed to encode map e449 with expected crc 2023-09-23T23:06:57.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[180979]: failed to encode map e449 with expected crc 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1205574763' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[180979]: osdmap e449: 8 total, 8 up, 5 in 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[180979]: failed to encode map e449 with expected crc 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[187743]: failed to encode map e449 with expected crc 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1205574763' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:06:57.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[187743]: osdmap e449: 8 total, 8 up, 5 in 2023-09-23T23:06:57.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:56 smithi052 ceph-mon[187743]: failed to encode map e449 with expected crc 2023-09-23T23:06:57.725 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:06:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:57.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:06:57.829 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:57.866 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:57 smithi052 ceph-mon[187743]: pgmap v1401: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 185 op/s; 65240/195726 objects degraded (33.332%); 3287/195726 objects misplaced (1.679%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:06:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:57 smithi052 ceph-mon[180979]: pgmap v1401: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 185 op/s; 65240/195726 objects degraded (33.332%); 3287/195726 objects misplaced (1.679%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:06:58.015 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:58.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:58.139 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:58.187 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:58.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:57 smithi151 ceph-mon[159210]: pgmap v1401: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 185 op/s; 65240/195726 objects degraded (33.332%); 3287/195726 objects misplaced (1.679%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T23:06:58.301 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:58.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:58.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:58.341 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:06:58.364 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:06:58.396 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:06:58.407 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:58.427 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:06:58.458 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:06:58.464 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:58.481 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:06:58.515 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:06:58.539 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.542 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:06:58.550 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:58.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:58.584 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:06:58.584 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:06:58.584 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:06:58.584 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [] 2023-09-23T23:06:58.584 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:06:58.655 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:06:58.729 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:58.789 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:moved 0 / 297 (0%) 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:avg 59.4 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 8.54634 -> 8.54634 (expected baseline 6.89348) 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.2 with 50 -> 50 pgs (0.841751 -> 0.841751 * mean) 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.1 with 73 -> 73 pgs (1.22896 -> 1.22896 * mean) 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 1 2023-09-23T23:06:58.794 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T23:06:58.795 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0277 2023-09-23T23:06:58.795 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.2773 2023-09-23T23:06:58.797 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T23:06:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:58.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:06:58.899 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:06:58.926 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:58.978 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:06:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:58.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:06:59.082 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:06:59.108 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/811179043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:59.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/811179043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:59.112 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:59.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/811179043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:06:59.419 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:06:59.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:59.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:06:59.493 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:06:59.523 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:06:59.566 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:06:59.588 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:06:59.958 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:06:59.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:06:59.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:06:59.966 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:59 smithi052 ceph-mon[180979]: pgmap v1402: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 220 op/s; 64706/194016 objects degraded (33.351%); 3287/194016 objects misplaced (1.694%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:06:59.966 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:06:59 smithi052 ceph-mon[180979]: from='client.56977 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:06:59.969 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T23:06:59.969 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T23:07:00.065 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:00.087 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:00.109 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:00.163 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:00.171 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:00.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:00.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:00.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:59 smithi052 ceph-mon[187743]: pgmap v1402: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 220 op/s; 64706/194016 objects degraded (33.351%); 3287/194016 objects misplaced (1.694%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:07:00.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:06:59 smithi052 ceph-mon[187743]: from='client.56977 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:00.286 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:00.335 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:59 smithi151 ceph-mon[159210]: pgmap v1402: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 220 op/s; 64706/194016 objects degraded (33.351%); 3287/194016 objects misplaced (1.694%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T23:07:00.335 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:06:59 smithi151 ceph-mon[159210]: from='client.56977 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:00.339 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:00.418 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:00.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:00.529 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:00.565 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:00.789 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:00.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:00.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:00.957 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:01.049 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:01.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:01.161 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:01.190 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:01.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 64706/194016 objects degraded (33.351%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:01.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 64706/194016 objects degraded (33.351%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 64706/194016 objects degraded (33.351%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:01.592 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:01.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:01.700 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:01.735 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:01.744 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:07:01.807 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:01.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:01.914 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:01.958 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:01.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:01 smithi052 ceph-mon[180979]: pgmap v1403: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 219 op/s; 64706/193569 objects degraded (33.428%); 3180/193569 objects misplaced (1.643%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:07:02.045 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:02.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:02.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:02.180 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:02.236 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:01 smithi151 ceph-mon[159210]: pgmap v1403: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 219 op/s; 64706/193569 objects degraded (33.428%); 3180/193569 objects misplaced (1.643%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:07:02.238 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:02.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:01 smithi052 ceph-mon[187743]: pgmap v1403: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 219 op/s; 64706/193569 objects degraded (33.428%); 3180/193569 objects misplaced (1.643%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T23:07:02.302 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:02.330 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:02.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:02.442 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:02.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:02.629 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:02.644 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:02.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:02.751 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:02.802 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:02.962 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:02.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:02.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:02 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/927098975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:02.999 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:02 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:02.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:02 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/927098975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:02.999 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:02 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:03.000 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:07:03.067 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:03.116 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:03.247 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:03.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:03.340 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:07:03.340 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:07:03.340 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T23:07:03.341 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:07:03.351 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:03.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:02 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/927098975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:03.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:02 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:03.391 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:03.458 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:03.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:03.563 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:03.596 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:07:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:07:03] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.33.4" 2023-09-23T23:07:03.600 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:03.984 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[180979]: pgmap v1404: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 247 op/s; 64075/190095 objects degraded (33.707%); 3180/190095 objects misplaced (1.673%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T23:07:03.984 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:07:03.984 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:03.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[180979]: osdmap e450: 8 total, 8 up, 5 in 2023-09-23T23:07:03.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[180979]: failed to encode map e450 with expected crc 2023-09-23T23:07:03.985 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[180979]: failed to encode map e450 with expected crc 2023-09-23T23:07:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[187743]: pgmap v1404: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 247 op/s; 64075/190095 objects degraded (33.707%); 3180/190095 objects misplaced (1.673%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T23:07:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:07:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[187743]: osdmap e450: 8 total, 8 up, 5 in 2023-09-23T23:07:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[187743]: failed to encode map e450 with expected crc 2023-09-23T23:07:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:03 smithi052 ceph-mon[187743]: failed to encode map e450 with expected crc 2023-09-23T23:07:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:03 smithi151 ceph-mon[159210]: pgmap v1404: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 247 op/s; 64075/190095 objects degraded (33.707%); 3180/190095 objects misplaced (1.673%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T23:07:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:03 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:07:04.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:03 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:04.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:03 smithi151 ceph-mon[159210]: osdmap e450: 8 total, 8 up, 5 in 2023-09-23T23:07:04.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:03 smithi151 ceph-mon[159210]: failed to encode map e450 with expected crc 2023-09-23T23:07:04.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:03 smithi151 ceph-mon[159210]: failed to encode map e450 with expected crc 2023-09-23T23:07:05.158 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:05.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:05.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:05.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:05.304 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:05.388 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:05.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:05.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:05.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:05.578 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:05.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:05.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:05 smithi151 ceph-mon[159210]: pgmap v1406: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 64075/190095 objects degraded (33.707%); 3180/190095 objects misplaced (1.673%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T23:07:05.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 64075/190095 objects degraded (33.707%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:05.584 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:05.677 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:05.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:05.796 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:05.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:05.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:05.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[180979]: pgmap v1406: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 64075/190095 objects degraded (33.707%); 3180/190095 objects misplaced (1.673%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T23:07:05.828 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 64075/190095 objects degraded (33.707%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:05.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:05.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:05.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[187743]: pgmap v1406: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 64075/190095 objects degraded (33.707%); 3180/190095 objects misplaced (1.673%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T23:07:05.829 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 64075/190095 objects degraded (33.707%), 24 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:05.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:05.902 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:07:05.913 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:05.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:06.027 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:06.076 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:06.175 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:06.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:06.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:06.281 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:06.334 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:06.571 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:06.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:06.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:06.638 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:06.666 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:07:06.689 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:06.714 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:07:06.745 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:07:06.759 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:06.790 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:07:06.826 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:07:06.853 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:07:06.855 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:06.858 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/613283378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:06.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/613283378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:06.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:06.874 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:07:06.899 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.901 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:07:06.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/613283378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:06.960 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:07.011 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:07.169 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:07.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:07.246 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:07:07.272 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:07.318 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:moved 7 / 297 (2.3569%) 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:avg 59.4 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 8.54634 -> 6.3435 (expected baseline 6.89348) 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.2 with 50 -> 52 pgs (0.841751 -> 0.875421 * mean) 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.1 with 73 -> 69 pgs (1.22896 -> 1.16162 * mean) 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:oload 110 2023-09-23T23:07:07.707 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 0.05 2023-09-23T23:07:07.708 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T23:07:07.708 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0272 2023-09-23T23:07:07.708 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.0300 2023-09-23T23:07:07.708 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1 weight 1.0000 -> 0.9500 2023-09-23T23:07:07.708 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T23:07:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:07 smithi151 ceph-mon[159210]: pgmap v1407: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 259 op/s; 64075/187932 objects degraded (34.095%); 3070/187932 objects misplaced (1.634%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T23:07:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:07 smithi151 ceph-mon[159210]: from='client.47183 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:07:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:07 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:07.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[187743]: pgmap v1407: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 259 op/s; 64075/187932 objects degraded (34.095%); 3070/187932 objects misplaced (1.634%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T23:07:07.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[187743]: from='client.47183 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:07.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:07:07.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[180979]: pgmap v1407: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 259 op/s; 64075/187932 objects degraded (34.095%); 3070/187932 objects misplaced (1.634%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T23:07:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[180979]: from='client.47183 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260'}"}]: dispatch 2023-09-23T23:07:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:07 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:07:08.027 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:08.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:08.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:08.134 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:08.231 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:08.341 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:08.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:08.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:08.459 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:08.496 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:08.696 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260'}"}]': finished 2023-09-23T23:07:08.696 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: osdmap e451: 8 total, 8 up, 5 in 2023-09-23T23:07:08.696 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.696 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.697 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.697 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.697 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.697 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[180979]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.700 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:08.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:08.804 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:08.834 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:08.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260'}"}]': finished 2023-09-23T23:07:08.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: osdmap e451: 8 total, 8 up, 5 in 2023-09-23T23:07:08.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:08 smithi151 ceph-mon[159210]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260'}"}]': finished 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: osdmap e451: 8 total, 8 up, 5 in 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.960 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:08 smithi052 ceph-mon[187743]: failed to encode map e451 with expected crc 2023-09-23T23:07:08.961 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:07:08.961 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T23:07:09.170 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:09.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:09.274 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:09.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:09.351 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:09.436 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:09.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:09.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:09.637 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:09.752 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:09.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:09.788 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:09.865 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:09.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[180979]: osdmap e452: 8 total, 8 up, 5 in 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[180979]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[180979]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[180979]: pgmap v1410: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 63416/185811 objects degraded (34.129%); 3070/185811 objects misplaced (1.652%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[180979]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[187743]: osdmap e452: 8 total, 8 up, 5 in 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[187743]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[187743]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[187743]: pgmap v1410: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 63416/185811 objects degraded (34.129%); 3070/185811 objects misplaced (1.652%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:07:09.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:09 smithi052 ceph-mon[187743]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.914 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:09.934 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:07:09.934 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:07:09.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:09 smithi151 ceph-mon[159210]: osdmap e452: 8 total, 8 up, 5 in 2023-09-23T23:07:09.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:09 smithi151 ceph-mon[159210]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:09 smithi151 ceph-mon[159210]: failed to encode map e452 with expected crc 2023-09-23T23:07:09.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:09 smithi151 ceph-mon[159210]: pgmap v1410: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 63416/185811 objects degraded (34.129%); 3070/185811 objects misplaced (1.652%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:07:09.951 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:09 smithi151 ceph-mon[159210]: failed to encode map e452 with expected crc 2023-09-23T23:07:10.432 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:07:10.675 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:10.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:10.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:10 smithi052 ceph-mon[187743]: failed to encode map e452 with expected crc 2023-09-23T23:07:10.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 63416/185811 objects degraded (34.129%), 24 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:10.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:10 smithi052 ceph-mon[180979]: failed to encode map e452 with expected crc 2023-09-23T23:07:10.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 63416/185811 objects degraded (34.129%), 24 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:10.779 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:10.822 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:10.878 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:10.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:10.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:10.986 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:11.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:10 smithi151 ceph-mon[159210]: failed to encode map e452 with expected crc 2023-09-23T23:07:11.012 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 63416/185811 objects degraded (34.129%), 24 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:11.015 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:11.148 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:11.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:11.253 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:11.290 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:11.409 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:11.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:11.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:11.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:11.563 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:11.727 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:11 smithi052 ceph-mon[187743]: pgmap v1411: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 206 op/s; 63416/185289 objects degraded (34.225%); 2954/185289 objects misplaced (1.594%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:07:11.729 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:11 smithi052 ceph-mon[180979]: pgmap v1411: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 206 op/s; 63416/185289 objects degraded (34.225%); 2954/185289 objects misplaced (1.594%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:07:12.036 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:12.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:12.141 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:12.177 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:11 smithi151 ceph-mon[159210]: pgmap v1411: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 206 op/s; 63416/185289 objects degraded (34.225%); 2954/185289 objects misplaced (1.594%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T23:07:12.346 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:07:12.512 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:12.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:12.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:12.617 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:12.675 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:12.752 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:12.813 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:12.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:12.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:12.926 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:12.957 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:13.031 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:13.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:13.141 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:13.178 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:13.274 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:13.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:13.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:13 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:13.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:13 smithi151 ceph-mon[159210]: pgmap v1412: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 331 op/s; 62831/182139 objects degraded (34.496%); 2954/182139 objects misplaced (1.622%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:07:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:13.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:13.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:13.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:13 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:13.345 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:13 smithi052 ceph-mon[187743]: pgmap v1412: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 331 op/s; 62831/182139 objects degraded (34.496%); 2954/182139 objects misplaced (1.622%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:07:13.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:13.345 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:13 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:13.346 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:13 smithi052 ceph-mon[180979]: pgmap v1412: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 331 op/s; 62831/182139 objects degraded (34.496%); 2954/182139 objects misplaced (1.622%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:07:13.400 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:13.444 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:13.519 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:13.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:13.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:13.622 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:07:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:07:13] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.33.4" 2023-09-23T23:07:13.632 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:13.661 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:13.688 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:13.787 INFO:teuthology.orchestra.run.smithi052.stdout:moved 0 / 297 (0%) 2023-09-23T23:07:13.787 INFO:teuthology.orchestra.run.smithi052.stdout:avg 59.4 2023-09-23T23:07:13.787 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 6.3435 -> 6.3435 (expected baseline 6.89348) 2023-09-23T23:07:13.787 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.2 with 52 -> 52 pgs (0.875421 -> 0.875421 * mean) 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.1 with 69 -> 69 pgs (1.16162 -> 1.16162 * mean) 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 0.05 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0266 2023-09-23T23:07:13.788 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.2663 2023-09-23T23:07:13.795 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T23:07:13.817 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:13.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:13.923 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:13.963 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:14.027 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:14.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:14.106 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:07:14.132 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:14.162 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:14.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:14 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/275104186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:07:14.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:14 smithi151 ceph-mon[159210]: from='client.37092 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:14.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:14 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/275104186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:07:14.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:14 smithi052 ceph-mon[187743]: from='client.37092 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:14.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:14 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/275104186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:07:14.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:14 smithi052 ceph-mon[180979]: from='client.37092 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:15.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:15.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:15.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:15.144 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:15.178 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:15.195 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T23:07:15.195 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T23:07:15.278 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:15.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:15.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:15.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/275104186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:07:15.302 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: osdmap e453: 8 total, 8 up, 5 in 2023-09-23T23:07:15.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.303 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[180979]: pgmap v1414: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 173 op/s; 62831/182139 objects degraded (34.496%); 2954/182139 objects misplaced (1.622%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/275104186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: osdmap e453: 8 total, 8 up, 5 in 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.305 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:15 smithi052 ceph-mon[187743]: pgmap v1414: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 173 op/s; 62831/182139 objects degraded (34.496%); 2954/182139 objects misplaced (1.622%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T23:07:15.392 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/275104186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: osdmap e453: 8 total, 8 up, 5 in 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: failed to encode map e453 with expected crc 2023-09-23T23:07:15.431 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:15 smithi151 ceph-mon[159210]: pgmap v1414: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 173 op/s; 62831/182139 objects degraded (34.496%); 2954/182139 objects misplaced (1.622%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T23:07:15.433 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:15.499 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:15.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:15.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:15.605 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:15.631 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:15.694 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:15.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:15.814 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:15.847 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:16.092 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:16 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 62831/182139 objects degraded (34.496%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:16.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:16 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 62831/182139 objects degraded (34.496%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:16.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:16 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 62831/182139 objects degraded (34.496%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:16.700 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:16.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:16.805 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:16.837 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:16.925 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:16.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:16.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:17.040 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:17.087 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:17.107 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:17 smithi052 ceph-mon[180979]: pgmap v1415: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 232 op/s; 62831/180222 objects degraded (34.863%); 2841/180222 objects misplaced (1.576%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:07:17.192 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:17.230 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:07:17.256 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:07:17.281 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:07:17.305 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:07:17.315 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:17.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:17.333 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:07:17.356 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:07:17.381 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:07:17.406 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.408 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.409 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:07:17.421 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:17.434 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:17 smithi052 ceph-mon[187743]: pgmap v1415: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 232 op/s; 62831/180222 objects degraded (34.863%); 2841/180222 objects misplaced (1.576%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:07:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:17 smithi151 ceph-mon[159210]: pgmap v1415: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 232 op/s; 62831/180222 objects degraded (34.863%); 2841/180222 objects misplaced (1.576%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T23:07:17.458 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:17.579 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:17.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:17.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:17.684 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:17.716 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:17.860 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:07:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:07:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:07:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:07:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.860 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114198 10.5 10:a53fd4ad:::benchmark_data_smithi052_461816_object17397:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.12080652099996, 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047289519, 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114198 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.861 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735122+0000", 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735120+0000", 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735128+0000", 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6639999999999998e-06 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735140+0000", 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1888999999999999e-05 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.735151+0000", 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1241e-05 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736424+0000", 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001272942 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736451+0000", 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7029000000000002e-05 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772313+0000", 2023-09-23T23:07:17.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.035862128 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772461+0000", 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014821499999999999 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.772545+0000", 2023-09-23T23:07:17.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3183000000000006e-05 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779124+0000", 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0065792929999999999 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782207+0000", 2023-09-23T23:07:17.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030828169999999998 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782247+0000", 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0198999999999999e-05 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782304+0000", 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7172000000000001e-05 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782412+0000", 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000107539 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114209 10.5 10:a8019dd6:::benchmark_data_smithi052_461816_object17408:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.869 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.11961206399997, 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.052215573, 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114209 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736317+0000", 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736316+0000", 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736321+0000", 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6640000000000003e-06 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736333+0000", 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1854e-05 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736343+0000", 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7399999999999999e-06 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736479+0000", 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013588000000000001 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736486+0000", 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3150000000000003e-06 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774671+0000", 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0381846910000001 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774856+0000", 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000185179 2023-09-23T23:07:17.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.774979+0000", 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000122843 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782160+0000", 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071812610000000004 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.875 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782451+0000", 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029058300000000002 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788381+0000", 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059298370000000003 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788422+0000", 2023-09-23T23:07:17.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1409999999999998e-05 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788532+0000", 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110103 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114217 10.5 10:a151ffa4:::benchmark_data_smithi052_461816_object17416:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:07:17.877 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.11957958300002, 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068861074, 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114217 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736349+0000", 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:17.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736352+0000", 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9399999999999998e-06 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736358+0000", 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5599999999999999e-06 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736363+0000", 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3199999999999999e-06 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737029+0000", 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000666032 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737038+0000", 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5490000000000006e-06 2023-09-23T23:07:17.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779153+0000", 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0421153320000001 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779339+0000", 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000186016 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.779477+0000", 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000137923 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782455+0000", 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029779519999999999 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790613+0000", 2023-09-23T23:07:17.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081575689999999999 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805085+0000", 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014472021 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805126+0000", 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1619000000000003e-05 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805210+0000", 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3728999999999997e-05 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114228 10.5 10:abf9b074:::benchmark_data_smithi052_461816_object17427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.11914354299995, 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068634885, 2023-09-23T23:07:17.885 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114228 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736785+0000", 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736784+0000", 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736797+0000", 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2558000000000001e-05 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736807+0000", 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7669999999999998e-06 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.890 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736815+0000", 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4789999999999996e-06 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737073+0000", 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000257502 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737079+0000", 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5180000000000001e-06 2023-09-23T23:07:17.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782471+0000", 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0453921399999999 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782629+0000", 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015792200000000001 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.782710+0000", 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0499000000000003e-05 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788710+0000", 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060006640000000002 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796703+0000", 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079926860000000006 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805261+0000", 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0085578570000000003 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805317+0000", 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5649000000000002e-05 2023-09-23T23:07:17.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805420+0000", 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000103467 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114233 10.5 10:adc2278f:::benchmark_data_smithi052_461816_object17432:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:07:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.11909828600005, 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068750884, 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114233 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736830+0000", 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736833+0000", 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0469999999999998e-06 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736840+0000", 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2130000000000002e-06 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736846+0000", 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1099999999999999e-06 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737097+0000", 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025043299999999999 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737103+0000", 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8150000000000002e-06 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785090+0000", 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.047987419 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785226+0000", 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013601299999999999 2023-09-23T23:07:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.785376+0000", 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000150407 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790542+0000", 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051658429999999998 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.796752+0000", 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0062099620000000003 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805468+0000", 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0087155189999999997 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805509+0000", 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1789000000000003e-05 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805581+0000", 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1771000000000001e-05 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114234 10.5 10:a1bb5c32:::benchmark_data_smithi052_461816_object17433:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.119077662, 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.068915523, 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114234 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736851+0000", 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736853+0000", 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3379999999999999e-06 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736859+0000", 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0499999999999997e-06 2023-09-23T23:07:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736870+0000", 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1021e-05 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737132+0000", 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00026153199999999998 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737138+0000", 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9839999999999998e-06 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786800+0000", 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.049662885 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.786974+0000", 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017351000000000001 2023-09-23T23:07:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.787107+0000", 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013272100000000001 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790552+0000", 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034448650000000001 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797758+0000", 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0072065200000000001 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805623+0000", 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0078654350000000005 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805667+0000", 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3665000000000003e-05 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.805766+0000", 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9365999999999997e-05 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114235 10.5 10:a3d19bc6:::benchmark_data_smithi052_461816_object17434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.11905404799995, 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081414576, 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114235 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736875+0000", 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736874+0000", 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736877+0000", 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8949999999999998e-06 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736883+0000", 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3459999999999999e-06 2023-09-23T23:07:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736891+0000", 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8050000000000007e-06 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737166+0000", 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027465399999999999 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737172+0000", 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.207e-06 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788737+0000", 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0515644399999999 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788877+0000", 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014082 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.788968+0000", 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.0167999999999995e-05 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792267+0000", 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003299256 2023-09-23T23:07:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797797+0000", 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.005530274 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818146+0000", 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020349151999999999 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818185+0000", 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8575000000000003e-05 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818289+0000", 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010435899999999999 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:114248 10.5 10:a5b9c0ca:::benchmark_data_smithi052_461816_object17447:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e353)", 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 552.11903288799999, 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.081611235, 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 114248 2023-09-23T23:07:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736896+0000", 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736895+0000", 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736898+0000", 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4250000000000001e-06 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736904+0000", 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3509999999999998e-06 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.736909+0000", 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2310000000000004e-06 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737193+0000", 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000283885 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:05.737199+0000", 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9610000000000002e-06 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790684+0000", 2023-09-23T23:07:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.053484358 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790817+0000", 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000133693 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 3,5", 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.790895+0000", 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8218000000000002e-05 2023-09-23T23:07:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.792358+0000", 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001462758 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.797833+0000", 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0054744759999999998 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818343+0000", 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020510347000000002 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818398+0000", 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4530999999999997e-05 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:06.818507+0000", 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010940100000000001 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:116534 10.6 10:62c2756c:::benchmark_data_smithi052_461816_object19733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 510.97639784299997, 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18822530200000001, 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 116534 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879531+0000", 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879530+0000", 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879535+0000", 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1139999999999997e-06 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879547+0000", 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1918e-05 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879555+0000", 2023-09-23T23:07:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1459999999999996e-06 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879602+0000", 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6768999999999999e-05 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879810+0000", 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00020844799999999999 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.879951+0000", 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014103400000000001 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.881979+0000", 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020274709999999999 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:46.883452+0000", 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001472985 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067651+0000", 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18419902199999999 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067709+0000", 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8180000000000002e-05 2023-09-23T23:07:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T22:58:47.067756+0000", 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7275999999999999e-05 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138140 10.8 10:10bb8653:::benchmark_data_smithi052_461816_object41339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:07:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 410.24319575499999, 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.272336093, 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138140 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612731+0000", 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612741+0000", 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4800000000000007e-06 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612762+0000", 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1761000000000001e-05 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612772+0000", 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6229999999999992e-06 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612837+0000", 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4789999999999995e-05 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613071+0000", 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023441400000000001 2023-09-23T23:07:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613293+0000", 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022140800000000001 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.616187+0000", 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028945080000000001 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.618246+0000", 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020593949999999999 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883729+0000", 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265482459 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883802+0000", 2023-09-23T23:07:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3008000000000003e-05 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.885069+0000", 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012670579999999999 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138142 10.8 10:10a11927:::benchmark_data_smithi052_461816_object41341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:07:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 410.22910429000001, 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26049065999999998, 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138142 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626822+0000", 2023-09-23T23:07:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626834+0000", 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626866+0000", 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1934999999999998e-05 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626881+0000", 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5491000000000001e-05 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626950+0000", 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8701000000000005e-05 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627110+0000", 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000160347 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627220+0000", 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110009 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.629339+0000", 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021183280000000001 2023-09-23T23:07:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.635341+0000", 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060026009999999998 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886154+0000", 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25081289200000001 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886210+0000", 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5321000000000001e-05 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.887315+0000", 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001105396 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:140723 10.6 10:68c22ebc:::benchmark_data_smithi052_461816_object43922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 398.64090534299999, 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.195061394, 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140723 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215022+0000", 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215028+0000", 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215039+0000", 2023-09-23T23:07:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0998e-05 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215044+0000", 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6380000000000001e-06 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215089+0000", 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4929000000000003e-05 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215188+0000", 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8741999999999998e-05 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215268+0000", 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0049000000000006e-05 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217087+0000", 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018184539999999999 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217880+0000", 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079313599999999995 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410012+0000", 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19213219200000001 2023-09-23T23:07:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410050+0000", 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8262000000000001e-05 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410085+0000", 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4588000000000002e-05 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:188923 10.d e403/354 10:ba1021e3:::benchmark_data_smithi052_461816_object92122:head v 403'11913, mlcod=344'7206)", 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 247.10148701700001, 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.205832343, 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754440+0000", 2023-09-23T23:07:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754450+0000", 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3270000000000007e-06 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754462+0000", 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2313999999999999e-05 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754470+0000", 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2209999999999997e-06 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754510+0000", 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9437999999999997e-05 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754531+0000", 2023-09-23T23:07:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1557999999999999e-05 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.960234+0000", 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20570290699999999 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.960274+0000", 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9780999999999997e-05 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:44802 11.6 11:6ab43c8b:::benchmark_data_smithi052_606809_object44801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 169.93800330900001, 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22081410400000001, 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 44802 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917924+0000", 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917950+0000", 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5999e-05 2023-09-23T23:07:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917964+0000", 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4412e-05 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917972+0000", 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2800000000000003e-06 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918102+0000", 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012992200000000001 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918229+0000", 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012657599999999999 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918357+0000", 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012783599999999999 2023-09-23T23:07:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.920436+0000", 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020790959999999999 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.921317+0000", 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088133199999999997 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138645+0000", 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21732747999999999 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138699+0000", 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4261000000000001e-05 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138739+0000", 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0448000000000003e-05 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:48637 11.6 11:6a40531f:::benchmark_data_smithi052_606809_object48636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e418)", 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 165.581418138, 2023-09-23T23:07:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18496931799999999, 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48637 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:07:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274531+0000", 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1537000000000001e-05 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274548+0000", 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6458000000000001e-05 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274555+0000", 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0389999999999999e-06 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.278791+0000", 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042367860000000002 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.278955+0000", 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016336700000000001 2023-09-23T23:07:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.279146+0000", 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019095900000000001 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283306+0000", 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041598900000000003 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283525+0000", 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021912200000000001 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459382+0000", 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17585730099999999 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459433+0000", 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1341999999999997e-05 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459480+0000", 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6394999999999999e-05 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:63348 11.6 11:69e6f9e0:::benchmark_data_smithi052_606809_object63347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e419)", 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:50.778092+0000", 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 147.07783687599999, 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19583917300000001, 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 63348 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778092+0000", 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778092+0000", 2023-09-23T23:07:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778090+0000", 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778116+0000", 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5840999999999999e-05 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778134+0000", 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8145e-05 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778143+0000", 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7329999999999992e-06 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778197+0000", 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4280999999999997e-05 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778334+0000", 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013696300000000001 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778463+0000", 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012883999999999999 2023-09-23T23:07:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790115+0000", 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011652199 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791359+0000", 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012437889999999999 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.973800+0000", 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18244153799999999 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.973876+0000", 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5632999999999999e-05 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.973931+0000", 2023-09-23T23:07:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4935999999999997e-05 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:63357 11.6 11:631bdb38:::benchmark_data_smithi052_606809_object63356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e419)", 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:50.790896+0000", 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 147.06503217700001, 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.186466243, 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 63357 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790896+0000", 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790896+0000", 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790895+0000", 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790919+0000", 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3969e-05 2023-09-23T23:07:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790934+0000", 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.539e-05 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790942+0000", 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791391+0000", 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00044866900000000001 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791559+0000", 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168643 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791665+0000", 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105408 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.793549+0000", 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00188408 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.793803+0000", 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025422800000000001 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.977275+0000", 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18347145100000001 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.977326+0000", 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1465000000000003e-05 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.977363+0000", 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6607999999999999e-05 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66508 11.6 11:6896caef:::benchmark_data_smithi052_606809_object66507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.685256+0000", 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 143.17067221100001, 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.233501501, 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66508 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685256+0000", 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685256+0000", 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685255+0000", 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685281+0000", 2023-09-23T23:07:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6123e-05 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685295+0000", 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4019000000000001e-05 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685303+0000", 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0209999999999999e-06 2023-09-23T23:07:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.687147+0000", 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001843988 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.687286+0000", 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139083 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.687419+0000", 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013247500000000001 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.690234+0000", 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028158459999999999 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.693191+0000", 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029565989999999999 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918666+0000", 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22547516100000001 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918721+0000", 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5090999999999998e-05 2023-09-23T23:07:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918758+0000", 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6711000000000001e-05 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72298 11.6 11:6e6f37f0:::benchmark_data_smithi052_606809_object72297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-09-23T23:07:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.400540+0000", 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 132.45538842900001, 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20392650300000001, 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72298 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400540+0000", 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400540+0000", 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400539+0000", 2023-09-23T23:07:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400561+0000", 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2129000000000001e-05 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400574+0000", 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3552e-05 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400582+0000", 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4749999999999996e-06 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400637+0000", 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5077000000000001e-05 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400792+0000", 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015541700000000001 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400939+0000", 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000146766 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.404192+0000", 2023-09-23T23:07:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003252376 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.406041+0000", 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001849313 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.604379+0000", 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19833788699999999 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.604427+0000", 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8018e-05 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.604467+0000", 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9919000000000002e-05 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:85554 11.6 11:6191145a:::benchmark_data_smithi052_606809_object3400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:34.193299+0000", 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 103.662629391, 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.260360021, 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 85554 2023-09-23T23:07:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193299+0000", 2023-09-23T23:07:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193299+0000", 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193297+0000", 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193319+0000", 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2223000000000001e-05 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193330+0000", 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0623e-05 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193336+0000", 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9410000000000004e-06 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193385+0000", 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9261999999999997e-05 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193470+0000", 2023-09-23T23:07:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5123000000000005e-05 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193534+0000", 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4336000000000002e-05 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.194964+0000", 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014301019999999999 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.195157+0000", 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019253299999999999 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.453544+0000", 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25838723800000002 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.453609+0000", 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5129999999999995e-05 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.453659+0000", 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9976999999999999e-05 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:18.013 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:07:18.439 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:18 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3555408746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:18.439 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:18 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3555408746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:18.448 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:18 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3555408746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:18.557 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:07:18.583 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:18.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:18.690 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:18.719 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:19.106 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:19.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:19.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:19.212 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:19.265 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:19.313 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:19.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:19.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:19.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:19.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:19 smithi151 ceph-mon[159210]: pgmap v1416: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 62216/178287 objects degraded (34.897%); 2841/178287 objects misplaced (1.593%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:07:19.482 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:19.489 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:19 smithi052 ceph-mon[187743]: pgmap v1416: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 62216/178287 objects degraded (34.897%); 2841/178287 objects misplaced (1.593%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:07:19.489 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:19 smithi052 ceph-mon[180979]: pgmap v1416: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 62216/178287 objects degraded (34.897%); 2841/178287 objects misplaced (1.593%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:07:19.557 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:19.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:19.660 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:19.693 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:19.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:07:19.919 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:19.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:20.028 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:20.052 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:20.097 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:20.182 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:20.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:20.308 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:20.406 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:20.485 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:20.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:20.602 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:20.660 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:20.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 62216/178287 objects degraded (34.897%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:20.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:20.765 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:20.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 62216/178287 objects degraded (34.897%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:20.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:20.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:20.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 62216/178287 objects degraded (34.897%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:20.872 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:20.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:20.913 INFO:teuthology.orchestra.run.smithi052.stdout:moved 0 / 297 (0%) 2023-09-23T23:07:20.913 INFO:teuthology.orchestra.run.smithi052.stdout:avg 59.4 2023-09-23T23:07:20.913 INFO:teuthology.orchestra.run.smithi052.stdout:stddev 6.3435 -> 6.3435 (expected baseline 6.89348) 2023-09-23T23:07:20.913 INFO:teuthology.orchestra.run.smithi052.stdout:min osd.2 with 52 -> 52 pgs (0.875421 -> 0.875421 * mean) 2023-09-23T23:07:20.913 INFO:teuthology.orchestra.run.smithi052.stdout:max osd.1 with 69 -> 69 pgs (1.16162 -> 1.16162 * mean) 2023-09-23T23:07:20.913 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:07:20.914 INFO:teuthology.orchestra.run.smithi052.stdout:oload 1000 2023-09-23T23:07:20.914 INFO:teuthology.orchestra.run.smithi052.stdout:max_change 0.05 2023-09-23T23:07:20.914 INFO:teuthology.orchestra.run.smithi052.stdout:max_change_osds 4 2023-09-23T23:07:20.914 INFO:teuthology.orchestra.run.smithi052.stdout:average_utilization 0.0262 2023-09-23T23:07:20.914 INFO:teuthology.orchestra.run.smithi052.stdout:overload_utilization 0.2622 2023-09-23T23:07:20.918 INFO:teuthology.orchestra.run.smithi052.stderr:no change 2023-09-23T23:07:20.979 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:21.016 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:21.089 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:21.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:21.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:21.203 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:21.265 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:21.366 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:21.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:21.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:21.475 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:21.508 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:21.514 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:07:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:21 smithi151 ceph-mon[159210]: pgmap v1417: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 62216/177828 objects degraded (34.987%); 2730/177828 objects misplaced (1.535%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:07:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:21 smithi151 ceph-mon[159210]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:21.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:21 smithi052 ceph-mon[187743]: pgmap v1417: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 62216/177828 objects degraded (34.987%); 2730/177828 objects misplaced (1.535%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:07:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:21 smithi052 ceph-mon[187743]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:21 smithi052 ceph-mon[180979]: pgmap v1417: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 62216/177828 objects degraded (34.987%); 2730/177828 objects misplaced (1.535%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T23:07:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:21 smithi052 ceph-mon[180979]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T23:07:22.083 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:22.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:22.188 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:22.217 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:22.265 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:22.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:22.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:22.373 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:22.422 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:22.611 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:22.684 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:22.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:22.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:22.792 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:22.840 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:22.964 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.060 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:23.104 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:23.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:23.214 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.247 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:23.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:07:23.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:22 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.258 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:23.436 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:23.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:22 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:22 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:22 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:07:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:22 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:07:23.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:22 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:23.540 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:23.565 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:07:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:07:23] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.33.4" 2023-09-23T23:07:23.568 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:23.615 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:23.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:23.720 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:23.758 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:23.982 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:23.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:23.997 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:07:24.094 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:24.134 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:24.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:23 smithi052 ceph-mon[187743]: pgmap v1418: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 61865/174897 objects degraded (35.372%); 2730/174897 objects misplaced (1.561%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T23:07:24.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3253135041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:24.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:24.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:23 smithi052 ceph-mon[180979]: pgmap v1418: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 61865/174897 objects degraded (35.372%); 2730/174897 objects misplaced (1.561%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T23:07:24.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3253135041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:24.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:24.350 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:24.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:24.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:23 smithi151 ceph-mon[159210]: pgmap v1418: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 61865/174897 objects degraded (35.372%); 2730/174897 objects misplaced (1.561%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T23:07:24.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3253135041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:24.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:24.457 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:24.494 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:24.548 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:24.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:24.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:24.656 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:24.685 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:24.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:24.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[180979]: failed to encode map e454 with expected crc 2023-09-23T23:07:24.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:24.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[180979]: osdmap e454: 8 total, 8 up, 5 in 2023-09-23T23:07:24.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[180979]: failed to encode map e454 with expected crc 2023-09-23T23:07:24.972 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[180979]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.209 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:25.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:25.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:25.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[187743]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:25.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[187743]: osdmap e454: 8 total, 8 up, 5 in 2023-09-23T23:07:25.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[187743]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.223 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:24 smithi052 ceph-mon[187743]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.324 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:25.403 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:24 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:25.404 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:24 smithi151 ceph-mon[159210]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.404 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:24 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:25.404 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:24 smithi151 ceph-mon[159210]: osdmap e454: 8 total, 8 up, 5 in 2023-09-23T23:07:25.404 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:24 smithi151 ceph-mon[159210]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.404 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:24 smithi151 ceph-mon[159210]: failed to encode map e454 with expected crc 2023-09-23T23:07:25.406 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:25.498 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:25.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:25.612 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:25.646 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:26.000 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:26.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:26.104 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:26.138 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:07:26.138 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:07:26.138 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.138 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:107427 11.d 11:bfad620a:::benchmark_data_smithi052_606809_object25273:head [delete] snapc 0=[] ondisk+write+known_if_redirected e454)", 2023-09-23T23:07:26.138 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:07:26.130627+0000", 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 0.0063253889999999998, 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0063488479999999998, 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "waiting for sub ops", 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 107427 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:26.139 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130627+0000", 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130627+0000", 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:26.140 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130627+0000", 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130631+0000", 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6310000000000002e-06 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130642+0000", 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0774999999999999e-05 2023-09-23T23:07:26.141 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130647+0000", 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1499999999999998e-06 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130707+0000", 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9435999999999998e-05 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.142 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.130904+0000", 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000197324 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.131115+0000", 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021088500000000001 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:26.143 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.133812+0000", 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002697554 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:26.135067+0000", 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012550910000000001 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: ], 2023-09-23T23:07:26.144 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 1 2023-09-23T23:07:26.145 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:07:26.145 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:26.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:25 smithi052 ceph-mon[187743]: pgmap v1420: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 264 op/s; 61549/174207 objects degraded (35.331%); 2730/174207 objects misplaced (1.567%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:26.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 61549/174207 objects degraded (35.331%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:26.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:25 smithi052 ceph-mon[180979]: pgmap v1420: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 264 op/s; 61549/174207 objects degraded (35.331%); 2730/174207 objects misplaced (1.567%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:26.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 61549/174207 objects degraded (35.331%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:26.456 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:26.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:25 smithi151 ceph-mon[159210]: pgmap v1420: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 264 op/s; 61549/174207 objects degraded (35.331%); 2730/174207 objects misplaced (1.567%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:26.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 61549/174207 objects degraded (35.331%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:26.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:26.563 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:26.588 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:26.658 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:26.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:26.764 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:26.789 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:26.832 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:26.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:26.937 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:26.962 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:27.006 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:27.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:27.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:27.023 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:07:27.023 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:07:27.023 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:07:27.024 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:07:27.024 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:07:27.024 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:07:27.058 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:07:27.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/539558432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:27.081 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/539558432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:27.082 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.084 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:07:27.104 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:07:27.113 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:27.148 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:27.250 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:07:27.272 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/539558432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:27.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:27.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:27.382 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:27.412 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:27.641 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:07:27.641 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:07:27.641 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-23T23:07:27.641 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:07:27.667 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:27.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:27.772 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:27.824 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:27.893 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:27.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:27.920 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:27.998 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:28.007 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:28.042 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:28.110 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:28.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:28.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:28.120 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:27 smithi151 ceph-mon[159210]: pgmap v1421: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 61549/172302 objects degraded (35.722%); 2620/172302 objects misplaced (1.521%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:28.220 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:28.236 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:27 smithi052 ceph-mon[180979]: pgmap v1421: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 61549/172302 objects degraded (35.722%); 2620/172302 objects misplaced (1.521%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:28.236 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:27 smithi052 ceph-mon[187743]: pgmap v1421: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 61549/172302 objects degraded (35.722%); 2620/172302 objects misplaced (1.521%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:28.265 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:28.403 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:28.695 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:28.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:28.800 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:28.828 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:29.216 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:29.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:29.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:29.321 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:29.354 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:29.581 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:29.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:29.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:29.686 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:29.715 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:29.770 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:29.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:29.818 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:07:29.874 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:29.931 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:30.037 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:30.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:30.148 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:29 smithi052 ceph-mon[187743]: pgmap v1422: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 60961/170400 objects degraded (35.775%); 2620/170400 objects misplaced (1.538%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:30.148 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:30.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:29 smithi052 ceph-mon[180979]: pgmap v1422: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 60961/170400 objects degraded (35.775%); 2620/170400 objects misplaced (1.538%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:30.185 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:29 smithi151 ceph-mon[159210]: pgmap v1422: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 60961/170400 objects degraded (35.775%); 2620/170400 objects misplaced (1.538%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:30.459 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:30.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:30.526 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:07:30.527 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:07:30.565 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:30.621 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:30.688 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:30.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:30.803 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:30.848 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:31.212 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 60961/170400 objects degraded (35.775%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:31.213 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 60961/170400 objects degraded (35.775%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:31.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 60961/170400 objects degraded (35.775%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:31.948 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:31.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:32.017 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:07:32.051 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:32.088 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:07:32.097 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:32.202 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:32.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:31 smithi052 ceph-mon[180979]: pgmap v1423: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 265 op/s; 60961/169872 objects degraded (35.886%); 2512/169872 objects misplaced (1.479%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:32.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/754661131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:07:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:31 smithi052 ceph-mon[187743]: pgmap v1423: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 265 op/s; 60961/169872 objects degraded (35.886%); 2512/169872 objects misplaced (1.479%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:32.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/754661131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:07:32.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:32.387 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:31 smithi151 ceph-mon[159210]: pgmap v1423: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 265 op/s; 60961/169872 objects degraded (35.886%); 2512/169872 objects misplaced (1.479%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T23:07:32.388 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/754661131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:07:32.392 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:32.455 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:32.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:32.563 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:32.591 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:33.000 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:33.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:33.109 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:33.137 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:33.152 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:33.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/754661131' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: osdmap e455: 8 total, 8 up, 5 in 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:32 smithi052 ceph-mon[180979]: pgmap v1425: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 280 op/s; 60561/166827 objects degraded (36.302%); 2512/166827 objects misplaced (1.506%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/754661131' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: osdmap e455: 8 total, 8 up, 5 in 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.238 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:33 smithi052 ceph-mon[187743]: pgmap v1425: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 280 op/s; 60561/166827 objects degraded (36.302%); 2512/166827 objects misplaced (1.506%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:07:33.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:33.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:33.340 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:33.395 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/754661131' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:07:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: osdmap e455: 8 total, 8 up, 5 in 2023-09-23T23:07:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: failed to encode map e455 with expected crc 2023-09-23T23:07:33.396 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:32 smithi151 ceph-mon[159210]: pgmap v1425: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 280 op/s; 60561/166827 objects degraded (36.302%); 2512/166827 objects misplaced (1.506%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:07:33.398 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:33.416 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:33.461 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:33.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:33.489 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:07:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:07:33] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.33.4" 2023-09-23T23:07:33.568 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:33.619 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:33.727 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:07:33.744 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:07:33.847 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:33.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:33.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:34.028 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:34.131 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:34 smithi052 ceph-mon[187743]: from='client.47231 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:07:34.132 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:34 smithi052 ceph-mon[180979]: from='client.47231 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:07:34.139 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:34.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:34.262 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:34.288 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:34 smithi151 ceph-mon[159210]: from='client.47231 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:07:34.293 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:34.369 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:34.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:34.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:34.478 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:34.523 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:34.640 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:34.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:34.750 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:34.797 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:34.873 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:34.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:34.981 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:35.004 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:07:35.038 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:35.114 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:35.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:35.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:35.228 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:35.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:35.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:35 smithi052 ceph-mon[187743]: pgmap v1426: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 268 op/s; 60269/166155 objects degraded (36.273%); 2512/166155 objects misplaced (1.512%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T23:07:35.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:35.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:35 smithi052 ceph-mon[180979]: pgmap v1426: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 268 op/s; 60269/166155 objects degraded (36.273%); 2512/166155 objects misplaced (1.512%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T23:07:35.265 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:35.333 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:35.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:35.339 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:35.339 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:35 smithi151 ceph-mon[159210]: pgmap v1426: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 268 op/s; 60269/166155 objects degraded (36.273%); 2512/166155 objects misplaced (1.512%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T23:07:35.439 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:35.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:35.506 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1425,"stamp":"2023-09-23T23:07:32.746937+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3643348920,"num_objects":55609,"num_object_clones":0,"num_object_copies":166827,"num_objects_missing_on_primary":7730,"num_objects_missing":7730,"num_objects_degraded":60561,"num_objects_misplaced":2512,"num_objects_unfound":0,"num_objects_dirty":55609,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":108979,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18858,"num_bytes_recovered":1204965824,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":153608,"ondisk_log_size":153608,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":11834908,"kb_used_data":8173876,"kb_used_omap":99,"kb_used_meta":3660892,"kb_avail":456890852,"statfs":{"total":479975178240,"available":467856232448,"internally_reserved":0,"allocated":8370049024,"data_stored":8362443781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3748753430},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,0,2,1,4,7,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-161218560,"num_objects":-2460,"num_object_clones":0,"num_object_copies":-7380,"num_objects_missing_on_primary":-209,"num_objects_missing":-209,"num_objects_degraded":-988,"num_objects_misplaced":-218,"num_objects_unfound":0,"num_objects_dirty":-2460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2460,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":967,"num_bytes_recovered":63373312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.771956"},"pg_stats":[{"pgid":"11.12","version":"454'6075","reported_seq":6149,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.627663+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:07:31.627663+0000","last_peered":"2023-09-23T23:07:31.627663+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:07:31.627663+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113278976,"num_objects":1729,"num_object_clones":0,"num_object_copies":5187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3412,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1760},{"shards":"2,3","objects":-728}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":466,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102767+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:07:14.102767+0000","last_peered":"2023-09-23T23:07:14.102767+0000","last_clean":"2023-09-23T23:07:14.102767+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:07:14.102767+0000","last_undegraded":"2023-09-23T23:07:14.102767+0000","last_fullsized":"2023-09-23T23:07:14.102767+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":126,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:07:23.981826+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:07:23.981826+0000","last_peered":"2023-09-23T23:07:23.981826+0000","last_clean":"2023-09-23T23:07:23.981826+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:07:23.981826+0000","last_undegraded":"2023-09-23T23:07:23.981826+0000","last_fullsized":"2023-09-23T23:07:23.981826+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"454'6071","reported_seq":7440,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:07:31.980930+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:07:31.980930+0000","last_peered":"2023-09-23T23:07:31.980930+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:07:31.980930+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:07:31.980930+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112230400,"num_objects":1713,"num_object_clones":0,"num_object_copies":5139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1999,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3418,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":644,"num_bytes_recovered":42139648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":1386},{"shards":"0,3,6","objects":270},{"shards":"2,3,6","objects":57}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102170+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:07:14.102170+0000","last_peered":"2023-09-23T23:07:14.102170+0000","last_clean":"2023-09-23T23:07:14.102170+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:07:14.102170+0000","last_undegraded":"2023-09-23T23:07:14.102170+0000","last_fullsized":"2023-09-23T23:07:14.102170+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":471,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102166+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:07:14.102166+0000","last_peered":"2023-09-23T23:07:14.102166+0000","last_clean":"2023-09-23T23:07:14.102166+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:07:14.102166+0000","last_undegraded":"2023-09-23T23:07:14.102166+0000","last_fullsized":"2023-09-23T23:07:14.102166+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"454'6073","reported_seq":10109,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:07:31.684555+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:07:31.684555+0000","last_peered":"2023-09-23T23:07:31.684555+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:07:31.684555+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:07:31.684555+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110755840,"num_objects":1690,"num_object_clones":0,"num_object_copies":5070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3417,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":163},{"shards":"1,3,6","objects":1527}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":32,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.099657+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:07:14.099657+0000","last_peered":"2023-09-23T23:07:14.099657+0000","last_clean":"2023-09-23T23:07:14.099657+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:07:14.099657+0000","last_undegraded":"2023-09-23T23:07:14.099657+0000","last_fullsized":"2023-09-23T23:07:14.099657+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.091302+0000","last_change":"2023-09-23T23:07:08.745477+0000","last_active":"2023-09-23T23:07:14.091302+0000","last_peered":"2023-09-23T23:07:14.091302+0000","last_clean":"2023-09-23T23:07:14.091302+0000","last_became_active":"2023-09-23T23:07:08.745150+0000","last_became_peered":"2023-09-23T23:07:08.745150+0000","last_unstale":"2023-09-23T23:07:14.091302+0000","last_undegraded":"2023-09-23T23:07:14.091302+0000","last_fullsized":"2023-09-23T23:07:14.091302+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:06:11.046568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":397,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100304+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:07:14.100304+0000","last_peered":"2023-09-23T23:07:14.100304+0000","last_clean":"2023-09-23T23:07:14.100304+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:07:14.100304+0000","last_undegraded":"2023-09-23T23:07:14.100304+0000","last_fullsized":"2023-09-23T23:07:14.100304+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":144,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102978+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:07:14.102978+0000","last_peered":"2023-09-23T23:07:14.102978+0000","last_clean":"2023-09-23T23:07:14.102978+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:07:14.102978+0000","last_undegraded":"2023-09-23T23:07:14.102978+0000","last_fullsized":"2023-09-23T23:07:14.102978+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":259,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102329+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:07:14.102329+0000","last_peered":"2023-09-23T23:07:14.102329+0000","last_clean":"2023-09-23T23:07:14.102329+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:07:14.102329+0000","last_undegraded":"2023-09-23T23:07:14.102329+0000","last_fullsized":"2023-09-23T23:07:14.102329+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":158,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.099929+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:07:14.099929+0000","last_peered":"2023-09-23T23:07:14.099929+0000","last_clean":"2023-09-23T23:07:14.099929+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:07:14.099929+0000","last_undegraded":"2023-09-23T23:07:14.099929+0000","last_fullsized":"2023-09-23T23:07:14.099929+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":191,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101178+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:07:14.101178+0000","last_peered":"2023-09-23T23:07:14.101178+0000","last_clean":"2023-09-23T23:07:14.101178+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:07:14.101178+0000","last_undegraded":"2023-09-23T23:07:14.101178+0000","last_fullsized":"2023-09-23T23:07:14.101178+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":199,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.092167+0000","last_change":"2023-09-23T23:07:08.762606+0000","last_active":"2023-09-23T23:07:14.092167+0000","last_peered":"2023-09-23T23:07:14.092167+0000","last_clean":"2023-09-23T23:07:14.092167+0000","last_became_active":"2023-09-23T23:07:08.762252+0000","last_became_peered":"2023-09-23T23:07:08.762252+0000","last_unstale":"2023-09-23T23:07:14.092167+0000","last_undegraded":"2023-09-23T23:07:14.092167+0000","last_fullsized":"2023-09-23T23:07:14.092167+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:11.193211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":544,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103243+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:07:14.103243+0000","last_peered":"2023-09-23T23:07:14.103243+0000","last_clean":"2023-09-23T23:07:14.103243+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:07:14.103243+0000","last_undegraded":"2023-09-23T23:07:14.103243+0000","last_fullsized":"2023-09-23T23:07:14.103243+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":437,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102872+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:07:14.102872+0000","last_peered":"2023-09-23T23:07:14.102872+0000","last_clean":"2023-09-23T23:07:14.102872+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:07:14.102872+0000","last_undegraded":"2023-09-23T23:07:14.102872+0000","last_fullsized":"2023-09-23T23:07:14.102872+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":369,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102953+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:07:14.102953+0000","last_peered":"2023-09-23T23:07:14.102953+0000","last_clean":"2023-09-23T23:07:14.102953+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:07:14.102953+0000","last_undegraded":"2023-09-23T23:07:14.102953+0000","last_fullsized":"2023-09-23T23:07:14.102953+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":137,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100461+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:07:14.100461+0000","last_peered":"2023-09-23T23:07:14.100461+0000","last_clean":"2023-09-23T23:07:14.100461+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:07:14.100461+0000","last_undegraded":"2023-09-23T23:07:14.100461+0000","last_fullsized":"2023-09-23T23:07:14.100461+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":455,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102620+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:07:14.102620+0000","last_peered":"2023-09-23T23:07:14.102620+0000","last_clean":"2023-09-23T23:07:14.102620+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:07:14.102620+0000","last_undegraded":"2023-09-23T23:07:14.102620+0000","last_fullsized":"2023-09-23T23:07:14.102620+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":360,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101087+0000","last_change":"2023-09-23T23:07:08.758074+0000","last_active":"2023-09-23T23:07:14.101087+0000","last_peered":"2023-09-23T23:07:14.101087+0000","last_clean":"2023-09-23T23:07:14.101087+0000","last_became_active":"2023-09-23T23:07:08.757779+0000","last_became_peered":"2023-09-23T23:07:08.757779+0000","last_unstale":"2023-09-23T23:07:14.101087+0000","last_undegraded":"2023-09-23T23:07:14.101087+0000","last_fullsized":"2023-09-23T23:07:14.101087+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:19:13.524487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"454'5987","reported_seq":11135,"reported_epoch":454,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:07:31.256155+0000","last_change":"2023-09-23T23:06:32.889539+0000","last_active":"2023-09-23T23:07:31.256155+0000","last_peered":"2023-09-23T23:07:31.256155+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:06:32.875369+0000","last_became_peered":"2023-09-23T23:06:32.875369+0000","last_unstale":"2023-09-23T23:07:31.256155+0000","last_undegraded":"2023-09-23T23:07:31.256155+0000","last_fullsized":"2023-09-23T23:07:31.256155+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115113984,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3338,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":1757}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":357,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102467+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:07:14.102467+0000","last_peered":"2023-09-23T23:07:14.102467+0000","last_clean":"2023-09-23T23:07:14.102467+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:07:14.102467+0000","last_undegraded":"2023-09-23T23:07:14.102467+0000","last_fullsized":"2023-09-23T23:07:14.102467+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":479,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:16.982089+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:07:16.982089+0000","last_peered":"2023-09-23T23:07:16.982089+0000","last_clean":"2023-09-23T23:07:16.982089+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:07:16.982089+0000","last_undegraded":"2023-09-23T23:07:16.982089+0000","last_fullsized":"2023-09-23T23:07:16.982089+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"454'6127","reported_seq":8309,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.894272+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:07:31.894272+0000","last_peered":"2023-09-23T23:07:31.894272+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:07:31.894272+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113868800,"num_objects":1738,"num_object_clones":0,"num_object_copies":5214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3437,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":749,"num_bytes_recovered":48889856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1394},{"shards":"3,5,6","objects":317},{"shards":"3,6","objects":27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":67,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.099757+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:07:14.099757+0000","last_peered":"2023-09-23T23:07:14.099757+0000","last_clean":"2023-09-23T23:07:14.099757+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:07:14.099757+0000","last_undegraded":"2023-09-23T23:07:14.099757+0000","last_fullsized":"2023-09-23T23:07:14.099757+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":150,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100389+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:07:14.100389+0000","last_peered":"2023-09-23T23:07:14.100389+0000","last_clean":"2023-09-23T23:07:14.100389+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:07:14.100389+0000","last_undegraded":"2023-09-23T23:07:14.100389+0000","last_fullsized":"2023-09-23T23:07:14.100389+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"454'6039","reported_seq":6118,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:30.385506+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:07:30.385506+0000","last_peered":"2023-09-23T23:07:30.385506+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:07:30.385506+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6039,"log_dups_size":0,"ondisk_log_size":6039,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111902720,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-658},{"shards":"1,4,5","objects":1749}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":364,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.106464+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:07:14.106464+0000","last_peered":"2023-09-23T23:07:14.106464+0000","last_clean":"2023-09-23T23:07:14.106464+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:07:14.106464+0000","last_undegraded":"2023-09-23T23:07:14.106464+0000","last_fullsized":"2023-09-23T23:07:14.106464+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":320,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101929+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:07:14.101929+0000","last_peered":"2023-09-23T23:07:14.101929+0000","last_clean":"2023-09-23T23:07:14.101929+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:07:14.101929+0000","last_undegraded":"2023-09-23T23:07:14.101929+0000","last_fullsized":"2023-09-23T23:07:14.101929+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"454'5995","reported_seq":10982,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:07:31.548551+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:07:31.548551+0000","last_peered":"2023-09-23T23:07:31.548551+0000","last_clean":"2023-09-23T23:07:31.548551+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:07:31.548551+0000","last_undegraded":"2023-09-23T23:07:31.548551+0000","last_fullsized":"2023-09-23T23:07:31.548551+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5995,"log_dups_size":0,"ondisk_log_size":5995,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111378432,"num_objects":1699,"num_object_clones":0,"num_object_copies":5097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3363,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":82,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.095136+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:07:14.095136+0000","last_peered":"2023-09-23T23:07:14.095136+0000","last_clean":"2023-09-23T23:07:14.095136+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:07:14.095136+0000","last_undegraded":"2023-09-23T23:07:14.095136+0000","last_fullsized":"2023-09-23T23:07:14.095136+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":461,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102383+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:07:14.102383+0000","last_peered":"2023-09-23T23:07:14.102383+0000","last_clean":"2023-09-23T23:07:14.102383+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:07:14.102383+0000","last_undegraded":"2023-09-23T23:07:14.102383+0000","last_fullsized":"2023-09-23T23:07:14.102383+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"454'6029","reported_seq":8185,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:27.543748+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:07:27.543748+0000","last_peered":"2023-09-23T23:07:27.543748+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:07:27.543748+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6029,"log_dups_size":0,"ondisk_log_size":6029,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111771648,"num_objects":1705,"num_object_clones":0,"num_object_copies":5115,"num_objects_missing_on_primary":1538,"num_objects_missing":1538,"num_objects_degraded":3513,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3380,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":732,"num_bytes_recovered":47972352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1443},{"shards":"0,3,5,6","objects":1},{"shards":"1,6","objects":-47},{"shards":"3,5,6","objects":308}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":111,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101000+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:07:14.101000+0000","last_peered":"2023-09-23T23:07:14.101000+0000","last_clean":"2023-09-23T23:07:14.101000+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:07:14.101000+0000","last_undegraded":"2023-09-23T23:07:14.101000+0000","last_fullsized":"2023-09-23T23:07:14.101000+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":50,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101148+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:07:14.101148+0000","last_peered":"2023-09-23T23:07:14.101148+0000","last_clean":"2023-09-23T23:07:14.101148+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:07:14.101148+0000","last_undegraded":"2023-09-23T23:07:14.101148+0000","last_fullsized":"2023-09-23T23:07:14.101148+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"454'6152","reported_seq":6570,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.269736+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:07:31.269736+0000","last_peered":"2023-09-23T23:07:31.269736+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:07:31.269736+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6152,"log_dups_size":0,"ondisk_log_size":6152,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117080064,"num_objects":1786,"num_object_clones":0,"num_object_copies":5358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3436,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-258},{"shards":"1,2,3","objects":1495}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":144,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100391+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:07:14.100391+0000","last_peered":"2023-09-23T23:07:14.100391+0000","last_clean":"2023-09-23T23:07:14.100391+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:07:14.100391+0000","last_undegraded":"2023-09-23T23:07:14.100391+0000","last_fullsized":"2023-09-23T23:07:14.100391+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":276,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102801+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:07:14.102801+0000","last_peered":"2023-09-23T23:07:14.102801+0000","last_clean":"2023-09-23T23:07:14.102801+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:07:14.102801+0000","last_undegraded":"2023-09-23T23:07:14.102801+0000","last_fullsized":"2023-09-23T23:07:14.102801+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"454'6047","reported_seq":7504,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:27.670814+0000","last_change":"2023-09-23T23:06:13.792666+0000","last_active":"2023-09-23T23:07:27.670814+0000","last_peered":"2023-09-23T23:07:27.670814+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:06:13.017017+0000","last_became_peered":"2023-09-23T23:06:13.017017+0000","last_unstale":"2023-09-23T23:07:27.670814+0000","last_undegraded":"2023-09-23T23:06:10.970550+0000","last_fullsized":"2023-09-23T23:06:09.601294+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6047,"log_dups_size":0,"ondisk_log_size":6047,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120586240,"num_objects":1840,"num_object_clones":0,"num_object_copies":5520,"num_objects_missing_on_primary":1525,"num_objects_missing":1525,"num_objects_degraded":1646,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3364,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":682,"num_bytes_recovered":44597248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[6,1],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":1452},{"shards":"1,2,5,6","objects":3},{"shards":"1,6","objects":385}],"blocked_by":[],"up_primary":2,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":158,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101576+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:07:14.101576+0000","last_peered":"2023-09-23T23:07:14.101576+0000","last_clean":"2023-09-23T23:07:14.101576+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:07:14.101576+0000","last_undegraded":"2023-09-23T23:07:14.101576+0000","last_fullsized":"2023-09-23T23:07:14.101576+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":456,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103154+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:07:14.103154+0000","last_peered":"2023-09-23T23:07:14.103154+0000","last_clean":"2023-09-23T23:07:14.103154+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:07:14.103154+0000","last_undegraded":"2023-09-23T23:07:14.103154+0000","last_fullsized":"2023-09-23T23:07:14.103154+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"454'6061","reported_seq":6136,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.495374+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:07:31.495374+0000","last_peered":"2023-09-23T23:07:31.495374+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:07:31.495374+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114196480,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3397,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1711},{"shards":"2,3","objects":-653}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":227,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102917+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:07:14.102917+0000","last_peered":"2023-09-23T23:07:14.102917+0000","last_clean":"2023-09-23T23:07:14.102917+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:07:14.102917+0000","last_undegraded":"2023-09-23T23:07:14.102917+0000","last_fullsized":"2023-09-23T23:07:14.102917+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":464,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103748+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:07:14.103748+0000","last_peered":"2023-09-23T23:07:14.103748+0000","last_clean":"2023-09-23T23:07:14.103748+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:07:14.103748+0000","last_undegraded":"2023-09-23T23:07:14.103748+0000","last_fullsized":"2023-09-23T23:07:14.103748+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"454'6049","reported_seq":7615,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:27.579687+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:07:27.579687+0000","last_peered":"2023-09-23T23:07:27.579687+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:07:27.579687+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113672192,"num_objects":1734,"num_object_clones":0,"num_object_copies":5202,"num_objects_missing_on_primary":1576,"num_objects_missing":1576,"num_objects_degraded":2007,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3395,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":742,"num_bytes_recovered":48496640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1489},{"shards":"0,3,6","objects":300},{"shards":"2,6","objects":-55}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":477,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102167+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:07:14.102167+0000","last_peered":"2023-09-23T23:07:14.102167+0000","last_clean":"2023-09-23T23:07:14.102167+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:07:14.102167+0000","last_undegraded":"2023-09-23T23:07:14.102167+0000","last_fullsized":"2023-09-23T23:07:14.102167+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":473,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.104219+0000","last_change":"2023-09-23T23:07:08.756307+0000","last_active":"2023-09-23T23:07:14.104219+0000","last_peered":"2023-09-23T23:07:14.104219+0000","last_clean":"2023-09-23T23:07:14.104219+0000","last_became_active":"2023-09-23T23:07:08.755986+0000","last_became_peered":"2023-09-23T23:07:08.755986+0000","last_unstale":"2023-09-23T23:07:14.104219+0000","last_undegraded":"2023-09-23T23:07:14.104219+0000","last_fullsized":"2023-09-23T23:07:14.104219+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"454'5993","reported_seq":7429,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:07:31.908932+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:07:31.908932+0000","last_peered":"2023-09-23T23:07:31.908932+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:07:31.908932+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:07:31.908932+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5993,"log_dups_size":0,"ondisk_log_size":5993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3337,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":681,"num_bytes_recovered":44630016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1406},{"shards":"1,3,6","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":126,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.104621+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:07:14.104621+0000","last_peered":"2023-09-23T23:07:14.104621+0000","last_clean":"2023-09-23T23:07:14.104621+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:07:14.104621+0000","last_undegraded":"2023-09-23T23:07:14.104621+0000","last_fullsized":"2023-09-23T23:07:14.104621+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":360,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:21.659250+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:07:21.659250+0000","last_peered":"2023-09-23T23:07:21.659250+0000","last_clean":"2023-09-23T23:07:21.659250+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:07:21.659250+0000","last_undegraded":"2023-09-23T23:07:21.659250+0000","last_fullsized":"2023-09-23T23:07:21.659250+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"454'6912","reported_seq":8359,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.481067+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:07:31.481067+0000","last_peered":"2023-09-23T23:07:31.481067+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:07:31.481067+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"415'1074","ondisk_log_start":"415'1074","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232194072,"num_objects":3544,"num_object_clones":0,"num_object_copies":10632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3544,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6912,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":2868},{"shards":"2,4","objects":-322}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":373,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.094626+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:07:14.094626+0000","last_peered":"2023-09-23T23:07:14.094626+0000","last_clean":"2023-09-23T23:07:14.094626+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:07:14.094626+0000","last_undegraded":"2023-09-23T23:07:14.094626+0000","last_fullsized":"2023-09-23T23:07:14.094626+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":291,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100836+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:07:14.100836+0000","last_peered":"2023-09-23T23:07:14.100836+0000","last_clean":"2023-09-23T23:07:14.100836+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:07:14.100836+0000","last_undegraded":"2023-09-23T23:07:14.100836+0000","last_fullsized":"2023-09-23T23:07:14.100836+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":195,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102688+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:07:14.102688+0000","last_peered":"2023-09-23T23:07:14.102688+0000","last_clean":"2023-09-23T23:07:14.102688+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:07:14.102688+0000","last_undegraded":"2023-09-23T23:07:14.102688+0000","last_fullsized":"2023-09-23T23:07:14.102688+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"454'6810","reported_seq":6907,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.915707+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:07:31.915707+0000","last_peered":"2023-09-23T23:07:31.915707+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:07:31.915707+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"416'2267","ondisk_log_start":"416'2267","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4543,"log_dups_size":0,"ondisk_log_size":4543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227147776,"num_objects":3466,"num_object_clones":0,"num_object_copies":10398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6810,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":3466},{"shards":"1,3","objects":-1359}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":50,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102531+0000","last_change":"2023-09-23T23:06:08.607076+0000","last_active":"2023-09-23T23:07:14.102531+0000","last_peered":"2023-09-23T23:07:14.102531+0000","last_clean":"2023-09-23T23:07:14.102531+0000","last_became_active":"2023-09-23T23:06:08.606649+0000","last_became_peered":"2023-09-23T23:06:08.606649+0000","last_unstale":"2023-09-23T23:07:14.102531+0000","last_undegraded":"2023-09-23T23:07:14.102531+0000","last_fullsized":"2023-09-23T23:07:14.102531+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:47.677948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":492,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103096+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:07:14.103096+0000","last_peered":"2023-09-23T23:07:14.103096+0000","last_clean":"2023-09-23T23:07:14.103096+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:07:14.103096+0000","last_undegraded":"2023-09-23T23:07:14.103096+0000","last_fullsized":"2023-09-23T23:07:14.103096+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":358,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.094578+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:07:14.094578+0000","last_peered":"2023-09-23T23:07:14.094578+0000","last_clean":"2023-09-23T23:07:14.094578+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:07:14.094578+0000","last_undegraded":"2023-09-23T23:07:14.094578+0000","last_fullsized":"2023-09-23T23:07:14.094578+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"454'6157","reported_seq":8407,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.842975+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:07:31.842975+0000","last_peered":"2023-09-23T23:07:31.842975+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:07:31.842975+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6157,"log_dups_size":0,"ondisk_log_size":6157,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111902720,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3524,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3466,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":777,"num_bytes_recovered":50855936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1419},{"shards":"3,5,6","objects":315},{"shards":"3,6","objects":-27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":433,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.089414+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:07:14.089414+0000","last_peered":"2023-09-23T23:07:14.089414+0000","last_clean":"2023-09-23T23:07:14.089414+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:07:14.089414+0000","last_undegraded":"2023-09-23T23:07:14.089414+0000","last_fullsized":"2023-09-23T23:07:14.089414+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":285,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.088992+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:07:14.088992+0000","last_peered":"2023-09-23T23:07:14.088992+0000","last_clean":"2023-09-23T23:07:14.088992+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:07:14.088992+0000","last_undegraded":"2023-09-23T23:07:14.088992+0000","last_fullsized":"2023-09-23T23:07:14.088992+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":381,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100214+0000","last_change":"2023-09-23T23:07:08.730132+0000","last_active":"2023-09-23T23:07:14.100214+0000","last_peered":"2023-09-23T23:07:14.100214+0000","last_clean":"2023-09-23T23:07:14.100214+0000","last_became_active":"2023-09-23T23:07:08.729741+0000","last_became_peered":"2023-09-23T23:07:08.729741+0000","last_unstale":"2023-09-23T23:07:14.100214+0000","last_undegraded":"2023-09-23T23:07:14.100214+0000","last_fullsized":"2023-09-23T23:07:14.100214+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:47.433362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"454'6816","reported_seq":10277,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.943133+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:07:31.943133+0000","last_peered":"2023-09-23T23:07:31.943133+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:07:31.943133+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"416'2334","ondisk_log_start":"416'2334","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4482,"log_dups_size":0,"ondisk_log_size":4482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224657408,"num_objects":3428,"num_object_clones":0,"num_object_copies":10284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3886,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6816,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1694,"num_bytes_recovered":111017984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":2808},{"shards":"3,5","objects":1},{"shards":"3,5,6","objects":619}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":364,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.089938+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:07:14.089938+0000","last_peered":"2023-09-23T23:07:14.089938+0000","last_clean":"2023-09-23T23:07:14.089938+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:07:14.089938+0000","last_undegraded":"2023-09-23T23:07:14.089938+0000","last_fullsized":"2023-09-23T23:07:14.089938+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":565,"reported_epoch":453,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:07:14.089282+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:07:14.089282+0000","last_peered":"2023-09-23T23:07:14.089282+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:07:14.089282+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:07:14.089282+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":144,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.101229+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:07:14.101229+0000","last_peered":"2023-09-23T23:07:14.101229+0000","last_clean":"2023-09-23T23:07:14.101229+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:07:14.101229+0000","last_undegraded":"2023-09-23T23:07:14.101229+0000","last_fullsized":"2023-09-23T23:07:14.101229+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":363,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103928+0000","last_change":"2023-09-23T23:07:08.758086+0000","last_active":"2023-09-23T23:07:14.103928+0000","last_peered":"2023-09-23T23:07:14.103928+0000","last_clean":"2023-09-23T23:07:14.103928+0000","last_became_active":"2023-09-23T23:07:08.757837+0000","last_became_peered":"2023-09-23T23:07:08.757837+0000","last_unstale":"2023-09-23T23:07:14.103928+0000","last_undegraded":"2023-09-23T23:07:14.103928+0000","last_fullsized":"2023-09-23T23:07:14.103928+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"454'6040","reported_seq":6245,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.679934+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:07:31.679934+0000","last_peered":"2023-09-23T23:07:31.679934+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:07:31.679934+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6040,"log_dups_size":0,"ondisk_log_size":6040,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111640576,"num_objects":1703,"num_object_clones":0,"num_object_copies":5109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4030464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":1498},{"shards":"0,3,6","objects":321},{"shards":"1,3","objects":-795}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":377,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100136+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:07:14.100136+0000","last_peered":"2023-09-23T23:07:14.100136+0000","last_clean":"2023-09-23T23:07:14.100136+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:07:14.100136+0000","last_undegraded":"2023-09-23T23:07:14.100136+0000","last_fullsized":"2023-09-23T23:07:14.100136+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":348,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102075+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:07:14.102075+0000","last_peered":"2023-09-23T23:07:14.102075+0000","last_clean":"2023-09-23T23:07:14.102075+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:07:14.102075+0000","last_undegraded":"2023-09-23T23:07:14.102075+0000","last_fullsized":"2023-09-23T23:07:14.102075+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":313,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.090193+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:07:14.090193+0000","last_peered":"2023-09-23T23:07:14.090193+0000","last_clean":"2023-09-23T23:07:14.090193+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:07:14.090193+0000","last_undegraded":"2023-09-23T23:07:14.090193+0000","last_fullsized":"2023-09-23T23:07:14.090193+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"454'6755","reported_seq":6809,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:07:31.427672+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:07:31.427672+0000","last_peered":"2023-09-23T23:07:31.427672+0000","last_clean":"2023-09-23T23:07:31.427672+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:07:31.427672+0000","last_undegraded":"2023-09-23T23:07:31.427672+0000","last_fullsized":"2023-09-23T23:07:31.427672+0000","mapping_epoch":410,"log_start":"415'911","ondisk_log_start":"415'911","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5844,"log_dups_size":0,"ondisk_log_size":5844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227868672,"num_objects":3477,"num_object_clones":0,"num_object_copies":10431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6755,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":313,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103669+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:07:14.103669+0000","last_peered":"2023-09-23T23:07:14.103669+0000","last_clean":"2023-09-23T23:07:14.103669+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:07:14.103669+0000","last_undegraded":"2023-09-23T23:07:14.103669+0000","last_fullsized":"2023-09-23T23:07:14.103669+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":310,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.094762+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:07:14.094762+0000","last_peered":"2023-09-23T23:07:14.094762+0000","last_clean":"2023-09-23T23:07:14.094762+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:07:14.094762+0000","last_undegraded":"2023-09-23T23:07:14.094762+0000","last_fullsized":"2023-09-23T23:07:14.094762+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":275,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.089727+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:07:14.089727+0000","last_peered":"2023-09-23T23:07:14.089727+0000","last_clean":"2023-09-23T23:07:14.089727+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:07:14.089727+0000","last_undegraded":"2023-09-23T23:07:14.089727+0000","last_fullsized":"2023-09-23T23:07:14.089727+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"454'6744","reported_seq":6806,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.432711+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:07:31.432711+0000","last_peered":"2023-09-23T23:07:31.432711+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:07:31.432711+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"415'883","ondisk_log_start":"415'883","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221904896,"num_objects":3386,"num_object_clones":0,"num_object_copies":10158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6744,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":3386},{"shards":"1,2","objects":-1383}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":176,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.100530+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:07:14.100530+0000","last_peered":"2023-09-23T23:07:14.100530+0000","last_clean":"2023-09-23T23:07:14.100530+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:07:14.100530+0000","last_undegraded":"2023-09-23T23:07:14.100530+0000","last_fullsized":"2023-09-23T23:07:14.100530+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":152,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.104142+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:07:14.104142+0000","last_peered":"2023-09-23T23:07:14.104142+0000","last_clean":"2023-09-23T23:07:14.104142+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:07:14.104142+0000","last_undegraded":"2023-09-23T23:07:14.104142+0000","last_fullsized":"2023-09-23T23:07:14.104142+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-09-23T23:07:23.983042+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:07:23.983042+0000","last_peered":"2023-09-23T23:07:23.983042+0000","last_clean":"2023-09-23T23:07:23.983042+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:07:23.983042+0000","last_undegraded":"2023-09-23T23:07:23.983042+0000","last_fullsized":"2023-09-23T23:07:23.983042+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"454'6824","reported_seq":11255,"reported_epoch":454,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:30.421511+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:07:30.421511+0000","last_peered":"2023-09-23T23:07:30.421511+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:07:30.421511+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"415'1120","ondisk_log_start":"415'1120","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229900288,"num_objects":3508,"num_object_clones":0,"num_object_copies":10524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":2512,"num_objects_unfound":0,"num_objects_dirty":3508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6824,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2185,"num_bytes_recovered":143196160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":2512},{"shards":"1,4","objects":812}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":402,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.093548+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:07:14.093548+0000","last_peered":"2023-09-23T23:07:14.093548+0000","last_clean":"2023-09-23T23:07:14.093548+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:07:14.093548+0000","last_undegraded":"2023-09-23T23:07:14.093548+0000","last_fullsized":"2023-09-23T23:07:14.093548+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":75,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.104322+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:07:14.104322+0000","last_peered":"2023-09-23T23:07:14.104322+0000","last_clean":"2023-09-23T23:07:14.104322+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:07:14.104322+0000","last_undegraded":"2023-09-23T23:07:14.104322+0000","last_fullsized":"2023-09-23T23:07:14.104322+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":446,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102246+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:07:14.102246+0000","last_peered":"2023-09-23T23:07:14.102246+0000","last_clean":"2023-09-23T23:07:14.102246+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:07:14.102246+0000","last_undegraded":"2023-09-23T23:07:14.102246+0000","last_fullsized":"2023-09-23T23:07:14.102246+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"454'6110","reported_seq":7696,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:27.722524+0000","last_change":"2023-09-23T23:06:13.782895+0000","last_active":"2023-09-23T23:07:27.722524+0000","last_peered":"2023-09-23T23:07:27.722524+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:06:13.017993+0000","last_became_peered":"2023-09-23T23:06:13.017993+0000","last_unstale":"2023-09-23T23:07:27.722524+0000","last_undegraded":"2023-09-23T23:06:12.255966+0000","last_fullsized":"2023-09-23T23:06:10.977105+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6110,"log_dups_size":0,"ondisk_log_size":6110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116457472,"num_objects":1777,"num_object_clones":0,"num_object_copies":5331,"num_objects_missing_on_primary":1557,"num_objects_missing":1557,"num_objects_degraded":1698,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3427,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":745,"num_bytes_recovered":48791552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[6,1],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":1468},{"shards":"1,2,5,6","objects":5},{"shards":"1,6","objects":304}],"blocked_by":[],"up_primary":2,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":36,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102667+0000","last_change":"2023-09-23T23:07:08.756443+0000","last_active":"2023-09-23T23:07:14.102667+0000","last_peered":"2023-09-23T23:07:14.102667+0000","last_clean":"2023-09-23T23:07:14.102667+0000","last_became_active":"2023-09-23T23:07:08.756086+0000","last_became_peered":"2023-09-23T23:07:08.756086+0000","last_unstale":"2023-09-23T23:07:14.102667+0000","last_undegraded":"2023-09-23T23:07:14.102667+0000","last_fullsized":"2023-09-23T23:07:14.102667+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":204,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.094813+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:07:14.094813+0000","last_peered":"2023-09-23T23:07:14.094813+0000","last_clean":"2023-09-23T23:07:14.094813+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:07:14.094813+0000","last_undegraded":"2023-09-23T23:07:14.094813+0000","last_fullsized":"2023-09-23T23:07:14.094813+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"454'6163","reported_seq":6566,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.405268+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:07:31.405268+0000","last_peered":"2023-09-23T23:07:31.405268+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:07:31.405268+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116359168,"num_objects":1776,"num_object_clones":0,"num_object_copies":5328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3448,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-199},{"shards":"1,2,3","objects":1406}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":276,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103078+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:07:14.103078+0000","last_peered":"2023-09-23T23:07:14.103078+0000","last_clean":"2023-09-23T23:07:14.103078+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:07:14.103078+0000","last_undegraded":"2023-09-23T23:07:14.103078+0000","last_fullsized":"2023-09-23T23:07:14.103078+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":477,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103166+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:07:14.103166+0000","last_peered":"2023-09-23T23:07:14.103166+0000","last_clean":"2023-09-23T23:07:14.103166+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:07:14.103166+0000","last_undegraded":"2023-09-23T23:07:14.103166+0000","last_fullsized":"2023-09-23T23:07:14.103166+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"454'5973","reported_seq":7972,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:27.725016+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:07:27.725016+0000","last_peered":"2023-09-23T23:07:27.725016+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:07:27.725016+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115441664,"num_objects":1762,"num_object_clones":0,"num_object_copies":5286,"num_objects_missing_on_primary":1532,"num_objects_missing":1532,"num_objects_degraded":3475,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3325,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":676,"num_bytes_recovered":44236800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1416},{"shards":"3,5,6","objects":298},{"shards":"3,6","objects":48}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":303,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.103845+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:07:14.103845+0000","last_peered":"2023-09-23T23:07:14.103845+0000","last_clean":"2023-09-23T23:07:14.103845+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:07:14.103845+0000","last_undegraded":"2023-09-23T23:07:14.103845+0000","last_fullsized":"2023-09-23T23:07:14.103845+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":373,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102842+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:07:14.102842+0000","last_peered":"2023-09-23T23:07:14.102842+0000","last_clean":"2023-09-23T23:07:14.102842+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:07:14.102842+0000","last_undegraded":"2023-09-23T23:07:14.102842+0000","last_fullsized":"2023-09-23T23:07:14.102842+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"454'6005","reported_seq":6998,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:31.404640+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:07:31.404640+0000","last_peered":"2023-09-23T23:07:31.404640+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:07:31.404640+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6005,"log_dups_size":0,"ondisk_log_size":6005,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110723072,"num_objects":1690,"num_object_clones":0,"num_object_copies":5070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3374,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":1768},{"shards":"1,2","objects":-297}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":28,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102429+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:07:14.102429+0000","last_peered":"2023-09-23T23:07:14.102429+0000","last_clean":"2023-09-23T23:07:14.102429+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:07:14.102429+0000","last_undegraded":"2023-09-23T23:07:14.102429+0000","last_fullsized":"2023-09-23T23:07:14.102429+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":171,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102294+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:07:14.102294+0000","last_peered":"2023-09-23T23:07:14.102294+0000","last_clean":"2023-09-23T23:07:14.102294+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:07:14.102294+0000","last_undegraded":"2023-09-23T23:07:14.102294+0000","last_fullsized":"2023-09-23T23:07:14.102294+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"454'5996","reported_seq":6065,"reported_epoch":454,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:07:30.151148+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:07:30.151148+0000","last_peered":"2023-09-23T23:07:30.151148+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:07:30.151148+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5996,"log_dups_size":0,"ondisk_log_size":5996,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114720768,"num_objects":1751,"num_object_clones":0,"num_object_copies":5253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3346,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-505},{"shards":"1,4,5","objects":1709}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":221,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.093771+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:07:14.093771+0000","last_peered":"2023-09-23T23:07:14.093771+0000","last_clean":"2023-09-23T23:07:14.093771+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:07:14.093771+0000","last_undegraded":"2023-09-23T23:07:14.093771+0000","last_fullsized":"2023-09-23T23:07:14.093771+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":448,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T23:07:14.102012+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:07:14.102012+0000","last_peered":"2023-09-23T23:07:14.102012+0000","last_clean":"2023-09-23T23:07:14.102012+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:07:14.102012+0000","last_undegraded":"2023-09-23T23:07:14.102012+0000","last_fullsized":"2023-09-23T23:07:14.102012+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":3642753048,"num_objects":55585,"num_object_clones":0,"num_object_copies":166755,"num_objects_missing_on_primary":7728,"num_objects_missing":7728,"num_objects_degraded":60559,"num_objects_misplaced":2512,"num_objects_unfound":0,"num_objects_dirty":55585,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108719,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18788,"num_bytes_recovered":1204944896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12788445184,"data_stored":12788432968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":153414,"ondisk_log_size":153414,"up":78,"acting":58,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430694,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626918,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1436144,"kb_used_data":541608,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92309008,"statfs":{"total":95995035648,"available":94524424192,"internally_reserved":0,"allocated":554606592,"data_stored":553619964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,2,1,4,7,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:07:24 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":4.0599999999999996,"15min":5.3140000000000001},"min":{"1min":0.309,"5min":0.249,"15min":0.249},"max":{"1min":8.8800000000000008,"5min":115.32899999999999,"15min":115.32899999999999},"last":0.752},{"interface":"front","average":{"1min":1.385,"5min":3.976,"15min":5.3250000000000002},"min":{"1min":0.27600000000000002,"5min":0.26900000000000002,"15min":0.26200000000000001},"max":{"1min":4.9039999999999999,"5min":115.348,"15min":115.348},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":3.7690000000000001,"15min":3.8159999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":2.3439999999999999,"5min":149.947,"15min":254.62100000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.89200000000000002,"5min":4.0279999999999996,"15min":4.2229999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.24199999999999999},"max":{"1min":8.7859999999999996,"5min":150.22499999999999,"15min":255.20099999999999},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:06:44 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":5.625,"15min":2.5600000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":41.359000000000002,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.395,"5min":5.5119999999999996,"15min":2.669},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":41.412999999999997,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.67700000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":5.2439999999999998,"15min":3.2069999999999999},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":4.6680000000000001,"5min":142.24700000000001,"15min":142.24700000000001},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.091,"5min":5.2720000000000002,"15min":3.2389999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.309},"max":{"1min":8.7439999999999998,"5min":143.86699999999999,"15min":143.86699999999999},"last":0.71599999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":2.367,"15min":1.9530000000000001},"min":{"1min":0.39000000000000001,"5min":0.28599999999999998,"15min":0.23999999999999999},"max":{"1min":8.8520000000000003,"5min":119.804,"15min":119.804},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.2989999999999999,"15min":1.9650000000000001},"min":{"1min":0.41899999999999998,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":7.5679999999999996,"5min":122.31999999999999,"15min":122.31999999999999},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Sep 23 23:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":3.0310000000000001,"15min":2.605},"min":{"1min":0.46100000000000002,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":11.862,"5min":115.595,"15min":186.27600000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":2.7429999999999999,"5min":3.0470000000000002,"15min":2.6659999999999999},"min":{"1min":0.46800000000000003,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":10.254,"5min":115.55,"15min":191.28700000000001},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":2.0409999999999999,"15min":1.498},"min":{"1min":0.41799999999999998,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":8.5299999999999994,"5min":115.61,"15min":115.61},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.0710000000000002,"15min":1.7050000000000001},"min":{"1min":0.38300000000000001,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":5.968,"5min":115.578,"15min":115.578},"last":0.79000000000000004}]}]},{"osd":1,"up_from":84,"seq":360777253232,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3621324,"kb_used_data":2704196,"kb_used_omap":9,"kb_used_meta":917110,"kb_avail":90123828,"statfs":{"total":95995035648,"available":92286799872,"internally_reserved":0,"allocated":2769096704,"data_stored":2767290206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":939121641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":5.5519999999999996,"15min":2.5329999999999999},"min":{"1min":0.20999999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.90600000000000003,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":5.7190000000000003,"15min":2.883},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":3.4529999999999998,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.314}]},{"osd":2,"last update":"Sat Sep 23 23:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":4.8010000000000002,"15min":1.9390000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":14.445,"5min":161.61699999999999,"15min":161.61699999999999},"last":0.62},{"interface":"front","average":{"1min":1.1359999999999999,"5min":5.1470000000000002,"15min":2.1720000000000002},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":14.558999999999999,"5min":162.61699999999999,"15min":162.61699999999999},"last":0.35299999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":2.4649999999999999,"15min":1.6000000000000001},"min":{"1min":0.32400000000000001,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":1.859,"5min":59.424999999999997,"15min":59.424999999999997},"last":1.4059999999999999},{"interface":"front","average":{"1min":0.68999999999999995,"5min":2.3439999999999999,"15min":1.6120000000000001},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":2.4900000000000002,"5min":55.174999999999997,"15min":55.174999999999997},"last":0.501}]},{"osd":4,"last update":"Sat Sep 23 23:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.675,"15min":2.1019999999999999},"min":{"1min":0.34200000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.3839999999999999,"5min":67.738,"15min":151.036},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.628,"5min":2.085,"15min":2.4340000000000002},"min":{"1min":0.38800000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.774,"5min":67.670000000000002,"15min":151.06899999999999},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":2.3010000000000002,"15min":2.4009999999999998},"min":{"1min":0.39500000000000002,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":18.228999999999999,"5min":68.259,"15min":116.098},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.996,"5min":1.954,"15min":2.1989999999999998},"min":{"1min":0.35399999999999998,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":6.585,"5min":70.057000000000002,"15min":116.08499999999999},"last":0.68000000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":2.2389999999999999,"15min":2.0470000000000002},"min":{"1min":0.34100000000000003,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":8.2680000000000007,"5min":67.349000000000004,"15min":67.349000000000004},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.1850000000000001,"5min":2.141,"15min":1.867},"min":{"1min":0.38100000000000001,"5min":0.34999999999999998,"15min":0.33300000000000002},"max":{"1min":6.6989999999999998,"5min":67.366,"15min":67.366},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":0.95999999999999996,"15min":1.2689999999999999},"min":{"1min":0.39200000000000002,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":7.0960000000000001,"5min":7.0960000000000001,"15min":8.702},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.4770000000000001,"15min":1.6970000000000001},"min":{"1min":0.48399999999999999,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":9.016,"5min":9.016,"15min":20.902999999999999},"last":0.66100000000000003}]}]},{"osd":0,"up_from":0,"seq":1516123455602,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259375,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2053456,"kb_used_data":1785928,"kb_used_omap":83,"kb_used_meta":267436,"kb_avail":91691696,"statfs":{"total":95995035648,"available":93892296704,"internally_reserved":0,"allocated":1828790272,"data_stored":1826858489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":273854917},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":2.6560000000000001,"15min":1.163},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.1960000000000002,"5min":74.241,"15min":74.241},"last":1.4910000000000001},{"interface":"front","average":{"1min":0.624,"5min":3.0270000000000001,"15min":1.4670000000000001},"min":{"1min":0.24399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.7679999999999998,"5min":74.430000000000007,"15min":74.430000000000007},"last":1.716}]},{"osd":1,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.669,"15min":0.97799999999999998},"min":{"1min":0.26100000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.8100000000000001,"5min":18.396999999999998,"15min":18.396999999999998},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5960000000000001,"15min":1.2569999999999999},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.625,"5min":12.744999999999999,"15min":12.744999999999999},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":2.6379999999999999,"15min":1.355},"min":{"1min":0.29499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.8460000000000001,"5min":110.31699999999999,"15min":110.31699999999999},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":2.5979999999999999,"15min":1.421},"min":{"1min":0.28499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.3799999999999999,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.2769999999999999,"15min":0.873},"min":{"1min":0.38,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.1709999999999998,"5min":33.476999999999997,"15min":33.476999999999997},"last":1.5329999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.538,"15min":1.083},"min":{"1min":0.36899999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1500000000000004,"5min":33.497999999999998,"15min":33.497999999999998},"last":1.123}]},{"osd":5,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":2.9449999999999998,"15min":1.621},"min":{"1min":0.34499999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.258,"5min":139.405,"15min":139.405},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.3100000000000001,"5min":2.8570000000000002,"15min":1.456},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":6.8600000000000003,"5min":139.42099999999999,"15min":139.42099999999999},"last":1.6080000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":4.6289999999999996,"5min":2.028,"15min":1.3680000000000001},"min":{"1min":0.39500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":71.105999999999995,"5min":71.105999999999995,"15min":71.105999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":4.6230000000000002,"5min":1.7529999999999999,"15min":1.077},"min":{"1min":0.47399999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":71.087999999999994,"5min":71.087999999999994,"15min":71.087999999999994},"last":1.575}]},{"osd":7,"last update":"Sat Sep 23 23:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.2010000000000001,"15min":1.115},"min":{"1min":0.46500000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":7.8639999999999999,"5min":10.164,"15min":10.164},"last":1.637},{"interface":"front","average":{"1min":1.329,"5min":1.276,"15min":1.1819999999999999},"min":{"1min":0.45700000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.1989999999999998,"5min":7.7830000000000004,"15min":7.7830000000000004},"last":1.1950000000000001}]}]},{"osd":3,"up_from":94,"seq":403726926179,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2805408,"kb_used_data":2131032,"kb_used_omap":7,"kb_used_meta":674360,"kb_avail":90939744,"statfs":{"total":95995035648,"available":93122297856,"internally_reserved":0,"allocated":2182176768,"data_stored":2180283584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":690545256},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":1.6080000000000001,"15min":1.3680000000000001},"min":{"1min":0.222,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":1.577,"5min":17.015000000000001,"15min":122.289},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.0409999999999999,"15min":1.1220000000000001},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.17799999999999999},"max":{"1min":3.1080000000000001,"5min":11.382999999999999,"15min":122.041},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.198,"15min":2.2130000000000001},"min":{"1min":0.224,"5min":0.22,"15min":0.187},"max":{"1min":4.5339999999999998,"5min":23.969000000000001,"15min":123.866},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":1.9670000000000001,"15min":2.1859999999999999},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.20699999999999999},"max":{"1min":4.2910000000000004,"5min":16.986000000000001,"15min":122.265},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 23:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.522,"15min":0.95699999999999996},"min":{"1min":0.217,"5min":0.217,"15min":0.20200000000000001},"max":{"1min":3.069,"5min":18.588000000000001,"15min":18.588000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.621,"5min":1.835,"15min":1.1890000000000001},"min":{"1min":0.253,"5min":0.215,"15min":0.215},"max":{"1min":3.3820000000000001,"5min":10.353999999999999,"15min":10.353999999999999},"last":0.84499999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:07:04 2023","interfaces":[{"interface":"back","average":{"1min":5.4779999999999998,"5min":2.5329999999999999,"15min":2.169},"min":{"1min":0.31,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":115.69,"5min":115.69,"15min":115.69},"last":0.46800000000000003},{"interface":"front","average":{"1min":5.0110000000000001,"5min":2.4020000000000001,"15min":2.1110000000000002},"min":{"1min":0.36099999999999999,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":115.617,"5min":115.617,"15min":115.617},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 23:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.653,"15min":1.4299999999999999},"min":{"1min":0.35299999999999998,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":10.422000000000001,"5min":16.959,"15min":16.959},"last":0.498},{"interface":"front","average":{"1min":1.573,"5min":1.369,"15min":1.51},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.27100000000000002},"max":{"1min":10.545,"5min":10.545,"15min":10.545},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:07:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.6200000000000001,"15min":1.6819999999999999},"min":{"1min":0.372,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":3.7669999999999999,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.202,"5min":1.2470000000000001,"15min":1.5920000000000001},"min":{"1min":0.41799999999999998,"5min":0.372,"15min":0.32400000000000001},"max":{"1min":3.794,"5min":8.1999999999999993,"15min":12.887},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:07:04 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.288,"15min":1.4139999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":4.5410000000000004,"5min":8.1620000000000008,"15min":8.8160000000000007},"last":0.997},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.792,"15min":1.5489999999999999},"min":{"1min":0.40799999999999997,"5min":0.40300000000000002,"15min":0.33200000000000002},"max":{"1min":6.516,"5min":10.278,"15min":10.278},"last":1.004}]}]},{"osd":4,"up_from":30,"seq":128849019376,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1918576,"kb_used_data":1011112,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":91826576,"statfs":{"total":95995035648,"available":94030413824,"internally_reserved":0,"allocated":1035378688,"data_stored":1034391538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":3.306,"15min":2.181},"min":{"1min":0.27400000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":11.004,"5min":156.375,"15min":156.375},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.579,"5min":3.472,"15min":2.347},"min":{"1min":0.32400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.9489999999999998,"5min":157.02199999999999,"15min":157.02199999999999},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":3.4849999999999999,"15min":3.8919999999999999},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.4770000000000003,"5min":95.591999999999999,"15min":132.304},"last":0.309},{"interface":"front","average":{"1min":1.252,"5min":3.2749999999999999,"15min":3.819},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":10.956,"5min":97.929000000000002,"15min":132.453},"last":1.5860000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.0649999999999999,"15min":1.403},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.6970000000000001,"5min":56.531999999999996,"15min":56.531999999999996},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.0740000000000001,"5min":2.1680000000000001,"15min":1.246},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.5510000000000002,"5min":56.204000000000001,"15min":56.204000000000001},"last":5.04}]},{"osd":3,"last update":"Sat Sep 23 23:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":2.0619999999999998,"15min":1.9530000000000001},"min":{"1min":0.35399999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.08,"5min":60.274000000000001,"15min":74.138999999999996},"last":1.6619999999999999},{"interface":"front","average":{"1min":1.655,"5min":2.2330000000000001,"15min":2.0289999999999999},"min":{"1min":0.39800000000000002,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":6.532,"5min":60.677999999999997,"15min":75.099999999999994},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:06:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":0.93500000000000005,"15min":1.7290000000000001},"min":{"1min":0.33200000000000002,"5min":0.22,"15min":0.214},"max":{"1min":10.130000000000001,"5min":10.130000000000001,"15min":148.61600000000001},"last":5.093},{"interface":"front","average":{"1min":1.1819999999999999,"5min":0.80100000000000005,"15min":1.78},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.247},"max":{"1min":6.6699999999999999,"5min":8.7810000000000006,"15min":151.905},"last":0.73699999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:07:12 2023","interfaces":[{"interface":"back","average":{"1min":3.222,"5min":1.2609999999999999,"15min":1.401},"min":{"1min":0.34999999999999998,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":35.896000000000001,"5min":35.896000000000001,"15min":35.896000000000001},"last":5.1189999999999998},{"interface":"front","average":{"1min":4.1310000000000002,"5min":1.4930000000000001,"15min":1.361},"min":{"1min":0.40699999999999997,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":35.908000000000001,"5min":35.908000000000001,"15min":35.908000000000001},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":1.0660000000000001,"15min":1.228},"min":{"1min":0.36299999999999999,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":4.1239999999999997,"5min":7.1050000000000004,"15min":12.84},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.72899999999999998,"15min":1.0660000000000001},"min":{"1min":0.40600000000000003,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":2.7400000000000002,"5min":3.6539999999999999,"15min":10.587},"last":6.6180000000000003}]}]},{"osd":5,"up_from":0,"seq":154618823150,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2934833152,"data_stored":2934833152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2760638464,"data_stored":2760638464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1817055232,"data_stored":1817051160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2168717312,"data_stored":2168717312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1026166784,"data_stored":1026162712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":548999168,"data_stored":548995096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:07:35.513 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 26 2023-09-23T23:07:35.593 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:35.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:35.703 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:35.737 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:35.796 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:35.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:35.906 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:35.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:36.095 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:36.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:36.199 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:36.239 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:36 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 60269/166155 objects degraded (36.273%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:36.240 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:36 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 60269/166155 objects degraded (36.273%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:36.249 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:36.313 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:36.318 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:36 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 60269/166155 objects degraded (36.273%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:36.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:36.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:36.421 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:36.465 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:36.539 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:07:36.554 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:07:36.555 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:36.589 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:07:36.623 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:07:36.659 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:07:36.701 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:07:36.706 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:36.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:36.714 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:36.746 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:07:36.778 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.788 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.789 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:07:36.815 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:41359 11.3 e417/410 11:cd5108ed:::benchmark_data_smithi052_606809_object41358:head v 417'2595, mlcod=417'2590)", 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.158364+0000", 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.66236996200001, 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23886327299999999, 2023-09-23T23:07:36.832 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158364+0000", 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158364+0000", 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.833 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158363+0000", 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158407+0000", 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3223999999999998e-05 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.834 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158421+0000", 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4093e-05 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158429+0000", 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9969999999999998e-06 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.835 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158493+0000", 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3806000000000005e-05 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.158516+0000", 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3459999999999999e-05 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.836 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.397172+0000", 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23865600100000001 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.397228+0000", 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5553000000000002e-05 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.837 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:41357 11.f 11:f99afbff:::benchmark_data_smithi052_606809_object41356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.159108+0000", 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.661626622, 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23755769099999999, 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 41357 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.838 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159108+0000", 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159108+0000", 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.839 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159106+0000", 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159133+0000", 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6750999999999998e-05 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159145+0000", 2023-09-23T23:07:36.840 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2359e-05 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159153+0000", 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8269999999999998e-06 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159204+0000", 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1050000000000001e-05 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.841 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159329+0000", 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012519399999999999 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159505+0000", 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017603399999999999 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.842 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.283664+0000", 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12415902500000001 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395574+0000", 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.111909323 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.396576+0000", 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001002537 2023-09-23T23:07:36.843 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.396629+0000", 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2593999999999998e-05 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.396665+0000", 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6476000000000001e-05 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.844 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:41361 11.2 11:47bb5080:::benchmark_data_smithi052_606809_object41360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.66156289, 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23663515700000001, 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 41361 2023-09-23T23:07:36.845 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.846 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159171+0000", 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.847 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159194+0000", 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3439999999999999e-05 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159202+0000", 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2790000000000001e-06 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.159207+0000", 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4609999999999999e-06 2023-09-23T23:07:36.848 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160385+0000", 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00117815 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160518+0000", 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013272699999999999 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.849 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.160643+0000", 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000125251 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.285496+0000", 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.124852347 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.850 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395674+0000", 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.11017845 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395719+0000", 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4589000000000003e-05 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395761+0000", 2023-09-23T23:07:36.851 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2354000000000001e-05 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:24.395807+0000", 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5643000000000003e-05 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.852 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66493 11.4 11:2854aa07:::benchmark_data_smithi052_606809_object66492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.667304+0000", 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 162.15343072900001, 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.248592391, 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66493 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.853 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667304+0000", 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667304+0000", 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.854 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667303+0000", 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667322+0000", 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9454999999999999e-05 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.855 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667329+0000", 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0049999999999998e-06 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667335+0000", 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6500000000000001e-06 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.856 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667382+0000", 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7197000000000002e-05 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667501+0000", 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119177 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:07:36.857 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.667635+0000", 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013359600000000001 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.675585+0000", 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0079498629999999997 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682693+0000", 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0071082100000000002 2023-09-23T23:07:36.858 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915822+0000", 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23312934099999999 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915860+0000", 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7058000000000003e-05 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.859 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915896+0000", 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6412000000000003e-05 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66496 11.d 11:bbe513fe:::benchmark_data_smithi052_606809_object66495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.676916+0000", 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 162.143818275, 2023-09-23T23:07:36.860 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239258154, 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66496 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676916+0000", 2023-09-23T23:07:36.861 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676916+0000", 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676915+0000", 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.862 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676940+0000", 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5650999999999999e-05 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676951+0000", 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0159e-05 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.863 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676958+0000", 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7260000000000002e-06 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677258+0000", 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00029930899999999999 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677369+0000", 2023-09-23T23:07:36.864 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011102700000000001 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677498+0000", 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012934200000000001 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682271+0000", 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004772844 2023-09-23T23:07:36.865 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682278+0000", 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5519999999999999e-06 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916127+0000", 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23384854799999999 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.866 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916153+0000", 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5812e-05 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916174+0000", 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1461e-05 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.867 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66498 11.f 11:f0c9ffe3:::benchmark_data_smithi052_606809_object66497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.676974+0000", 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 162.14376062900001, 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23892223100000001, 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66498 2023-09-23T23:07:36.868 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676974+0000", 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676974+0000", 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.869 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.676973+0000", 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677006+0000", 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.2792e-05 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.870 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677014+0000", 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5860000000000005e-06 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677021+0000", 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5930000000000002e-06 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.678110+0000", 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001088756 2023-09-23T23:07:36.871 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.678230+0000", 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012026799999999999 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.678360+0000", 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013007399999999999 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.872 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.681128+0000", 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027677370000000001 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915745+0000", 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.234617042 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915810+0000", 2023-09-23T23:07:36.873 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5512999999999994e-05 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915856+0000", 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5179999999999998e-05 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915896+0000", 2023-09-23T23:07:36.874 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0346000000000003e-05 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66499 11.d 11:baa8f49a:::benchmark_data_smithi052_606809_object66498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 162.14369980999999, 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.239213228, 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.875 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66499 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.876 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677034+0000", 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677069+0000", 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4826999999999997e-05 2023-09-23T23:07:36.877 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677077+0000", 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7479999999999999e-06 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677083+0000", 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9499999999999998e-06 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.878 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.679753+0000", 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026696469999999998 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.679881+0000", 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000128369 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.879 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680012+0000", 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000130987 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682407+0000", 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0023948989999999998 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.682482+0000", 2023-09-23T23:07:36.880 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4179999999999998e-05 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916204+0000", 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.23372264200000001 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916229+0000", 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5060000000000001e-05 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.881 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.916248+0000", 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8491e-05 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66501 11.c 11:338ca9b7:::benchmark_data_smithi052_606809_object66500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.677089+0000", 2023-09-23T23:07:36.882 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 162.14364538300001, 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.240935287, 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66501 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.883 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677089+0000", 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677089+0000", 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677088+0000", 2023-09-23T23:07:36.884 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677111+0000", 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2957999999999999e-05 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677118+0000", 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5710000000000003e-06 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.885 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.677123+0000", 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1699999999999996e-06 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680249+0000", 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031262730000000002 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.886 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680354+0000", 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0001044 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.680473+0000", 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011943 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.887 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.684728+0000", 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004254982 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.915325+0000", 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.230596933 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917951+0000", 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026257350000000001 2023-09-23T23:07:36.888 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.917989+0000", 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7696e-05 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918024+0000", 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5638e-05 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.889 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72288 11.2 11:44178e5e:::benchmark_data_smithi052_606809_object72287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.394288+0000", 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 151.42644607400001, 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.355785346, 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.890 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72288 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394288+0000", 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394288+0000", 2023-09-23T23:07:36.891 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394287+0000", 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394315+0000", 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8002999999999999e-05 2023-09-23T23:07:36.892 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394324+0000", 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7780000000000002e-06 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394331+0000", 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9120000000000001e-06 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.893 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394497+0000", 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000165167 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394798+0000", 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00030159799999999998 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:07:36.894 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394917+0000", 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011914800000000001 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.403946+0000", 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0090287289999999992 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.404128+0000", 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018199700000000001 2023-09-23T23:07:36.895 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.749969+0000", 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34584069200000001 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.750027+0000", 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8128999999999999e-05 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.750074+0000", 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6765999999999999e-05 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75762 11.2 11:402b2427:::benchmark_data_smithi052_606809_object75761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.316624+0000", 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 145.504110677, 2023-09-23T23:07:36.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.37235110799999999, 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75762 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316624+0000", 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316624+0000", 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316623+0000", 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316646+0000", 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2690000000000001e-05 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316652+0000", 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4999999999999996e-06 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316657+0000", 2023-09-23T23:07:36.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0499999999999999e-06 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316746+0000", 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8312000000000004e-05 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316851+0000", 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105086 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.316971+0000", 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000119861 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.321893+0000", 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049227719999999997 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.902 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.321963+0000", 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9827999999999996e-05 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688891+0000", 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36692815499999998 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688940+0000", 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8717999999999997e-05 2023-09-23T23:07:36.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688975+0000", 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4561999999999999e-05 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPushReply(11.d 431/422 [PushReplyOp(11:b11d2e70:::benchmark_data_smithi052_606809_object8462:head)])", 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:34.164832+0000", 2023-09-23T23:07:36.904 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 122.65590229999999, 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28677324300000001, 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164832+0000", 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164832+0000", 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164832+0000", 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164835+0000", 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.653e-06 2023-09-23T23:07:36.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164842+0000", 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2910000000000004e-06 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164847+0000", 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1970000000000004e-06 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.164897+0000", 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0071000000000002e-05 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.451605+0000", 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28670836 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:85549 11.12 11:486b86f3:::benchmark_data_smithi052_606809_object3395:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:34.168648+0000", 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 122.652086286, 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.29057180399999999, 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 85549 2023-09-23T23:07:36.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168648+0000", 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168648+0000", 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168647+0000", 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168651+0000", 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3409999999999996e-06 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168659+0000", 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5719999999999997e-06 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168664+0000", 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8029999999999996e-06 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168701+0000", 2023-09-23T23:07:36.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6869000000000002e-05 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168779+0000", 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8770000000000006e-05 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.168850+0000", 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1035e-05 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.457332+0000", 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28848126299999999 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459118+0000", 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017865159999999999 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459126+0000", 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5070000000000003e-06 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459177+0000", 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1220000000000001e-05 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459220+0000", 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2880000000000003e-05 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.46892.0:85549 11.12 e431/428 ondisk, result = 0)", 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:34.170043+0000", 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 122.650691057, 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28729146, 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.916 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.170043+0000", 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.170043+0000", 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.170043+0000", 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.170047+0000", 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1790000000000001e-06 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.451483+0000", 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28143660700000001 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.451505+0000", 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2099e-05 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.457318+0000", 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058120710000000003 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.457326+0000", 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.9750000000000007e-06 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.457335+0000", 2023-09-23T23:07:36.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.183e-06 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.46892.0:85549 11.12 e431/428 ondisk, result = 0)", 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:34.171797+0000", 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 122.648937753, 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28742603500000002, 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:07:36.921 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.171797+0000", 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.171797+0000", 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.171796+0000", 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.171801+0000", 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8489999999999997e-06 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.455945+0000", 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.28414437399999998 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.455954+0000", 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3829999999999998e-06 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459105+0000", 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003150501 2023-09-23T23:07:36.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459112+0000", 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5310000000000004e-06 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.459223+0000", 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011009 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.925 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:86859 11.12 11:4e37b948:::benchmark_data_smithi052_606809_object4705:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:38.887406+0000", 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 117.933328006, 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2643445079999998, 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.926 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.929 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.929 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 86859 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.887406+0000", 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.887406+0000", 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.887405+0000", 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.887411+0000", 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3280000000000002e-06 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.931 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.887423+0000", 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2044e-05 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.887432+0000", 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7700000000000007e-06 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.890349+0000", 2023-09-23T23:07:36.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029167170000000001 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.890360+0000", 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1168999999999999e-05 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.870867+0000", 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.98050695300000001 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.870879+0000", 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2388000000000001e-05 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.137349+0000", 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2664693810000001 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.137511+0000", 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000162268 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.137668+0000", 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000157131 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.142091+0000", 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.004423026 2023-09-23T23:07:36.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.149853+0000", 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0077623789999999998 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.150127+0000", 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027387700000000001 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.150199+0000", 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.2118000000000003e-05 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.151751+0000", 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001551471 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:86871 11.16 11:6d3f3df1:::benchmark_data_smithi052_606809_object4717:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:38.967636+0000", 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 117.853098308, 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1440925559999999, 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 86871 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.938 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967636+0000", 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967636+0000", 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967635+0000", 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967641+0000", 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.139e-06 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967653+0000", 2023-09-23T23:07:36.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2217e-05 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967659+0000", 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2729999999999999e-06 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967723+0000", 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3519e-05 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.967739+0000", 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6274e-05 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.868909+0000", 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.90117012100000005 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.868923+0000", 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3385e-05 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.087165+0000", 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2182423529999999 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.087347+0000", 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00018178700000000001 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.087513+0000", 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016640599999999999 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.092222+0000", 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047091069999999997 2023-09-23T23:07:36.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.092849+0000", 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00062658899999999999 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.109417+0000", 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.016567703 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.109475+0000", 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8192000000000001e-05 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.111729+0000", 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002253835 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:86878 11.2 11:4458b66f:::benchmark_data_smithi052_606809_object4724:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:39.042786+0000", 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 117.777948421, 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1074456420000001, 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 86878 2023-09-23T23:07:36.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042786+0000", 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042786+0000", 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042785+0000", 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042792+0000", 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8530000000000003e-06 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042804+0000", 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2948999999999999e-05 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042812+0000", 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0709999999999997e-06 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042872+0000", 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0529999999999998e-05 2023-09-23T23:07:36.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.042883+0000", 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0655999999999999e-05 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.871087+0000", 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.82820419999999995 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.871097+0000", 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9080000000000007e-06 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.127096+0000", 2023-09-23T23:07:36.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2559994919999999 2023-09-23T23:07:36.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.127261+0000", 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000164778 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.127418+0000", 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015682399999999999 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.142079+0000", 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.014661110999999999 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.143198+0000", 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0011190550000000001 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.148338+0000", 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0051397860000000004 2023-09-23T23:07:36.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.148402+0000", 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4350000000000006e-05 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.150232+0000", 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001829289 2023-09-23T23:07:36.956 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:89665 11.d 11:b4df92a6:::benchmark_data_smithi052_606809_object7511:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:59.012223+0000", 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 97.808510788999996, 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5635003169999999, 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.957 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 89665 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012223+0000", 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012223+0000", 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012222+0000", 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012232+0000", 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5179999999999993e-06 2023-09-23T23:07:36.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012256+0000", 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4144000000000001e-05 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012267+0000", 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1219e-05 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012331+0000", 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4533e-05 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012511+0000", 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017961299999999999 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.012668+0000", 2023-09-23T23:07:36.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000157462 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.015947+0000", 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003278773 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.016627+0000", 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00067949799999999995 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.573502+0000", 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5568747419999998 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.573570+0000", 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8739999999999996e-05 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.575724+0000", 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021535690000000001 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:89701 11.d 11:b8619b1e:::benchmark_data_smithi052_606809_object7547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:59.226248+0000", 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 97.594485793000004, 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3513164899999999, 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.964 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 89701 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226248+0000", 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226248+0000", 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226247+0000", 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226253+0000", 2023-09-23T23:07:36.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1700000000000002e-06 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226273+0000", 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9734000000000001e-05 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226280+0000", 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9199999999999998e-06 2023-09-23T23:07:36.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226341+0000", 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1325e-05 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226540+0000", 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000198906 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:07:36.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.226720+0000", 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000179533 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.229573+0000", 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002852884 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.229626+0000", 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3313999999999998e-05 2023-09-23T23:07:36.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.575765+0000", 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.346138737 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.575815+0000", 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0090000000000003e-05 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.577565+0000", 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0017501089999999999 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:89722 11.d 11:bfa4ed94:::benchmark_data_smithi052_606809_object7568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:59.370013+0000", 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 97.450721697000006, 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2088915, 2023-09-23T23:07:36.971 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 89722 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370013+0000", 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370013+0000", 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370011+0000", 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:36.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370019+0000", 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.554e-06 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370031+0000", 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2646000000000001e-05 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370040+0000", 2023-09-23T23:07:36.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4190000000000002e-06 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370111+0000", 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.1517e-05 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370218+0000", 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000106285 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,6", 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.370310+0000", 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.2596000000000006e-05 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.372878+0000", 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002567433 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:59.376573+0000", 2023-09-23T23:07:36.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036956049999999998 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.577645+0000", 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2010721489999998 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.577680+0000", 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4539000000000003e-05 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:36.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:06:01.578904+0000", 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001224248 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:36.978 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:07:36.979 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:37.117 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:37.122 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:37 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/13177487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:37.122 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:37 smithi052 ceph-mon[187743]: pgmap v1427: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 276 op/s; 60269/163992 objects degraded (36.751%); 2399/163992 objects misplaced (1.463%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T23:07:37.122 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:37 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/13177487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:37.122 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:37 smithi052 ceph-mon[180979]: pgmap v1427: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 276 op/s; 60269/163992 objects degraded (36.751%); 2399/163992 objects misplaced (1.463%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T23:07:37.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:37.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:37.224 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:37.270 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:37.348 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:37.352 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:37 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/13177487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:37.352 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:37 smithi151 ceph-mon[159210]: pgmap v1427: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 276 op/s; 60269/163992 objects degraded (36.751%); 2399/163992 objects misplaced (1.463%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T23:07:37.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:37.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:37.454 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:37.512 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:37.572 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:37.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:37.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:37.689 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:37.760 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:37.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:37.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:37.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:37.950 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:37.989 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:38.318 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:38.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:38.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:38.424 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:38.451 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:38.523 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:38.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:38.630 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:38.662 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:38.925 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:38.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:38.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:38.930 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:07:39.030 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:39.059 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:39.892 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:39 smithi052 ceph-mon[187743]: pgmap v1428: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 294 op/s; 59492/161547 objects degraded (36.826%); 2399/161547 objects misplaced (1.485%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:07:39.892 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:39 smithi052 ceph-mon[180979]: pgmap v1428: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 294 op/s; 59492/161547 objects degraded (36.826%); 2399/161547 objects misplaced (1.485%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:07:40.068 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:40.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:40.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:40.100 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:07:40.173 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:40.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:39 smithi151 ceph-mon[159210]: pgmap v1428: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 294 op/s; 59492/161547 objects degraded (36.826%); 2399/161547 objects misplaced (1.485%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:07:40.218 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:40.293 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:40.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:40.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:40.405 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:40.461 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:40.494 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:40.659 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:40.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:40.767 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:40.826 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:40.924 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:40.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:40.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:41.032 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:41.075 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 59492/161547 objects degraded (36.826%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:41.078 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:41.114 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 59492/161547 objects degraded (36.826%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:41.116 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 59492/161547 objects degraded (36.826%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:41.153 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:41.157 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:41.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:41.265 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:41.305 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:41.827 INFO:teuthology.orchestra.run.smithi052.stderr:set pool 11 pgp_num to 26 2023-09-23T23:07:41.829 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:07:42.129 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[187743]: pgmap v1429: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 292 op/s; 59492/161094 objects degraded (36.930%); 2286/161094 objects misplaced (1.419%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2087461806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1176766579' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[180979]: pgmap v1429: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 292 op/s; 59492/161094 objects degraded (36.930%); 2286/161094 objects misplaced (1.419%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2087461806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:42.130 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1176766579' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T23:07:42.131 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T23:07:42.133 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:42.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:42.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:41 smithi151 ceph-mon[159210]: pgmap v1429: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 292 op/s; 59492/161094 objects degraded (36.930%); 2286/161094 objects misplaced (1.419%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:07:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2087461806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1176766579' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T23:07:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T23:07:42.238 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:42.278 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:07:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T23:07:43.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: osdmap e456: 8 total, 8 up, 5 in 2023-09-23T23:07:43.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:42 smithi151 ceph-mon[159210]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: osdmap e456: 8 total, 8 up, 5 in 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[180979]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:07:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T23:07:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: osdmap e456: 8 total, 8 up, 5 in 2023-09-23T23:07:43.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:42 smithi052 ceph-mon[187743]: failed to encode map e456 with expected crc 2023-09-23T23:07:43.478 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:43.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:43.583 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:43.613 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:07:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:07:43] "GET /metrics HTTP/1.1" 200 36653 "" "Prometheus/2.33.4" 2023-09-23T23:07:43.634 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:43.868 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:43 smithi052 ceph-mon[180979]: pgmap v1431: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 297 op/s; 59135/157896 objects degraded (37.452%); 2286/157896 objects misplaced (1.448%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:43.868 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:43 smithi052 ceph-mon[187743]: pgmap v1431: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 297 op/s; 59135/157896 objects degraded (37.452%); 2286/157896 objects misplaced (1.448%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:43.913 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:43.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:43.941 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:07:43.941 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:07:43.941 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T23:07:43.941 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:07:44.018 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:44.058 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:44.161 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:44.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:44.171 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:43 smithi151 ceph-mon[159210]: pgmap v1431: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 297 op/s; 59135/157896 objects degraded (37.452%); 2286/157896 objects misplaced (1.448%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:44.272 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:44.305 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:45.165 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:45.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:45.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:07:45.273 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:45.309 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:45.432 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:45.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:45.537 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:45.593 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:45.674 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:07:45.692 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:45.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:45.702 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:07:45.728 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:07:45.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:07:45.778 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:45.785 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:07:45.800 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:45.820 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:07:45.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:45 smithi151 ceph-mon[159210]: pgmap v1432: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 296 op/s; 58826/157245 objects degraded (37.410%); 2286/157245 objects misplaced (1.454%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:45.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 58826/157245 objects degraded (37.410%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:45.851 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1899959188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:45.861 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:07:45.870 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:07:45.873 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:45.896 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.899 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:07:45.961 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:45.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:46.084 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:46.115 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:45 smithi052 ceph-mon[187743]: pgmap v1432: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 296 op/s; 58826/157245 objects degraded (37.410%); 2286/157245 objects misplaced (1.454%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:46.115 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 58826/157245 objects degraded (37.410%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:46.115 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1899959188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:46.116 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:45 smithi052 ceph-mon[180979]: pgmap v1432: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 296 op/s; 58826/157245 objects degraded (37.410%); 2286/157245 objects misplaced (1.454%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:46.116 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 58826/157245 objects degraded (37.410%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:46.116 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1899959188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:46.126 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:46.886 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:46.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:46.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:46.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:47.026 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:47.294 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:47.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:47.399 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:47.452 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:47.542 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:47.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:47.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:47.654 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:47.691 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:47.792 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:47.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:47.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:47.839 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:47.897 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:47.979 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:48.086 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:48.093 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:47 smithi151 ceph-mon[159210]: pgmap v1433: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 303 op/s; 58826/154887 objects degraded (37.980%); 2174/154887 objects misplaced (1.404%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:48.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:48.195 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:48.207 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:47 smithi052 ceph-mon[187743]: pgmap v1433: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 303 op/s; 58826/154887 objects degraded (37.980%); 2174/154887 objects misplaced (1.404%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:48.208 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:47 smithi052 ceph-mon[180979]: pgmap v1433: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 303 op/s; 58826/154887 objects degraded (37.980%); 2174/154887 objects misplaced (1.404%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:48.251 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:48.327 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:48.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:48.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:48.432 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:48.471 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:48.614 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:07:48.614 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:07:48.614 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T23:07:48.614 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.068177 2023-09-23T23:07:48.615 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0.06817713979324092 2023-09-23T23:07:48.963 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:07:48.963 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:07:48.985 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:48.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:48.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:49.090 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:49.095 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:07:49.127 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:49.219 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:49.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:49.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:49.343 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:49.403 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:49.509 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:49.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:49.617 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:49.660 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:49.998 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:50.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:50.017 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:07:50.094 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:49 smithi052 ceph-mon[180979]: pgmap v1434: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 300 op/s; 58079/152514 objects degraded (38.081%); 2174/152514 objects misplaced (1.425%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T23:07:50.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:50.095 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:50.095 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:49 smithi052 ceph-mon[187743]: pgmap v1434: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 300 op/s; 58079/152514 objects degraded (38.081%); 2174/152514 objects misplaced (1.425%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T23:07:50.095 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:50.095 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:50.109 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:50.124 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:49 smithi151 ceph-mon[159210]: pgmap v1434: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 300 op/s; 58079/152514 objects degraded (38.081%); 2174/152514 objects misplaced (1.425%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T23:07:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:07:50.165 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:07:50.166 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:50.258 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:50.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:50.371 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:50.415 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:50.443 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:50.600 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:50.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:50.707 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:50.793 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:50.884 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:50.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:50.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:50.993 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:51.039 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 58079/152514 objects degraded (38.081%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:51.040 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 58079/152514 objects degraded (38.081%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:51.045 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:51.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 58079/152514 objects degraded (38.081%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:51.881 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:07:51.970 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:51.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:52.076 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:52.115 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:51 smithi052 ceph-mon[180979]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 307 op/s; 58079/151875 objects degraded (38.241%); 2067/151875 objects misplaced (1.361%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T23:07:52.115 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2728542393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:07:52.115 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:51 smithi052 ceph-mon[187743]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 307 op/s; 58079/151875 objects degraded (38.241%); 2067/151875 objects misplaced (1.361%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T23:07:52.116 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2728542393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:07:52.120 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:52.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:51 smithi151 ceph-mon[159210]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 307 op/s; 58079/151875 objects degraded (38.241%); 2067/151875 objects misplaced (1.361%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T23:07:52.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2728542393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:07:53.004 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:53.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:53.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:53.101 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:53.108 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[180979]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2728542393' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[180979]: osdmap e457: 8 total, 8 up, 5 in 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[180979]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[180979]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:53.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[187743]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2728542393' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:07:53.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[187743]: osdmap e457: 8 total, 8 up, 5 in 2023-09-23T23:07:53.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[187743]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.150 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:52 smithi052 ceph-mon[187743]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.157 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:52 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:07:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:52 smithi151 ceph-mon[159210]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2728542393' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:07:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:52 smithi151 ceph-mon[159210]: osdmap e457: 8 total, 8 up, 5 in 2023-09-23T23:07:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:52 smithi151 ceph-mon[159210]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:52 smithi151 ceph-mon[159210]: failed to encode map e457 with expected crc 2023-09-23T23:07:53.390 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:53.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:53.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:53.495 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:07:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:07:53] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.33.4" 2023-09-23T23:07:53.539 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:53.633 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:53.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:53.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:53.743 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:53.795 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:53.841 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:07:53.861 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:07:53.865 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:53.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:53.899 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:07:53.929 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:07:53.963 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:07:53.977 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:53.989 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:07:54.008 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:54.016 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:07:54.056 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:07:54.058 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.058 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.058 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.059 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.059 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.059 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.059 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.059 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:07:54.074 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:54.080 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:53 smithi151 ceph-mon[159210]: pgmap v1437: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 313 op/s; 57687/148500 objects degraded (38.846%); 2067/148500 objects misplaced (1.392%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T23:07:54.080 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/518691763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:54.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:54.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:54.182 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:54.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:53 smithi052 ceph-mon[187743]: pgmap v1437: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 313 op/s; 57687/148500 objects degraded (38.846%); 2067/148500 objects misplaced (1.392%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T23:07:54.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/518691763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:54.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:53 smithi052 ceph-mon[180979]: pgmap v1437: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 313 op/s; 57687/148500 objects degraded (38.846%); 2067/148500 objects misplaced (1.392%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T23:07:54.223 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/518691763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:07:54.233 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:54.329 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:54.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:54.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:54.448 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:07:54.492 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:54.549 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:07:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:54.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:07:54.656 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:54.706 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:54.798 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:54.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:54.890 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.1 primary-affinity to 0.0681771 (844682) 2023-09-23T23:07:54.906 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:54.939 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:55.007 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:07:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1380004253' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]: dispatch 2023-09-23T23:07:55.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:54 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]: dispatch 2023-09-23T23:07:55.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1380004253' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]: dispatch 2023-09-23T23:07:55.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:54 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]: dispatch 2023-09-23T23:07:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1380004253' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]: dispatch 2023-09-23T23:07:55.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:54 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]: dispatch 2023-09-23T23:07:55.505 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:55.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:55.610 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:07:55.636 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:55.679 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:07:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:55.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:07:55.783 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:55.813 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: pgmap v1438: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 315 op/s; 57351/147783 objects degraded (38.808%); 2067/147783 objects misplaced (1.399%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]': finished 2023-09-23T23:07:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: osdmap e458: 8 total, 8 up, 5 in 2023-09-23T23:07:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: failed to encode map e458 with expected crc 2023-09-23T23:07:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: failed to encode map e458 with expected crc 2023-09-23T23:07:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: failed to encode map e458 with expected crc 2023-09-23T23:07:55.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: failed to encode map e458 with expected crc 2023-09-23T23:07:55.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 57351/147783 objects degraded (38.808%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:55.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:55 smithi151 ceph-mon[159210]: osdmap e459: 8 total, 8 up, 5 in 2023-09-23T23:07:56.015 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:56.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:56.118 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:56.121 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:56.146 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:56.189 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:56.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: pgmap v1438: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 315 op/s; 57351/147783 objects degraded (38.808%); 2067/147783 objects misplaced (1.399%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]': finished 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: osdmap e458: 8 total, 8 up, 5 in 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 57351/147783 objects degraded (38.808%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:56.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[187743]: osdmap e459: 8 total, 8 up, 5 in 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: pgmap v1438: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 315 op/s; 57351/147783 objects degraded (38.808%); 2067/147783 objects misplaced (1.399%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.06817713979324092}]': finished 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: osdmap e458: 8 total, 8 up, 5 in 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 57351/147783 objects degraded (38.808%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:07:56.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:55 smithi052 ceph-mon[180979]: osdmap e459: 8 total, 8 up, 5 in 2023-09-23T23:07:56.294 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:56.326 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:56.476 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:56.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:56.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:56.582 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:07:56.646 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18869 11.f e415/410 11:ff878455:::benchmark_data_smithi052_606809_object18868:head v 415'1185, mlcod=415'1183)", 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.018560+0000", 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.66434547200001, 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123187853, 2023-09-23T23:07:56.691 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018560+0000", 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018560+0000", 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.692 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018559+0000", 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018588+0000", 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8861999999999999e-05 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.693 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018600+0000", 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2295e-05 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018609+0000", 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1680000000000004e-06 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.694 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018655+0000", 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5895e-05 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018676+0000", 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1279000000000001e-05 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141706+0000", 2023-09-23T23:07:56.695 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12302988199999999 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141748+0000", 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2484000000000002e-05 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18874 11.c e415/410 11:32037efb:::benchmark_data_smithi052_606809_object18873:head v 415'1134, mlcod=415'1132)", 2023-09-23T23:07:56.696 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.018626+0000", 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.66428014900001, 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.121485887, 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018626+0000", 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.697 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018626+0000", 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018625+0000", 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.698 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018650+0000", 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4698999999999998e-05 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018658+0000", 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4409999999999993e-06 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.699 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018664+0000", 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4070000000000001e-06 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018896+0000", 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000232067 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.700 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018909+0000", 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3356999999999999e-05 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019955+0000", 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001045299 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.701 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140112+0000", 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.120156981 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18872 11.d e415/410 11:b7e9ee71:::benchmark_data_smithi052_606809_object18871:head v 415'1191, mlcod=415'1189)", 2023-09-23T23:07:56.702 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.018678+0000", 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.66422820899999, 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122316303, 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018678+0000", 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.703 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018678+0000", 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018677+0000", 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.704 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018703+0000", 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6233000000000001e-05 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018711+0000", 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4970000000000004e-06 2023-09-23T23:07:56.705 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018716+0000", 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4580000000000001e-06 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018762+0000", 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.5433999999999999e-05 2023-09-23T23:07:56.706 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018781+0000", 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9020000000000001e-05 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140959+0000", 2023-09-23T23:07:56.707 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122177981 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140994+0000", 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4989e-05 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.708 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18871 11.f e415/410 11:f8b42f1e:::benchmark_data_smithi052_606809_object18870:head v 415'1186, mlcod=415'1183)", 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.018730+0000", 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.664176174, 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12304536100000001, 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.709 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018730+0000", 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018730+0000", 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.710 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018729+0000", 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.711 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018752+0000", 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2201e-05 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018758+0000", 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8619999999999997e-06 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018764+0000", 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.468e-06 2023-09-23T23:07:56.712 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019531+0000", 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00076726599999999995 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019552+0000", 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1248000000000001e-05 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.713 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141753+0000", 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122201025 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141775+0000", 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1577e-05 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.714 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18877 11.d e415/410 11:bc69900b:::benchmark_data_smithi052_606809_object18876:head v 415'1192, mlcod=415'1189)", 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.018769+0000", 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.664136638, 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122533489, 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018769+0000", 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.715 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018769+0000", 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018769+0000", 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.716 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018790+0000", 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1047999999999999e-05 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018797+0000", 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6800000000000004e-06 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.018801+0000", 2023-09-23T23:07:56.717 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.352e-06 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019613+0000", 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00081232899999999998 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140084+0000", 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.120470458 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.718 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141277+0000", 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001193632 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141303+0000", 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.527e-05 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.719 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18878 11.7 e415/410 11:e92c6813:::benchmark_data_smithi052_606809_object18877:head v 415'1169, mlcod=415'1167)", 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.019655+0000", 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.66325027900001, 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.122157926, 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019655+0000", 2023-09-23T23:07:56.720 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019655+0000", 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019655+0000", 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.721 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.019708+0000", 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3003000000000001e-05 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140105+0000", 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12039715500000001 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.722 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140114+0000", 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3170000000000008e-06 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140165+0000", 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0315000000000002e-05 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.723 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140187+0000", 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2235999999999999e-05 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141783+0000", 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0015960659999999999 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.141813+0000", 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0576e-05 2023-09-23T23:07:56.724 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:18879 11.6 e415/410 11:67005c97:::benchmark_data_smithi052_606809_object18878:head v 415'1164, mlcod=415'1162)", 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:04.020706+0000", 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 232.66219952500001, 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.121650387, 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.725 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.020706+0000", 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.726 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.020706+0000", 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.020705+0000", 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.020729+0000", 2023-09-23T23:07:56.727 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3759000000000001e-05 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140135+0000", 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.119406199 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140145+0000", 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3570000000000004e-06 2023-09-23T23:07:56.728 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140192+0000", 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7409e-05 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.140207+0000", 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4572999999999999e-05 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.729 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.142317+0000", 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002110631 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:04.142357+0000", 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9212000000000003e-05 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.730 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26944 11.f e416/410 11:f14a1a11:::benchmark_data_smithi052_606809_object26943:head v 416'1685, mlcod=416'1683)", 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.506813+0000", 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.176092595, 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12853473400000001, 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.731 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506813+0000", 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506813+0000", 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.732 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506813+0000", 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506835+0000", 2023-09-23T23:07:56.733 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2425999999999998e-05 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506845+0000", 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8339999999999994e-06 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506851+0000", 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2709999999999998e-06 2023-09-23T23:07:56.734 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506954+0000", 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00010315300000000001 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.506973+0000", 2023-09-23T23:07:56.735 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8454e-05 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.635307+0000", 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12833402799999999 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.635348+0000", 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.1109000000000002e-05 2023-09-23T23:07:56.736 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26948 11.e e416/410 11:75f56cd8:::benchmark_data_smithi052_606809_object26947:head v 416'1692, mlcod=416'1690)", 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.508539+0000", 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.17436725799999, 2023-09-23T23:07:56.737 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.124711618, 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.508539+0000", 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.738 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.508539+0000", 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.508538+0000", 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.739 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509778+0000", 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012403340000000001 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509792+0000", 2023-09-23T23:07:56.740 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3560999999999999e-05 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509800+0000", 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8869999999999992e-06 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.741 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509861+0000", 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1056000000000006e-05 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509883+0000", 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2120000000000002e-05 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.742 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633213+0000", 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123330255 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633250+0000", 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7171e-05 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.743 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26950 11.4 e416/410 11:26c74ea6:::benchmark_data_smithi052_606809_object26949:head v 416'1670, mlcod=416'1668)", 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.509828+0000", 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.173077662, 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123458921, 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.744 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509828+0000", 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509828+0000", 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.745 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509828+0000", 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509850+0000", 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2580999999999999e-05 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509858+0000", 2023-09-23T23:07:56.746 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5159999999999997e-06 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509863+0000", 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0839999999999999e-06 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509902+0000", 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9589000000000003e-05 2023-09-23T23:07:56.747 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509920+0000", 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7791999999999998e-05 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633252+0000", 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123332436 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.748 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633287+0000", 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4532000000000001e-05 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26949 11.f e416/410 11:f2b35dc1:::benchmark_data_smithi052_606809_object26948:head v 416'1686, mlcod=416'1683)", 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.509876+0000", 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.17302984299999, 2023-09-23T23:07:56.749 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12550117199999999, 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509876+0000", 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509876+0000", 2023-09-23T23:07:56.750 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509876+0000", 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509897+0000", 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1192e-05 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.751 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509903+0000", 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.3459999999999999e-06 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509908+0000", 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4100000000000001e-06 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.752 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.510009+0000", 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000101432 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.510029+0000", 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9663000000000001e-05 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.635354+0000", 2023-09-23T23:07:56.753 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.125325412 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.635377+0000", 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2994999999999999e-05 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26952 11.f e416/410 11:f79f1353:::benchmark_data_smithi052_606809_object26951:head v 416'1687, mlcod=416'1683)", 2023-09-23T23:07:56.754 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.509920+0000", 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.17298593000001, 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12638785799999999, 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509920+0000", 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.755 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509920+0000", 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509920+0000", 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.756 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509940+0000", 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0078e-05 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509946+0000", 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2330000000000003e-06 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.509950+0000", 2023-09-23T23:07:56.757 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3259999999999997e-06 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.634354+0000", 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12440379 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.634375+0000", 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0758e-05 2023-09-23T23:07:56.758 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.636280+0000", 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001905084 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.636308+0000", 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.779e-05 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.759 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.760 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.760 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26953 11.9 e416/410 11:92330c24:::benchmark_data_smithi052_606809_object26952:head v 416'1645, mlcod=416'1641)", 2023-09-23T23:07:56.760 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.512171+0000", 2023-09-23T23:07:56.760 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.170734474, 2023-09-23T23:07:56.760 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123637914, 2023-09-23T23:07:56.760 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.512171+0000", 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.512171+0000", 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.761 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.512170+0000", 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.762 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.512194+0000", 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4232000000000001e-05 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633182+0000", 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12098745800000001 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633197+0000", 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.522e-05 2023-09-23T23:07:56.763 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633250+0000", 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3127999999999997e-05 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633274+0000", 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3455000000000002e-05 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.764 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.635773+0000", 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002499603 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.635809+0000", 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5890999999999999e-05 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.765 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:26955 11.7 e416/410 11:ea04cb06:::benchmark_data_smithi052_606809_object26954:head v 416'1673, mlcod=416'1671)", 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:10.513184+0000", 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 226.16972137100001, 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.123091887, 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.766 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.513184+0000", 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.513184+0000", 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.767 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.768 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.768 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.513184+0000", 2023-09-23T23:07:56.768 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.768 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.768 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.513217+0000", 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.3420999999999997e-05 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633197+0000", 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.119979876 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.769 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.633207+0000", 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7620000000000007e-06 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.634282+0000", 2023-09-23T23:07:56.770 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0010753449999999999 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.634300+0000", 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7850999999999999e-05 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.636245+0000", 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001945359 2023-09-23T23:07:56.771 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:10.636276+0000", 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1021e-05 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.772 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:63359 11.7 e419/410 11:e59b5d85:::benchmark_data_smithi052_606809_object63358:head v 419'3940, mlcod=419'3937)", 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:50.794996+0000", 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 185.88791006899999, 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19420020800000001, 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.773 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.794996+0000", 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.794996+0000", 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.774 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.794995+0000", 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.795072+0000", 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.7016999999999998e-05 2023-09-23T23:07:56.775 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.795080+0000", 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0239999999999997e-06 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.776 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.795085+0000", 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9989999999999999e-06 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.984515+0000", 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18942983699999999 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.777 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.984540+0000", 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.461e-05 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.989161+0000", 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046212780000000004 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.778 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.989196+0000", 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4978999999999998e-05 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:72293 11.b e424/410 11:d68ae8c4:::benchmark_data_smithi052_606809_object72292:head v 424'4509, mlcod=423'4507)", 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.390556+0000", 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.29234998699999, 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.210600275, 2023-09-23T23:07:56.779 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390556+0000", 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390556+0000", 2023-09-23T23:07:56.780 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390556+0000", 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967296 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390579+0000", 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3207999999999999e-05 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.781 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390588+0000", 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6369999999999994e-06 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390596+0000", 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3699999999999997e-06 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.782 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390642+0000", 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6312e-05 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.390660+0000", 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7445e-05 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.394166+0000", 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035067420000000002 2023-09-23T23:07:56.783 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.601156+0000", 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.206989798 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:72289 11.c e423/410 11:3ab98c7d:::benchmark_data_smithi052_606809_object72288:head v 423'4508, mlcod=423'4506)", 2023-09-23T23:07:56.784 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.398507+0000", 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 171.28439845299999, 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20873881, 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.398507+0000", 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.785 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.398507+0000", 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.398506+0000", 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.786 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.398535+0000", 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8870999999999999e-05 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.601092+0000", 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.202557131 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.601107+0000", 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4297999999999999e-05 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.787 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.601382+0000", 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00027535 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.601407+0000", 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4926000000000002e-05 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.788 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.607221+0000", 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058144290000000003 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.607246+0000", 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4825999999999999e-05 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.789 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:100697 11.5 e449/441 11:a4984fa5:::benchmark_data_smithi052_606809_object18543:head v 449'5810, mlcod=415'1186)", 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:07:01.097965+0000", 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.584940791999998, 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12516001500000001, 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.097965+0000", 2023-09-23T23:07:56.790 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.097965+0000", 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.097964+0000", 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.791 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.097970+0000", 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4380000000000003e-06 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.097976+0000", 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6540000000000002e-06 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.792 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.097981+0000", 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0450000000000002e-06 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.098009+0000", 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.8293000000000002e-05 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.098026+0000", 2023-09-23T23:07:56.793 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7263000000000001e-05 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.223040+0000", 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.12501390600000001 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:01.223125+0000", 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4988999999999996e-05 2023-09-23T23:07:56.794 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPush(11.15 456/440 [PushOp(11:ae4facb7:::benchmark_data_smithi052_606809_object29977:head, version: 416'1874, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:ae4facb7:::benchmark_data_smithi052_606809_object29977:head@416'1874, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:07:42.570337+0000", 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.112569055, 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.084227645000000004, 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.795 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570337+0000", 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570337+0000", 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.796 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570336+0000", 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570379+0000", 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2648000000000002e-05 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570391+0000", 2023-09-23T23:07:56.797 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2405000000000001e-05 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570401+0000", 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0018e-05 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.798 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570451+0000", 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0268000000000001e-05 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.570460+0000", 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1699999999999997e-06 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.799 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.654564+0000", 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.084104813 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:112155 11.16 e456/433 11:69326b1b:::benchmark_data_smithi052_606809_object30001:head v 456'6110, mlcod=415'1486)", 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:07:42.571998+0000", 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.110907881999999, 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.084999274, 2023-09-23T23:07:56.800 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.571998+0000", 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.801 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.571998+0000", 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.571997+0000", 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:07:56.802 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.572009+0000", 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1654000000000001e-05 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.652697+0000", 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.080688626999999999 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.803 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.652728+0000", 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0190000000000001e-05 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.652814+0000", 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6429999999999997e-05 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:07:56.804 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.652839+0000", 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4471999999999999e-05 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.656961+0000", 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041227629999999998 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:07:56.805 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:07:42.656997+0000", 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.5818000000000002e-05 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:07:56.806 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:07:56.885 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[180979]: failed to encode map e459 with expected crc 2023-09-23T23:07:56.885 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[180979]: failed to encode map e459 with expected crc 2023-09-23T23:07:56.885 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[180979]: failed to encode map e459 with expected crc 2023-09-23T23:07:56.885 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[180979]: failed to encode map e458 with expected crc 2023-09-23T23:07:56.885 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[180979]: osdmap e460: 8 total, 8 up, 5 in 2023-09-23T23:07:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:56 smithi151 ceph-mon[159210]: failed to encode map e459 with expected crc 2023-09-23T23:07:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:56 smithi151 ceph-mon[159210]: failed to encode map e459 with expected crc 2023-09-23T23:07:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:56 smithi151 ceph-mon[159210]: failed to encode map e459 with expected crc 2023-09-23T23:07:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:56 smithi151 ceph-mon[159210]: failed to encode map e458 with expected crc 2023-09-23T23:07:57.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:56 smithi151 ceph-mon[159210]: osdmap e460: 8 total, 8 up, 5 in 2023-09-23T23:07:57.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[187743]: failed to encode map e459 with expected crc 2023-09-23T23:07:57.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[187743]: failed to encode map e459 with expected crc 2023-09-23T23:07:57.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[187743]: failed to encode map e459 with expected crc 2023-09-23T23:07:57.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[187743]: failed to encode map e458 with expected crc 2023-09-23T23:07:57.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:56 smithi052 ceph-mon[187743]: osdmap e460: 8 total, 8 up, 5 in 2023-09-23T23:07:57.345 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:07:57.489 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:07:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:57.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:07:57.593 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:57.625 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:57.994 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:58.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:58.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:07:58.026 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:07:58.116 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:07:58.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[187743]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[187743]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[187743]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[187743]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.153 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[187743]: pgmap v1442: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 291 op/s; 57351/145848 objects degraded (39.322%); 1960/145848 objects misplaced (1.344%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:07:58.153 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[180979]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[180979]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[180979]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[180979]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.154 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:57 smithi052 ceph-mon[180979]: pgmap v1442: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 291 op/s; 57351/145848 objects degraded (39.322%); 1960/145848 objects misplaced (1.344%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:07:58.159 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:57 smithi151 ceph-mon[159210]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:57 smithi151 ceph-mon[159210]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:57 smithi151 ceph-mon[159210]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:57 smithi151 ceph-mon[159210]: failed to encode map e460 with expected crc 2023-09-23T23:07:58.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:57 smithi151 ceph-mon[159210]: pgmap v1442: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 291 op/s; 57351/145848 objects degraded (39.322%); 1960/145848 objects misplaced (1.344%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T23:07:58.288 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:07:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:58.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:07:58.395 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:07:58.411 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:58.439 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:58.524 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:07:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:58.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:07:58.632 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:07:58.664 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:58.718 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:07:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:58.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:07:58.824 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:07:58.854 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:07:58.854 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:58.915 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:07:58.922 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:07:58.928 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3918089197' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:58.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:07:58.946 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3918089197' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:58.946 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3918089197' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:07:59.030 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:07:59.085 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:07:59.981 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:07:59.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:07:59.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:00.086 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:00.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: pgmap v1443: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 252 op/s; 56807/143961 objects degraded (39.460%); 1960/143961 objects misplaced (1.361%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T23:08:00.146 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3918089197' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: osdmap e461: 8 total, 8 up, 5 in 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[187743]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: pgmap v1443: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 252 op/s; 56807/143961 objects degraded (39.460%); 1960/143961 objects misplaced (1.361%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3918089197' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:00.147 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: osdmap e461: 8 total, 8 up, 5 in 2023-09-23T23:08:00.148 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:07:59 smithi052 ceph-mon[180979]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.151 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: pgmap v1443: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 252 op/s; 56807/143961 objects degraded (39.460%); 1960/143961 objects misplaced (1.361%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3918089197' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: osdmap e461: 8 total, 8 up, 5 in 2023-09-23T23:08:00.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:07:59 smithi151 ceph-mon[159210]: failed to encode map e461 with expected crc 2023-09-23T23:08:00.281 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:00.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:00.389 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:00.439 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:00.534 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:00.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:00.646 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:00.693 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:00.721 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:00.747 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:00.775 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:00.775 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:00.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:00.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:00.804 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:00.832 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:00.863 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:00.885 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:00.897 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:00.928 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:00.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.930 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.931 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:08:00.938 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 56807/143961 objects degraded (39.460%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3298233469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:00.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:00.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:01.036 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:08:01.036 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:08:01.037 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T23:08:01.037 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:08:01.037 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:08:01.037 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T23:08:01.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 56807/143961 objects degraded (39.460%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:01.062 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3298233469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:01.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 56807/143961 objects degraded (39.460%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:01.062 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3298233469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:01.091 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:01.122 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:01.212 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:01.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:01.330 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:01.395 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:01.473 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:01.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:01.579 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:01.606 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:02.085 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:02.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:02.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:02.132 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:08:02.175 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:01 smithi052 ceph-mon[187743]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 56807/143505 objects degraded (39.585%); 1848/143505 objects misplaced (1.288%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T23:08:02.175 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:01 smithi052 ceph-mon[180979]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 56807/143505 objects degraded (39.585%); 1848/143505 objects misplaced (1.288%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T23:08:02.191 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:02.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:01 smithi151 ceph-mon[159210]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 242 op/s; 56807/143505 objects degraded (39.585%); 1848/143505 objects misplaced (1.288%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T23:08:02.231 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:02.306 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:02.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:02.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:02.414 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:02.438 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:02.447 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:02.514 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:02.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:02.583 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:08:02.584 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:08:02.623 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:02.661 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:03.248 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:03.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:03.353 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:03.414 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:03.495 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:03.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:03.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:03.606 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:03.609 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:08:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:08:03] "GET /metrics HTTP/1.1" 200 36658 "" "Prometheus/2.33.4" 2023-09-23T23:08:03.656 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:03.715 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:03.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:03.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:03.825 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:03.873 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:04.069 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:08:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:03 smithi151 ceph-mon[159210]: pgmap v1446: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 56328/139488 objects degraded (40.382%); 1848/139488 objects misplaced (1.325%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T23:08:04.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:03 smithi052 ceph-mon[180979]: pgmap v1446: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 56328/139488 objects degraded (40.382%); 1848/139488 objects misplaced (1.325%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T23:08:04.218 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:03 smithi052 ceph-mon[187743]: pgmap v1446: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 56328/139488 objects degraded (40.382%); 1848/139488 objects misplaced (1.325%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T23:08:04.254 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:08:04.430 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:04.496 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:04.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:04.605 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:04.659 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:04.724 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:04.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:04.832 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:04.876 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:05.222 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:05.372 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:05.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:05.478 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:05.508 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:05.520 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:08:05.520 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "" 2023-09-23T23:08:05.520 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:08:05.559 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:05.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:05.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:05.663 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[187743]: pgmap v1447: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 55836/138147 objects degraded (40.418%); 1848/138147 objects misplaced (1.338%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T23:08:05.688 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 56328/139488 objects degraded (40.382%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:05.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:05.688 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:05.689 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[180979]: pgmap v1447: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 55836/138147 objects degraded (40.418%); 1848/138147 objects misplaced (1.338%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T23:08:05.689 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 56328/139488 objects degraded (40.382%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:05.690 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:05.745 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:05.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:05.851 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:05.887 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:05 smithi151 ceph-mon[159210]: pgmap v1447: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 55836/138147 objects degraded (40.418%); 1848/138147 objects misplaced (1.338%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T23:08:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 56328/139488 objects degraded (40.382%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:06.915 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:06.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:06.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:07.033 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:07.098 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:07.290 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:07.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:07.298 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:07.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:07.399 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:07.443 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:07.540 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:07.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:07.647 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:07.683 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:07.766 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:07.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:07.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:07.878 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:07.904 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:07 smithi052 ceph-mon[187743]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 344 op/s; 55836/135519 objects degraded (41.202%); 1739/135519 objects misplaced (1.283%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:08:07.904 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:07 smithi052 ceph-mon[180979]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 344 op/s; 55836/135519 objects degraded (41.202%); 1739/135519 objects misplaced (1.283%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:08:07.907 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:07 smithi151 ceph-mon[159210]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 344 op/s; 55836/135519 objects degraded (41.202%); 1739/135519 objects misplaced (1.283%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T23:08:08.328 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:08.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:08.433 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:08.485 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:08.564 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:08.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:08.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:08.672 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:08.705 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:08.757 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:08.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:08.849 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:08:08.862 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:08.878 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3288885777' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:08.878 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:08.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3288885777' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:08.878 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:08.902 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:08.955 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:08.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:09.062 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:09.096 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3288885777' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:09.465 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:09.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:09.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:09.570 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:09.599 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:09.975 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:09.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:10.071 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: pgmap v1449: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 55484/134628 objects degraded (41.213%); 1739/134628 objects misplaced (1.292%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: osdmap e462: 8 total, 8 up, 5 in 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.072 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[187743]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: pgmap v1449: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 55484/134628 objects degraded (41.213%); 1739/134628 objects misplaced (1.292%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: osdmap e462: 8 total, 8 up, 5 in 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:09 smithi052 ceph-mon[180979]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.081 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:10.142 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: pgmap v1449: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 55484/134628 objects degraded (41.213%); 1739/134628 objects misplaced (1.292%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: osdmap e462: 8 total, 8 up, 5 in 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:09 smithi151 ceph-mon[159210]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.248 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:10.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:10.355 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:10.412 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:10.506 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:10.514 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:10.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:10.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:10.539 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:10.562 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:10.588 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:10.613 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:10.620 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:10.644 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:10.669 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:10.692 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:10.726 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:10.727 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.728 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:08:10.760 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:08:10.760 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:08:10.760 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:08:10.760 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:08:10.761 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:08:10.766 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:10.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:10.873 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:10.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:10 smithi052 ceph-mon[187743]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 55484/134628 objects degraded (41.213%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:10.929 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2817469227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:10.930 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:10 smithi052 ceph-mon[180979]: failed to encode map e462 with expected crc 2023-09-23T23:08:10.930 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 55484/134628 objects degraded (41.213%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:10.930 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2817469227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:10.946 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:11.041 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:11.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:11.150 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:11.200 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:11.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:10 smithi151 ceph-mon[159210]: failed to encode map e462 with expected crc 2023-09-23T23:08:11.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 55484/134628 objects degraded (41.213%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:11.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2817469227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:11.265 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:11.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:11.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:11.375 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:11.410 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:11.736 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:11.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:11.849 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:11.877 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:11.933 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:11.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:11.958 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:08:11.959 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:08:11.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:11 smithi052 ceph-mon[180979]: pgmap v1451: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.7 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 385 op/s; 55006/131940 objects degraded (41.690%); 1632/131940 objects misplaced (1.237%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T23:08:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:11 smithi052 ceph-mon[187743]: pgmap v1451: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.7 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 385 op/s; 55006/131940 objects degraded (41.690%); 1632/131940 objects misplaced (1.237%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T23:08:12.045 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:12.103 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:08:12.104 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:11 smithi151 ceph-mon[159210]: pgmap v1451: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.7 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 385 op/s; 55006/131940 objects degraded (41.690%); 1632/131940 objects misplaced (1.237%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T23:08:12.108 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:12.176 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:12.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:12.279 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:12.376 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:08:12.391 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:12.483 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:12.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:12.596 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:12.698 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:12.744 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:12.774 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:12.784 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:12.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:12.894 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:12.931 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:13.429 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:13.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:08:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:08:13] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.33.4" 2023-09-23T23:08:13.563 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:13.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:13.670 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:13.712 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:13.766 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:13.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:13.858 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:13 smithi052 ceph-mon[180979]: pgmap v1452: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 346 op/s; 54941/129084 objects degraded (42.562%); 1632/129084 objects misplaced (1.264%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T23:08:13.872 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:13.925 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:14.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:13 smithi151 ceph-mon[159210]: pgmap v1452: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 346 op/s; 54941/129084 objects degraded (42.562%); 1632/129084 objects misplaced (1.264%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T23:08:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:13 smithi052 ceph-mon[187743]: pgmap v1452: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 346 op/s; 54941/129084 objects degraded (42.562%); 1632/129084 objects misplaced (1.264%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T23:08:14.336 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:08:15.062 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:15.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:15.167 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:15.238 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:15.627 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:15.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:15.737 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:15.774 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:15.826 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:15.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:15.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:15 smithi052 ceph-mon[180979]: pgmap v1453: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 390 op/s; 54141/126417 objects degraded (42.827%); 1632/126417 objects misplaced (1.291%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:08:15.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:15 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 54141/126417 objects degraded (42.827%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:15.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:15 smithi052 ceph-mon[187743]: pgmap v1453: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 390 op/s; 54141/126417 objects degraded (42.827%); 1632/126417 objects misplaced (1.291%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:08:15.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:15 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 54141/126417 objects degraded (42.827%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:15.931 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:16.003 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:15 smithi151 ceph-mon[159210]: pgmap v1453: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 390 op/s; 54141/126417 objects degraded (42.827%); 1632/126417 objects misplaced (1.291%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:08:16.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:15 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 54141/126417 objects degraded (42.827%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:16.815 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:17.003 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:17.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:17.115 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:17.171 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:17.268 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:17.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:17.383 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:17.447 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:17.607 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:17.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:17.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:17.711 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:17.787 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:17.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[180979]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 328 op/s; 54141/125664 objects degraded (43.084%); 1519/125664 objects misplaced (1.209%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T23:08:17.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/929093776' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:17.882 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3079976547' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:08:17.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:17.883 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:08:17.889 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:08:17.891 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:08:17.894 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72274 11.0 11:091c96c6:::benchmark_data_smithi052_606809_object72273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.359155+0000", 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.5313247, 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35951398299999998, 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.895 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72274 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359155+0000", 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.896 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359155+0000", 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359154+0000", 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359176+0000", 2023-09-23T23:08:17.897 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2184999999999999e-05 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359185+0000", 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3670000000000003e-06 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359192+0000", 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.6579999999999996e-06 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.898 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.365273+0000", 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060809260000000004 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.365469+0000", 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019644899999999999 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:17.899 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.365656+0000", 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000186126 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371407+0000", 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057510319999999997 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400824+0000", 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.029417499 2023-09-23T23:08:17.900 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400866+0000", 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.2184000000000001e-05 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400935+0000", 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8869999999999996e-05 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.901 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.718669+0000", 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.31773385799999998 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72275 11.0 11:038c1362:::benchmark_data_smithi052_606809_object72274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.359198+0000", 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.531281691, 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36280767000000003, 2023-09-23T23:08:17.902 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72275 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359198+0000", 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.903 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359198+0000", 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359198+0000", 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.904 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359217+0000", 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.959e-05 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359225+0000", 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.413e-06 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.359230+0000", 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5949999999999998e-06 2023-09-23T23:08:17.905 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.368430+0000", 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0091995719999999996 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.368586+0000", 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015664499999999999 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.906 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.368702+0000", 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000115161 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371712+0000", 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030108969999999998 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.718730+0000", 2023-09-23T23:08:17.907 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.347017925 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.718768+0000", 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.7174e-05 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.718812+0000", 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4696e-05 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.908 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.722006+0000", 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0031934419999999999 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72279 11.8 11:19c81c5a:::benchmark_data_smithi052_606809_object72278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.371780+0000", 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.51869995600001, 2023-09-23T23:08:17.909 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35401158300000002, 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72279 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371780+0000", 2023-09-23T23:08:17.910 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371780+0000", 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371778+0000", 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.911 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371809+0000", 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.0899999999999999e-05 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371827+0000", 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8111e-05 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.912 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371837+0000", 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7480000000000005e-06 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371892+0000", 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5220000000000003e-05 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.372100+0000", 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000207655 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.913 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.372290+0000", 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019020199999999999 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.378203+0000", 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0059133370000000003 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.914 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.380682+0000", 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002478619 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.722066+0000", 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34138423499999998 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.722123+0000", 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6947000000000002e-05 2023-09-23T23:08:17.915 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.725791+0000", 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0036681629999999999 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72283 11.0 11:036003fb:::benchmark_data_smithi052_606809_object72282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.371851+0000", 2023-09-23T23:08:17.916 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.51862882699999, 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35357732400000003, 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72283 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371851+0000", 2023-09-23T23:08:17.917 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371851+0000", 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371850+0000", 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.918 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371875+0000", 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4581000000000001e-05 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371901+0000", 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6274999999999998e-05 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371912+0000", 2023-09-23T23:08:17.919 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.06e-05 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.372126+0000", 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021405099999999999 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.372282+0000", 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015645599999999999 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.920 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.372459+0000", 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017638199999999999 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.378215+0000", 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0057561499999999998 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.921 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.722388+0000", 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34417360699999999 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.722425+0000", 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6508999999999999e-05 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.722481+0000", 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6416000000000003e-05 2023-09-23T23:08:17.922 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.725428+0000", 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002946878 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72284 11.8 11:18627106:::benchmark_data_smithi052_606809_object72283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.923 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.371924+0000", 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.51855574199999, 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35772018500000002, 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72284 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.924 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371924+0000", 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371924+0000", 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371923+0000", 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:08:17.925 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371955+0000", 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1662000000000003e-05 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371967+0000", 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2557e-05 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.926 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.371973+0000", 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0100000000000001e-06 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.375147+0000", 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003173291 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.375324+0000", 2023-09-23T23:08:17.927 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000177598 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.375520+0000", 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019568499999999999 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.378221+0000", 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027012579999999998 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.928 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400770+0000", 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.022548651999999999 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.725848+0000", 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.325077587 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.929 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.725900+0000", 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2932000000000001e-05 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.729644+0000", 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0037436750000000001 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.930 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72287 11.0 11:083fc723:::benchmark_data_smithi052_606809_object72286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.388364+0000", 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.50211610599999, 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36052509399999999, 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72287 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.931 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388364+0000", 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388364+0000", 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.932 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388362+0000", 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388389+0000", 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7197999999999999e-05 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388403+0000", 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3828999999999999e-05 2023-09-23T23:08:17.933 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388411+0000", 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.3299999999999999e-06 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.729684+0000", 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.341273149 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.934 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.729844+0000", 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015985899999999999 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.730003+0000", 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015847499999999999 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.737307+0000", 2023-09-23T23:08:17.935 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073038369999999997 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.738699+0000", 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0013924079999999999 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.746048+0000", 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0073492760000000001 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.936 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.746118+0000", 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.004e-05 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.748889+0000", 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027703179999999999 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.937 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72291 11.0 11:0122111c:::benchmark_data_smithi052_606809_object72290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e423)", 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.388458+0000", 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 192.50202209599999, 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36334318100000002, 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72291 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.938 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388458+0000", 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388458+0000", 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.939 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388457+0000", 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388475+0000", 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8227999999999999e-05 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388481+0000", 2023-09-23T23:08:17.940 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8289999999999996e-06 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.388488+0000", 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5429999999999999e-06 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.732656+0000", 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.344168482 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.941 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.732876+0000", 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000219337 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.733087+0000", 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021187800000000001 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.942 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.737317+0000", 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042291230000000004 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.739013+0000", 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001696712 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.748990+0000", 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0099767450000000004 2023-09-23T23:08:17.943 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.749057+0000", 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7218000000000006e-05 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.751801+0000", 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0027435049999999998 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.944 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.46892.0:75762 11.2 e425/410 11:402b2427:::benchmark_data_smithi052_606809_object75761:head v 425'4725, mlcod=425'4723)", 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.319889+0000", 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.57059057500001, 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36877252999999999, 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.945 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.319889+0000", 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.319889+0000", 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.319888+0000", 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:08:17.946 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.319916+0000", 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.7297999999999999e-05 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.319927+0000", 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1459e-05 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.947 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.319935+0000", 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.1599999999999998e-06 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.325215+0000", 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052800010000000003 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.325237+0000", 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1338000000000001e-05 2023-09-23T23:08:17.948 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688618+0000", 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36338143499999997 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688662+0000", 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3707e-05 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.949 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75764 11.a 11:51ad2263:::benchmark_data_smithi052_606809_object75763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.320866+0000", 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.56961396400001, 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.37099696900000001, 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75764 2023-09-23T23:08:17.950 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.320866+0000", 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.320866+0000", 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.951 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.320864+0000", 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.320908+0000", 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.3467000000000002e-05 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.952 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.320916+0000", 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.4589999999999998e-06 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.320922+0000", 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5879999999999997e-06 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.327953+0000", 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070309730000000003 2023-09-23T23:08:17.953 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.328167+0000", 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021408500000000001 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.328372+0000", 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000204935 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.954 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688673+0000", 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36030150100000002 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688722+0000", 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8764000000000003e-05 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.955 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688836+0000", 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000113571 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688896+0000", 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.0009000000000001e-05 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.691863+0000", 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002966982 2023-09-23T23:08:17.956 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75772 11.a 11:53845015:::benchmark_data_smithi052_606809_object75771:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.330935+0000", 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.55954491599999, 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.38737729799999998, 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.957 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75772 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330935+0000", 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330935+0000", 2023-09-23T23:08:17.958 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330934+0000", 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330957+0000", 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3666999999999999e-05 2023-09-23T23:08:17.959 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330967+0000", 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5170000000000005e-06 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330974+0000", 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.661e-06 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.960 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.338578+0000", 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.007603144 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.338745+0000", 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016700499999999999 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.338908+0000", 2023-09-23T23:08:17.961 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016303699999999999 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688733+0000", 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34982576599999998 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.700128+0000", 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011394118999999999 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.962 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.715355+0000", 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.015227030000000001 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.715410+0000", 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5730999999999999e-05 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.718312+0000", 2023-09-23T23:08:17.963 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002901799 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75774 11.8 11:1514fd8d:::benchmark_data_smithi052_606809_object75773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.330996+0000", 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.55948360400001, 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.364793121, 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.964 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75774 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330996+0000", 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.965 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330996+0000", 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.330996+0000", 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.966 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331016+0000", 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0781e-05 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331025+0000", 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5029999999999997e-06 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331030+0000", 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7219999999999999e-06 2023-09-23T23:08:17.967 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336250+0000", 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052206889999999997 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336442+0000", 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019190599999999999 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.968 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336605+0000", 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016248099999999999 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.691576+0000", 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35497179400000001 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.691602+0000", 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5181000000000002e-05 2023-09-23T23:08:17.969 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.691622+0000", 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0327e-05 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.691662+0000", 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0253000000000002e-05 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.970 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.695789+0000", 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041270109999999999 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.46892.0:75764 11.a e425/422 ondisk, result = 0)", 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.331012+0000", 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.55946731899999, 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35772241500000002, 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.971 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331012+0000", 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331012+0000", 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.972 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331012+0000", 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331017+0000", 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6720000000000001e-06 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.973 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331028+0000", 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0971e-05 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331035+0000", 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7460000000000003e-06 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688706+0000", 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35767043599999998 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.974 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688715+0000", 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8239999999999995e-06 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688735+0000", 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9446e-05 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.975 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75776 11.9 11:975bf0a3:::benchmark_data_smithi052_606809_object75775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.331037+0000", 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.55944277500001, 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35913467599999999, 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75776 2023-09-23T23:08:17.976 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331037+0000", 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331037+0000", 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.977 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331036+0000", 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331090+0000", 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4132999999999998e-05 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331097+0000", 2023-09-23T23:08:17.978 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.2580000000000001e-06 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331101+0000", 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9310000000000001e-06 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331135+0000", 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4137999999999997e-05 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.979 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331284+0000", 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00014942199999999999 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.331399+0000", 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00011493999999999999 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.980 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.334209+0000", 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028103630000000002 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.338433+0000", 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042234079999999997 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.338507+0000", 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3935000000000003e-05 2023-09-23T23:08:17.981 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.338563+0000", 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6651999999999999e-05 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.690172+0000", 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.351608211 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.982 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75777 11.8 11:12c8dd34:::benchmark_data_smithi052_606809_object75776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.336396+0000", 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.55408332100001, 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.36713948299999999, 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.983 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75777 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336396+0000", 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336396+0000", 2023-09-23T23:08:17.984 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336389+0000", 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999924 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336458+0000", 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8697999999999999e-05 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.985 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336469+0000", 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1212e-05 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336477+0000", 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0339999999999997e-06 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.986 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.339661+0000", 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003184227 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688546+0000", 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34888484199999997 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688763+0000", 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021718900000000001 2023-09-23T23:08:17.987 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.695849+0000", 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0070861309999999999 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.699082+0000", 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003232885 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.988 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.699457+0000", 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00037461500000000002 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.699630+0000", 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00017296 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.703536+0000", 2023-09-23T23:08:17.989 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0039062340000000002 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75781 11.a 11:57e03fba:::benchmark_data_smithi052_606809_object75780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.336483+0000", 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.553996571, 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.38499652400000001, 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:17.990 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75781 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336483+0000", 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.991 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336483+0000", 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336483+0000", 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336521+0000", 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8736999999999999e-05 2023-09-23T23:08:17.992 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336528+0000", 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5050000000000004e-06 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336533+0000", 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.0200000000000002e-06 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.993 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.693705+0000", 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35717244300000001 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.693885+0000", 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000179345 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 5,6", 2023-09-23T23:08:17.994 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.694086+0000", 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000201385 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.700136+0000", 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060500390000000001 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.700176+0000", 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9711e-05 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.995 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.718403+0000", 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.018227259999999999 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.718454+0000", 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1107999999999999e-05 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:17.996 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.721480+0000", 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003025467 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop_reply(client.46892.0:75764 11.a e425/422 ondisk, result = 0)", 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.336566+0000", 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.553913338, 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35531109799999999, 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:17.997 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "started", 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336566+0000", 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336566+0000", 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.998 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336566+0000", 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336569+0000", 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6600000000000001e-06 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:17.999 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336576+0000", 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7730000000000002e-06 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.336581+0000", 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7700000000000001e-06 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688819+0000", 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.35223862 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.000 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688829+0000", 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1800000000000002e-06 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.691877+0000", 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0030488379999999999 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.001 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:75783 11.0 11:07908526:::benchmark_data_smithi052_606809_object75782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:11.337179+0000", 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 186.553300479, 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.37018284600000001, 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 75783 2023-09-23T23:08:18.002 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.337179+0000", 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.337179+0000", 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.003 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.337178+0000", 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.337226+0000", 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.7324999999999998e-05 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:18.004 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.337235+0000", 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.5459999999999997e-06 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.337242+0000", 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7939999999999997e-06 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.339672+0000", 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0024300319999999999 2023-09-23T23:08:18.005 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688541+0000", 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.34886901100000001 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.688841+0000", 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00030024199999999999 2023-09-23T23:08:18.006 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.699338+0000", 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.010496449999999999 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.703764+0000", 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0044262700000000004 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.007 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.704328+0000", 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00056407199999999999 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.704390+0000", 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1686999999999994e-05 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:11.707362+0000", 2023-09-23T23:08:18.008 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029722860000000002 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:85351 11.9 11:90383392:::benchmark_data_smithi052_606809_object3197:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:32.005329+0000", 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 165.88515092700001, 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.4916702, 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:18.009 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 85351 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005329+0000", 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.010 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005329+0000", 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005327+0000", 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:18.011 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005333+0000", 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9660000000000001e-06 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005346+0000", 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2452e-05 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005352+0000", 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.7689999999999999e-06 2023-09-23T23:08:18.012 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005405+0000", 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.2330000000000002e-05 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.005416+0000", 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1722000000000001e-05 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.013 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.386236+0000", 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.38082004000000003 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for degraded object", 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.386493+0000", 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000256555 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.472077+0000", 2023-09-23T23:08:18.014 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.085584297000000004 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.472147+0000", 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.9419999999999996e-05 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 0,6", 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.472223+0000", 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.6349999999999999e-05 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.015 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.474839+0000", 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0026157089999999999 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.474847+0000", 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2120000000000003e-06 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.016 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.495473+0000", 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.020625569 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.495512+0000", 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9589999999999999e-05 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:32.496999+0000", 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014868030000000001 2023-09-23T23:08:18.017 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:86862 11.b 11:d64703cf:::benchmark_data_smithi052_606809_object4708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:38.908228+0000", 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 158.982251236, 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2592133329999999, 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:18.018 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 86862 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908228+0000", 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:18.019 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908228+0000", 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908227+0000", 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908236+0000", 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8869999999999998e-06 2023-09-23T23:08:18.020 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908255+0000", 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8943000000000001e-05 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908263+0000", 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0199999999999994e-06 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.021 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908323+0000", 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9862999999999999e-05 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.908331+0000", 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.8060000000000001e-06 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.873612+0000", 2023-09-23T23:08:18.022 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.96528023900000004 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.873623+0000", 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.1849e-05 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.138336+0000", 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2647130289999999 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.023 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.138491+0000", 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000154122 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.139031+0000", 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00054005100000000005 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:18.024 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.151109+0000", 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.01207855 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.154049+0000", 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.002939695 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.163944+0000", 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0098954820000000006 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.025 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.164006+0000", 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1451000000000005e-05 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.167442+0000", 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0034359830000000001 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.026 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:86872 11.b 11:d02cdde1:::benchmark_data_smithi052_606809_object4718:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:38.976478+0000", 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 158.91400125999999, 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1944338970000001, 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 86872 2023-09-23T23:08:18.027 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976478+0000", 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976478+0000", 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.028 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976477+0000", 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976483+0000", 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.3600000000000004e-06 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:08:18.029 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976495+0000", 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2567e-05 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976501+0000", 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8030000000000002e-06 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976550+0000", 2023-09-23T23:08:18.030 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8619999999999999e-05 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:38.976558+0000", 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.6079999999999996e-06 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.874036+0000", 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.89747762499999995 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.031 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for peered", 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:39.874048+0000", 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2275e-05 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.147800+0000", 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2737519609999999 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.147965+0000", 2023-09-23T23:08:18.032 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016504700000000001 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,4", 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.148147+0000", 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000181602 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.154005+0000", 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0058587819999999999 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.033 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.154169+0000", 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000163673 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.167501+0000", 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.013331813 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:08:18.034 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.167560+0000", 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8947000000000003e-05 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:41.170912+0000", 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0033524430000000001 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:08:18.035 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:08:18.070 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:18.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:18.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:18.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[187743]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 328 op/s; 54141/125664 objects degraded (43.084%); 1519/125664 objects misplaced (1.209%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T23:08:18.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/929093776' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:18.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3079976547' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:08:18.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:18.197 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:08:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:17 smithi151 ceph-mon[159210]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.6 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 328 op/s; 54141/125664 objects degraded (43.084%); 1519/125664 objects misplaced (1.209%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T23:08:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/929093776' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:17 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3079976547' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:08:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:18.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:08:18.249 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:18.825 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:18.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:18.934 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:18.980 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:19.042 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:19.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:19.158 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:19.185 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:08:19.185 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[180979]: failed to encode map e463 with expected crc 2023-09-23T23:08:19.185 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[180979]: failed to encode map e463 with expected crc 2023-09-23T23:08:19.185 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[180979]: osdmap e463: 8 total, 8 up, 5 in 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[187743]: failed to encode map e463 with expected crc 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[187743]: failed to encode map e463 with expected crc 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:08:19.186 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:18 smithi052 ceph-mon[187743]: osdmap e463: 8 total, 8 up, 5 in 2023-09-23T23:08:19.188 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:19.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:18 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:08:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:18 smithi151 ceph-mon[159210]: failed to encode map e463 with expected crc 2023-09-23T23:08:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:18 smithi151 ceph-mon[159210]: failed to encode map e463 with expected crc 2023-09-23T23:08:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:18 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:18 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:08:19.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:18 smithi151 ceph-mon[159210]: osdmap e463: 8 total, 8 up, 5 in 2023-09-23T23:08:19.901 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.5 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 412 op/s; 53764/122325 objects degraded (43.952%); 1519/122325 objects misplaced (1.242%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T23:08:19.901 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: osdmap e464: 8 total, 8 up, 5 in 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: failed to encode map e464 with expected crc 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: failed to encode map e464 with expected crc 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: failed to encode map e464 with expected crc 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.5 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 412 op/s; 53764/122325 objects degraded (43.952%); 1519/122325 objects misplaced (1.242%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: osdmap e464: 8 total, 8 up, 5 in 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: failed to encode map e464 with expected crc 2023-09-23T23:08:19.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: failed to encode map e464 with expected crc 2023-09-23T23:08:19.903 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: failed to encode map e464 with expected crc 2023-09-23T23:08:19.903 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:19.903 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.5 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 412 op/s; 53764/122325 objects degraded (43.952%); 1519/122325 objects misplaced (1.242%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: osdmap e464: 8 total, 8 up, 5 in 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: failed to encode map e464 with expected crc 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: failed to encode map e464 with expected crc 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: failed to encode map e464 with expected crc 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:20.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:20.540 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:20.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:20.651 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:08:20.655 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:20.689 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:08:20.704 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:20.891 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:20.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:20.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:20 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 53764/122325 objects degraded (43.952%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:21.009 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:21.061 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:21.153 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:21.156 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:20 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 53764/122325 objects degraded (43.952%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:21.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:21.157 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:20 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 53764/122325 objects degraded (43.952%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:21.212 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:21.250 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:21.258 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:21.281 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:21.283 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:21.324 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:21.326 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:21.342 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:21.362 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:21.414 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:21.431 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:21.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:21.464 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:21.499 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.513 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:08:21.536 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:21.585 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:21.697 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:21.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:21.805 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:21.869 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:21.955 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:21.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:21.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:21 smithi052 ceph-mon[180979]: pgmap v1458: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.4 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 401 op/s; 53256/119433 objects degraded (44.591%); 1406/119433 objects misplaced (1.177%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:08:21.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:21 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4075100524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:21 smithi052 ceph-mon[187743]: pgmap v1458: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.4 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 401 op/s; 53256/119433 objects degraded (44.591%); 1406/119433 objects misplaced (1.177%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:08:21.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:21 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4075100524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:22.063 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:22.097 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:22.167 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:22.169 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:21 smithi151 ceph-mon[159210]: pgmap v1458: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.4 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 401 op/s; 53256/119433 objects degraded (44.591%); 1406/119433 objects misplaced (1.177%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:08:22.169 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:21 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4075100524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:22.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:22.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:22.270 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:22.308 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:22.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:08:23.186 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:23.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:23.293 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:23.364 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:23.442 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:08:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:08:23] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.33.4" 2023-09-23T23:08:23.643 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:23.739 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:23.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:23.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:23.849 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:23.902 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:23.979 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:23 smithi151 ceph-mon[159210]: pgmap v1459: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 399 op/s; 53197/116817 objects degraded (45.539%); 1406/116817 objects misplaced (1.204%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T23:08:23.979 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:23 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:23 smithi052 ceph-mon[187743]: pgmap v1459: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 399 op/s; 53197/116817 objects degraded (45.539%); 1406/116817 objects misplaced (1.204%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T23:08:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:23 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:23.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:23 smithi052 ceph-mon[180979]: pgmap v1459: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 399 op/s; 53197/116817 objects degraded (45.539%); 1406/116817 objects misplaced (1.204%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T23:08:23.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:23 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:08:24.132 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:24.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:24.238 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:24.268 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:08:24.268 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:08:24.284 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:24.459 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:24.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:24.564 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:24.628 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:24.946 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:08:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/476788222' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:08:25.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/476788222' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:08:25.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/476788222' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:08:25.448 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:08:25.481 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:08:25.933 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: pgmap v1460: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 473 op/s; 52389/114288 objects degraded (45.839%); 1406/114288 objects misplaced (1.230%); 11 MiB/s, 170 objects/s recovering 2023-09-23T23:08:25.933 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: failed to encode map e465 with expected crc 2023-09-23T23:08:25.933 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: failed to encode map e465 with expected crc 2023-09-23T23:08:25.933 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/476788222' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:08:25.933 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: osdmap e465: 8 total, 8 up, 5 in 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: failed to encode map e465 with expected crc 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 52389/114288 objects degraded (45.839%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: pgmap v1460: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 473 op/s; 52389/114288 objects degraded (45.839%); 1406/114288 objects misplaced (1.230%); 11 MiB/s, 170 objects/s recovering 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: failed to encode map e465 with expected crc 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: failed to encode map e465 with expected crc 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/476788222' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:08:25.934 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: osdmap e465: 8 total, 8 up, 5 in 2023-09-23T23:08:25.935 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: failed to encode map e465 with expected crc 2023-09-23T23:08:25.935 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:25 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 52389/114288 objects degraded (45.839%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:26.025 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:26.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:26.077 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:26.129 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:26.169 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: pgmap v1460: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 473 op/s; 52389/114288 objects degraded (45.839%); 1406/114288 objects misplaced (1.230%); 11 MiB/s, 170 objects/s recovering 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: failed to encode map e465 with expected crc 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: failed to encode map e465 with expected crc 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/476788222' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: osdmap e465: 8 total, 8 up, 5 in 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: failed to encode map e465 with expected crc 2023-09-23T23:08:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:25 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 52389/114288 objects degraded (45.839%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:26.323 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T23:08:26.376 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:26.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:26.480 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:26.526 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:26.555 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:27.046 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:27.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:27.150 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:27.216 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:27.320 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:27.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:27.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:27.435 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:27.480 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:27.939 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:27 smithi151 ceph-mon[159210]: pgmap v1462: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 52389/113625 objects degraded (46.107%); 1296/113625 objects misplaced (1.141%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T23:08:27.939 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2794269196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:08:27.940 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:27 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:08:27.965 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:08:28.218 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:28.221 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:27 smithi052 ceph-mon[180979]: pgmap v1462: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 52389/113625 objects degraded (46.107%); 1296/113625 objects misplaced (1.141%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T23:08:28.221 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2794269196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:08:28.221 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:27 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:08:28.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:27 smithi052 ceph-mon[187743]: pgmap v1462: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 52389/113625 objects degraded (46.107%); 1296/113625 objects misplaced (1.141%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T23:08:28.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2794269196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:08:28.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:27 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:08:28.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:28.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:28.305 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:08:28.306 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:08:28.323 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:28.364 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:28.381 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:29.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:28 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:29.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:28 smithi151 ceph-mon[159210]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.204 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:28 smithi151 ceph-mon[159210]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:28 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:08:29.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:28 smithi151 ceph-mon[159210]: osdmap e466: 8 total, 8 up, 5 in 2023-09-23T23:08:29.205 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:28 smithi151 ceph-mon[159210]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:29.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[187743]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[187743]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[187743]: osdmap e466: 8 total, 8 up, 5 in 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[187743]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[180979]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[180979]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:08:29.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[180979]: osdmap e466: 8 total, 8 up, 5 in 2023-09-23T23:08:29.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:28 smithi052 ceph-mon[180979]: failed to encode map e466 with expected crc 2023-09-23T23:08:29.284 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:29.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:29.394 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:29.466 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:29.486 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:08:29.539 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:29.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:29.650 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:29.724 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:29.751 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:29.790 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:29.820 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:29.851 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:29.888 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:29.926 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:29.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:29 smithi052 ceph-mon[180979]: pgmap v1464: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 52041/110877 objects degraded (46.936%); 1296/110877 objects misplaced (1.169%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:08:29.951 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:29 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2786120063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:29.961 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:29.961 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:29.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:29.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:29.994 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:29.994 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.994 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.994 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.995 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.995 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.995 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.995 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:08:29.995 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:08:30.064 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:30.154 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:30.180 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:30.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:29 smithi052 ceph-mon[187743]: pgmap v1464: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 52041/110877 objects degraded (46.936%); 1296/110877 objects misplaced (1.169%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:08:30.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:29 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2786120063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:30.244 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:30.257 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:29 smithi151 ceph-mon[159210]: pgmap v1464: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 52041/110877 objects degraded (46.936%); 1296/110877 objects misplaced (1.169%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:08:30.257 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:29 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2786120063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:30.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:30.359 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:30.434 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:30.529 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:30.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:30.634 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:30.682 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:30.778 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:30.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:30.890 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:30.938 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:31.044 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:31.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:31.151 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:31.176 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:31.219 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:31.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:31.323 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:31.365 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:31 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:31.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:31 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:31.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:31 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 52041/110877 objects degraded (46.936%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:31.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:31 smithi151 ceph-mon[159210]: pgmap v1465: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.2 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 340 op/s; 51651/108642 objects degraded (47.542%); 1182/108642 objects misplaced (1.088%); 10 MiB/s, 161 objects/s recovering 2023-09-23T23:08:31.366 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:31.422 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:31.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:31.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:31.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:31.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:31.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 52041/110877 objects degraded (46.936%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:31.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[180979]: pgmap v1465: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.2 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 340 op/s; 51651/108642 objects degraded (47.542%); 1182/108642 objects misplaced (1.088%); 10 MiB/s, 161 objects/s recovering 2023-09-23T23:08:31.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:31.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:31.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 52041/110877 objects degraded (46.936%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:31.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:31 smithi052 ceph-mon[187743]: pgmap v1465: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.2 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 340 op/s; 51651/108642 objects degraded (47.542%); 1182/108642 objects misplaced (1.088%); 10 MiB/s, 161 objects/s recovering 2023-09-23T23:08:31.526 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:31.595 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:31.688 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:08:31.715 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:31.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:31.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:31.825 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:31.856 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:31.989 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:31.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:32.097 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:32.149 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:32.200 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:08:32.200 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:08:32.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:08:32.200 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:08:32.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.200 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:17355 11.e 11:7857a061:::benchmark_data_smithi052_606809_object17354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 269.58801008199998, 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15692616000000001, 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 17355 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607819+0000", 2023-09-23T23:08:32.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3510999999999995e-05 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607892+0000", 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0942e-05 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607903+0000", 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8921999999999999e-05 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607922+0000", 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4350000000000001e-05 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607956+0000", 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6907000000000001e-05 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608033+0000", 2023-09-23T23:08:32.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8525999999999999e-05 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608122+0000", 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00055712700000000003 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608679+0000", 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049612479999999997 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.613640+0000", 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15104679400000001 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764687+0000", 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5262999999999998e-05 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764722+0000", 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3643999999999999e-05 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764746+0000", 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15692616000000001 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:17367 11.e 11:71b796f5:::benchmark_data_smithi052_606809_object17366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:08:32.207 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:02.615149+0000", 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 269.58068102800001, 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.152643889, 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 17367 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.208 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615149+0000", 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615149+0000", 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615148+0000", 2023-09-23T23:08:32.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1823999999999999e-05 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615180+0000", 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4660000000000005e-06 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615189+0000", 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8840000000000002e-06 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615194+0000", 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8225e-05 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615242+0000", 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4358999999999997e-05 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615317+0000", 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9893000000000001e-05 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.615407+0000", 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00063447099999999995 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.616041+0000", 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00090008299999999998 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.616941+0000", 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15079795700000001 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.767739+0000", 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1355e-05 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.767770+0000", 2023-09-23T23:08:32.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2345e-05 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.767793+0000", 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.152643889 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:21315 11.e 11:7fea961b:::benchmark_data_smithi052_606809_object21314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:08:32.214 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.667901+0000", 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 266.527928847, 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13239036300000001, 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 21315 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.215 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667901+0000", 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667901+0000", 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667899+0000", 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.4780999999999999e-05 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.667994+0000", 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9510000000000001e-06 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668004+0000", 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0350000000000001e-06 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668009+0000", 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3237e-05 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668052+0000", 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5195e-05 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668138+0000", 2023-09-23T23:08:32.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000100846 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.668238+0000", 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00083182799999999995 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.669070+0000", 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13107804100000001 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800148+0000", 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5852000000000001e-05 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800194+0000", 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2657000000000001e-05 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800247+0000", 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4549000000000003e-05 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800291+0000", 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13239036300000001 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.221 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:21320 11.c e415/410 11:3aa1cab5:::benchmark_data_smithi052_606809_object21319:head v 415'1301, mlcod=415'1298)", 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 266.52469859799999, 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12977512799999999, 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.222 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671131+0000", 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1396000000000001e-05 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671172+0000", 2023-09-23T23:08:32.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2270000000000001e-06 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671178+0000", 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1e-06 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671181+0000", 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00070157599999999998 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671882+0000", 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1986e-05 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671894+0000", 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.128977016 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800871+0000", 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5169999999999997e-05 2023-09-23T23:08:32.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800906+0000", 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12977512799999999 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:21321 11.c e415/410 11:30848661:::benchmark_data_smithi052_606809_object21320:head v 415'1302, mlcod=415'1298)", 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.671193+0000", 2023-09-23T23:08:32.226 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 266.52463722900001, 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13194534299999999, 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671193+0000", 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671193+0000", 2023-09-23T23:08:32.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671192+0000", 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2735000000000003e-05 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671235+0000", 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3810000000000004e-06 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671241+0000", 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0230000000000001e-06 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.671245+0000", 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010633789999999999 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672308+0000", 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3658e-05 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672322+0000", 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13077090799999999 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.803092+0000", 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5525000000000002e-05 2023-09-23T23:08:32.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.803138+0000", 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13194534299999999 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop_reply(client.46892.0:21315 11.e e415/410 ondisk, result = 0)", 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:05.672760+0000", 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 266.52306977199999, 2023-09-23T23:08:32.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12739600200000001, 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672760+0000", 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672760+0000", 2023-09-23T23:08:32.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672759+0000", 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1360000000000002e-06 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.672765+0000", 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12722345900000001 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.799989+0000", 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9403e-05 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800018+0000", 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010932 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800128+0000", 2023-09-23T23:08:32.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1924e-05 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800140+0000", 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6569999999999999e-05 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:05.800156+0000", 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12739600200000001 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.235 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:61498 11.e 11:74d9cd66:::benchmark_data_smithi052_606809_object61497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e419)", 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:48.235116+0000", 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 223.96071389700001, 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.141686063, 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.236 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 61498 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235116+0000", 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235116+0000", 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235115+0000", 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7705999999999999e-05 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235172+0000", 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2746e-05 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235185+0000", 2023-09-23T23:08:32.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2739999999999998e-06 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235190+0000", 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9527000000000003e-05 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235230+0000", 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9593999999999995e-05 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.239 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235330+0000", 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000116106 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.235446+0000", 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001788544 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.240 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.237234+0000", 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00053666399999999996 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.237771+0000", 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.13892012300000001 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.376691+0000", 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1877000000000004e-05 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.376753+0000", 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9186e-05 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:48.376802+0000", 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.141686063 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.242 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:74473 11.e 11:7aba8dc9:::benchmark_data_smithi052_606809_object74472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e425)", 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:08.997352+0000", 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 203.198477935, 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.127978069, 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 74473 2023-09-23T23:08:32.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997352+0000", 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997352+0000", 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997349+0000", 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1848999999999996e-05 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997421+0000", 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1778999999999999e-05 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997443+0000", 2023-09-23T23:08:32.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1619999999999997e-06 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997452+0000", 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6990000000000002e-05 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997509+0000", 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000128899 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997638+0000", 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012929900000000001 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:08.997767+0000", 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003043263 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.000811+0000", 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001496559 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.002307+0000", 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.12291558800000001 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.125223+0000", 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1513000000000005e-05 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.125284+0000", 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5707999999999997e-05 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:09.125330+0000", 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.127978069 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:86860 11.e 11:75ab6736:::benchmark_data_smithi052_606809_object4706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.31252078099999, 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3618323549999998, 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 86860 2023-09-23T23:08:32.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883307+0000", 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3599999999999996e-06 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883316+0000", 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7278e-05 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883333+0000", 2023-09-23T23:08:32.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2119999999999992e-06 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883342+0000", 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011255879999999999 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.894598+0000", 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2694000000000001e-05 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.894611+0000", 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99447565000000004 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889087+0000", 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2336e-05 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:08:32.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889099+0000", 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3388404170000001 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.227939+0000", 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017411099999999999 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.228113+0000", 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012214100000000001 2023-09-23T23:08:32.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.228236+0000", 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.01031818 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238554+0000", 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9231000000000002e-05 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.256 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238623+0000", 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039840279999999997 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.242607+0000", 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5633999999999997e-05 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.242653+0000", 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024888699999999998 2023-09-23T23:08:32.257 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.245141+0000", 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3618323549999998 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:86874 11.e 11:77be3fa7:::benchmark_data_smithi052_606809_object4720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.17293021699999, 2023-09-23T23:08:32.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.23339197, 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 86874 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:08:32.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999943 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022894+0000", 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5319e-05 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022909+0000", 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3163000000000002e-05 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022932+0000", 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0350000000000007e-06 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022941+0000", 2023-09-23T23:08:32.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3570000000000002e-05 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.023015+0000", 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4547e-05 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.023030+0000", 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86609229700000001 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889122+0000", 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8590000000000005e-06 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889130+0000", 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3413657240000001 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230495+0000", 2023-09-23T23:08:32.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000186393 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230682+0000", 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196875 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230879+0000", 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076860059999999996 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238565+0000", 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8522000000000004e-05 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238653+0000", 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013606922 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.252260+0000", 2023-09-23T23:08:32.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8609000000000006e-05 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.252339+0000", 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039528990000000002 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.256292+0000", 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.23339197 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.266 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:98148 11.17 11:ea01d5f4:::benchmark_data_smithi052_606809_object15994:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:50.079069+0000", 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 102.11676052999999, 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15118050699999999, 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.267 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 98148 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079069+0000", 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079069+0000", 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079068+0000", 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.2570000000000001e-06 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079075+0000", 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1517e-05 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079086+0000", 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5969999999999999e-06 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079092+0000", 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6631999999999997e-05 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079139+0000", 2023-09-23T23:08:32.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112699 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079251+0000", 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000109639 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.079361+0000", 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012242100000000001 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.080585+0000", 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00032680200000000001 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.080912+0000", 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.147299438 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.228211+0000", 2023-09-23T23:08:32.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3800999999999994e-05 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.228275+0000", 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019746059999999998 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:50.230250+0000", 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15118050699999999 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.273 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:100250 11.e 11:7a9e86a3:::benchmark_data_smithi052_606809_object18096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:58.864702+0000", 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 93.331127640999995, 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20398459899999999, 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.274 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100250 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864702+0000", 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864702+0000", 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864700+0000", 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9250000000000001e-06 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864709+0000", 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3275000000000001e-05 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864722+0000", 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0099999999999995e-06 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864730+0000", 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4678000000000002e-05 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864775+0000", 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010582999999999999 2023-09-23T23:08:32.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864881+0000", 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143514 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.865024+0000", 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081216210000000007 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873146+0000", 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1297999999999995e-05 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873207+0000", 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.193264456 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.066472+0000", 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2669000000000001e-05 2023-09-23T23:08:32.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.066514+0000", 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021723770000000001 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.068687+0000", 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20398459899999999 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.280 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:100251 11.e 11:7f2f4774:::benchmark_data_smithi052_606809_object18097:head [delete] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:58.873564+0000", 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 93.322266135000007, 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19695320399999999, 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.281 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100251 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873564+0000", 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873564+0000", 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873562+0000", 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8020999999999999e-05 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873580+0000", 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7269e-05 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873597+0000", 2023-09-23T23:08:32.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7150000000000004e-06 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873606+0000", 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1041000000000002e-05 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873647+0000", 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000108952 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873756+0000", 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120344 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873876+0000", 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002459316 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.876335+0000", 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015686000000000001 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.876492+0000", 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19224075400000001 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.068733+0000", 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8087999999999998e-05 2023-09-23T23:08:32.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.068781+0000", 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001736044 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.070517+0000", 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19695320399999999 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:105570 11.e 11:70bbfdb0:::benchmark_data_smithi052_606809_object23416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:19.238660+0000", 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 72.957170160000004, 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19364197, 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 105570 2023-09-23T23:08:32.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238660+0000", 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238660+0000", 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238658+0000", 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9029999999999999e-06 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238668+0000", 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.912e-05 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238687+0000", 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8269999999999993e-06 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238697+0000", 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0084551339999999996 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.247152+0000", 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000156324 2023-09-23T23:08:32.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.247308+0000", 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000194666 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.247503+0000", 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0061626679999999996 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.253665+0000", 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1190000000000003e-05 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.253717+0000", 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17570802499999999 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.429425+0000", 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8717e-05 2023-09-23T23:08:32.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.429483+0000", 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028183470000000001 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.432302+0000", 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19364197 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.294 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:105576 11.e 11:79dd3146:::benchmark_data_smithi052_606809_object23422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:19.259234+0000", 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 72.936596227999999, 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.175814893, 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.295 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 105576 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259234+0000", 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259234+0000", 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259232+0000", 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9440000000000005e-06 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259240+0000", 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4470999999999999e-05 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259254+0000", 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8029999999999999e-06 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259261+0000", 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9159000000000001e-05 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259320+0000", 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016387500000000001 2023-09-23T23:08:32.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259484+0000", 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020229199999999999 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259686+0000", 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030865490000000001 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.262773+0000", 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1262999999999999e-05 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.262844+0000", 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16952422 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.432368+0000", 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4986000000000003e-05 2023-09-23T23:08:32.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.432443+0000", 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026052240000000002 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.435049+0000", 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.175814893 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.301 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:106756 11.e 11:752baf3e:::benchmark_data_smithi052_606809_object24602:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:23.502244+0000", 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 68.693586077000006, 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.185223678, 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.302 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 106756 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502244+0000", 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502244+0000", 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502242+0000", 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7110000000000001e-06 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502251+0000", 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4238e-05 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502265+0000", 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.798e-06 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502272+0000", 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5841999999999998e-05 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502328+0000", 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016212 2023-09-23T23:08:32.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502490+0000", 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000199478 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502689+0000", 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031355469999999998 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.505825+0000", 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019053399999999999 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.506015+0000", 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17894664900000001 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.684962+0000", 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8783000000000003e-05 2023-09-23T23:08:32.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.685021+0000", 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024467629999999998 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.687467+0000", 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.185223678 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.308 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:117157 11.e 11:7bd89a1e:::benchmark_data_smithi052_606809_object35003:head [delete] snapc 0=[] ondisk+write+known_if_redirected e461)", 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:59.566001+0000", 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 32.629828983000003, 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.114215708, 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 117157 2023-09-23T23:08:32.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566001+0000", 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566001+0000", 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566000+0000", 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8369999999999999e-06 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566006+0000", 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1481000000000001e-05 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566018+0000", 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9529999999999996e-06 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566024+0000", 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5785999999999998e-05 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566070+0000", 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000109413 2023-09-23T23:08:32.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566179+0000", 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012726799999999999 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.566306+0000", 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024635630000000002 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.568770+0000", 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00021930300000000001 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.568989+0000", 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.109373975 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.678363+0000", 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.0770000000000002e-05 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.678434+0000", 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001782711 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:59.680217+0000", 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.114215708 2023-09-23T23:08:32.315 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:122791 11.e 11:7e277a63:::benchmark_data_smithi052_606809_object40637:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:13.957962+0000", 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 18.237867926, 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.097776812000000005, 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.316 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 122791 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.957962+0000", 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.957962+0000", 2023-09-23T23:08:32.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.957960+0000", 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7180000000000008e-06 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.957970+0000", 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4586999999999999e-05 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.957984+0000", 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8699999999999992e-06 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.957992+0000", 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6665e-05 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.958049+0000", 2023-09-23T23:08:32.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00013973699999999999 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.958188+0000", 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014061100000000001 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.958329+0000", 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002930916 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.961260+0000", 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010012879999999999 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:13.962261+0000", 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.092001669999999994 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:14.054263+0000", 2023-09-23T23:08:32.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000113985 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:14.054377+0000", 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001361807 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:14.055739+0000", 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.097776812000000005 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.322 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:124400 11.e 11:7954a7ab:::benchmark_data_smithi052_606809_object42246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:17.849445+0000", 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 14.346384514, 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17804371399999999, 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.323 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 124400 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849445+0000", 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849445+0000", 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:32.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849444+0000", 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0339999999999997e-06 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849452+0000", 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0784999999999999e-05 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849462+0000", 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.652e-06 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849469+0000", 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0318999999999997e-05 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849519+0000", 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017331900000000001 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849693+0000", 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019187800000000001 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849885+0000", 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00303418 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.852919+0000", 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112779 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.853032+0000", 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17280121000000001 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.025833+0000", 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4613e-05 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.025897+0000", 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015916750000000001 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.027489+0000", 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17804371399999999 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.329 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:124487 11.7 11:e57a7346:::benchmark_data_smithi052_606809_object42333:head [delete] snapc 0=[] ondisk+write+known_if_redirected e463)", 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:18.235406+0000", 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 13.960424151, 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.093835871000000001, 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 124487 2023-09-23T23:08:32.330 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235406+0000", 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235406+0000", 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.331 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235404+0000", 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1740000000000003e-06 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235411+0000", 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2177999999999999e-05 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235424+0000", 2023-09-23T23:08:32.332 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9880000000000001e-06 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235430+0000", 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3235000000000002e-05 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235473+0000", 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014172300000000001 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.333 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.235615+0000", 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00041311899999999999 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.236028+0000", 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001676922 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.237705+0000", 2023-09-23T23:08:32.334 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030676599999999998 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.240772+0000", 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.08698699 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.327759+0000", 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.6946000000000004e-05 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.335 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.327846+0000", 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001395383 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.329242+0000", 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.093835871000000001 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:32.336 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:08:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:32.338 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:32.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:32.364 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:32.438 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:32.496 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:32.569 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:32.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:32.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:32.674 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:32.707 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:32.893 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:32.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:32.998 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:33.045 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:33.076 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:08:33.185 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:33.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:33.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:33.291 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:33.319 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T23:08:33.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:33.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:33 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:33.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:33 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1690232033' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:08:33.385 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:33 smithi151 ceph-mon[159210]: pgmap v1466: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 326 op/s; 51601/106452 objects degraded (48.473%); 1182/106452 objects misplaced (1.110%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:08:33.399 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:33.465 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:08:33.487 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:33.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:33.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:08:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:08:33] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.33.4" 2023-09-23T23:08:33.493 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1690232033' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:08:33.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[187743]: pgmap v1466: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 326 op/s; 51601/106452 objects degraded (48.473%); 1182/106452 objects misplaced (1.110%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:08:33.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:33.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:33.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1690232033' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:08:33.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:33 smithi052 ceph-mon[180979]: pgmap v1466: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 326 op/s; 51601/106452 objects degraded (48.473%); 1182/106452 objects misplaced (1.110%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T23:08:33.593 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:33.632 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:33.658 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:34.314 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:34.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:34.420 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1690232033' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: osdmap e467: 8 total, 8 up, 5 in 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:34.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:34.451 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1690232033' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: osdmap e467: 8 total, 8 up, 5 in 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.591 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:34.592 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:34.593 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.593 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.593 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1690232033' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:08:34.593 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: osdmap e467: 8 total, 8 up, 5 in 2023-09-23T23:08:34.594 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.594 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.594 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: failed to encode map e467 with expected crc 2023-09-23T23:08:34.594 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:34.594 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:34.619 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:08:34.666 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:34.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:34.771 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:34.880 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:35.063 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:35.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:35.170 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:35.183 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:35.233 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:35.376 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:35.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:35.404 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:35.483 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:35.498 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:08:35.498 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:08:35.498 INFO:teuthology.orchestra.run.smithi151.stdout: "num_ops": 0 2023-09-23T23:08:35.498 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:08:35.527 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:35 smithi151 ceph-mon[159210]: pgmap v1468: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 50936/104301 objects degraded (48.836%); 1182/104301 objects misplaced (1.133%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:08:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 51601/106452 objects degraded (48.473%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[187743]: pgmap v1468: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 50936/104301 objects degraded (48.836%); 1182/104301 objects misplaced (1.133%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 51601/106452 objects degraded (48.473%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[180979]: pgmap v1468: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 50936/104301 objects degraded (48.836%); 1182/104301 objects misplaced (1.133%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T23:08:35.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 51601/106452 objects degraded (48.473%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:36.117 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:36.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:36.226 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:36.292 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:36.501 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:36.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:36.552 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:08:36.606 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:08:36.609 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:36.635 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/606135191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:36.635 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:36 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:36.635 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/606135191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:36.635 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:36 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:36.638 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/606135191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:36 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:37.195 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:37.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: osdmap e468: 8 total, 8 up, 5 in 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: pgmap v1470: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 299 op/s; 50936/103683 objects degraded (49.127%); 1072/103683 objects misplaced (1.034%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T23:08:37.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[180979]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: osdmap e468: 8 total, 8 up, 5 in 2023-09-23T23:08:37.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: pgmap v1470: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 299 op/s; 50936/103683 objects degraded (49.127%); 1072/103683 objects misplaced (1.034%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T23:08:37.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:37 smithi052 ceph-mon[187743]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: osdmap e468: 8 total, 8 up, 5 in 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.801 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: pgmap v1470: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.1 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 299 op/s; 50936/103683 objects degraded (49.127%); 1072/103683 objects misplaced (1.034%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T23:08:37.802 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:37 smithi151 ceph-mon[159210]: failed to encode map e468 with expected crc 2023-09-23T23:08:37.858 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:37.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:37.963 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:37.995 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:38.148 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:38.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:38.251 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:38.278 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:38.764 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:38.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:38.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:38.871 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:38.947 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:39.048 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:39.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:39.144 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:39.162 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:39.169 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:39.173 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:08:39.174 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:08:39.174 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:08:39.174 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:08:39.174 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:08:39.195 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:39.210 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:39.217 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:39.236 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:39.259 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:39.291 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:39.321 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:39.322 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.323 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:08:39.742 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:39.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:39.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:39.847 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:39.912 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:40.168 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:39 smithi151 ceph-mon[159210]: pgmap v1471: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.0 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 330 op/s; 50547/100713 objects degraded (50.189%); 1072/100713 objects misplaced (1.064%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:08:40.168 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/492562077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:40.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:08:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:39 smithi052 ceph-mon[187743]: pgmap v1471: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.0 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 330 op/s; 50547/100713 objects degraded (50.189%); 1072/100713 objects misplaced (1.064%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:08:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/492562077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:39 smithi052 ceph-mon[180979]: pgmap v1471: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.0 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 330 op/s; 50547/100713 objects degraded (50.189%); 1072/100713 objects misplaced (1.064%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T23:08:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/492562077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:40.497 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:40.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:40.600 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:40.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:08:40.664 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:40.697 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:40.753 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:40.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:40.859 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:40.920 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:41.016 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:40 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 50547/100713 objects degraded (50.189%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:41.020 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 50547/100713 objects degraded (50.189%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:41.066 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:41.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:41.168 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 50547/100713 objects degraded (50.189%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:41.177 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:41.222 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:41.396 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:41.406 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:41.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:41.511 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:41.561 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:41.623 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:41.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:41.727 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:41.762 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:42.038 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:41 smithi151 ceph-mon[159210]: pgmap v1472: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 50013/97896 objects degraded (51.088%); 961/97896 objects misplaced (0.982%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:08:42.038 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:41 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1381057622' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:08:42.038 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:41 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:08:42.068 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:42.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:42.177 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:42.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:41 smithi052 ceph-mon[187743]: pgmap v1472: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 50013/97896 objects degraded (51.088%); 961/97896 objects misplaced (0.982%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:08:42.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:41 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1381057622' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:08:42.191 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:41 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:08:42.191 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:41 smithi052 ceph-mon[180979]: pgmap v1472: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 2.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 50013/97896 objects degraded (51.088%); 961/97896 objects misplaced (0.982%); 11 MiB/s, 168 objects/s recovering 2023-09-23T23:08:42.191 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:41 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1381057622' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:08:42.192 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:41 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:08:42.225 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:42.398 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:42.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:42.471 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:08:42.502 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:42.568 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:42.646 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:08:42.646 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:08:42.666 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:42.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:42.678 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:08:42.773 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:42.812 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:42.889 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:42.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:42.993 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:43.018 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:43.059 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:43.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:43.163 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:43.190 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:43.390 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:43.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:43.493 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:08:43.494 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:43.540 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:43.602 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: osdmap e469: 8 total, 8 up, 5 in 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[180979]: pgmap v1474: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 49949/95235 objects degraded (52.448%); 961/95235 objects misplaced (1.009%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:08:43.603 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.604 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: osdmap e469: 8 total, 8 up, 5 in 2023-09-23T23:08:43.604 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.604 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.604 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.604 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:43 smithi052 ceph-mon[187743]: pgmap v1474: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 49949/95235 objects degraded (52.448%); 961/95235 objects misplaced (1.009%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:08:43.605 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:08:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:08:43] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.33.4" 2023-09-23T23:08:43.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:43.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:43.808 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:43.862 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:08:43.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: osdmap e469: 8 total, 8 up, 5 in 2023-09-23T23:08:43.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: failed to encode map e469 with expected crc 2023-09-23T23:08:43.863 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:43 smithi151 ceph-mon[159210]: pgmap v1474: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 49949/95235 objects degraded (52.448%); 961/95235 objects misplaced (1.009%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T23:08:43.864 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:43.948 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:43.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:44.057 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:44.106 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:44.140 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:44.296 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:44.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:44.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T23:08:44.405 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:44.462 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:44.611 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:44.616 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:44.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:44.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: osdmap e470: 8 total, 8 up, 5 in 2023-09-23T23:08:44.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:08:44.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:44 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.726 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:44.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: osdmap e470: 8 total, 8 up, 5 in 2023-09-23T23:08:44.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: osdmap e470: 8 total, 8 up, 5 in 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: failed to encode map e470 with expected crc 2023-09-23T23:08:44.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:08:44.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:08:44.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:44 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:08:44.768 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:44.857 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:44.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:44.962 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:44.986 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:45.030 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:45.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:45.135 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:45.174 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:45.477 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:08:45.546 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:45.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:45.655 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:45.709 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:45 smithi052 ceph-mon[187743]: pgmap v1476: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 452 op/s; 49166/92814 objects degraded (52.973%); 961/92814 objects misplaced (1.035%); 11 MiB/s, 170 objects/s recovering 2023-09-23T23:08:45.710 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4102131445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:45.710 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 49949/95235 objects degraded (52.448%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:45.711 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:45 smithi052 ceph-mon[180979]: pgmap v1476: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 452 op/s; 49166/92814 objects degraded (52.973%); 961/92814 objects misplaced (1.035%); 11 MiB/s, 170 objects/s recovering 2023-09-23T23:08:45.711 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4102131445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:45.711 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 49949/95235 objects degraded (52.448%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:45.712 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:45.768 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:45 smithi151 ceph-mon[159210]: pgmap v1476: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 452 op/s; 49166/92814 objects degraded (52.973%); 961/92814 objects misplaced (1.035%); 11 MiB/s, 170 objects/s recovering 2023-09-23T23:08:45.768 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4102131445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:08:45.768 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 49949/95235 objects degraded (52.448%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:46.133 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:08:46.133 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:08:46.203 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:46.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:46.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:46.311 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:46.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:46.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:46.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:46.517 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:46.577 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:46.584 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:46.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:46.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4102131445' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:08:46.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: osdmap e471: 8 total, 8 up, 5 in 2023-09-23T23:08:46.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:46 smithi151 ceph-mon[159210]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:46.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4102131445' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: osdmap e471: 8 total, 8 up, 5 in 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[187743]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4102131445' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: osdmap e471: 8 total, 8 up, 5 in 2023-09-23T23:08:46.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:46 smithi052 ceph-mon[180979]: failed to encode map e471 with expected crc 2023-09-23T23:08:46.829 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:46.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:46.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:47.019 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:47.058 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:08:47.266 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:47.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:47.373 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:47.440 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:47.634 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:47.647 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:47.647 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:47.647 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:47.682 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:47.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:47.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:47.687 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:47 smithi052 ceph-mon[180979]: pgmap v1478: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 317 op/s; 49166/92178 objects degraded (53.338%); 849/92178 objects misplaced (0.921%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:08:47.689 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:47 smithi052 ceph-mon[187743]: pgmap v1478: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 317 op/s; 49166/92178 objects degraded (53.338%); 849/92178 objects misplaced (0.921%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:08:47.698 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:47.788 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:47.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:47 smithi151 ceph-mon[159210]: pgmap v1478: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 317 op/s; 49166/92178 objects degraded (53.338%); 849/92178 objects misplaced (0.921%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T23:08:47.821 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.855 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.856 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.856 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.856 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.856 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.856 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:08:47.937 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:47.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:48.041 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:48.093 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:48.314 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:48.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:48.419 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:48.448 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:48.502 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:48.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:48.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:48.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:48.632 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:48.676 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:48.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:48.706 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:48 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/412870261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:48.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:48 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/412870261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:48.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:48 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/412870261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:48.784 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:48.838 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:08:48.856 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:49.120 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:49.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:49.235 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:49.303 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:49.436 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:49.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:49.523 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:49.541 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:49.597 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:49.675 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:08:49.676 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:08:49.676 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:08:49.676 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:08:49.676 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:08:49.692 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:49.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:49.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:49.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:49 smithi151 ceph-mon[159210]: pgmap v1479: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 342 op/s; 48801/89082 objects degraded (54.782%); 849/89082 objects misplaced (0.953%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T23:08:49.771 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:49 smithi052 ceph-mon[180979]: pgmap v1479: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 342 op/s; 48801/89082 objects degraded (54.782%); 849/89082 objects misplaced (0.953%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T23:08:49.771 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:49 smithi052 ceph-mon[187743]: pgmap v1479: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 342 op/s; 48801/89082 objects degraded (54.782%); 849/89082 objects misplaced (0.953%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T23:08:49.799 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:49.865 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:49.945 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:49.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:50.051 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:50.081 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:50.132 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:50.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:50.236 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:50.273 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:50.340 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:50.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:50.445 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:50.478 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:50.521 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T23:08:50.576 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:50.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:50.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:50.619 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:08:50.683 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:50.743 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3049270076' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:08:50.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:50 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 48801/89082 objects degraded (54.782%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3049270076' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:08:50.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:50 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 48801/89082 objects degraded (54.782%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:50.870 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:50.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:50.880 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:08:50.880 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3049270076' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:08:50.880 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:50 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 48801/89082 objects degraded (54.782%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:50.941 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:08:50.980 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:51.038 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:51.180 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:51.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:51.202 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:51.285 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:51.355 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3049270076' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: osdmap e472: 8 total, 8 up, 5 in 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: pgmap v1481: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 384 op/s; 48275/86181 objects degraded (56.016%); 739/86181 objects misplaced (0.857%); 10 MiB/s, 162 objects/s recovering 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[187743]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.612 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3049270076' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:08:51.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: osdmap e472: 8 total, 8 up, 5 in 2023-09-23T23:08:51.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: pgmap v1481: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 384 op/s; 48275/86181 objects degraded (56.016%); 739/86181 objects misplaced (0.857%); 10 MiB/s, 162 objects/s recovering 2023-09-23T23:08:51.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.613 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:51 smithi052 ceph-mon[180979]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.648 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:08:51.648 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:08:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3049270076' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:08:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: osdmap e472: 8 total, 8 up, 5 in 2023-09-23T23:08:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: pgmap v1481: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 384 op/s; 48275/86181 objects degraded (56.016%); 739/86181 objects misplaced (0.857%); 10 MiB/s, 162 objects/s recovering 2023-09-23T23:08:51.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:51 smithi151 ceph-mon[159210]: failed to encode map e472 with expected crc 2023-09-23T23:08:51.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:51.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:51.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:51.808 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:51.926 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:52.068 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:52.108 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:52.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:52.201 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:52.213 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:52.296 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:52.415 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:52.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:52.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:52.522 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:52.593 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:52.645 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:08:52.684 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:52.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:52.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:52.789 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:52.852 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1655128749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:52.855 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1655128749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:52.856 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1655128749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:08:52.867 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85301 11.4 11:20006e33:::benchmark_data_smithi052_606809_object3147:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.923239+0000", 2023-09-23T23:08:53.159 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 202.227745414, 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2027856080000001, 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85301 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.160 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923239+0000", 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923239+0000", 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923238+0000", 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8920000000000003e-06 2023-09-23T23:08:53.161 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923245+0000", 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1908e-05 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923257+0000", 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.443e-06 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.162 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923263+0000", 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8831311619999997 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.806394+0000", 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037811699999999999 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.810175+0000", 2023-09-23T23:08:53.163 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017708903000000002 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.827884+0000", 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029209840000000002 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.830805+0000", 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.25806233699999997 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.164 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.088868+0000", 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030479840000000001 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.091916+0000", 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068492040000000002 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.165 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.098765+0000", 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062520520000000001 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.105017+0000", 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011661407 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.116678+0000", 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001103129 2023-09-23T23:08:53.166 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.117781+0000", 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015897769999999999 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.119371+0000", 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066538880000000002 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.167 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.126025+0000", 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2027856080000001 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85304 11.4 11:25d25a9c:::benchmark_data_smithi052_606809_object3150:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 202.227715356, 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2751230859999998, 2023-09-23T23:08:53.168 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85304 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.169 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923269+0000", 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.4080000000000001e-06 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.170 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923271+0000", 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1699999999999996e-06 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923277+0000", 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4489999999999999e-06 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.923280+0000", 2023-09-23T23:08:53.171 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.8877396659999999 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.811020+0000", 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038907109999999998 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.814910+0000", 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.030186995000000001 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.172 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.845097+0000", 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031564060000000001 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.848254+0000", 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.297196077 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.145450+0000", 2023-09-23T23:08:53.173 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004836478 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.150286+0000", 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0097873660000000005 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.160074+0000", 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062974859999999997 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.174 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.166371+0000", 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00091250000000000001 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.167284+0000", 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.024166043000000002 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.175 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.191450+0000", 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014562430000000001 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.192906+0000", 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.005486505 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.198393+0000", 2023-09-23T23:08:53.176 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2751230859999998 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85312 11.1 11:80e4190c:::benchmark_data_smithi052_606809_object3158:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.947654+0000", 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 202.20333039900001, 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7308309500000001, 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.177 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85312 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947654+0000", 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.178 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947654+0000", 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947652+0000", 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1599e-05 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947664+0000", 2023-09-23T23:08:53.179 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7906000000000001e-05 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947682+0000", 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1664999999999999e-05 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.947693+0000", 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.513713555 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.180 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.461407+0000", 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003630253 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.465037+0000", 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015287227 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.181 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.480324+0000", 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003515991 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.483840+0000", 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16787706099999999 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.651717+0000", 2023-09-23T23:08:53.182 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035492700000000002 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.655267+0000", 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077437690000000002 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.663010+0000", 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066746690000000003 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.183 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.669685+0000", 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00093855499999999999 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.670624+0000", 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00061731399999999999 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.671241+0000", 2023-09-23T23:08:53.184 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015721909999999999 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.672813+0000", 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0056723030000000001 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.678485+0000", 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7308309500000001 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.185 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85313 11.11 11:88ebcce4:::benchmark_data_smithi052_606809_object3159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:30.949419+0000", 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 202.20156621800001, 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7675347939999999, 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.186 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85313 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949419+0000", 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.187 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949419+0000", 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949416+0000", 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3740000000000004e-06 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949426+0000", 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4061e-05 2023-09-23T23:08:53.188 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949440+0000", 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0450000000000001e-06 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:30.949448+0000", 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5184956220000001 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.189 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.467943+0000", 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043664400000000001 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.472310+0000", 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023586055000000002 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.495896+0000", 2023-09-23T23:08:53.190 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043012090000000003 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.500197+0000", 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19441752200000001 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.694615+0000", 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0023650899999999998 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.191 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.696980+0000", 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050274810000000003 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,3", 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.702007+0000", 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046063249999999997 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.192 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.706613+0000", 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001047886 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.707661+0000", 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00031287699999999998 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.707974+0000", 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020656289999999998 2023-09-23T23:08:53.193 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.710040+0000", 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069135209999999997 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.716953+0000", 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7675347939999999 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.194 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85369 11.4 11:24b22a8c:::benchmark_data_smithi052_606809_object3215:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.515955+0000", 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 200.63502958999999, 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.953101872, 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.195 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85369 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515955+0000", 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515955+0000", 2023-09-23T23:08:53.196 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515953+0000", 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1839999999999999e-06 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515961+0000", 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.428e-05 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.197 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515976+0000", 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.148e-06 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.515983+0000", 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.333705964 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.198 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.849689+0000", 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038120300000000001 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.853501+0000", 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021648292999999999 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.875149+0000", 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0045914110000000001 2023-09-23T23:08:53.199 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.879741+0000", 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.30051507500000002 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.180256+0000", 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026806270000000001 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.200 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.182936+0000", 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0048667820000000001 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.187803+0000", 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011472008000000001 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.199275+0000", 2023-09-23T23:08:53.201 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.010707579 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.209983+0000", 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.24838902900000001 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.458372+0000", 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020854889999999998 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.202 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.460457+0000", 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0085999939999999997 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.469057+0000", 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.953101872 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.203 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85403 11.4 11:26ba6595:::benchmark_data_smithi052_606809_object3249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.632315+0000", 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 200.51866964499999, 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.84570751, 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85403 2023-09-23T23:08:53.204 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632315+0000", 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632315+0000", 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.205 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632314+0000", 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.2310000000000004e-06 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632319+0000", 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2821e-05 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.206 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632332+0000", 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4320000000000004e-06 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.632338+0000", 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.221519861 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.853858+0000", 2023-09-23T23:08:53.207 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044097279999999999 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.858268+0000", 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.038489075999999997 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.896757+0000", 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035420310000000002 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.208 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.900299+0000", 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31082930399999997 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.211128+0000", 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0038503690000000002 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.209 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.214979+0000", 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075619830000000004 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.222541+0000", 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069424860000000003 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.229483+0000", 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0078532020000000001 2023-09-23T23:08:53.210 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.237336+0000", 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.23273159299999999 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.470068+0000", 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014063319999999999 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.211 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.471474+0000", 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.006548262 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.478023+0000", 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.84570751 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.212 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85405 11.14 11:2a8d78d3:::benchmark_data_smithi052_606809_object3251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.634161+0000", 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 200.516823883, 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.832578955, 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85405 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.213 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634161+0000", 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634161+0000", 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.214 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634160+0000", 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0370000000000002e-06 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634165+0000", 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.52e-06 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634173+0000", 2023-09-23T23:08:53.215 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7840000000000001e-06 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.634177+0000", 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.263104448 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.897281+0000", 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039633519999999998 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.216 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.901244+0000", 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.015875415 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.917120+0000", 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003554771 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.217 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.920674+0000", 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.29039737500000001 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.211072+0000", 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039948229999999998 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.215067+0000", 2023-09-23T23:08:53.218 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0079402069999999995 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.223007+0000", 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069677610000000003 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.229975+0000", 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081174709999999994 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.219 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.238092+0000", 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.22017930799999999 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.458271+0000", 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015077059999999999 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.220 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.459779+0000", 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0069607460000000003 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.466740+0000", 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.832578955 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.221 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85433 11.4 11:2052dad0:::benchmark_data_smithi052_606809_object3279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.737596+0000", 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 200.413389153, 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7512567720000001, 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85433 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.222 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737596+0000", 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737596+0000", 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.223 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737594+0000", 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8739999999999995e-06 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737602+0000", 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2435999999999999e-05 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737614+0000", 2023-09-23T23:08:53.224 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.677e-06 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.737620+0000", 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1630549939999999 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.900675+0000", 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031074190000000002 2023-09-23T23:08:53.225 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.903782+0000", 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013217001000000001 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.916999+0000", 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036349080000000001 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.226 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.920634+0000", 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.31714880899999998 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.237783+0000", 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002564555 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.227 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.240348+0000", 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0050599310000000002 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.245408+0000", 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023625132 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.269033+0000", 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00037696600000000001 2023-09-23T23:08:53.228 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.269410+0000", 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.209935662 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.479345+0000", 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021585160000000001 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.229 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.481504+0000", 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0073485110000000003 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.488852+0000", 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7512567720000001 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.230 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:85464 11.14 11:289df81e:::benchmark_data_smithi052_606809_object3310:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:32.835884+0000", 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 200.31510063900001, 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6773388709999999, 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.231 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 85464 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835884+0000", 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835884+0000", 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:53.232 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835882+0000", 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.679e-06 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835892+0000", 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1365999999999999e-05 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.233 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835903+0000", 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.2380000000000002e-06 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:32.835909+0000", 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0858108289999999 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.234 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.921720+0000", 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043400929999999997 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.926060+0000", 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.023379476999999999 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.949440+0000", 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031490149999999998 2023-09-23T23:08:53.235 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:33.952589+0000", 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.32618674399999997 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.278776+0000", 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0033116529999999999 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.236 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.282087+0000", 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0059453020000000004 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.288033+0000", 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0044534190000000001 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.292486+0000", 2023-09-23T23:08:53.237 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00032079800000000001 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.292807+0000", 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20849826399999999 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.501305+0000", 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002349056 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.238 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.239 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.503654+0000", 2023-09-23T23:08:53.239 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095689150000000008 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:34.513223+0000", 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6773388709999999 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.241 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89654 11.15 11:afe848ca:::benchmark_data_smithi052_606809_object7500:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:58.974110+0000", 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 174.17687449799999, 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2612470760000001, 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89654 2023-09-23T23:08:53.242 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974110+0000", 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974110+0000", 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.243 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974109+0000", 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5589999999999996e-06 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974115+0000", 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9350000000000006e-06 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.244 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974125+0000", 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.0460000000000002e-06 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:58.974131+0000", 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7024555349999999 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.676586+0000", 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.004985817 2023-09-23T23:08:53.245 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.681572+0000", 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.036358366000000003 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.717930+0000", 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0046866579999999998 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.246 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.722617+0000", 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.481659956 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.204277+0000", 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037195980000000002 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.207997+0000", 2023-09-23T23:08:53.247 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0075747210000000004 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.215571+0000", 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066158290000000002 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.222187+0000", 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0012446479999999999 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.248 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.223432+0000", 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00045506700000000003 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.223887+0000", 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020853339999999999 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.249 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.225972+0000", 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0093852569999999993 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.235357+0000", 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.2612470760000001 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.250 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:89665 11.d e438/422 11:b4df92a6:::benchmark_data_smithi052_606809_object7511:head v 438'5708, mlcod=415'892)", 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.012900+0000", 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 174.138084733, 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5605278060000001, 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012900+0000", 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.251 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012900+0000", 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012899+0000", 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0849000000000001e-05 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.252 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012909+0000", 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.1338999999999999e-05 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012921+0000", 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4239999999999998e-06 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.012927+0000", 2023-09-23T23:08:53.253 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.542952552 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.555880+0000", 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5443999999999999e-05 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.555905+0000", 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.017479251000000001 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.254 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573384+0000", 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3417999999999997e-05 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573428+0000", 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.5605278060000001 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.255 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89672 11.5 11:a3f2d77a:::benchmark_data_smithi052_606809_object7518:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.061551+0000", 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 174.089433826, 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0987285240000002, 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89672 2023-09-23T23:08:53.256 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.257 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.257 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.257 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.257 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061551+0000", 2023-09-23T23:08:53.257 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061551+0000", 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061549+0000", 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8250000000000005e-06 2023-09-23T23:08:53.258 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061557+0000", 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2894000000000001e-05 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061570+0000", 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2576e-05 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.259 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.061583+0000", 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6154805059999999 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.677063+0000", 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0047144320000000002 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.681778+0000", 2023-09-23T23:08:53.260 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031613074999999997 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.713391+0000", 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039974870000000001 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.717388+0000", 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.416516314 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.261 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.133904+0000", 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024337619999999999 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.136338+0000", 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0059120209999999999 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.262 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.142250+0000", 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049678300000000003 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.147218+0000", 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001389562 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.148608+0000", 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039206070000000004 2023-09-23T23:08:53.263 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.152528+0000", 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015177439999999999 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.154046+0000", 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062334870000000002 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.264 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.160280+0000", 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0987285240000002 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:89701 11.d e438/422 11:b8619b1e:::benchmark_data_smithi052_606809_object7547:head v 438'5709, mlcod=415'892)", 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.226962+0000", 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.924022452, 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3464912619999998, 2023-09-23T23:08:53.265 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226962+0000", 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226962+0000", 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:08:53.266 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226961+0000", 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1880000000000008e-06 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226971+0000", 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9430000000000006e-06 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.267 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226980+0000", 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9900000000000002e-06 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.226986+0000", 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3334172569999998 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.560403+0000", 2023-09-23T23:08:53.268 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7268999999999999e-05 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.560430+0000", 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013000843 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573431+0000", 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2730000000000001e-05 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.269 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573454+0000", 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3464912619999998 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89719 11.5 11:a1dc5e4d:::benchmark_data_smithi052_606809_object7565:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.354009+0000", 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.796976215, 2023-09-23T23:08:53.270 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9292337449999999, 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89719 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354009+0000", 2023-09-23T23:08:53.271 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354009+0000", 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354007+0000", 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3188e-05 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.272 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354020+0000", 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2751e-05 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354033+0000", 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0747000000000001e-05 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.273 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.354043+0000", 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3637451550000002 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.717788+0000", 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0032941609999999999 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.721083+0000", 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.033079843999999997 2023-09-23T23:08:53.274 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.754162+0000", 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002939999 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.757102+0000", 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.47205112399999999 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.275 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.229154+0000", 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0029339689999999998 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.232088+0000", 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0072230259999999996 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.239311+0000", 2023-09-23T23:08:53.276 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.012806079 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.252117+0000", 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001104449 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.253221+0000", 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.020816254999999999 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.277 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.274037+0000", 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002526169 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.276564+0000", 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0066788419999999999 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.278 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.283242+0000", 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9292337449999999 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:89722 11.d e438/422 11:bfa4ed94:::benchmark_data_smithi052_606809_object7568:head v 438'5710, mlcod=415'892)", 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.370512+0000", 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.78047267400001, 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.202966194, 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.279 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370512+0000", 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370512+0000", 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.280 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370511+0000", 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.453e-05 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370526+0000", 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7280999999999999e-05 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.281 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370543+0000", 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9209999999999998e-06 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.370552+0000", 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.193936415 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.564488+0000", 2023-09-23T23:08:53.282 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.2923000000000001e-05 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.564511+0000", 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0089455330000000003 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573457+0000", 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1611999999999999e-05 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.283 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.573478+0000", 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.202966194 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89724 11.15 11:a935a078:::benchmark_data_smithi052_606809_object7570:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.376891+0000", 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.774093916, 2023-09-23T23:08:53.284 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9619000199999999, 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89724 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376891+0000", 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.285 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376891+0000", 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376889+0000", 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3060000000000003e-06 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.286 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376898+0000", 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3233e-05 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376911+0000", 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3850000000000002e-06 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.287 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.376920+0000", 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.347168988 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.724089+0000", 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0035878080000000001 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.727677+0000", 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.021852636000000002 2023-09-23T23:08:53.288 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.749529+0000", 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041318489999999999 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.753661+0000", 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.54881714199999998 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.289 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.302478+0000", 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0043085609999999998 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.306787+0000", 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0085451180000000009 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.290 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.315332+0000", 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0083029590000000004 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.323635+0000", 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0019104199999999999 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.325545+0000", 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00072450300000000004 2023-09-23T23:08:53.291 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.326270+0000", 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0036638439999999999 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.329934+0000", 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0088572140000000004 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.292 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.338791+0000", 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.9619000199999999 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89726 11.15 11:a840aef0:::benchmark_data_smithi052_606809_object7572:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.395826+0000", 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.75515918599999, 2023-09-23T23:08:53.293 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1034079110000001, 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89726 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395826+0000", 2023-09-23T23:08:53.294 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395826+0000", 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999971 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395823+0000", 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0997e-05 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.295 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395834+0000", 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8947999999999999e-05 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395853+0000", 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3639999999999999e-06 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.296 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.395861+0000", 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3322495170000002 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.728111+0000", 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0041596289999999998 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.732270+0000", 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.063996094000000003 2023-09-23T23:08:53.297 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.796266+0000", 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053571390000000003 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.801624+0000", 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.65385371299999995 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.298 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.455477+0000", 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034231679999999999 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.458900+0000", 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081103259999999993 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.467011+0000", 2023-09-23T23:08:53.299 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.019588700000000001 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.486599+0000", 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00034858699999999997 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.486948+0000", 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001419462 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.300 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.488368+0000", 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0018342149999999999 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.490202+0000", 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0090318140000000009 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.301 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.499234+0000", 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.1034079110000001 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:89728 11.5 11:a12ca5da:::benchmark_data_smithi052_606809_object7574:head [delete] snapc 0=[] ondisk+write+known_if_redirected e438)", 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:59.431675+0000", 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 173.71930955600001, 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0659816869999998, 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.302 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 89728 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431675+0000", 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.303 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431675+0000", 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431673+0000", 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9670000000000001e-06 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.304 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431681+0000", 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.6407000000000001e-05 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431698+0000", 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7449999999999995e-06 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:59.431705+0000", 2023-09-23T23:08:53.305 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3256917989999999 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.757397+0000", 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002954422 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.760352+0000", 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.031998626000000002 2023-09-23T23:08:53.306 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.792350+0000", 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030543110000000001 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:01.795404+0000", 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.66251963899999999 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.307 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.457924+0000", 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0034388040000000002 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.461363+0000", 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076716960000000004 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.308 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.469035+0000", 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.018468185000000002 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.487503+0000", 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00090221400000000003 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.309 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.488405+0000", 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00091449000000000001 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.489319+0000", 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014892130000000001 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.490809+0000", 2023-09-23T23:08:53.310 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0068482480000000004 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:02.497657+0000", 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.0659816869999998 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:136295 11.15 11:ad1c76c1:::benchmark_data_smithi052_606809_object54141:head [delete] snapc 0=[] ondisk+write+known_if_redirected e471)", 2023-09-23T23:08:53.311 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:49.889325+0000", 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 3.261659635, 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.65801516500000001, 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 136295 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.312 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.889325+0000", 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.889325+0000", 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.889324+0000", 2023-09-23T23:08:53.313 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.4750000000000001e-06 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.889329+0000", 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0208e-05 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.889340+0000", 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.9559999999999996e-06 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.314 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.889345+0000", 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.056947902000000002 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.946292+0000", 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001565213 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.315 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.947858+0000", 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.008171691 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.956029+0000", 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015287720000000001 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for rw locks", 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.957558+0000", 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.52358468000000002 2023-09-23T23:08:53.316 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.481143+0000", 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001154698 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.482298+0000", 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026370500000000002 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.317 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.484935+0000", 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0077712520000000002 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.492706+0000", 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0054392520000000003 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.318 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.498145+0000", 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.044988943000000003 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.543134+0000", 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00074142900000000005 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.543875+0000", 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003464885 2023-09-23T23:08:53.319 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.547340+0000", 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.65801516500000001 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:136307 11.5 11:a7555c6c:::benchmark_data_smithi052_606809_object54153:head [delete] snapc 0=[] ondisk+write+known_if_redirected e471)", 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:49.925768+0000", 2023-09-23T23:08:53.320 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 3.2252170410000001, 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.61653544599999999, 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 136307 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:08:53.321 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.925768+0000", 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.925768+0000", 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.925766+0000", 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1744999999999999e-05 2023-09-23T23:08:53.322 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.925788+0000", 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4054000000000002e-05 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.925832+0000", 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.059e-05 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.323 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.925842+0000", 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.029868724999999999 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.955711+0000", 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0014975699999999999 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for missing object", 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.957209+0000", 2023-09-23T23:08:53.324 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013078154999999999 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.970287+0000", 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000868113 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for degraded object", 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:49.971155+0000", 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.50178362300000001 2023-09-23T23:08:53.325 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.472939+0000", 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015181459999999999 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.474457+0000", 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0037887279999999999 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.326 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.478245+0000", 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.014124017000000001 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.492369+0000", 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0055861790000000001 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.497956+0000", 2023-09-23T23:08:53.327 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.040414147999999997 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.538370+0000", 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00071014699999999995 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.539080+0000", 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.003223273 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:08:53.328 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:50.542303+0000", 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.61653544599999999 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:08:53.329 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:08:53.663 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:08:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:08:53] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1655128749' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: osdmap e473: 8 total, 8 up, 5 in 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:53 smithi151 ceph-mon[159210]: pgmap v1483: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.7 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 425 op/s; 48203/83484 objects degraded (57.739%); 739/83484 objects misplaced (0.885%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T23:08:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:08:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1655128749' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: osdmap e473: 8 total, 8 up, 5 in 2023-09-23T23:08:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[187743]: pgmap v1483: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.7 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 425 op/s; 48203/83484 objects degraded (57.739%); 739/83484 objects misplaced (0.885%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1655128749' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: osdmap e473: 8 total, 8 up, 5 in 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: failed to encode map e473 with expected crc 2023-09-23T23:08:53.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:53 smithi052 ceph-mon[180979]: pgmap v1483: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.7 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 425 op/s; 48203/83484 objects degraded (57.739%); 739/83484 objects misplaced (0.885%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T23:08:54.062 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:08:54.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:54.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:54.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:54.192 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:54.232 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:54.296 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:54.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:54.402 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:54.469 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:54.636 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:55.197 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:55.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:55.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:55.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:55.372 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:55.562 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:55.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:55.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:55.709 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:55.853 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:55 smithi052 ceph-mon[180979]: pgmap v1484: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 476 op/s; 47387/80727 objects degraded (58.700%); 739/80727 objects misplaced (0.915%); 11 MiB/s, 171 objects/s recovering 2023-09-23T23:08:55.854 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:55 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 47387/80727 objects degraded (58.700%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:55.944 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:08:55.991 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:08:56.018 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:08:56.053 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:55 smithi151 ceph-mon[159210]: pgmap v1484: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 476 op/s; 47387/80727 objects degraded (58.700%); 739/80727 objects misplaced (0.915%); 11 MiB/s, 171 objects/s recovering 2023-09-23T23:08:56.053 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:55 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 47387/80727 objects degraded (58.700%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:56.057 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:08:56.070 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:56.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:56.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:56.087 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:08:56.125 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:08:56.164 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:08:56.187 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:56.208 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.210 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:08:56.235 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:55 smithi052 ceph-mon[187743]: pgmap v1484: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 476 op/s; 47387/80727 objects degraded (58.700%); 739/80727 objects misplaced (0.915%); 11 MiB/s, 171 objects/s recovering 2023-09-23T23:08:56.235 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:55 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 47387/80727 objects degraded (58.700%), 24 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T23:08:56.253 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:56.360 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:08:56.376 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:08:56.391 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:56.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:56.498 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:08:56.570 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:56.653 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:08:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:56.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:08:56.665 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:08:56.666 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:08:56.760 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:56.796 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1913154341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:57.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1913154341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:57.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1913154341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:08:57.250 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:57.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:57.325 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:08:57.358 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:08:57.426 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:57.648 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:08:57.790 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:08:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:57.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:08:57.879 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1484,"stamp":"2023-09-23T23:08:54.788891+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1762465720,"num_objects":26909,"num_object_clones":0,"num_object_copies":80727,"num_objects_missing_on_primary":1837,"num_objects_missing":1837,"num_objects_degraded":47387,"num_objects_misplaced":739,"num_objects_unfound":0,"num_objects_dirty":26909,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":137679,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30108,"num_bytes_recovered":1942245824,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170228,"ondisk_log_size":170228,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8210832,"kb_used_data":4485800,"kb_used_omap":99,"kb_used_meta":3724892,"kb_avail":460514928,"statfs":{"total":479975178240,"available":471567286272,"internally_reserved":0,"allocated":4593459200,"data_stored":4585339039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3814289430},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,5,3,0,4,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-250150912,"num_objects":-3817,"num_object_clones":0,"num_object_copies":-11451,"num_objects_missing_on_primary":-367,"num_objects_missing":-367,"num_objects_degraded":-1779,"num_objects_misplaced":-110,"num_objects_unfound":0,"num_objects_dirty":-3817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3817,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1373,"num_bytes_recovered":89980928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003930"},"pg_stats":[{"pgid":"11.12","version":"472'6943","reported_seq":7035,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.752076+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:08:51.752076+0000","last_peered":"2023-09-23T23:08:51.752076+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:08:51.752076+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6943,"log_dups_size":0,"ondisk_log_size":6943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56393728,"num_objects":861,"num_object_clones":0,"num_object_copies":2583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4280,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":892},{"shards":"2,3","objects":-1596}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":483,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.472520+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:08:43.472520+0000","last_peered":"2023-09-23T23:08:43.472520+0000","last_clean":"2023-09-23T23:08:43.472520+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:08:43.472520+0000","last_undegraded":"2023-09-23T23:08:43.472520+0000","last_fullsized":"2023-09-23T23:08:43.472520+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531645+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:08:50.531645+0000","last_peered":"2023-09-23T23:08:50.531645+0000","last_clean":"2023-09-23T23:08:50.531645+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:08:50.531645+0000","last_undegraded":"2023-09-23T23:08:50.531645+0000","last_fullsized":"2023-09-23T23:08:50.531645+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"473'6969","reported_seq":10154,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:54.201081+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:08:54.201081+0000","last_peered":"2023-09-23T23:08:54.201081+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:08:54.201081+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:08:54.201081+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6969,"log_dups_size":0,"ondisk_log_size":6969,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53379072,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1101,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4316,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1542,"num_bytes_recovered":100990976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":488},{"shards":"0,3,6","objects":270},{"shards":"2,3,6","objects":57}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":144,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525163+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:08:50.525163+0000","last_peered":"2023-09-23T23:08:50.525163+0000","last_clean":"2023-09-23T23:08:50.525163+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:08:50.525163+0000","last_undegraded":"2023-09-23T23:08:50.525163+0000","last_fullsized":"2023-09-23T23:08:50.525163+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":491,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531788+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:08:50.531788+0000","last_peered":"2023-09-23T23:08:50.531788+0000","last_clean":"2023-09-23T23:08:50.531788+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:08:50.531788+0000","last_undegraded":"2023-09-23T23:08:50.531788+0000","last_fullsized":"2023-09-23T23:08:50.531788+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"473'6938","reported_seq":10994,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:54.180237+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:08:54.180237+0000","last_peered":"2023-09-23T23:08:54.180237+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:08:54.180237+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:08:54.180237+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6938,"log_dups_size":0,"ondisk_log_size":6938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54067200,"num_objects":825,"num_object_clones":0,"num_object_copies":2475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4282,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":163},{"shards":"1,3,6","objects":662}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526439+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:08:50.526439+0000","last_peered":"2023-09-23T23:08:50.526439+0000","last_clean":"2023-09-23T23:08:50.526439+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:08:50.526439+0000","last_undegraded":"2023-09-23T23:08:50.526439+0000","last_fullsized":"2023-09-23T23:08:50.526439+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522389+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:08:50.522389+0000","last_peered":"2023-09-23T23:08:50.522389+0000","last_clean":"2023-09-23T23:08:50.522389+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:08:50.522389+0000","last_undegraded":"2023-09-23T23:08:50.522389+0000","last_fullsized":"2023-09-23T23:08:50.522389+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":417,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522621+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:08:50.522621+0000","last_peered":"2023-09-23T23:08:50.522621+0000","last_clean":"2023-09-23T23:08:50.522621+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:08:50.522621+0000","last_undegraded":"2023-09-23T23:08:50.522621+0000","last_fullsized":"2023-09-23T23:08:50.522621+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":163,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.529334+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:08:50.529334+0000","last_peered":"2023-09-23T23:08:50.529334+0000","last_clean":"2023-09-23T23:08:50.529334+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:08:50.529334+0000","last_undegraded":"2023-09-23T23:08:50.529334+0000","last_fullsized":"2023-09-23T23:08:50.529334+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":279,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531892+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:08:50.531892+0000","last_peered":"2023-09-23T23:08:50.531892+0000","last_clean":"2023-09-23T23:08:50.531892+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:08:50.531892+0000","last_undegraded":"2023-09-23T23:08:50.531892+0000","last_fullsized":"2023-09-23T23:08:50.531892+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":177,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523846+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:08:50.523846+0000","last_peered":"2023-09-23T23:08:50.523846+0000","last_clean":"2023-09-23T23:08:50.523846+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:08:50.523846+0000","last_undegraded":"2023-09-23T23:08:50.523846+0000","last_fullsized":"2023-09-23T23:08:50.523846+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":210,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531603+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:08:50.531603+0000","last_peered":"2023-09-23T23:08:50.531603+0000","last_clean":"2023-09-23T23:08:50.531603+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:08:50.531603+0000","last_undegraded":"2023-09-23T23:08:50.531603+0000","last_fullsized":"2023-09-23T23:08:50.531603+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":227,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.669174+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:08:45.669174+0000","last_peered":"2023-09-23T23:08:45.669174+0000","last_clean":"2023-09-23T23:08:45.669174+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:08:45.669174+0000","last_undegraded":"2023-09-23T23:08:45.669174+0000","last_fullsized":"2023-09-23T23:08:45.669174+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":561,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473026+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:08:43.473026+0000","last_peered":"2023-09-23T23:08:43.473026+0000","last_clean":"2023-09-23T23:08:43.473026+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:08:43.473026+0000","last_undegraded":"2023-09-23T23:08:43.473026+0000","last_fullsized":"2023-09-23T23:08:43.473026+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":454,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473321+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:08:43.473321+0000","last_peered":"2023-09-23T23:08:43.473321+0000","last_clean":"2023-09-23T23:08:43.473321+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:08:43.473321+0000","last_undegraded":"2023-09-23T23:08:43.473321+0000","last_fullsized":"2023-09-23T23:08:43.473321+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":386,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473400+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:08:43.473400+0000","last_peered":"2023-09-23T23:08:43.473400+0000","last_clean":"2023-09-23T23:08:43.473400+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:08:43.473400+0000","last_undegraded":"2023-09-23T23:08:43.473400+0000","last_fullsized":"2023-09-23T23:08:43.473400+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522305+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:08:50.522305+0000","last_peered":"2023-09-23T23:08:50.522305+0000","last_clean":"2023-09-23T23:08:50.522305+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:08:50.522305+0000","last_undegraded":"2023-09-23T23:08:50.522305+0000","last_fullsized":"2023-09-23T23:08:50.522305+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":474,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523650+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:08:50.523650+0000","last_peered":"2023-09-23T23:08:50.523650+0000","last_clean":"2023-09-23T23:08:50.523650+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:08:50.523650+0000","last_undegraded":"2023-09-23T23:08:50.523650+0000","last_fullsized":"2023-09-23T23:08:50.523650+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":389,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531548+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:08:50.531548+0000","last_peered":"2023-09-23T23:08:50.531548+0000","last_clean":"2023-09-23T23:08:50.531548+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:08:50.531548+0000","last_undegraded":"2023-09-23T23:08:50.531548+0000","last_fullsized":"2023-09-23T23:08:50.531548+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"473'6868","reported_seq":12036,"reported_epoch":473,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:08:54.191829+0000","last_change":"2023-09-23T23:06:32.889539+0000","last_active":"2023-09-23T23:08:54.191829+0000","last_peered":"2023-09-23T23:08:54.191829+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:06:32.875369+0000","last_became_peered":"2023-09-23T23:06:32.875369+0000","last_unstale":"2023-09-23T23:08:54.191829+0000","last_undegraded":"2023-09-23T23:08:54.191829+0000","last_fullsized":"2023-09-23T23:08:54.191829+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6868,"log_dups_size":0,"ondisk_log_size":6868,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57376768,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4219,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":876}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":374,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469892+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:08:43.469892+0000","last_peered":"2023-09-23T23:08:43.469892+0000","last_clean":"2023-09-23T23:08:43.469892+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:08:43.469892+0000","last_undegraded":"2023-09-23T23:08:43.469892+0000","last_fullsized":"2023-09-23T23:08:43.469892+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":498,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531745+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:08:50.531745+0000","last_peered":"2023-09-23T23:08:50.531745+0000","last_clean":"2023-09-23T23:08:50.531745+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:08:50.531745+0000","last_undegraded":"2023-09-23T23:08:50.531745+0000","last_fullsized":"2023-09-23T23:08:50.531745+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"473'7048","reported_seq":12015,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:54.196765+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:08:54.196765+0000","last_peered":"2023-09-23T23:08:54.196765+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:08:54.196765+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53510144,"num_objects":817,"num_object_clones":0,"num_object_copies":2451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1593,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4358,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1671,"num_bytes_recovered":109314048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":472},{"shards":"3,5,6","objects":317},{"shards":"3,6","objects":28}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":87,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526902+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:08:50.526902+0000","last_peered":"2023-09-23T23:08:50.526902+0000","last_clean":"2023-09-23T23:08:50.526902+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:08:50.526902+0000","last_undegraded":"2023-09-23T23:08:50.526902+0000","last_fullsized":"2023-09-23T23:08:50.526902+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":169,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522699+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:08:50.522699+0000","last_peered":"2023-09-23T23:08:50.522699+0000","last_clean":"2023-09-23T23:08:50.522699+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:08:50.522699+0000","last_undegraded":"2023-09-23T23:08:50.522699+0000","last_fullsized":"2023-09-23T23:08:50.522699+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"471'6947","reported_seq":7043,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:49.916059+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:08:49.916059+0000","last_peered":"2023-09-23T23:08:49.916059+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:08:49.916059+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6947,"log_dups_size":0,"ondisk_log_size":6947,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52396032,"num_objects":799,"num_object_clones":0,"num_object_copies":2397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4296,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1566},{"shards":"1,4,5","objects":841}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":381,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.475019+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:08:43.475019+0000","last_peered":"2023-09-23T23:08:43.475019+0000","last_clean":"2023-09-23T23:08:43.475019+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:08:43.475019+0000","last_undegraded":"2023-09-23T23:08:43.475019+0000","last_fullsized":"2023-09-23T23:08:43.475019+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":337,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469386+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:08:43.469386+0000","last_peered":"2023-09-23T23:08:43.469386+0000","last_clean":"2023-09-23T23:08:43.469386+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:08:43.469386+0000","last_undegraded":"2023-09-23T23:08:43.469386+0000","last_fullsized":"2023-09-23T23:08:43.469386+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"472'6809","reported_seq":11815,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:51.758237+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:08:51.758237+0000","last_peered":"2023-09-23T23:08:51.758237+0000","last_clean":"2023-09-23T23:08:51.758237+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:08:51.758237+0000","last_undegraded":"2023-09-23T23:08:51.758237+0000","last_fullsized":"2023-09-23T23:08:51.758237+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6809,"log_dups_size":0,"ondisk_log_size":6809,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58032128,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4177,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":101,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.673223+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:08:45.673223+0000","last_peered":"2023-09-23T23:08:45.673223+0000","last_clean":"2023-09-23T23:08:45.673223+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:08:45.673223+0000","last_undegraded":"2023-09-23T23:08:45.673223+0000","last_fullsized":"2023-09-23T23:08:45.673223+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":478,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469822+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:08:43.469822+0000","last_peered":"2023-09-23T23:08:43.469822+0000","last_clean":"2023-09-23T23:08:43.469822+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:08:43.469822+0000","last_undegraded":"2023-09-23T23:08:43.469822+0000","last_fullsized":"2023-09-23T23:08:43.469822+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"473'6966","reported_seq":11954,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:53.154571+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:08:53.154571+0000","last_peered":"2023-09-23T23:08:53.154571+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:08:53.154571+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6966,"log_dups_size":0,"ondisk_log_size":6966,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50364416,"num_objects":768,"num_object_clones":0,"num_object_copies":2304,"num_objects_missing_on_primary":599,"num_objects_missing":599,"num_objects_degraded":1637,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4317,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1669,"num_bytes_recovered":109379584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":504},{"shards":"0,3,5,6","objects":3},{"shards":"1,6","objects":-47},{"shards":"3,5,6","objects":308}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":131,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531471+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:08:50.531471+0000","last_peered":"2023-09-23T23:08:50.531471+0000","last_clean":"2023-09-23T23:08:50.531471+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:08:50.531471+0000","last_undegraded":"2023-09-23T23:08:50.531471+0000","last_fullsized":"2023-09-23T23:08:50.531471+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522164+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:08:50.522164+0000","last_peered":"2023-09-23T23:08:50.522164+0000","last_clean":"2023-09-23T23:08:50.522164+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:08:50.522164+0000","last_undegraded":"2023-09-23T23:08:50.522164+0000","last_fullsized":"2023-09-23T23:08:50.522164+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"472'7085","reported_seq":7522,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.794845+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:08:51.794845+0000","last_peered":"2023-09-23T23:08:51.794845+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:08:51.794845+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7085,"log_dups_size":0,"ondisk_log_size":7085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55934976,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4369,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1191},{"shards":"1,2,3","objects":562}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":164,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522214+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:08:50.522214+0000","last_peered":"2023-09-23T23:08:50.522214+0000","last_clean":"2023-09-23T23:08:50.522214+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:08:50.522214+0000","last_undegraded":"2023-09-23T23:08:50.522214+0000","last_fullsized":"2023-09-23T23:08:50.522214+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":293,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473255+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:08:43.473255+0000","last_peered":"2023-09-23T23:08:43.473255+0000","last_clean":"2023-09-23T23:08:43.473255+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:08:43.473255+0000","last_undegraded":"2023-09-23T23:08:43.473255+0000","last_fullsized":"2023-09-23T23:08:43.473255+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"473'7014","reported_seq":10417,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:52.644867+0000","last_change":"2023-09-23T23:07:56.507184+0000","last_active":"2023-09-23T23:08:52.644867+0000","last_peered":"2023-09-23T23:08:52.644867+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.488393+0000","last_became_peered":"2023-09-23T23:07:56.488393+0000","last_unstale":"2023-09-23T23:08:52.644867+0000","last_undegraded":"2023-09-23T23:07:56.465702+0000","last_fullsized":"2023-09-23T23:07:56.443434+0000","mapping_epoch":459,"log_start":"414'569","ondisk_log_start":"414'569","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":682,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4331,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1642,"num_bytes_recovered":107511808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":488},{"shards":"1,6","objects":385}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":178,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.528863+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:08:50.528863+0000","last_peered":"2023-09-23T23:08:50.528863+0000","last_clean":"2023-09-23T23:08:50.528863+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:08:50.528863+0000","last_undegraded":"2023-09-23T23:08:50.528863+0000","last_fullsized":"2023-09-23T23:08:50.528863+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":473,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.472945+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:08:43.472945+0000","last_peered":"2023-09-23T23:08:43.472945+0000","last_clean":"2023-09-23T23:08:43.472945+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:08:43.472945+0000","last_undegraded":"2023-09-23T23:08:43.472945+0000","last_fullsized":"2023-09-23T23:08:43.472945+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"472'6946","reported_seq":7040,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.354981+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:08:51.354981+0000","last_peered":"2023-09-23T23:08:51.354981+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:08:51.354981+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6946,"log_dups_size":0,"ondisk_log_size":6946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56197120,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4282,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":826},{"shards":"2,3","objects":-1538}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":244,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.472695+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:08:43.472695+0000","last_peered":"2023-09-23T23:08:43.472695+0000","last_clean":"2023-09-23T23:08:43.472695+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:08:43.472695+0000","last_undegraded":"2023-09-23T23:08:43.472695+0000","last_fullsized":"2023-09-23T23:08:43.472695+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":481,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.474323+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:08:43.474323+0000","last_peered":"2023-09-23T23:08:43.474323+0000","last_clean":"2023-09-23T23:08:43.474323+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:08:43.474323+0000","last_undegraded":"2023-09-23T23:08:43.474323+0000","last_fullsized":"2023-09-23T23:08:43.474323+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"473'6995","reported_seq":10473,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:53.167645+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:08:53.167645+0000","last_peered":"2023-09-23T23:08:53.167645+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:08:53.167645+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51675136,"num_objects":788,"num_object_clones":0,"num_object_copies":2364,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":1060,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4341,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1688,"num_bytes_recovered":110493696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,2,3","objects":542},{"shards":"0,2,3,6","objects":1},{"shards":"0,3,6","objects":300},{"shards":"2,6","objects":-55}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":494,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469628+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:08:43.469628+0000","last_peered":"2023-09-23T23:08:43.469628+0000","last_clean":"2023-09-23T23:08:43.469628+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:08:43.469628+0000","last_undegraded":"2023-09-23T23:08:43.469628+0000","last_fullsized":"2023-09-23T23:08:43.469628+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":499,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.474888+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:08:43.474888+0000","last_peered":"2023-09-23T23:08:43.474888+0000","last_clean":"2023-09-23T23:08:43.474888+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:08:43.474888+0000","last_undegraded":"2023-09-23T23:08:43.474888+0000","last_fullsized":"2023-09-23T23:08:43.474888+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"473'6913","reported_seq":10209,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:54.073032+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:08:54.073032+0000","last_peered":"2023-09-23T23:08:54.073032+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:08:54.073032+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:08:54.073032+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4257,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":486},{"shards":"1,3,6","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":146,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.530014+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:08:50.530014+0000","last_peered":"2023-09-23T23:08:50.530014+0000","last_clean":"2023-09-23T23:08:50.530014+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:08:50.530014+0000","last_undegraded":"2023-09-23T23:08:50.530014+0000","last_fullsized":"2023-09-23T23:08:50.530014+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":378,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.672143+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:08:45.672143+0000","last_peered":"2023-09-23T23:08:45.672143+0000","last_clean":"2023-09-23T23:08:45.672143+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:08:45.672143+0000","last_undegraded":"2023-09-23T23:08:45.672143+0000","last_fullsized":"2023-09-23T23:08:45.672143+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"472'8692","reported_seq":10158,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.676407+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:08:51.676407+0000","last_peered":"2023-09-23T23:08:51.676407+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:08:51.676407+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"417'2874","ondisk_log_start":"417'2874","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5818,"log_dups_size":0,"ondisk_log_size":5818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115539992,"num_objects":1764,"num_object_clones":0,"num_object_copies":5292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1764,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8692,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":1088},{"shards":"2,4","objects":-2102}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":392,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.537746+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:08:50.537746+0000","last_peered":"2023-09-23T23:08:50.537746+0000","last_clean":"2023-09-23T23:08:50.537746+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:08:50.537746+0000","last_undegraded":"2023-09-23T23:08:50.537746+0000","last_fullsized":"2023-09-23T23:08:50.537746+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531309+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:08:50.531309+0000","last_peered":"2023-09-23T23:08:50.531309+0000","last_clean":"2023-09-23T23:08:50.531309+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:08:50.531309+0000","last_undegraded":"2023-09-23T23:08:50.531309+0000","last_fullsized":"2023-09-23T23:08:50.531309+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":212,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.472427+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:08:43.472427+0000","last_peered":"2023-09-23T23:08:43.472427+0000","last_clean":"2023-09-23T23:08:43.472427+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:08:43.472427+0000","last_undegraded":"2023-09-23T23:08:43.472427+0000","last_fullsized":"2023-09-23T23:08:43.472427+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"473'8610","reported_seq":8727,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:54.147362+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:08:54.147362+0000","last_peered":"2023-09-23T23:08:54.147362+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:08:54.147362+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"419'4067","ondisk_log_start":"419'4067","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4543,"log_dups_size":0,"ondisk_log_size":4543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8610,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1666},{"shards":"1,3","objects":-3159}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":209,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.512389+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:08:50.512389+0000","last_peered":"2023-09-23T23:08:50.512389+0000","last_clean":"2023-09-23T23:08:50.512389+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:08:50.512389+0000","last_undegraded":"2023-09-23T23:08:50.512389+0000","last_fullsized":"2023-09-23T23:08:50.512389+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":509,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473548+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:08:43.473548+0000","last_peered":"2023-09-23T23:08:43.473548+0000","last_clean":"2023-09-23T23:08:43.473548+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:08:43.473548+0000","last_undegraded":"2023-09-23T23:08:43.473548+0000","last_fullsized":"2023-09-23T23:08:43.473548+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":377,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:46.827738+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:08:46.827738+0000","last_peered":"2023-09-23T23:08:46.827738+0000","last_clean":"2023-09-23T23:08:46.827738+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:08:46.827738+0000","last_undegraded":"2023-09-23T23:08:46.827738+0000","last_fullsized":"2023-09-23T23:08:46.827738+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"473'7065","reported_seq":12059,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:54.139318+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:08:54.139318+0000","last_peered":"2023-09-23T23:08:54.139318+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:08:54.139318+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52396032,"num_objects":799,"num_object_clones":0,"num_object_copies":2397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4374,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1685,"num_bytes_recovered":110362624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":511},{"shards":"3,5,6","objects":315},{"shards":"3,6","objects":-27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":452,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:46.827648+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:08:46.827648+0000","last_peered":"2023-09-23T23:08:46.827648+0000","last_clean":"2023-09-23T23:08:46.827648+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:08:46.827648+0000","last_undegraded":"2023-09-23T23:08:46.827648+0000","last_fullsized":"2023-09-23T23:08:46.827648+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":304,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.668717+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:08:45.668717+0000","last_peered":"2023-09-23T23:08:45.668717+0000","last_clean":"2023-09-23T23:08:45.668717+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:08:45.668717+0000","last_undegraded":"2023-09-23T23:08:45.668717+0000","last_fullsized":"2023-09-23T23:08:45.668717+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":410,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522533+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:08:50.522533+0000","last_peered":"2023-09-23T23:08:50.522533+0000","last_clean":"2023-09-23T23:08:50.522533+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:08:50.522533+0000","last_undegraded":"2023-09-23T23:08:50.522533+0000","last_fullsized":"2023-09-23T23:08:50.522533+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"473'8640","reported_seq":15769,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:54.169733+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:08:54.169733+0000","last_peered":"2023-09-23T23:08:54.169733+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:08:54.169733+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"420'4134","ondisk_log_start":"420'4134","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8640,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3518,"num_bytes_recovered":230555648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":984},{"shards":"3,5","objects":1},{"shards":"3,5,6","objects":619}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":383,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.668815+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:08:45.668815+0000","last_peered":"2023-09-23T23:08:45.668815+0000","last_clean":"2023-09-23T23:08:45.668815+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:08:45.668815+0000","last_undegraded":"2023-09-23T23:08:45.668815+0000","last_fullsized":"2023-09-23T23:08:45.668815+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":584,"reported_epoch":471,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:45.668919+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:08:45.668919+0000","last_peered":"2023-09-23T23:08:45.668919+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:08:45.668919+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:08:45.668919+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":164,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522253+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:08:50.522253+0000","last_peered":"2023-09-23T23:08:50.522253+0000","last_clean":"2023-09-23T23:08:50.522253+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:08:50.522253+0000","last_undegraded":"2023-09-23T23:08:50.522253+0000","last_fullsized":"2023-09-23T23:08:50.522253+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":389,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.483886+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:08:43.483886+0000","last_peered":"2023-09-23T23:08:43.483886+0000","last_clean":"2023-09-23T23:08:43.483886+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:08:43.483886+0000","last_undegraded":"2023-09-23T23:08:43.483886+0000","last_fullsized":"2023-09-23T23:08:43.483886+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"473'7030","reported_seq":7255,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:54.035368+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:08:54.035368+0000","last_peered":"2023-09-23T23:08:54.035368+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:08:54.035368+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"414'19","ondisk_log_start":"414'19","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46759936,"num_objects":713,"num_object_clones":0,"num_object_copies":2139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4380,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4030464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":508},{"shards":"0,3,6","objects":321},{"shards":"1,3","objects":-1785}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":397,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.528210+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:08:50.528210+0000","last_peered":"2023-09-23T23:08:50.528210+0000","last_clean":"2023-09-23T23:08:50.528210+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:08:50.528210+0000","last_undegraded":"2023-09-23T23:08:50.528210+0000","last_fullsized":"2023-09-23T23:08:50.528210+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":365,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469550+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:08:43.469550+0000","last_peered":"2023-09-23T23:08:43.469550+0000","last_clean":"2023-09-23T23:08:43.469550+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:08:43.469550+0000","last_undegraded":"2023-09-23T23:08:43.469550+0000","last_fullsized":"2023-09-23T23:08:43.469550+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":332,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.671195+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:08:45.671195+0000","last_peered":"2023-09-23T23:08:45.671195+0000","last_clean":"2023-09-23T23:08:45.671195+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:08:45.671195+0000","last_undegraded":"2023-09-23T23:08:45.671195+0000","last_fullsized":"2023-09-23T23:08:45.671195+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"472'8516","reported_seq":8589,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:51.678603+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:08:51.678603+0000","last_peered":"2023-09-23T23:08:51.678603+0000","last_clean":"2023-09-23T23:08:51.678603+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:08:51.678603+0000","last_undegraded":"2023-09-23T23:08:51.678603+0000","last_fullsized":"2023-09-23T23:08:51.678603+0000","mapping_epoch":410,"log_start":"417'2711","ondisk_log_start":"417'2711","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112459776,"num_objects":1716,"num_object_clones":0,"num_object_copies":5148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8516,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":330,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.474255+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:08:43.474255+0000","last_peered":"2023-09-23T23:08:43.474255+0000","last_clean":"2023-09-23T23:08:43.474255+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:08:43.474255+0000","last_undegraded":"2023-09-23T23:08:43.474255+0000","last_fullsized":"2023-09-23T23:08:43.474255+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":329,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:46.827770+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:08:46.827770+0000","last_peered":"2023-09-23T23:08:46.827770+0000","last_clean":"2023-09-23T23:08:46.827770+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:08:46.827770+0000","last_undegraded":"2023-09-23T23:08:46.827770+0000","last_fullsized":"2023-09-23T23:08:46.827770+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":294,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.668568+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:08:45.668568+0000","last_peered":"2023-09-23T23:08:45.668568+0000","last_clean":"2023-09-23T23:08:45.668568+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:08:45.668568+0000","last_undegraded":"2023-09-23T23:08:45.668568+0000","last_fullsized":"2023-09-23T23:08:45.668568+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"472'8470","reported_seq":8551,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.694260+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:08:51.694260+0000","last_peered":"2023-09-23T23:08:51.694260+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:08:51.694260+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"417'2683","ondisk_log_start":"417'2683","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108789760,"num_objects":1660,"num_object_clones":0,"num_object_copies":4980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8470,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":1660},{"shards":"1,2","objects":-3109}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":196,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522849+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:08:50.522849+0000","last_peered":"2023-09-23T23:08:50.522849+0000","last_clean":"2023-09-23T23:08:50.522849+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:08:50.522849+0000","last_undegraded":"2023-09-23T23:08:50.522849+0000","last_fullsized":"2023-09-23T23:08:50.522849+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":169,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.474813+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:08:43.474813+0000","last_peered":"2023-09-23T23:08:43.474813+0000","last_clean":"2023-09-23T23:08:43.474813+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:08:43.474813+0000","last_undegraded":"2023-09-23T23:08:43.474813+0000","last_fullsized":"2023-09-23T23:08:43.474813+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.529381+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:08:50.529381+0000","last_peered":"2023-09-23T23:08:50.529381+0000","last_clean":"2023-09-23T23:08:50.529381+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:08:50.529381+0000","last_undegraded":"2023-09-23T23:08:50.529381+0000","last_fullsized":"2023-09-23T23:08:50.529381+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"471'8518","reported_seq":16512,"reported_epoch":471,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:50.519544+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:08:50.519544+0000","last_peered":"2023-09-23T23:08:50.519544+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:08:50.519544+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"417'2820","ondisk_log_start":"417'2820","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118882304,"num_objects":1814,"num_object_clones":0,"num_object_copies":5442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":739,"num_objects_unfound":0,"num_objects_dirty":1814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8518,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3958,"num_bytes_recovered":259391488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":739},{"shards":"1,4","objects":891}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":420,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.671971+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:08:45.671971+0000","last_peered":"2023-09-23T23:08:45.671971+0000","last_clean":"2023-09-23T23:08:45.671971+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:08:45.671971+0000","last_undegraded":"2023-09-23T23:08:45.671971+0000","last_fullsized":"2023-09-23T23:08:45.671971+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":92,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.474978+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:08:43.474978+0000","last_peered":"2023-09-23T23:08:43.474978+0000","last_clean":"2023-09-23T23:08:43.474978+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:08:43.474978+0000","last_undegraded":"2023-09-23T23:08:43.474978+0000","last_fullsized":"2023-09-23T23:08:43.474978+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":463,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469700+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:08:43.469700+0000","last_peered":"2023-09-23T23:08:43.469700+0000","last_clean":"2023-09-23T23:08:43.469700+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:08:43.469700+0000","last_undegraded":"2023-09-23T23:08:43.469700+0000","last_fullsized":"2023-09-23T23:08:43.469700+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"472'7035","reported_seq":9122,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:52.611204+0000","last_change":"2023-09-23T23:07:56.517407+0000","last_active":"2023-09-23T23:08:52.611204+0000","last_peered":"2023-09-23T23:08:52.611204+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.507318+0000","last_became_peered":"2023-09-23T23:07:56.507318+0000","last_unstale":"2023-09-23T23:08:52.611204+0000","last_undegraded":"2023-09-23T23:07:56.466720+0000","last_fullsized":"2023-09-23T23:07:56.443187+0000","mapping_epoch":459,"log_start":"414'792","ondisk_log_start":"414'792","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6243,"log_dups_size":0,"ondisk_log_size":6243,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55836672,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4352,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":64454656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2","objects":-379},{"shards":"1,2,5","objects":549}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":62,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473135+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:08:43.473135+0000","last_peered":"2023-09-23T23:08:43.473135+0000","last_clean":"2023-09-23T23:08:43.473135+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:08:43.473135+0000","last_undegraded":"2023-09-23T23:08:43.473135+0000","last_fullsized":"2023-09-23T23:08:43.473135+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":223,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.676288+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:08:45.676288+0000","last_peered":"2023-09-23T23:08:45.676288+0000","last_clean":"2023-09-23T23:08:45.676288+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:08:45.676288+0000","last_undegraded":"2023-09-23T23:08:45.676288+0000","last_fullsized":"2023-09-23T23:08:45.676288+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"472'7044","reported_seq":7466,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.796069+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:08:51.796069+0000","last_peered":"2023-09-23T23:08:51.796069+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:08:51.796069+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58621952,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4329,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1080},{"shards":"1,2,3","objects":525}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":293,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.472871+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:08:43.472871+0000","last_peered":"2023-09-23T23:08:43.472871+0000","last_clean":"2023-09-23T23:08:43.472871+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:08:43.472871+0000","last_undegraded":"2023-09-23T23:08:43.472871+0000","last_fullsized":"2023-09-23T23:08:43.472871+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":494,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.473632+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:08:43.473632+0000","last_peered":"2023-09-23T23:08:43.473632+0000","last_clean":"2023-09-23T23:08:43.473632+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:08:43.473632+0000","last_undegraded":"2023-09-23T23:08:43.473632+0000","last_fullsized":"2023-09-23T23:08:43.473632+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"473'6896","reported_seq":11685,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:53.176029+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:08:53.176029+0000","last_peered":"2023-09-23T23:08:53.176029+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:08:53.176029+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6896,"log_dups_size":0,"ondisk_log_size":6896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54951936,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":607,"num_objects_missing":607,"num_objects_degraded":1627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4248,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1599,"num_bytes_recovered":104726528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":491},{"shards":"0,3,5,6","objects":2},{"shards":"3,5,6","objects":298},{"shards":"3,6","objects":48}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":320,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.474400+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:08:43.474400+0000","last_peered":"2023-09-23T23:08:43.474400+0000","last_clean":"2023-09-23T23:08:43.474400+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:08:43.474400+0000","last_undegraded":"2023-09-23T23:08:43.474400+0000","last_fullsized":"2023-09-23T23:08:43.474400+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":390,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.472612+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:08:43.472612+0000","last_peered":"2023-09-23T23:08:43.472612+0000","last_clean":"2023-09-23T23:08:43.472612+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:08:43.472612+0000","last_undegraded":"2023-09-23T23:08:43.472612+0000","last_fullsized":"2023-09-23T23:08:43.472612+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"472'6903","reported_seq":7915,"reported_epoch":472,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:51.647949+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:08:51.647949+0000","last_peered":"2023-09-23T23:08:51.647949+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:08:51.647949+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51871744,"num_objects":792,"num_object_clones":0,"num_object_copies":2376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4272,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":870},{"shards":"1,2","objects":-1195}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":48,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531943+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:08:50.531943+0000","last_peered":"2023-09-23T23:08:50.531943+0000","last_clean":"2023-09-23T23:08:50.531943+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:08:50.531943+0000","last_undegraded":"2023-09-23T23:08:50.531943+0000","last_fullsized":"2023-09-23T23:08:50.531943+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":190,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525313+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:08:50.525313+0000","last_peered":"2023-09-23T23:08:50.525313+0000","last_clean":"2023-09-23T23:08:50.525313+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:08:50.525313+0000","last_undegraded":"2023-09-23T23:08:50.525313+0000","last_fullsized":"2023-09-23T23:08:50.525313+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"471'6843","reported_seq":6929,"reported_epoch":471,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:50.296979+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:08:50.296979+0000","last_peered":"2023-09-23T23:08:50.296979+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:08:50.296979+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59211776,"num_objects":904,"num_object_clones":0,"num_object_copies":2712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4193,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1352},{"shards":"1,4,5","objects":862}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":240,"reported_epoch":471,"state":"active+clean","last_fresh":"2023-09-23T23:08:45.672241+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:08:45.672241+0000","last_peered":"2023-09-23T23:08:45.672241+0000","last_clean":"2023-09-23T23:08:45.672241+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:08:45.672241+0000","last_undegraded":"2023-09-23T23:08:45.672241+0000","last_fullsized":"2023-09-23T23:08:45.672241+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":465,"reported_epoch":470,"state":"active+clean","last_fresh":"2023-09-23T23:08:43.469460+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:08:43.469460+0000","last_peered":"2023-09-23T23:08:43.469460+0000","last_clean":"2023-09-23T23:08:43.469460+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:08:43.469460+0000","last_undegraded":"2023-09-23T23:08:43.469460+0000","last_fullsized":"2023-09-23T23:08:43.469460+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1761869848,"num_objects":26885,"num_object_clones":0,"num_object_copies":80655,"num_objects_missing_on_primary":1835,"num_objects_missing":1835,"num_objects_degraded":47385,"num_objects_misplaced":739,"num_objects_unfound":0,"num_objects_dirty":26885,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137419,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30038,"num_bytes_recovered":1942224896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9019863040,"data_stored":9019850824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170034,"ondisk_log_size":170034,"up":78,"acting":58,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430710,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626935,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1331400,"kb_used_data":428672,"kb_used_omap":0,"kb_used_meta":902720,"kb_avail":92413752,"statfs":{"total":95995035648,"available":94631682048,"internally_reserved":0,"allocated":438960128,"data_stored":437905406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924385280},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.304,"15min":4.9790000000000001},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":6.4320000000000004,"5min":10.853,"15min":115.32899999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.371,"5min":1.1919999999999999,"15min":4.9989999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":115.348},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1559999999999997,"5min":3.2839999999999998,"15min":3.1099999999999999},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":38.259999999999998,"5min":149.947,"15min":149.947},"last":1.675},{"interface":"front","average":{"1min":3.5950000000000002,"5min":3.4940000000000002,"15min":3.1030000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":38.530000000000001,"5min":150.22499999999999,"15min":150.22499999999999},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:08:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9420000000000002,"5min":5.5709999999999997,"15min":2.9260000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":45.051000000000002,"5min":138.43299999999999,"15min":138.43299999999999},"last":2.9820000000000002},{"interface":"front","average":{"1min":4.8579999999999997,"5min":5.25,"15min":2.9969999999999999},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32500000000000001},"max":{"1min":45.258000000000003,"5min":138.45599999999999,"15min":138.45599999999999},"last":1.6459999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.831,"5min":4.5940000000000003,"15min":3.2719999999999998},"min":{"1min":0.39300000000000002,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":47.734000000000002,"5min":142.24700000000001,"15min":142.24700000000001},"last":4.9969999999999999},{"interface":"front","average":{"1min":3.1549999999999998,"5min":4.617,"15min":3.3210000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":47.162999999999997,"5min":143.86699999999999,"15min":143.86699999999999},"last":1.48}]},{"osd":4,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.1970000000000001,"15min":1.925},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.23999999999999999},"max":{"1min":7.3869999999999996,"5min":12.407999999999999,"15min":119.804},"last":0.371},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.117,"15min":1.95},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":8.8940000000000001,"5min":12.388,"15min":122.31999999999999},"last":3.016}]},{"osd":5,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.863,"15min":2.5600000000000001},"min":{"1min":0.42599999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":1.665,"5min":44.781999999999996,"15min":186.27600000000001},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.9930000000000001,"15min":2.6840000000000002},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":8.7720000000000002,"5min":44.601999999999997,"15min":191.28700000000001},"last":2.9430000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.224,"15min":1.5880000000000001},"min":{"1min":0.46700000000000003,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":8.8670000000000009,"5min":8.8670000000000009,"15min":115.61},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.698,"5min":0.97499999999999998,"15min":1.7070000000000001},"min":{"1min":0.39600000000000002,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":8.2609999999999992,"5min":8.2609999999999992,"15min":115.578},"last":1.6100000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253248,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2413344,"kb_used_data":1464216,"kb_used_omap":9,"kb_used_meta":949110,"kb_avail":91331808,"statfs":{"total":95995035648,"available":93523771392,"internally_reserved":0,"allocated":1499357184,"data_stored":1497427022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":971889641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":5.5030000000000001,"15min":2.544},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.389,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.81100000000000005,"5min":5.6820000000000004,"15min":2.8799999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":3.4500000000000002,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":7.96,"5min":6.2439999999999998,"15min":2.4449999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.527,"5min":161.61699999999999,"15min":161.61699999999999},"last":7.6470000000000002},{"interface":"front","average":{"1min":7.5199999999999996,"5min":6.4059999999999997,"15min":2.6059999999999999},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":118.29900000000001,"5min":162.61699999999999,"15min":162.61699999999999},"last":7.2069999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.4140000000000001,"15min":1.496},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":2.7949999999999999,"5min":59.424999999999997,"15min":59.424999999999997},"last":3.8039999999999998},{"interface":"front","average":{"1min":0.85199999999999998,"5min":2.4180000000000001,"15min":1.536},"min":{"1min":0.27100000000000002,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":6.4569999999999999,"5min":55.174999999999997,"15min":55.174999999999997},"last":7.1580000000000004}]},{"osd":4,"last update":"Sat Sep 23 23:07:52 2023","interfaces":[{"interface":"back","average":{"1min":7.7460000000000004,"5min":2.161,"15min":2.5249999999999999},"min":{"1min":0.36399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":129.69,"5min":129.69,"15min":151.036},"last":0.499},{"interface":"front","average":{"1min":7.8109999999999999,"5min":2.5049999999999999,"15min":2.794},"min":{"1min":0.309,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":129.93700000000001,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.5449999999999999,"15min":2.3839999999999999},"min":{"1min":0.438,"5min":0.32300000000000001,"15min":0.29899999999999999},"max":{"1min":6.242,"5min":18.228999999999999,"15min":116.098},"last":0.629},{"interface":"front","average":{"1min":0.84099999999999997,"5min":1.0109999999999999,"15min":2.1549999999999998},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":3.379,"5min":6.585,"15min":116.08499999999999},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.292,"15min":1.9770000000000001},"min":{"1min":0.45400000000000001,"5min":0.34100000000000003,"15min":0.29999999999999999},"max":{"1min":2.7879999999999998,"5min":8.2680000000000007,"15min":67.349000000000004},"last":0.72299999999999998},{"interface":"front","average":{"1min":2.6899999999999999,"5min":1.5589999999999999,"15min":1.919},"min":{"1min":0.53300000000000003,"5min":0.36299999999999999,"15min":0.33300000000000002},"max":{"1min":8.2530000000000001,"5min":8.2530000000000001,"15min":67.366},"last":0.66700000000000004}]},{"osd":7,"last update":"Sat Sep 23 23:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.163,"15min":1.27},"min":{"1min":0.48099999999999998,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":6.4089999999999998,"5min":7.0960000000000001,"15min":8.702},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.5720000000000001,"5min":1.6060000000000001,"15min":1.647},"min":{"1min":0.439,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":6.2830000000000004,"5min":9.016,"15min":20.902999999999999},"last":0.67800000000000005}]}]},{"osd":0,"up_from":0,"seq":1516123455618,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259391,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1168992,"kb_used_data":883416,"kb_used_omap":83,"kb_used_meta":285484,"kb_avail":92576160,"statfs":{"total":95995035648,"available":94797987840,"internally_reserved":0,"allocated":904617984,"data_stored":902565933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":292336069},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.9370000000000001,"15min":1.1890000000000001},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":4.7729999999999997,"5min":55.735999999999997,"15min":74.241},"last":0.248},{"interface":"front","average":{"1min":0.876,"5min":2.2679999999999998,"15min":1.478},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.4429999999999996,"5min":55.969999999999999,"15min":74.430000000000007},"last":0.33200000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.671,"15min":1.0069999999999999},"min":{"1min":0.221,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.8469999999999995,"5min":18.396999999999998,"15min":18.396999999999998},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.472,"15min":1.2370000000000001},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.22700000000000001},"max":{"1min":2.8639999999999999,"5min":12.744999999999999,"15min":12.744999999999999},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":2.5870000000000002,"15min":1.3720000000000001},"min":{"1min":0.29199999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.3600000000000003,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.873,"5min":2.5150000000000001,"15min":1.425},"min":{"1min":0.29099999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.887,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.27,"15min":0.91200000000000003},"min":{"1min":0.371,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.625,"5min":33.476999999999997,"15min":33.476999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":1.452,"15min":1.079},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.8180000000000001,"5min":33.497999999999998,"15min":33.497999999999998},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":2.9540000000000002,"15min":1.6559999999999999},"min":{"1min":0.36099999999999999,"5min":0.34499999999999997,"15min":0.32000000000000001},"max":{"1min":6.5279999999999996,"5min":139.405,"15min":139.405},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.167,"5min":2.8439999999999999,"15min":1.488},"min":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":4.6639999999999997,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":8.0739999999999998,"5min":3.3149999999999999,"15min":1.8420000000000001},"min":{"1min":0.36799999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":108.53700000000001,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":7.5190000000000001,"5min":3.0910000000000002,"15min":1.538},"min":{"1min":0.44900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":108.496,"5min":108.496,"15min":108.496},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.276,"15min":1.151},"min":{"1min":0.50900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":5.0700000000000003,"5min":10.164,"15min":10.164},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.2210000000000001,"15min":1.1639999999999999},"min":{"1min":0.40799999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4020000000000001,"5min":7.7830000000000004,"15min":7.7830000000000004},"last":0.61899999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926196,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1879180,"kb_used_data":1199044,"kb_used_omap":7,"kb_used_meta":680120,"kb_avail":91865972,"statfs":{"total":95995035648,"available":94070755328,"internally_reserved":0,"allocated":1227821056,"data_stored":1225789291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":696443496},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.534,"15min":1.4099999999999999},"min":{"1min":0.22600000000000001,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":8.1660000000000004,"5min":17.015000000000001,"15min":122.289},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.004,"15min":1.1419999999999999},"min":{"1min":0.25600000000000001,"5min":0.218,"15min":0.17799999999999999},"max":{"1min":4.1470000000000002,"5min":11.382999999999999,"15min":122.041},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":2.145,"15min":2.0230000000000001},"min":{"1min":0.22600000000000001,"5min":0.224,"15min":0.187},"max":{"1min":3.9380000000000002,"5min":23.969000000000001,"15min":123.866},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.90200000000000002,"5min":1.889,"15min":1.978},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.20699999999999999},"max":{"1min":3.8610000000000002,"5min":16.986000000000001,"15min":122.265},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:08:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8479999999999999,"5min":2.3370000000000002,"15min":1.232},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":55.311999999999998,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.33300000000000002},{"interface":"front","average":{"1min":5.532,"5min":2.7480000000000002,"15min":1.5389999999999999},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.215},"max":{"1min":57.308,"5min":57.308,"15min":57.308},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":2.5030000000000001,"15min":2.137},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":3.3130000000000002,"5min":115.69,"15min":115.69},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.83699999999999997,"5min":2.3250000000000002,"15min":2.0569999999999999},"min":{"1min":0.375,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":3.0670000000000002,"5min":115.617,"15min":115.617},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.645,"15min":1.4079999999999999},"min":{"1min":0.39600000000000002,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":3.5339999999999998,"5min":16.959,"15min":16.959},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.91000000000000003,"5min":1.319,"15min":1.5029999999999999},"min":{"1min":0.42299999999999999,"5min":0.34399999999999997,"15min":0.27100000000000002},"max":{"1min":3.3090000000000002,"5min":10.545,"15min":10.545},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.6519999999999999,"15min":1.647},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":7.1740000000000004,"5min":16.882000000000001,"15min":16.882000000000001},"last":77.707999999999998},{"interface":"front","average":{"1min":1.097,"5min":1.248,"15min":1.55},"min":{"1min":0.39900000000000002,"5min":0.372,"15min":0.32400000000000001},"max":{"1min":7.1870000000000003,"5min":8.1999999999999993,"15min":12.887},"last":77.721999999999994}]},{"osd":7,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.3580000000000001,"15min":1.413},"min":{"1min":0.50700000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":6.1820000000000004,"5min":8.1620000000000008,"15min":8.8160000000000007},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.6419999999999999,"5min":1.8720000000000001,"15min":1.542},"min":{"1min":0.435,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":6.1360000000000001,"5min":10.278,"15min":10.278},"last":0.52600000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019392,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1417916,"kb_used_data":510452,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92327236,"statfs":{"total":95995035648,"available":94543089664,"internally_reserved":0,"allocated":522702848,"data_stored":521651387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.0409999999999999,"15min":2.2360000000000002},"min":{"1min":0.27400000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.778,"5min":99.132999999999996,"15min":156.375},"last":9.8870000000000005},{"interface":"front","average":{"1min":0.95399999999999996,"5min":2.1829999999999998,"15min":2.371},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":2.9910000000000001,"5min":101.062,"15min":157.02199999999999},"last":4.4509999999999996}]},{"osd":1,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":3.4900000000000002,"15min":3.1970000000000001},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":6.915,"5min":95.591999999999999,"15min":98.846999999999994},"last":4.3840000000000003},{"interface":"front","average":{"1min":1.4710000000000001,"5min":3.4039999999999999,"15min":3.157},"min":{"1min":0.33800000000000002,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":15.728,"5min":97.929000000000002,"15min":98.938999999999993},"last":10.311}]},{"osd":2,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":11.628,"5min":4.2610000000000001,"15min":2.1459999999999999},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":124.819,"5min":124.819,"15min":124.819},"last":7.5179999999999998},{"interface":"front","average":{"1min":10.571999999999999,"5min":4.077,"15min":1.9850000000000001},"min":{"1min":0.36599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":122.086,"5min":122.086,"15min":122.086},"last":4.5350000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":2.5379999999999998,"15min":1.841},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":55.466999999999999,"5min":60.274000000000001,"15min":60.274000000000001},"last":8.7520000000000007},{"interface":"front","average":{"1min":3.3820000000000001,"5min":2.677,"15min":1.9259999999999999},"min":{"1min":0.38500000000000001,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":55.625999999999998,"5min":60.677999999999997,"15min":60.677999999999997},"last":4.4790000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.177,"15min":1.764},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.214},"max":{"1min":3.036,"5min":10.130000000000001,"15min":148.61600000000001},"last":10.282},{"interface":"front","average":{"1min":1.522,"5min":1.157,"15min":1.861},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":8.5489999999999995,"5min":8.5489999999999995,"15min":151.905},"last":17.02}]},{"osd":6,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.3899999999999999,"15min":1.4359999999999999},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.2590000000000003,"5min":35.896000000000001,"15min":35.896000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.5629999999999999,"15min":1.403},"min":{"1min":0.35499999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":7.008,"5min":35.908000000000001,"15min":35.908000000000001},"last":4.5620000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.161,"15min":1.288},"min":{"1min":0.373,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":3.1080000000000001,"5min":7.0789999999999997,"15min":12.84},"last":4.4219999999999997},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.28,"15min":1.232},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":15.808,"5min":15.808,"15min":15.808},"last":7.3179999999999996}]}]},{"osd":5,"up_from":0,"seq":154618823166,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2934833152,"data_stored":2934833152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1492320256,"data_stored":1492320256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":893784064,"data_stored":893779992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1218510848,"data_stored":1218510848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":516689920,"data_stored":516685848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":431689728,"data_stored":431685656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:08:57.885 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:08:57.897 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:57 smithi151 ceph-mon[159210]: from='client.57151 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:08:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:57 smithi151 ceph-mon[159210]: pgmap v1485: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 385 op/s; 47387/79830 objects degraded (59.360%); 628/79830 objects misplaced (0.787%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T23:08:57.976 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:58.073 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:08:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:58.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:08:58.149 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:58.191 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:08:58.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:57 smithi052 ceph-mon[187743]: from='client.57151 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:08:58.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:57 smithi052 ceph-mon[187743]: pgmap v1485: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 385 op/s; 47387/79830 objects degraded (59.360%); 628/79830 objects misplaced (0.787%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T23:08:58.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:57 smithi052 ceph-mon[180979]: from='client.57151 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:08:58.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:57 smithi052 ceph-mon[180979]: pgmap v1485: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 385 op/s; 47387/79830 objects degraded (59.360%); 628/79830 objects misplaced (0.787%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T23:08:58.253 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:58.354 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:08:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:58.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:08:58.399 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:08:58.464 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:08:58.521 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:58.701 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:08:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:58.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:08:58.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:58.850 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:59.009 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:59.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:59.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:59.117 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:08:59.161 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:59.216 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:08:59.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:59.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:08:59.320 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:08:59.347 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:59.472 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:08:59.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:59.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:08:59.581 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:08:59.621 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:08:59.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:08:59.741 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:08:59.821 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:08:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:08:59.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:08:59.876 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:08:59 smithi052 ceph-mon[180979]: pgmap v1486: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 429 op/s; 46954/76260 objects degraded (61.571%); 628/76260 objects misplaced (0.823%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T23:08:59.879 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:08:59 smithi151 ceph-mon[159210]: pgmap v1486: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 429 op/s; 46954/76260 objects degraded (61.571%); 628/76260 objects misplaced (0.823%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T23:08:59.880 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:08:59 smithi052 ceph-mon[187743]: pgmap v1486: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.6 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 429 op/s; 46954/76260 objects degraded (61.571%); 628/76260 objects misplaced (0.823%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T23:08:59.930 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:08:59.982 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:00.068 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:00.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:00.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:00.185 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:00.242 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:00.429 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:00.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:00.537 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:00.608 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:00.702 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:09:00.702 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:09:00.708 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:00.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:00.813 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:00.872 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:00.946 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:00.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:01.059 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:01.116 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:01.170 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:01.172 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:00 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 46954/76260 objects degraded (61.571%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:01.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:01.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:00 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 46954/76260 objects degraded (61.571%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:01.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:00 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 46954/76260 objects degraded (61.571%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:01.274 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:01.304 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:01.500 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:09:01.606 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:09:01.672 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:01.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:01.777 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:01.826 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:01.892 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:01 smithi151 ceph-mon[159210]: pgmap v1487: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.5 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 443 op/s; 46345/72870 objects degraded (63.600%); 515/72870 objects misplaced (0.707%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T23:09:01.892 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4122307507' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:09:01.892 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:01 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:01.892 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4122307507' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:09:01.892 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:01 smithi151 ceph-mon[159210]: osdmap e474: 8 total, 8 up, 5 in 2023-09-23T23:09:01.902 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:01.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:01.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[180979]: pgmap v1487: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.5 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 443 op/s; 46345/72870 objects degraded (63.600%); 515/72870 objects misplaced (0.707%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T23:09:01.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4122307507' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:09:01.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:01.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4122307507' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:09:01.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[180979]: osdmap e474: 8 total, 8 up, 5 in 2023-09-23T23:09:01.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[187743]: pgmap v1487: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.5 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 443 op/s; 46345/72870 objects degraded (63.600%); 515/72870 objects misplaced (0.707%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T23:09:01.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4122307507' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:09:01.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:01.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4122307507' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:09:01.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:01 smithi052 ceph-mon[187743]: osdmap e474: 8 total, 8 up, 5 in 2023-09-23T23:09:02.017 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:02.068 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:02.139 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:02.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:02.200 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:09:02.248 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:02.289 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:02.386 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:02.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:02.479 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:02.496 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:02.567 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:02.678 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:02.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:02.782 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:02.850 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:02.928 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:02.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:02.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:03.038 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:03.075 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:03.163 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:03.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:03.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:03.172 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:02 smithi151 ceph-mon[159210]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.172 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:02 smithi151 ceph-mon[159210]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.172 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:02 smithi151 ceph-mon[159210]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:02 smithi052 ceph-mon[187743]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:02 smithi052 ceph-mon[187743]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:02 smithi052 ceph-mon[187743]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:02 smithi052 ceph-mon[180979]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:02 smithi052 ceph-mon[180979]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:02 smithi052 ceph-mon[180979]: failed to encode map e474 with expected crc 2023-09-23T23:09:03.271 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:03.311 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:03.360 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:09:03.362 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:09:03.624 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:03.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:03.728 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:03.745 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:09:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:09:03] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-09-23T23:09:03.789 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:04.131 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:03 smithi052 ceph-mon[187743]: pgmap v1489: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 46266/69531 objects degraded (66.540%); 515/69531 objects misplaced (0.741%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:09:04.131 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:03 smithi052 ceph-mon[180979]: pgmap v1489: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 46266/69531 objects degraded (66.540%); 515/69531 objects misplaced (0.741%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:09:04.172 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:09:04.172 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:09:04.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:03 smithi151 ceph-mon[159210]: pgmap v1489: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 46266/69531 objects degraded (66.540%); 515/69531 objects misplaced (0.741%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T23:09:04.483 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:09:04.697 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:04.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:04.804 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:04.813 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1489,"stamp":"2023-09-23T23:09:02.792319+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1517885368,"num_objects":23177,"num_object_clones":0,"num_object_copies":69531,"num_objects_missing_on_primary":1624,"num_objects_missing":1624,"num_objects_degraded":46266,"num_objects_misplaced":515,"num_objects_unfound":0,"num_objects_dirty":23177,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":141411,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31141,"num_bytes_recovered":2009944512,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":171771,"ondisk_log_size":171771,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":7739164,"kb_used_data":3967412,"kb_used_omap":99,"kb_used_meta":3771612,"kb_avail":460986596,"statfs":{"total":479975178240,"available":472050274304,"internally_reserved":0,"allocated":4062629888,"data_stored":4054483293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3862130710},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-304807936,"num_objects":-4651,"num_object_clones":0,"num_object_copies":-13953,"num_objects_missing_on_primary":-405,"num_objects_missing":-405,"num_objects_degraded":-1937,"num_objects_misplaced":-224,"num_objects_unfound":0,"num_objects_dirty":-4651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4651,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1583,"num_bytes_recovered":103743488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004141"},"pg_stats":[{"pgid":"11.12","version":"474'7116","reported_seq":7211,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.748706+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:09:01.748706+0000","last_peered":"2023-09-23T23:09:01.748706+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:09:01.748706+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7116,"log_dups_size":0,"ondisk_log_size":7116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45056000,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4453,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":719},{"shards":"2,3","objects":-1769}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":485,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525687+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:08:50.525687+0000","last_peered":"2023-09-23T23:08:50.525687+0000","last_clean":"2023-09-23T23:08:50.525687+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:08:50.525687+0000","last_undegraded":"2023-09-23T23:08:50.525687+0000","last_fullsized":"2023-09-23T23:08:50.525687+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531645+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:08:50.531645+0000","last_peered":"2023-09-23T23:08:50.531645+0000","last_clean":"2023-09-23T23:08:50.531645+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:08:50.531645+0000","last_undegraded":"2023-09-23T23:08:50.531645+0000","last_fullsized":"2023-09-23T23:08:50.531645+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"473'7036","reported_seq":10353,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:59.123097+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:08:59.123097+0000","last_peered":"2023-09-23T23:08:59.123097+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:08:59.123097+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:08:59.123097+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48988160,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1035,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4383,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1608,"num_bytes_recovered":105316352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":422},{"shards":"0,3,6","objects":270},{"shards":"2,3,6","objects":56}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":144,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525163+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:08:50.525163+0000","last_peered":"2023-09-23T23:08:50.525163+0000","last_clean":"2023-09-23T23:08:50.525163+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:08:50.525163+0000","last_undegraded":"2023-09-23T23:08:50.525163+0000","last_fullsized":"2023-09-23T23:08:50.525163+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":491,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531788+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:08:50.531788+0000","last_peered":"2023-09-23T23:08:50.531788+0000","last_clean":"2023-09-23T23:08:50.531788+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:08:50.531788+0000","last_undegraded":"2023-09-23T23:08:50.531788+0000","last_fullsized":"2023-09-23T23:08:50.531788+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"473'7022","reported_seq":11078,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:59.028321+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:08:59.028321+0000","last_peered":"2023-09-23T23:08:59.028321+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:08:59.028321+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:08:59.028321+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48562176,"num_objects":741,"num_object_clones":0,"num_object_copies":2223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4366,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":163},{"shards":"1,3,6","objects":578}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526439+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:08:50.526439+0000","last_peered":"2023-09-23T23:08:50.526439+0000","last_clean":"2023-09-23T23:08:50.526439+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:08:50.526439+0000","last_undegraded":"2023-09-23T23:08:50.526439+0000","last_fullsized":"2023-09-23T23:08:50.526439+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522389+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:08:50.522389+0000","last_peered":"2023-09-23T23:08:50.522389+0000","last_clean":"2023-09-23T23:08:50.522389+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:08:50.522389+0000","last_undegraded":"2023-09-23T23:08:50.522389+0000","last_fullsized":"2023-09-23T23:08:50.522389+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":417,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522621+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:08:50.522621+0000","last_peered":"2023-09-23T23:08:50.522621+0000","last_clean":"2023-09-23T23:08:50.522621+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:08:50.522621+0000","last_undegraded":"2023-09-23T23:08:50.522621+0000","last_fullsized":"2023-09-23T23:08:50.522621+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":163,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.529334+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:08:50.529334+0000","last_peered":"2023-09-23T23:08:50.529334+0000","last_clean":"2023-09-23T23:08:50.529334+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:08:50.529334+0000","last_undegraded":"2023-09-23T23:08:50.529334+0000","last_fullsized":"2023-09-23T23:08:50.529334+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":279,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531892+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:08:50.531892+0000","last_peered":"2023-09-23T23:08:50.531892+0000","last_clean":"2023-09-23T23:08:50.531892+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:08:50.531892+0000","last_undegraded":"2023-09-23T23:08:50.531892+0000","last_fullsized":"2023-09-23T23:08:50.531892+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":177,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523846+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:08:50.523846+0000","last_peered":"2023-09-23T23:08:50.523846+0000","last_clean":"2023-09-23T23:08:50.523846+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:08:50.523846+0000","last_undegraded":"2023-09-23T23:08:50.523846+0000","last_fullsized":"2023-09-23T23:08:50.523846+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":210,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531603+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:08:50.531603+0000","last_peered":"2023-09-23T23:08:50.531603+0000","last_clean":"2023-09-23T23:08:50.531603+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:08:50.531603+0000","last_undegraded":"2023-09-23T23:08:50.531603+0000","last_fullsized":"2023-09-23T23:08:50.531603+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":229,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.636025+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:08:52.636025+0000","last_peered":"2023-09-23T23:08:52.636025+0000","last_clean":"2023-09-23T23:08:52.636025+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:08:52.636025+0000","last_undegraded":"2023-09-23T23:08:52.636025+0000","last_fullsized":"2023-09-23T23:08:52.636025+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":563,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526076+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:08:50.526076+0000","last_peered":"2023-09-23T23:08:50.526076+0000","last_clean":"2023-09-23T23:08:50.526076+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:08:50.526076+0000","last_undegraded":"2023-09-23T23:08:50.526076+0000","last_fullsized":"2023-09-23T23:08:50.526076+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":456,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526079+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:08:50.526079+0000","last_peered":"2023-09-23T23:08:50.526079+0000","last_clean":"2023-09-23T23:08:50.526079+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:08:50.526079+0000","last_undegraded":"2023-09-23T23:08:50.526079+0000","last_fullsized":"2023-09-23T23:08:50.526079+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":388,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526165+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:08:50.526165+0000","last_peered":"2023-09-23T23:08:50.526165+0000","last_clean":"2023-09-23T23:08:50.526165+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:08:50.526165+0000","last_undegraded":"2023-09-23T23:08:50.526165+0000","last_fullsized":"2023-09-23T23:08:50.526165+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522305+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:08:50.522305+0000","last_peered":"2023-09-23T23:08:50.522305+0000","last_clean":"2023-09-23T23:08:50.522305+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:08:50.522305+0000","last_undegraded":"2023-09-23T23:08:50.522305+0000","last_fullsized":"2023-09-23T23:08:50.522305+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":474,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523650+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:08:50.523650+0000","last_peered":"2023-09-23T23:08:50.523650+0000","last_clean":"2023-09-23T23:08:50.523650+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:08:50.523650+0000","last_undegraded":"2023-09-23T23:08:50.523650+0000","last_fullsized":"2023-09-23T23:08:50.523650+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":389,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531548+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:08:50.531548+0000","last_peered":"2023-09-23T23:08:50.531548+0000","last_clean":"2023-09-23T23:08:50.531548+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:08:50.531548+0000","last_undegraded":"2023-09-23T23:08:50.531548+0000","last_fullsized":"2023-09-23T23:08:50.531548+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"473'6934","reported_seq":12102,"reported_epoch":473,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:08:59.126827+0000","last_change":"2023-09-23T23:06:32.889539+0000","last_active":"2023-09-23T23:08:59.126827+0000","last_peered":"2023-09-23T23:08:59.126827+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:06:32.875369+0000","last_became_peered":"2023-09-23T23:06:32.875369+0000","last_unstale":"2023-09-23T23:08:59.126827+0000","last_undegraded":"2023-09-23T23:08:59.126827+0000","last_fullsized":"2023-09-23T23:08:59.126827+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6934,"log_dups_size":0,"ondisk_log_size":6934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53051392,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4285,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":810}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":376,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525507+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:08:50.525507+0000","last_peered":"2023-09-23T23:08:50.525507+0000","last_clean":"2023-09-23T23:08:50.525507+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:08:50.525507+0000","last_undegraded":"2023-09-23T23:08:50.525507+0000","last_fullsized":"2023-09-23T23:08:50.525507+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":498,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531745+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:08:50.531745+0000","last_peered":"2023-09-23T23:08:50.531745+0000","last_clean":"2023-09-23T23:08:50.531745+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:08:50.531745+0000","last_undegraded":"2023-09-23T23:08:50.531745+0000","last_fullsized":"2023-09-23T23:08:50.531745+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"473'7126","reported_seq":12325,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:59.089746+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:08:59.089746+0000","last_peered":"2023-09-23T23:08:59.089746+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:08:59.089746+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7126,"log_dups_size":0,"ondisk_log_size":7126,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48398336,"num_objects":739,"num_object_clones":0,"num_object_copies":2217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4436,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1748,"num_bytes_recovered":114360320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":395},{"shards":"3,5,6","objects":317},{"shards":"3,6","objects":27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":87,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526902+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:08:50.526902+0000","last_peered":"2023-09-23T23:08:50.526902+0000","last_clean":"2023-09-23T23:08:50.526902+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:08:50.526902+0000","last_undegraded":"2023-09-23T23:08:50.526902+0000","last_fullsized":"2023-09-23T23:08:50.526902+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":169,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522699+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:08:50.522699+0000","last_peered":"2023-09-23T23:08:50.522699+0000","last_clean":"2023-09-23T23:08:50.522699+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:08:50.522699+0000","last_undegraded":"2023-09-23T23:08:50.522699+0000","last_fullsized":"2023-09-23T23:08:50.522699+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"473'7080","reported_seq":7178,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:00.506628+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:09:00.506628+0000","last_peered":"2023-09-23T23:09:00.506628+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:09:00.506628+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7080,"log_dups_size":0,"ondisk_log_size":7080,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43679744,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4429,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1699},{"shards":"1,4,5","objects":708}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":383,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526887+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:08:50.526887+0000","last_peered":"2023-09-23T23:08:50.526887+0000","last_clean":"2023-09-23T23:08:50.526887+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:08:50.526887+0000","last_undegraded":"2023-09-23T23:08:50.526887+0000","last_fullsized":"2023-09-23T23:08:50.526887+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":339,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523433+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:08:50.523433+0000","last_peered":"2023-09-23T23:08:50.523433+0000","last_clean":"2023-09-23T23:08:50.523433+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:08:50.523433+0000","last_undegraded":"2023-09-23T23:08:50.523433+0000","last_fullsized":"2023-09-23T23:08:50.523433+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"474'6943","reported_seq":11951,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.726396+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:09:01.726396+0000","last_peered":"2023-09-23T23:09:01.726396+0000","last_clean":"2023-09-23T23:09:01.726396+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:09:01.726396+0000","last_undegraded":"2023-09-23T23:09:01.726396+0000","last_fullsized":"2023-09-23T23:09:01.726396+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6943,"log_dups_size":0,"ondisk_log_size":6943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49250304,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4311,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":103,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.639050+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:08:52.639050+0000","last_peered":"2023-09-23T23:08:52.639050+0000","last_clean":"2023-09-23T23:08:52.639050+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:08:52.639050+0000","last_undegraded":"2023-09-23T23:08:52.639050+0000","last_fullsized":"2023-09-23T23:08:52.639050+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":480,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525435+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:08:50.525435+0000","last_peered":"2023-09-23T23:08:50.525435+0000","last_clean":"2023-09-23T23:08:50.525435+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:08:50.525435+0000","last_undegraded":"2023-09-23T23:08:50.525435+0000","last_fullsized":"2023-09-23T23:08:50.525435+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"473'7046","reported_seq":12273,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:58.174090+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:08:58.174090+0000","last_peered":"2023-09-23T23:08:58.174090+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:08:58.174090+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7046,"log_dups_size":0,"ondisk_log_size":7046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45121536,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":520,"num_objects_missing":520,"num_objects_degraded":1478,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4397,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1749,"num_bytes_recovered":114622464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":425},{"shards":"0,3,5,6","objects":2},{"shards":"1,6","objects":-47},{"shards":"3,5,6","objects":308}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":131,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531471+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:08:50.531471+0000","last_peered":"2023-09-23T23:08:50.531471+0000","last_clean":"2023-09-23T23:08:50.531471+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:08:50.531471+0000","last_undegraded":"2023-09-23T23:08:50.531471+0000","last_fullsized":"2023-09-23T23:08:50.531471+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522164+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:08:50.522164+0000","last_peered":"2023-09-23T23:08:50.522164+0000","last_clean":"2023-09-23T23:08:50.522164+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:08:50.522164+0000","last_undegraded":"2023-09-23T23:08:50.522164+0000","last_fullsized":"2023-09-23T23:08:50.522164+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"474'7249","reported_seq":7688,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.677080+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:09:01.677080+0000","last_peered":"2023-09-23T23:09:01.677080+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:09:01.677080+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7249,"log_dups_size":0,"ondisk_log_size":7249,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45187072,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4533,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1355},{"shards":"1,2,3","objects":398}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":164,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522214+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:08:50.522214+0000","last_peered":"2023-09-23T23:08:50.522214+0000","last_clean":"2023-09-23T23:08:50.522214+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:08:50.522214+0000","last_undegraded":"2023-09-23T23:08:50.522214+0000","last_fullsized":"2023-09-23T23:08:50.522214+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":295,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526005+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:08:50.526005+0000","last_peered":"2023-09-23T23:08:50.526005+0000","last_clean":"2023-09-23T23:08:50.526005+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:08:50.526005+0000","last_undegraded":"2023-09-23T23:08:50.526005+0000","last_fullsized":"2023-09-23T23:08:50.526005+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"474'7162","reported_seq":10862,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:02.669106+0000","last_change":"2023-09-23T23:07:56.507184+0000","last_active":"2023-09-23T23:09:02.669106+0000","last_peered":"2023-09-23T23:09:02.669106+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.488393+0000","last_became_peered":"2023-09-23T23:07:56.488393+0000","last_unstale":"2023-09-23T23:09:02.669106+0000","last_undegraded":"2023-09-23T23:07:56.465702+0000","last_fullsized":"2023-09-23T23:07:56.443434+0000","mapping_epoch":459,"log_start":"415'861","ondisk_log_start":"415'861","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47513600,"num_objects":725,"num_object_clones":0,"num_object_copies":2175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":534,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4479,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1790,"num_bytes_recovered":117211136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":340},{"shards":"1,6","objects":385}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":178,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.528863+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:08:50.528863+0000","last_peered":"2023-09-23T23:08:50.528863+0000","last_clean":"2023-09-23T23:08:50.528863+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:08:50.528863+0000","last_undegraded":"2023-09-23T23:08:50.528863+0000","last_fullsized":"2023-09-23T23:08:50.528863+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":475,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525981+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:08:50.525981+0000","last_peered":"2023-09-23T23:08:50.525981+0000","last_clean":"2023-09-23T23:08:50.525981+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:08:50.525981+0000","last_undegraded":"2023-09-23T23:08:50.525981+0000","last_fullsized":"2023-09-23T23:08:50.525981+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"474'7081","reported_seq":7177,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.758565+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:09:01.758565+0000","last_peered":"2023-09-23T23:09:01.758565+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:09:01.758565+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7081,"log_dups_size":0,"ondisk_log_size":7081,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47349760,"num_objects":722,"num_object_clones":0,"num_object_copies":2166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4417,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":691},{"shards":"2,3","objects":-1673}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":246,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525774+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:08:50.525774+0000","last_peered":"2023-09-23T23:08:50.525774+0000","last_clean":"2023-09-23T23:08:50.525774+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:08:50.525774+0000","last_undegraded":"2023-09-23T23:08:50.525774+0000","last_fullsized":"2023-09-23T23:08:50.525774+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":483,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526282+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:08:50.526282+0000","last_peered":"2023-09-23T23:08:50.526282+0000","last_clean":"2023-09-23T23:08:50.526282+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:08:50.526282+0000","last_undegraded":"2023-09-23T23:08:50.526282+0000","last_fullsized":"2023-09-23T23:08:50.526282+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"473'7058","reported_seq":10662,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:58.161673+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:08:58.161673+0000","last_peered":"2023-09-23T23:08:58.161673+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:08:58.161673+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47546368,"num_objects":725,"num_object_clones":0,"num_object_copies":2175,"num_objects_missing_on_primary":566,"num_objects_missing":566,"num_objects_degraded":997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4404,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1751,"num_bytes_recovered":114622464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,2,3","objects":479},{"shards":"0,2,3,6","objects":1},{"shards":"0,3,6","objects":300},{"shards":"2,6","objects":-55}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":496,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523492+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:08:50.523492+0000","last_peered":"2023-09-23T23:08:50.523492+0000","last_clean":"2023-09-23T23:08:50.523492+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:08:50.523492+0000","last_undegraded":"2023-09-23T23:08:50.523492+0000","last_fullsized":"2023-09-23T23:08:50.523492+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":501,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526668+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:08:50.526668+0000","last_peered":"2023-09-23T23:08:50.526668+0000","last_clean":"2023-09-23T23:08:50.526668+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:08:50.526668+0000","last_undegraded":"2023-09-23T23:08:50.526668+0000","last_fullsized":"2023-09-23T23:08:50.526668+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"473'6990","reported_seq":10440,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:59.069373+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:08:59.069373+0000","last_peered":"2023-09-23T23:08:59.069373+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:08:59.069373+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:08:59.069373+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50659328,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":499,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4334,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1678,"num_bytes_recovered":109969408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":409},{"shards":"1,3,6","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":146,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.530014+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:08:50.530014+0000","last_peered":"2023-09-23T23:08:50.530014+0000","last_clean":"2023-09-23T23:08:50.530014+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:08:50.530014+0000","last_undegraded":"2023-09-23T23:08:50.530014+0000","last_fullsized":"2023-09-23T23:08:50.530014+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":380,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.640205+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:08:52.640205+0000","last_peered":"2023-09-23T23:08:52.640205+0000","last_clean":"2023-09-23T23:08:52.640205+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:08:52.640205+0000","last_undegraded":"2023-09-23T23:08:52.640205+0000","last_fullsized":"2023-09-23T23:08:52.640205+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"474'8977","reported_seq":10443,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.806244+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:09:01.806244+0000","last_peered":"2023-09-23T23:09:01.806244+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:09:01.806244+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"418'3174","ondisk_log_start":"418'3174","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96862232,"num_objects":1479,"num_object_clones":0,"num_object_copies":4437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1479,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8977,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":803},{"shards":"2,4","objects":-2387}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":394,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.637659+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:08:52.637659+0000","last_peered":"2023-09-23T23:08:52.637659+0000","last_clean":"2023-09-23T23:08:52.637659+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:08:52.637659+0000","last_undegraded":"2023-09-23T23:08:52.637659+0000","last_fullsized":"2023-09-23T23:08:52.637659+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531309+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:08:50.531309+0000","last_peered":"2023-09-23T23:08:50.531309+0000","last_clean":"2023-09-23T23:08:50.531309+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:08:50.531309+0000","last_undegraded":"2023-09-23T23:08:50.531309+0000","last_fullsized":"2023-09-23T23:08:50.531309+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":214,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525640+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:08:50.525640+0000","last_peered":"2023-09-23T23:08:50.525640+0000","last_clean":"2023-09-23T23:08:50.525640+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:08:50.525640+0000","last_undegraded":"2023-09-23T23:08:50.525640+0000","last_fullsized":"2023-09-23T23:08:50.525640+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"473'8757","reported_seq":8874,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:58.984960+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:08:58.984960+0000","last_peered":"2023-09-23T23:08:58.984960+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:08:58.984960+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"420'4267","ondisk_log_start":"420'4267","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4490,"log_dups_size":0,"ondisk_log_size":4490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99549184,"num_objects":1519,"num_object_clones":0,"num_object_copies":4557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8757,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1519},{"shards":"1,3","objects":-3306}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":210,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.641933+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:08:52.641933+0000","last_peered":"2023-09-23T23:08:52.641933+0000","last_clean":"2023-09-23T23:08:52.641933+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:08:52.641933+0000","last_undegraded":"2023-09-23T23:08:52.641933+0000","last_fullsized":"2023-09-23T23:08:52.641933+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":511,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526306+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:08:50.526306+0000","last_peered":"2023-09-23T23:08:50.526306+0000","last_clean":"2023-09-23T23:08:50.526306+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:08:50.526306+0000","last_undegraded":"2023-09-23T23:08:50.526306+0000","last_fullsized":"2023-09-23T23:08:50.526306+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":379,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.637550+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:08:52.637550+0000","last_peered":"2023-09-23T23:08:52.637550+0000","last_clean":"2023-09-23T23:08:52.637550+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:08:52.637550+0000","last_undegraded":"2023-09-23T23:08:52.637550+0000","last_fullsized":"2023-09-23T23:08:52.637550+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"473'7150","reported_seq":12399,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:59.138076+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:08:59.138076+0000","last_peered":"2023-09-23T23:08:59.138076+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:08:59.138076+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"414'31","ondisk_log_start":"414'31","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7119,"log_dups_size":0,"ondisk_log_size":7119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46825472,"num_objects":714,"num_object_clones":0,"num_object_copies":2142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4459,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1770,"num_bytes_recovered":115933184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":426},{"shards":"3,5,6","objects":315},{"shards":"3,6","objects":-27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":454,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.635542+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:08:52.635542+0000","last_peered":"2023-09-23T23:08:52.635542+0000","last_clean":"2023-09-23T23:08:52.635542+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:08:52.635542+0000","last_undegraded":"2023-09-23T23:08:52.635542+0000","last_fullsized":"2023-09-23T23:08:52.635542+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":306,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.635390+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:08:52.635390+0000","last_peered":"2023-09-23T23:08:52.635390+0000","last_clean":"2023-09-23T23:08:52.635390+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:08:52.635390+0000","last_undegraded":"2023-09-23T23:08:52.635390+0000","last_fullsized":"2023-09-23T23:08:52.635390+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":410,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522533+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:08:50.522533+0000","last_peered":"2023-09-23T23:08:50.522533+0000","last_clean":"2023-09-23T23:08:50.522533+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:08:50.522533+0000","last_undegraded":"2023-09-23T23:08:50.522533+0000","last_fullsized":"2023-09-23T23:08:50.522533+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"473'8782","reported_seq":16195,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:59.142050+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:08:59.142050+0000","last_peered":"2023-09-23T23:08:59.142050+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:08:59.142050+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"420'4234","ondisk_log_start":"420'4234","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4548,"log_dups_size":0,"ondisk_log_size":4548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95813632,"num_objects":1462,"num_object_clones":0,"num_object_copies":4386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1920,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8782,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3660,"num_bytes_recovered":239861760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":842},{"shards":"3,5","objects":1},{"shards":"3,5,6","objects":619}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":385,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.635844+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:08:52.635844+0000","last_peered":"2023-09-23T23:08:52.635844+0000","last_clean":"2023-09-23T23:08:52.635844+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:08:52.635844+0000","last_undegraded":"2023-09-23T23:08:52.635844+0000","last_fullsized":"2023-09-23T23:08:52.635844+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":586,"reported_epoch":473,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:08:52.635514+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:08:52.635514+0000","last_peered":"2023-09-23T23:08:52.635514+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:08:52.635514+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:08:52.635514+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":164,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522253+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:08:50.522253+0000","last_peered":"2023-09-23T23:08:50.522253+0000","last_clean":"2023-09-23T23:08:50.522253+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:08:50.522253+0000","last_undegraded":"2023-09-23T23:08:50.522253+0000","last_fullsized":"2023-09-23T23:08:50.522253+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":391,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526444+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:08:50.526444+0000","last_peered":"2023-09-23T23:08:50.526444+0000","last_clean":"2023-09-23T23:08:50.526444+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:08:50.526444+0000","last_undegraded":"2023-09-23T23:08:50.526444+0000","last_fullsized":"2023-09-23T23:08:50.526444+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"473'7112","reported_seq":7337,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:59.035069+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:08:59.035069+0000","last_peered":"2023-09-23T23:08:59.035069+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:08:59.035069+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"414'191","ondisk_log_start":"414'191","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6921,"log_dups_size":0,"ondisk_log_size":6921,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41385984,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4462,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4030464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":426},{"shards":"0,3,6","objects":321},{"shards":"1,3","objects":-1867}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":397,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.528210+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:08:50.528210+0000","last_peered":"2023-09-23T23:08:50.528210+0000","last_clean":"2023-09-23T23:08:50.528210+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:08:50.528210+0000","last_undegraded":"2023-09-23T23:08:50.528210+0000","last_fullsized":"2023-09-23T23:08:50.528210+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":367,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523460+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:08:50.523460+0000","last_peered":"2023-09-23T23:08:50.523460+0000","last_clean":"2023-09-23T23:08:50.523460+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:08:50.523460+0000","last_undegraded":"2023-09-23T23:08:50.523460+0000","last_fullsized":"2023-09-23T23:08:50.523460+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":334,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.636697+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:08:52.636697+0000","last_peered":"2023-09-23T23:08:52.636697+0000","last_clean":"2023-09-23T23:08:52.636697+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:08:52.636697+0000","last_undegraded":"2023-09-23T23:08:52.636697+0000","last_fullsized":"2023-09-23T23:08:52.636697+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"474'8837","reported_seq":8912,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.752556+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:09:01.752556+0000","last_peered":"2023-09-23T23:09:01.752556+0000","last_clean":"2023-09-23T23:09:01.752556+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:09:01.752556+0000","last_undegraded":"2023-09-23T23:09:01.752556+0000","last_fullsized":"2023-09-23T23:09:01.752556+0000","mapping_epoch":410,"log_start":"418'3011","ondisk_log_start":"418'3011","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5826,"log_dups_size":0,"ondisk_log_size":5826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91422720,"num_objects":1395,"num_object_clones":0,"num_object_copies":4185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8837,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":332,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526211+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:08:50.526211+0000","last_peered":"2023-09-23T23:08:50.526211+0000","last_clean":"2023-09-23T23:08:50.526211+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:08:50.526211+0000","last_undegraded":"2023-09-23T23:08:50.526211+0000","last_fullsized":"2023-09-23T23:08:50.526211+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":331,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.637971+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:08:52.637971+0000","last_peered":"2023-09-23T23:08:52.637971+0000","last_clean":"2023-09-23T23:08:52.637971+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:08:52.637971+0000","last_undegraded":"2023-09-23T23:08:52.637971+0000","last_fullsized":"2023-09-23T23:08:52.637971+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":296,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.635694+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:08:52.635694+0000","last_peered":"2023-09-23T23:08:52.635694+0000","last_clean":"2023-09-23T23:08:52.635694+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:08:52.635694+0000","last_undegraded":"2023-09-23T23:08:52.635694+0000","last_fullsized":"2023-09-23T23:08:52.635694+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"474'8752","reported_seq":8835,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.802711+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:09:01.802711+0000","last_peered":"2023-09-23T23:09:01.802711+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:09:01.802711+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"417'2883","ondisk_log_start":"417'2883","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5869,"log_dups_size":0,"ondisk_log_size":5869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90308608,"num_objects":1378,"num_object_clones":0,"num_object_copies":4134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8752,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":1378},{"shards":"1,2","objects":-3391}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":196,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.522849+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:08:50.522849+0000","last_peered":"2023-09-23T23:08:50.522849+0000","last_clean":"2023-09-23T23:08:50.522849+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:08:50.522849+0000","last_undegraded":"2023-09-23T23:08:50.522849+0000","last_fullsized":"2023-09-23T23:08:50.522849+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":171,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526621+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:08:50.526621+0000","last_peered":"2023-09-23T23:08:50.526621+0000","last_clean":"2023-09-23T23:08:50.526621+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:08:50.526621+0000","last_undegraded":"2023-09-23T23:08:50.526621+0000","last_fullsized":"2023-09-23T23:08:50.526621+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.529381+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:08:50.529381+0000","last_peered":"2023-09-23T23:08:50.529381+0000","last_clean":"2023-09-23T23:08:50.529381+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:08:50.529381+0000","last_undegraded":"2023-09-23T23:08:50.529381+0000","last_fullsized":"2023-09-23T23:08:50.529381+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"473'8827","reported_seq":17271,"reported_epoch":473,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:00.520254+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:09:00.520254+0000","last_peered":"2023-09-23T23:09:00.520254+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:09:00.520254+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"418'3120","ondisk_log_start":"418'3120","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5707,"log_dups_size":0,"ondisk_log_size":5707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":515,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8827,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4182,"num_bytes_recovered":274071552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":515},{"shards":"1,4","objects":806}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":422,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.640072+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:08:52.640072+0000","last_peered":"2023-09-23T23:08:52.640072+0000","last_clean":"2023-09-23T23:08:52.640072+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:08:52.640072+0000","last_undegraded":"2023-09-23T23:08:52.640072+0000","last_fullsized":"2023-09-23T23:08:52.640072+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":94,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526708+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:08:50.526708+0000","last_peered":"2023-09-23T23:08:50.526708+0000","last_clean":"2023-09-23T23:08:50.526708+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:08:50.526708+0000","last_undegraded":"2023-09-23T23:08:50.526708+0000","last_fullsized":"2023-09-23T23:08:50.526708+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":465,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523562+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:08:50.523562+0000","last_peered":"2023-09-23T23:08:50.523562+0000","last_clean":"2023-09-23T23:08:50.523562+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:08:50.523562+0000","last_undegraded":"2023-09-23T23:08:50.523562+0000","last_fullsized":"2023-09-23T23:08:50.523562+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"474'7189","reported_seq":9277,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:02.580268+0000","last_change":"2023-09-23T23:07:56.517407+0000","last_active":"2023-09-23T23:09:02.580268+0000","last_peered":"2023-09-23T23:09:02.580268+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.507318+0000","last_became_peered":"2023-09-23T23:07:56.507318+0000","last_unstale":"2023-09-23T23:09:02.580268+0000","last_undegraded":"2023-09-23T23:07:56.466720+0000","last_fullsized":"2023-09-23T23:07:56.443187+0000","mapping_epoch":459,"log_start":"415'1086","ondisk_log_start":"415'1086","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6103,"log_dups_size":0,"ondisk_log_size":6103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45744128,"num_objects":698,"num_object_clones":0,"num_object_copies":2094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4506,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":64454656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2","objects":-533},{"shards":"1,2,5","objects":395}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":64,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525873+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:08:50.525873+0000","last_peered":"2023-09-23T23:08:50.525873+0000","last_clean":"2023-09-23T23:08:50.525873+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:08:50.525873+0000","last_undegraded":"2023-09-23T23:08:50.525873+0000","last_fullsized":"2023-09-23T23:08:50.525873+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":225,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.638047+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:08:52.638047+0000","last_peered":"2023-09-23T23:08:52.638047+0000","last_clean":"2023-09-23T23:08:52.638047+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:08:52.638047+0000","last_undegraded":"2023-09-23T23:08:52.638047+0000","last_fullsized":"2023-09-23T23:08:52.638047+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"474'7195","reported_seq":7619,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.691954+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:09:01.691954+0000","last_peered":"2023-09-23T23:09:01.691954+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:09:01.691954+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7195,"log_dups_size":0,"ondisk_log_size":7195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48726016,"num_objects":744,"num_object_clones":0,"num_object_copies":2232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4480,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1231},{"shards":"1,2,3","objects":374}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":295,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525896+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:08:50.525896+0000","last_peered":"2023-09-23T23:08:50.525896+0000","last_clean":"2023-09-23T23:08:50.525896+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:08:50.525896+0000","last_undegraded":"2023-09-23T23:08:50.525896+0000","last_fullsized":"2023-09-23T23:08:50.525896+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":496,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526378+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:08:50.526378+0000","last_peered":"2023-09-23T23:08:50.526378+0000","last_clean":"2023-09-23T23:08:50.526378+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:08:50.526378+0000","last_undegraded":"2023-09-23T23:08:50.526378+0000","last_fullsized":"2023-09-23T23:08:50.526378+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"473'6968","reported_seq":11970,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:08:58.160157+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:08:58.160157+0000","last_peered":"2023-09-23T23:08:58.160157+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:08:58.160157+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6968,"log_dups_size":0,"ondisk_log_size":6968,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50233344,"num_objects":767,"num_object_clones":0,"num_object_copies":2301,"num_objects_missing_on_primary":536,"num_objects_missing":536,"num_objects_degraded":1485,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4320,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109379584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":420},{"shards":"0,3,5,6","objects":2},{"shards":"3,5,6","objects":298},{"shards":"3,6","objects":47}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":322,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.526367+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:08:50.526367+0000","last_peered":"2023-09-23T23:08:50.526367+0000","last_clean":"2023-09-23T23:08:50.526367+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:08:50.526367+0000","last_undegraded":"2023-09-23T23:08:50.526367+0000","last_fullsized":"2023-09-23T23:08:50.526367+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":392,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525732+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:08:50.525732+0000","last_peered":"2023-09-23T23:08:50.525732+0000","last_clean":"2023-09-23T23:08:50.525732+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:08:50.525732+0000","last_undegraded":"2023-09-23T23:08:50.525732+0000","last_fullsized":"2023-09-23T23:08:50.525732+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"474'7044","reported_seq":8058,"reported_epoch":474,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:01.785201+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:09:01.785201+0000","last_peered":"2023-09-23T23:09:01.785201+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:09:01.785201+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42631168,"num_objects":651,"num_object_clones":0,"num_object_copies":1953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4413,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":729},{"shards":"1,2","objects":-1336}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":48,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.531943+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:08:50.531943+0000","last_peered":"2023-09-23T23:08:50.531943+0000","last_clean":"2023-09-23T23:08:50.531943+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:08:50.531943+0000","last_undegraded":"2023-09-23T23:08:50.531943+0000","last_fullsized":"2023-09-23T23:08:50.531943+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":190,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.525313+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:08:50.525313+0000","last_peered":"2023-09-23T23:08:50.525313+0000","last_clean":"2023-09-23T23:08:50.525313+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:08:50.525313+0000","last_undegraded":"2023-09-23T23:08:50.525313+0000","last_fullsized":"2023-09-23T23:08:50.525313+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"473'7002","reported_seq":7090,"reported_epoch":473,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:00.334330+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:09:00.334330+0000","last_peered":"2023-09-23T23:09:00.334330+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:09:00.334330+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48791552,"num_objects":745,"num_object_clones":0,"num_object_copies":2235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4352,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1511},{"shards":"1,4,5","objects":703}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":242,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T23:08:52.640301+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:08:52.640301+0000","last_peered":"2023-09-23T23:08:52.640301+0000","last_clean":"2023-09-23T23:08:52.640301+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:08:52.640301+0000","last_undegraded":"2023-09-23T23:08:52.640301+0000","last_fullsized":"2023-09-23T23:08:52.640301+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":467,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T23:08:50.523438+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:08:50.523438+0000","last_peered":"2023-09-23T23:08:50.523438+0000","last_clean":"2023-09-23T23:08:50.523438+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:08:50.523438+0000","last_undegraded":"2023-09-23T23:08:50.523438+0000","last_fullsized":"2023-09-23T23:08:50.523438+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1517289496,"num_objects":23153,"num_object_clones":0,"num_object_copies":69459,"num_objects_missing_on_primary":1622,"num_objects_missing":1622,"num_objects_degraded":46264,"num_objects_misplaced":515,"num_objects_unfound":0,"num_objects_dirty":23153,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":141151,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31071,"num_bytes_recovered":2009923584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8465100800,"data_stored":8465088584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":171577,"ondisk_log_size":171577,"up":78,"acting":58,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430712,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626936,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1329940,"kb_used_data":419020,"kb_used_omap":0,"kb_used_meta":910912,"kb_avail":92415212,"statfs":{"total":95995035648,"available":94633177088,"internally_reserved":0,"allocated":429076480,"data_stored":428017975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932773888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.304,"15min":4.9790000000000001},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":6.4320000000000004,"5min":10.853,"15min":115.32899999999999},"last":2.0710000000000002},{"interface":"front","average":{"1min":1.371,"5min":1.1919999999999999,"15min":4.9989999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":115.348},"last":2.161}]},{"osd":1,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1559999999999997,"5min":3.2839999999999998,"15min":3.1099999999999999},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":38.259999999999998,"5min":149.947,"15min":149.947},"last":9.4060000000000006},{"interface":"front","average":{"1min":3.5950000000000002,"5min":3.4940000000000002,"15min":3.1030000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":38.530000000000001,"5min":150.22499999999999,"15min":150.22499999999999},"last":0.99399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:08:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9420000000000002,"5min":5.5709999999999997,"15min":2.9260000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":45.051000000000002,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.626},{"interface":"front","average":{"1min":4.8579999999999997,"5min":5.25,"15min":2.9969999999999999},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32500000000000001},"max":{"1min":45.258000000000003,"5min":138.45599999999999,"15min":138.45599999999999},"last":9.3629999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.831,"5min":4.5940000000000003,"15min":3.2719999999999998},"min":{"1min":0.39300000000000002,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":47.734000000000002,"5min":142.24700000000001,"15min":142.24700000000001},"last":9.4329999999999998},{"interface":"front","average":{"1min":3.1549999999999998,"5min":4.617,"15min":3.3210000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":47.162999999999997,"5min":143.86699999999999,"15min":143.86699999999999},"last":4.5789999999999997}]},{"osd":4,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.1970000000000001,"15min":1.925},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.23999999999999999},"max":{"1min":7.3869999999999996,"5min":12.407999999999999,"15min":119.804},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.117,"15min":1.95},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":8.8940000000000001,"5min":12.388,"15min":122.31999999999999},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.863,"15min":2.5600000000000001},"min":{"1min":0.42599999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":1.665,"5min":44.781999999999996,"15min":186.27600000000001},"last":0.505},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.9930000000000001,"15min":2.6840000000000002},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":8.7720000000000002,"5min":44.601999999999997,"15min":191.28700000000001},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.224,"15min":1.5880000000000001},"min":{"1min":0.46700000000000003,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":8.8670000000000009,"5min":8.8670000000000009,"15min":115.61},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.698,"5min":0.97499999999999998,"15min":1.7070000000000001},"min":{"1min":0.39600000000000002,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":8.2609999999999992,"5min":8.2609999999999992,"15min":115.578},"last":1.448}]}]},{"osd":1,"up_from":84,"seq":360777253250,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2231020,"kb_used_data":1267940,"kb_used_omap":9,"kb_used_meta":963062,"kb_avail":91514132,"statfs":{"total":95995035648,"available":93710471168,"internally_reserved":0,"allocated":1298370560,"data_stored":1296436615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":986176489},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":5.5030000000000001,"15min":2.544},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.389,"5min":279.30500000000001,"15min":279.30500000000001},"last":3.2189999999999999},{"interface":"front","average":{"1min":0.81100000000000005,"5min":5.6820000000000004,"15min":2.8799999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":3.4500000000000002,"5min":279.39800000000002,"15min":279.39800000000002},"last":2.5470000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":7.96,"5min":6.2439999999999998,"15min":2.4449999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.527,"5min":161.61699999999999,"15min":161.61699999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":7.5199999999999996,"5min":6.4059999999999997,"15min":2.6059999999999999},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":118.29900000000001,"5min":162.61699999999999,"15min":162.61699999999999},"last":0.437}]},{"osd":3,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.4510000000000001,"15min":1.389},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":3.8039999999999998,"5min":59.424999999999997,"15min":59.424999999999997},"last":2.7069999999999999},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.5680000000000001,"15min":1.454},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":7.1580000000000004,"5min":55.174999999999997,"15min":55.174999999999997},"last":3.2970000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2320000000000002,"5min":3.0569999999999999,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":67.813999999999993,"5min":129.69,"15min":151.036},"last":0.46999999999999997},{"interface":"front","average":{"1min":5.4039999999999999,"5min":3.427,"15min":2.9180000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":67.796999999999997,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.4419999999999999,"15min":2.2370000000000001},"min":{"1min":0.40300000000000002,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":1.522,"5min":18.228999999999999,"15min":116.098},"last":3.2719999999999998},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.0920000000000001,"15min":2.0419999999999998},"min":{"1min":0.38900000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.5800000000000001,"5min":6.585,"15min":116.08499999999999},"last":1.8520000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2759999999999998,"5min":2.1850000000000001,"15min":2.056},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":64.058000000000007,"5min":64.058000000000007,"15min":67.349000000000004},"last":4.2889999999999997},{"interface":"front","average":{"1min":4.7190000000000003,"5min":2.3690000000000002,"15min":2.0110000000000001},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.33300000000000002},"max":{"1min":64.239999999999995,"5min":64.239999999999995,"15min":67.366},"last":3.3159999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":1.337,"15min":1.1910000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":5.7549999999999999,"5min":7.0960000000000001,"15min":8.1400000000000006},"last":1.319},{"interface":"front","average":{"1min":1.472,"5min":1.7290000000000001,"15min":1.478},"min":{"1min":0.40699999999999997,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":8.8719999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.62}]}]},{"osd":0,"up_from":0,"seq":1516123455620,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259393,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1031612,"kb_used_data":733748,"kb_used_omap":83,"kb_used_meta":297772,"kb_avail":92713540,"statfs":{"total":95995035648,"available":94938664960,"internally_reserved":0,"allocated":751357952,"data_stored":749294599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":304918981},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.82999999999999996,"15min":1.212},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":4.1390000000000002,"5min":10.199999999999999,"15min":74.241},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.5,"5min":1.29,"15min":1.5309999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":14.944000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0670000000000002,"5min":2.5049999999999999,"15min":1.3009999999999999},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":83.897000000000006,"5min":83.897000000000006,"15min":83.897000000000006},"last":1.9770000000000001},{"interface":"front","average":{"1min":4.5289999999999999,"5min":2.1669999999999998,"15min":1.462},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":83.088999999999999,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7130000000000001,"5min":3.98,"15min":1.8380000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.23000000000000001},"max":{"1min":79.655000000000001,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":7.2039999999999997,"5min":3.8650000000000002,"15min":1.851},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26100000000000001},"max":{"1min":79.677999999999997,"5min":110.56999999999999,"15min":110.56999999999999},"last":3.2869999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":1.9279999999999999,"15min":1.1379999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":68.325000000000003,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":4.9020000000000001,"5min":2.2610000000000001,"15min":1.351},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":70.608000000000004,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":3.105,"15min":1.6839999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.6600000000000001,"5min":139.405,"15min":139.405},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.002,"5min":2.9129999999999998,"15min":1.5089999999999999},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":4.0499999999999998,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.64800000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":3.4180000000000001,"15min":1.8600000000000001},"min":{"1min":0.377,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.6989999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.432},{"interface":"front","average":{"1min":1.1439999999999999,"5min":3.1749999999999998,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34599999999999997,"15min":0.32200000000000001},"max":{"1min":4.0830000000000002,"5min":108.496,"15min":108.496},"last":2.6680000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.421,"15min":1.1890000000000001},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.1449999999999996,"5min":10.164,"15min":10.164},"last":1.4950000000000001},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.393,"15min":1.2050000000000001},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":15.037000000000001,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.75900000000000001}]}]},{"osd":3,"up_from":94,"seq":403726926197,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1803852,"kb_used_data":1119620,"kb_used_omap":7,"kb_used_meta":684216,"kb_avail":91941300,"statfs":{"total":95995035648,"available":94147891200,"internally_reserved":0,"allocated":1146490880,"data_stored":1144459115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":700637800},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.534,"15min":1.4099999999999999},"min":{"1min":0.22600000000000001,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":8.1660000000000004,"5min":17.015000000000001,"15min":122.289},"last":1.4530000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.004,"15min":1.1419999999999999},"min":{"1min":0.25600000000000001,"5min":0.218,"15min":0.17799999999999999},"max":{"1min":4.1470000000000002,"5min":11.382999999999999,"15min":122.041},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":2.145,"15min":2.0230000000000001},"min":{"1min":0.22600000000000001,"5min":0.224,"15min":0.187},"max":{"1min":3.9380000000000002,"5min":23.969000000000001,"15min":123.866},"last":2.5899999999999999},{"interface":"front","average":{"1min":0.90200000000000002,"5min":1.889,"15min":1.978},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.20699999999999999},"max":{"1min":3.8610000000000002,"5min":16.986000000000001,"15min":122.265},"last":0.36799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:08:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8479999999999999,"5min":2.3370000000000002,"15min":1.232},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":55.311999999999998,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":5.532,"5min":2.7480000000000002,"15min":1.5389999999999999},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.215},"max":{"1min":57.308,"5min":57.308,"15min":57.308},"last":2.5270000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":2.5030000000000001,"15min":2.137},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":3.3130000000000002,"5min":115.69,"15min":115.69},"last":2.548},{"interface":"front","average":{"1min":0.83699999999999997,"5min":2.3250000000000002,"15min":2.0569999999999999},"min":{"1min":0.375,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":3.0670000000000002,"5min":115.617,"15min":115.617},"last":0.41599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.645,"15min":1.4079999999999999},"min":{"1min":0.39600000000000002,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":3.5339999999999998,"5min":16.959,"15min":16.959},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":1.319,"15min":1.5029999999999999},"min":{"1min":0.42299999999999999,"5min":0.34399999999999997,"15min":0.27100000000000002},"max":{"1min":3.3090000000000002,"5min":10.545,"15min":10.545},"last":2.4630000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.6519999999999999,"15min":1.647},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":7.1740000000000004,"5min":16.882000000000001,"15min":16.882000000000001},"last":1.286},{"interface":"front","average":{"1min":1.097,"5min":1.248,"15min":1.55},"min":{"1min":0.39900000000000002,"5min":0.372,"15min":0.32400000000000001},"max":{"1min":7.1870000000000003,"5min":8.1999999999999993,"15min":12.887},"last":2.4380000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:08:09 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.3580000000000001,"15min":1.413},"min":{"1min":0.50700000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":6.1820000000000004,"5min":8.1620000000000008,"15min":8.8160000000000007},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.6419999999999999,"5min":1.8720000000000001,"15min":1.542},"min":{"1min":0.435,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":6.1360000000000001,"5min":10.278,"15min":10.278},"last":2.5720000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019394,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1342740,"kb_used_data":427084,"kb_used_omap":0,"kb_used_meta":915648,"kb_avail":92402412,"statfs":{"total":95995035648,"available":94620069888,"internally_reserved":0,"allocated":437334016,"data_stored":436274989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":937623552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.0409999999999999,"15min":2.2360000000000002},"min":{"1min":0.27400000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.778,"5min":99.132999999999996,"15min":156.375},"last":3.1779999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":2.1829999999999998,"15min":2.371},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":2.9910000000000001,"5min":101.062,"15min":157.02199999999999},"last":2.6280000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":3.4900000000000002,"15min":3.1970000000000001},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":6.915,"5min":95.591999999999999,"15min":98.846999999999994},"last":2.5699999999999998},{"interface":"front","average":{"1min":1.4710000000000001,"5min":3.4039999999999999,"15min":3.157},"min":{"1min":0.33800000000000002,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":15.728,"5min":97.929000000000002,"15min":98.938999999999993},"last":3.1299999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":11.628,"5min":4.2610000000000001,"15min":2.1459999999999999},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":124.819,"5min":124.819,"15min":124.819},"last":3.2130000000000001},{"interface":"front","average":{"1min":10.571999999999999,"5min":4.077,"15min":1.9850000000000001},"min":{"1min":0.36599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":122.086,"5min":122.086,"15min":122.086},"last":2.6099999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":2.5379999999999998,"15min":1.841},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":55.466999999999999,"5min":60.274000000000001,"15min":60.274000000000001},"last":3.2440000000000002},{"interface":"front","average":{"1min":3.3820000000000001,"5min":2.677,"15min":1.9259999999999999},"min":{"1min":0.38500000000000001,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":55.625999999999998,"5min":60.677999999999997,"15min":60.677999999999997},"last":2.593}]},{"osd":5,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.177,"15min":1.764},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.214},"max":{"1min":3.036,"5min":10.130000000000001,"15min":148.61600000000001},"last":2.661},{"interface":"front","average":{"1min":1.522,"5min":1.157,"15min":1.861},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":8.5489999999999995,"5min":8.5489999999999995,"15min":151.905},"last":3.1070000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.3899999999999999,"15min":1.4359999999999999},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.2590000000000003,"5min":35.896000000000001,"15min":35.896000000000001},"last":3.1579999999999999},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.5629999999999999,"15min":1.403},"min":{"1min":0.35499999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":7.008,"5min":35.908000000000001,"15min":35.908000000000001},"last":2.6749999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.161,"15min":1.288},"min":{"1min":0.373,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":3.1080000000000001,"5min":7.0789999999999997,"15min":12.84},"last":2.645},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.28,"15min":1.232},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":15.808,"5min":15.808,"15min":15.808},"last":3.2719999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823168,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2934833152,"data_stored":2934833152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1270546432,"data_stored":1270546432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":739381248,"data_stored":739377176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1136263168,"data_stored":1136263168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":429789184,"data_stored":429785112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":422252544,"data_stored":422248472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:09:04.820 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:09:04.891 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:04.980 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:09:05.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:04 smithi052 ceph-mon[180979]: from='client.37254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:05.057 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:05.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:04 smithi052 ceph-mon[187743]: from='client.37254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:05.057 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:05.075 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:05.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:05.195 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:05.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:04 smithi151 ceph-mon[159210]: from='client.37254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:05.227 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:05.310 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:05.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:05.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:05.419 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:05.467 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:09:05.471 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:05.500 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:05.507 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:09:05.577 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:09:05.616 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:09:05.673 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:09:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:05.695 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:05.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:05.710 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:09:05.746 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:09:05.755 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:05.780 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.787 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:09:05.797 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:05.801 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:05.878 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:06.047 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:06.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:06.158 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:05 smithi151 ceph-mon[159210]: pgmap v1490: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 482 op/s; 45258/66246 objects degraded (68.318%); 515/66246 objects misplaced (0.777%); 11 MiB/s, 173 objects/s recovering 2023-09-23T23:09:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/943349319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:05 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 45258/66246 objects degraded (68.318%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:06.216 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:06.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:05 smithi052 ceph-mon[180979]: pgmap v1490: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 482 op/s; 45258/66246 objects degraded (68.318%); 515/66246 objects misplaced (0.777%); 11 MiB/s, 173 objects/s recovering 2023-09-23T23:09:06.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/943349319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:06.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:05 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 45258/66246 objects degraded (68.318%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:05 smithi052 ceph-mon[187743]: pgmap v1490: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 482 op/s; 45258/66246 objects degraded (68.318%); 515/66246 objects misplaced (0.777%); 11 MiB/s, 173 objects/s recovering 2023-09-23T23:09:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/943349319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:05 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 45258/66246 objects degraded (68.318%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:06.308 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:06.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:06.418 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:06.453 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:06.501 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:06.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:06.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:06.668 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:06.783 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:06.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:06.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:06.894 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:06.957 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:07.053 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:07.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:07.162 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:07.206 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:07.992 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:07.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:08.098 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:08.105 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:07 smithi052 ceph-mon[180979]: pgmap v1491: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 481 op/s; 45258/65367 objects degraded (69.237%); 411/65367 objects misplaced (0.629%); 11 MiB/s, 172 objects/s recovering 2023-09-23T23:09:08.105 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:07 smithi052 ceph-mon[187743]: pgmap v1491: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 481 op/s; 45258/65367 objects degraded (69.237%); 411/65367 objects misplaced (0.629%); 11 MiB/s, 172 objects/s recovering 2023-09-23T23:09:08.136 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:08.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:07 smithi151 ceph-mon[159210]: pgmap v1491: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 481 op/s; 45258/65367 objects degraded (69.237%); 411/65367 objects misplaced (0.629%); 11 MiB/s, 172 objects/s recovering 2023-09-23T23:09:08.207 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:08.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:08.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:08.326 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:08.366 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:08.944 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:08.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:08.991 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:09:09.049 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:09.143 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:09.581 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:09.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:09.688 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:09.735 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:09.784 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:09.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:09.890 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:09.991 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:10.153 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:10.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:09 smithi151 ceph-mon[159210]: pgmap v1492: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 490 op/s; 44805/61530 objects degraded (72.818%); 411/61530 objects misplaced (0.668%); 11 MiB/s, 174 objects/s recovering 2023-09-23T23:09:10.222 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:09 smithi052 ceph-mon[187743]: pgmap v1492: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 490 op/s; 44805/61530 objects degraded (72.818%); 411/61530 objects misplaced (0.668%); 11 MiB/s, 174 objects/s recovering 2023-09-23T23:09:10.222 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:09 smithi052 ceph-mon[180979]: pgmap v1492: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 490 op/s; 44805/61530 objects degraded (72.818%); 411/61530 objects misplaced (0.668%); 11 MiB/s, 174 objects/s recovering 2023-09-23T23:09:10.259 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:10.330 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:10.435 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:10.495 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:10.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:10.565 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:09:10.565 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:09:10.566 INFO:teuthology.orchestra.run.smithi052.stdout: "num_ops": 0 2023-09-23T23:09:10.566 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:09:10.604 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:10.653 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:10.718 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:10.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:10.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:10.827 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:10.916 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:11.031 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:11.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:11.150 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:11.208 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:10 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 44805/61530 objects degraded (72.818%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:11.208 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3930317934' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:11.208 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:11.208 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:10 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 44805/61530 objects degraded (72.818%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:11.208 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3930317934' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:11.209 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:11.210 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:09:11.210 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 44805/61530 objects degraded (72.818%), 24 pgs degraded, 20 pgs undersized","count":44},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2083,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":474,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":5,"osd_in_since":1695510352,"num_remapped_pgs":20},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":74},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":19},{"state_name":"active+recovery_wait+degraded","count":4},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":20510,"data_bytes":1343100856,"bytes_used":7563710464,"bytes_avail":472411467776,"bytes_total":479975178240,"degraded_objects":44805,"degraded_total":61530,"degraded_ratio":0.7281813749390541,"misplaced_objects":411,"misplaced_total":61530,"misplaced_ratio":0.0066796684544124822,"recovering_objects_per_sec":174,"recovering_bytes_per_sec":11436938,"recovering_keys_per_sec":0,"num_objects_recovered":1747,"num_bytes_recovered":114491392,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":490},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T23:09:04.798859+0000","services":{}},"progress_events":{"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true},"fadc4ea3-ec6a-4d25-82eb-03b384d527e2":{"message":"Global Recovery Event (3m)\n [====================........] (remaining: 76s)","progress":0.74747473001480103,"add_to_ceph_s":true}}} 2023-09-23T23:09:11.215 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:11.321 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:11.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:11.426 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:10 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 44805/61530 objects degraded (72.818%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3930317934' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:11.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:11.454 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:11.482 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:09:11.503 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:11.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:11.607 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:11.692 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:12.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[180979]: pgmap v1493: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.2 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 494 op/s; 44214/58017 objects degraded (76.209%); 299/58017 objects misplaced (0.515%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:09:12.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/574626113' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:09:12.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:12.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:09:12.248 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[180979]: osdmap e475: 8 total, 8 up, 5 in 2023-09-23T23:09:12.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[187743]: pgmap v1493: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.2 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 494 op/s; 44214/58017 objects degraded (76.209%); 299/58017 objects misplaced (0.515%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:09:12.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/574626113' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:09:12.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:12.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:09:12.248 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:11 smithi052 ceph-mon[187743]: osdmap e475: 8 total, 8 up, 5 in 2023-09-23T23:09:12.419 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:12.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:11 smithi151 ceph-mon[159210]: pgmap v1493: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.2 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 494 op/s; 44214/58017 objects degraded (76.209%); 299/58017 objects misplaced (0.515%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:09:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/574626113' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:09:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:11 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:09:12.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:11 smithi151 ceph-mon[159210]: osdmap e475: 8 total, 8 up, 5 in 2023-09-23T23:09:12.526 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:12.556 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:12.980 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:12 smithi052 ceph-mon[180979]: failed to encode map e475 with expected crc 2023-09-23T23:09:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:12 smithi052 ceph-mon[187743]: failed to encode map e475 with expected crc 2023-09-23T23:09:13.451 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:12 smithi151 ceph-mon[159210]: failed to encode map e475 with expected crc 2023-09-23T23:09:13.748 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:09:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:09:13] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.33.4" 2023-09-23T23:09:13.823 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:09:13.823 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:09:13.824 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:09:13.986 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:13.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:13.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:14.046 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:13 smithi052 ceph-mon[187743]: pgmap v1495: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.1 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 493 op/s; 44147/54702 objects degraded (80.705%); 299/54702 objects misplaced (0.547%); 11 MiB/s, 173 objects/s recovering 2023-09-23T23:09:14.046 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:13 smithi052 ceph-mon[180979]: pgmap v1495: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.1 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 493 op/s; 44147/54702 objects degraded (80.705%); 299/54702 objects misplaced (0.547%); 11 MiB/s, 173 objects/s recovering 2023-09-23T23:09:14.097 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:14.132 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:14.209 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:14.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:14.318 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:14.366 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:13 smithi151 ceph-mon[159210]: pgmap v1495: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.1 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 493 op/s; 44147/54702 objects degraded (80.705%); 299/54702 objects misplaced (0.547%); 11 MiB/s, 173 objects/s recovering 2023-09-23T23:09:14.375 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:14.455 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:14.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:14.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:14.573 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:14.649 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:14.727 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:14.733 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:14.803 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:14.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:14.912 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:14.979 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:15.031 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:09:15.063 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:09:15.069 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:15.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:15.096 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:09:15.140 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:09:15.169 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2818162822' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:15.169 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2818162822' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:15.182 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:15.184 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:09:15.226 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:15.226 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2818162822' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:15.235 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:09:15.270 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:09:15.295 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:15.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:15.303 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:09:15.311 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.311 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.311 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.311 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.311 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.312 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.312 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.312 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:09:15.401 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:15.438 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:15.509 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:15.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:15.615 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:15.649 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:15.705 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:15.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:15.814 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:15.864 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:16.076 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:09:16.108 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:16 smithi052 ceph-mon[180979]: pgmap v1496: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 496 op/s; 43114/51348 objects degraded (83.964%); 299/51348 objects misplaced (0.582%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:09:16.109 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:16 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 43114/51348 objects degraded (83.964%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:16.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:16 smithi052 ceph-mon[187743]: pgmap v1496: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 496 op/s; 43114/51348 objects degraded (83.964%); 299/51348 objects misplaced (0.582%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:09:16.109 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:16 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 43114/51348 objects degraded (83.964%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:16.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:16 smithi151 ceph-mon[159210]: pgmap v1496: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 496 op/s; 43114/51348 objects degraded (83.964%); 299/51348 objects misplaced (0.582%); 11 MiB/s, 175 objects/s recovering 2023-09-23T23:09:16.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:16 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 43114/51348 objects degraded (83.964%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:16.717 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:16.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:16.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:16.824 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:16.850 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:16.896 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:16.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:17.002 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:17.066 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:18.093 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:18.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:18.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:18.146 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:18.206 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:18 smithi052 ceph-mon[187743]: pgmap v1497: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 502 op/s; 43114/50295 objects degraded (85.722%); 183/50295 objects misplaced (0.364%); 11 MiB/s, 176 objects/s recovering 2023-09-23T23:09:18.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:18 smithi052 ceph-mon[180979]: pgmap v1497: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 502 op/s; 43114/50295 objects degraded (85.722%); 183/50295 objects misplaced (0.364%); 11 MiB/s, 176 objects/s recovering 2023-09-23T23:09:18.268 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:18 smithi151 ceph-mon[159210]: pgmap v1497: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 1.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 502 op/s; 43114/50295 objects degraded (85.722%); 183/50295 objects misplaced (0.364%); 11 MiB/s, 176 objects/s recovering 2023-09-23T23:09:18.271 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:18.377 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:18.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:18.490 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:18.565 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:18.869 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:18.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:18.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:18.984 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:19.055 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:19.126 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:09:19.126 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:09:19.423 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:19.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:19.533 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:19.597 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:19.612 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:09:19.614 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:09:19.706 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:19.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:19.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:19.816 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:19.847 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:19.933 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:19.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:19.972 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:09:20.038 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:20.064 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:20.111 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:20.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:20.216 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:20.242 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:20 smithi052 ceph-mon[187743]: pgmap v1498: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 959 MiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 515 op/s; 42642/46068 objects degraded (92.563%); 183/46068 objects misplaced (0.397%); 11 MiB/s, 177 objects/s recovering 2023-09-23T23:09:20.242 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:20 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:20.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:20 smithi052 ceph-mon[180979]: pgmap v1498: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 959 MiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 515 op/s; 42642/46068 objects degraded (92.563%); 183/46068 objects misplaced (0.397%); 11 MiB/s, 177 objects/s recovering 2023-09-23T23:09:20.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:20 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:20.245 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:20.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:20 smithi151 ceph-mon[159210]: pgmap v1498: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 959 MiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 515 op/s; 42642/46068 objects degraded (92.563%); 183/46068 objects misplaced (0.397%); 11 MiB/s, 177 objects/s recovering 2023-09-23T23:09:20.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:20 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:21.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:21 smithi052 ceph-mon[187743]: from='client.57172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:21.149 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:21 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 42642/46068 objects degraded (92.563%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:21.150 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:21 smithi052 ceph-mon[180979]: from='client.57172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:21.150 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:21 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 42642/46068 objects degraded (92.563%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:21.407 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:21.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:21 smithi151 ceph-mon[159210]: from='client.57172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:21.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:21 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 42642/46068 objects degraded (92.563%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:21.513 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1498,"stamp":"2023-09-23T23:09:18.805648+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1005328312,"num_objects":15356,"num_object_clones":0,"num_object_copies":46068,"num_objects_missing_on_primary":687,"num_objects_missing":687,"num_objects_degraded":42642,"num_objects_misplaced":183,"num_objects_unfound":0,"num_objects_dirty":15356,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":149232,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34029,"num_bytes_recovered":2199212480,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173982,"ondisk_log_size":173982,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":6722436,"kb_used_data":2944924,"kb_used_omap":99,"kb_used_meta":3777372,"kb_avail":462003324,"statfs":{"total":479975178240,"available":473091403776,"internally_reserved":0,"allocated":3015602176,"data_stored":3007417845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3868028950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,2,4,0,0,3,1,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-337772544,"num_objects":-5154,"num_object_clones":0,"num_object_copies":-15462,"num_objects_missing_on_primary":-489,"num_objects_missing":-489,"num_objects_degraded":-2163,"num_objects_misplaced":-228,"num_objects_unfound":0,"num_objects_dirty":-5154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5154,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004514"},"pg_stats":[{"pgid":"11.12","version":"475'7363","reported_seq":7459,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.694969+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:09:16.694969+0000","last_peered":"2023-09-23T23:09:16.694969+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:09:16.694969+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28868608,"num_objects":441,"num_object_clones":0,"num_object_copies":1323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4700,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":472},{"shards":"2,3","objects":-2016}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":487,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506366+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:09:01.506366+0000","last_peered":"2023-09-23T23:09:01.506366+0000","last_clean":"2023-09-23T23:09:01.506366+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:09:01.506366+0000","last_undegraded":"2023-09-23T23:09:01.506366+0000","last_fullsized":"2023-09-23T23:09:01.506366+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":147,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493521+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:09:01.493521+0000","last_peered":"2023-09-23T23:09:01.493521+0000","last_clean":"2023-09-23T23:09:01.493521+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:09:01.493521+0000","last_undegraded":"2023-09-23T23:09:01.493521+0000","last_fullsized":"2023-09-23T23:09:01.493521+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"475'7262","reported_seq":11033,"reported_epoch":475,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:14.295931+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:09:14.295931+0000","last_peered":"2023-09-23T23:09:14.295931+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:09:14.295931+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:09:14.295931+0000","mapping_epoch":441,"log_start":"414'164","ondisk_log_start":"414'164","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7098,"log_dups_size":0,"ondisk_log_size":7098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34177024,"num_objects":522,"num_object_clones":0,"num_object_copies":1566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4609,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1834,"num_bytes_recovered":120127488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":196},{"shards":"0,3,6","objects":270},{"shards":"2,3,6","objects":56}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":146,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.494208+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:09:01.494208+0000","last_peered":"2023-09-23T23:09:01.494208+0000","last_clean":"2023-09-23T23:09:01.494208+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:09:01.494208+0000","last_undegraded":"2023-09-23T23:09:01.494208+0000","last_fullsized":"2023-09-23T23:09:01.494208+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":493,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493961+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:09:01.493961+0000","last_peered":"2023-09-23T23:09:01.493961+0000","last_clean":"2023-09-23T23:09:01.493961+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:09:01.493961+0000","last_undegraded":"2023-09-23T23:09:01.493961+0000","last_fullsized":"2023-09-23T23:09:01.493961+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"475'7256","reported_seq":11314,"reported_epoch":475,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:14.312417+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:09:14.312417+0000","last_peered":"2023-09-23T23:09:14.312417+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:09:14.312417+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:09:14.312417+0000","mapping_epoch":428,"log_start":"414'351","ondisk_log_start":"414'351","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4600,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":129892352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":163},{"shards":"1,3,6","objects":344}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.491195+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:09:01.491195+0000","last_peered":"2023-09-23T23:09:01.491195+0000","last_clean":"2023-09-23T23:09:01.491195+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:09:01.491195+0000","last_undegraded":"2023-09-23T23:09:01.491195+0000","last_fullsized":"2023-09-23T23:09:01.491195+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":64,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492590+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:09:01.492590+0000","last_peered":"2023-09-23T23:09:01.492590+0000","last_clean":"2023-09-23T23:09:01.492590+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:09:01.492590+0000","last_undegraded":"2023-09-23T23:09:01.492590+0000","last_fullsized":"2023-09-23T23:09:01.492590+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":419,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492824+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:09:01.492824+0000","last_peered":"2023-09-23T23:09:01.492824+0000","last_clean":"2023-09-23T23:09:01.492824+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:09:01.492824+0000","last_undegraded":"2023-09-23T23:09:01.492824+0000","last_fullsized":"2023-09-23T23:09:01.492824+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":166,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:09.355294+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:09:09.355294+0000","last_peered":"2023-09-23T23:09:09.355294+0000","last_clean":"2023-09-23T23:09:09.355294+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:09:09.355294+0000","last_undegraded":"2023-09-23T23:09:09.355294+0000","last_fullsized":"2023-09-23T23:09:09.355294+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":281,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.494097+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:09:01.494097+0000","last_peered":"2023-09-23T23:09:01.494097+0000","last_clean":"2023-09-23T23:09:01.494097+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:09:01.494097+0000","last_undegraded":"2023-09-23T23:09:01.494097+0000","last_fullsized":"2023-09-23T23:09:01.494097+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":180,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:09.355358+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:09:09.355358+0000","last_peered":"2023-09-23T23:09:09.355358+0000","last_clean":"2023-09-23T23:09:09.355358+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:09:09.355358+0000","last_undegraded":"2023-09-23T23:09:09.355358+0000","last_fullsized":"2023-09-23T23:09:09.355358+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":212,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493455+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:09:01.493455+0000","last_peered":"2023-09-23T23:09:01.493455+0000","last_clean":"2023-09-23T23:09:01.493455+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:09:01.493455+0000","last_undegraded":"2023-09-23T23:09:01.493455+0000","last_fullsized":"2023-09-23T23:09:01.493455+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":230,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.487478+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:09:01.487478+0000","last_peered":"2023-09-23T23:09:01.487478+0000","last_clean":"2023-09-23T23:09:01.487478+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:09:01.487478+0000","last_undegraded":"2023-09-23T23:09:01.487478+0000","last_fullsized":"2023-09-23T23:09:01.487478+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":565,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.507037+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:09:01.507037+0000","last_peered":"2023-09-23T23:09:01.507037+0000","last_clean":"2023-09-23T23:09:01.507037+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:09:01.507037+0000","last_undegraded":"2023-09-23T23:09:01.507037+0000","last_fullsized":"2023-09-23T23:09:01.507037+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506596+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:09:01.506596+0000","last_peered":"2023-09-23T23:09:01.506596+0000","last_clean":"2023-09-23T23:09:01.506596+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:09:01.506596+0000","last_undegraded":"2023-09-23T23:09:01.506596+0000","last_fullsized":"2023-09-23T23:09:01.506596+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":390,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506618+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:09:01.506618+0000","last_peered":"2023-09-23T23:09:01.506618+0000","last_clean":"2023-09-23T23:09:01.506618+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:09:01.506618+0000","last_undegraded":"2023-09-23T23:09:01.506618+0000","last_fullsized":"2023-09-23T23:09:01.506618+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":159,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:09.355467+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:09:09.355467+0000","last_peered":"2023-09-23T23:09:09.355467+0000","last_clean":"2023-09-23T23:09:09.355467+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:09:09.355467+0000","last_undegraded":"2023-09-23T23:09:09.355467+0000","last_fullsized":"2023-09-23T23:09:09.355467+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":476,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.494970+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:09:01.494970+0000","last_peered":"2023-09-23T23:09:01.494970+0000","last_clean":"2023-09-23T23:09:01.494970+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:09:01.494970+0000","last_undegraded":"2023-09-23T23:09:01.494970+0000","last_fullsized":"2023-09-23T23:09:01.494970+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":391,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493376+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:09:01.493376+0000","last_peered":"2023-09-23T23:09:01.493376+0000","last_clean":"2023-09-23T23:09:01.493376+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:09:01.493376+0000","last_undegraded":"2023-09-23T23:09:01.493376+0000","last_fullsized":"2023-09-23T23:09:01.493376+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"475'7164","reported_seq":12334,"reported_epoch":475,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:09:14.198200+0000","last_change":"2023-09-23T23:06:32.889539+0000","last_active":"2023-09-23T23:09:14.198200+0000","last_peered":"2023-09-23T23:09:14.198200+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:06:32.875369+0000","last_became_peered":"2023-09-23T23:06:32.875369+0000","last_unstale":"2023-09-23T23:09:14.198200+0000","last_undegraded":"2023-09-23T23:09:14.198200+0000","last_fullsized":"2023-09-23T23:09:14.198200+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7164,"log_dups_size":0,"ondisk_log_size":7164,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37978112,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4515,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":580}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":378,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506147+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:09:01.506147+0000","last_peered":"2023-09-23T23:09:01.506147+0000","last_clean":"2023-09-23T23:09:01.506147+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:09:01.506147+0000","last_undegraded":"2023-09-23T23:09:01.506147+0000","last_fullsized":"2023-09-23T23:09:01.506147+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":500,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493853+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:09:01.493853+0000","last_peered":"2023-09-23T23:09:01.493853+0000","last_clean":"2023-09-23T23:09:01.493853+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:09:01.493853+0000","last_undegraded":"2023-09-23T23:09:01.493853+0000","last_fullsized":"2023-09-23T23:09:01.493853+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"475'7343","reported_seq":13194,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:14.344180+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:09:14.344180+0000","last_peered":"2023-09-23T23:09:14.344180+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:09:14.344180+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"414'296","ondisk_log_start":"414'296","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7047,"log_dups_size":0,"ondisk_log_size":7047,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34177024,"num_objects":522,"num_object_clones":0,"num_object_copies":1566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1005,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4653,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1965,"num_bytes_recovered":128581632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":178},{"shards":"3,5,6","objects":317},{"shards":"3,6","objects":27}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.491780+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:09:01.491780+0000","last_peered":"2023-09-23T23:09:01.491780+0000","last_clean":"2023-09-23T23:09:01.491780+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:09:01.491780+0000","last_undegraded":"2023-09-23T23:09:01.491780+0000","last_fullsized":"2023-09-23T23:09:01.491780+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":171,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492906+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:09:01.492906+0000","last_peered":"2023-09-23T23:09:01.492906+0000","last_clean":"2023-09-23T23:09:01.492906+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:09:01.492906+0000","last_undegraded":"2023-09-23T23:09:01.492906+0000","last_fullsized":"2023-09-23T23:09:01.492906+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"475'7305","reported_seq":7405,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:15.338813+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:09:15.338813+0000","last_peered":"2023-09-23T23:09:15.338813+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:09:15.338813+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7305,"log_dups_size":0,"ondisk_log_size":7305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28934144,"num_objects":441,"num_object_clones":0,"num_object_copies":1323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4654,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1924},{"shards":"1,4,5","objects":483}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":385,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.507895+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:09:01.507895+0000","last_peered":"2023-09-23T23:09:01.507895+0000","last_clean":"2023-09-23T23:09:01.507895+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:09:01.507895+0000","last_undegraded":"2023-09-23T23:09:01.507895+0000","last_fullsized":"2023-09-23T23:09:01.507895+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":341,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.505875+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:09:01.505875+0000","last_peered":"2023-09-23T23:09:01.505875+0000","last_clean":"2023-09-23T23:09:01.505875+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:09:01.505875+0000","last_undegraded":"2023-09-23T23:09:01.505875+0000","last_fullsized":"2023-09-23T23:09:01.505875+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"475'7155","reported_seq":12164,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:16.670992+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:09:16.670992+0000","last_peered":"2023-09-23T23:09:16.670992+0000","last_clean":"2023-09-23T23:09:16.670992+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:09:16.670992+0000","last_undegraded":"2023-09-23T23:09:16.670992+0000","last_fullsized":"2023-09-23T23:09:16.670992+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35356672,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4523,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":104,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.488903+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:09:01.488903+0000","last_peered":"2023-09-23T23:09:01.488903+0000","last_clean":"2023-09-23T23:09:01.488903+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:09:01.488903+0000","last_undegraded":"2023-09-23T23:09:01.488903+0000","last_fullsized":"2023-09-23T23:09:01.488903+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":482,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506067+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:09:01.506067+0000","last_peered":"2023-09-23T23:09:01.506067+0000","last_clean":"2023-09-23T23:09:01.506067+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:09:01.506067+0000","last_undegraded":"2023-09-23T23:09:01.506067+0000","last_fullsized":"2023-09-23T23:09:01.506067+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"475'7353","reported_seq":13501,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:18.213364+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:09:18.213364+0000","last_peered":"2023-09-23T23:09:18.213364+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:09:18.213364+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25001984,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":215,"num_objects_missing":215,"num_objects_degraded":866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4704,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2056,"num_bytes_recovered":134742016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":120},{"shards":"1,6","objects":-47},{"shards":"3,5,6","objects":308}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":133,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493302+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:09:01.493302+0000","last_peered":"2023-09-23T23:09:01.493302+0000","last_clean":"2023-09-23T23:09:01.493302+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:09:01.493302+0000","last_undegraded":"2023-09-23T23:09:01.493302+0000","last_fullsized":"2023-09-23T23:09:01.493302+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492083+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:09:01.492083+0000","last_peered":"2023-09-23T23:09:01.492083+0000","last_clean":"2023-09-23T23:09:01.492083+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:09:01.492083+0000","last_undegraded":"2023-09-23T23:09:01.492083+0000","last_fullsized":"2023-09-23T23:09:01.492083+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"475'7479","reported_seq":7919,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.626773+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:09:16.626773+0000","last_peered":"2023-09-23T23:09:16.626773+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:09:16.626773+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7479,"log_dups_size":0,"ondisk_log_size":7479,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30113792,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4763,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1585},{"shards":"1,2,3","objects":168}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":166,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492845+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:09:01.492845+0000","last_peered":"2023-09-23T23:09:01.492845+0000","last_clean":"2023-09-23T23:09:01.492845+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:09:01.492845+0000","last_undegraded":"2023-09-23T23:09:01.492845+0000","last_fullsized":"2023-09-23T23:09:01.492845+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":297,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506453+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:09:01.506453+0000","last_peered":"2023-09-23T23:09:01.506453+0000","last_clean":"2023-09-23T23:09:01.506453+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:09:01.506453+0000","last_undegraded":"2023-09-23T23:09:01.506453+0000","last_fullsized":"2023-09-23T23:09:01.506453+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"475'7370","reported_seq":11487,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:17.738966+0000","last_change":"2023-09-23T23:07:56.507184+0000","last_active":"2023-09-23T23:09:17.738966+0000","last_peered":"2023-09-23T23:09:17.738966+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.488393+0000","last_became_peered":"2023-09-23T23:07:56.488393+0000","last_unstale":"2023-09-23T23:09:17.738966+0000","last_undegraded":"2023-09-23T23:07:56.465702+0000","last_fullsized":"2023-09-23T23:07:56.443434+0000","mapping_epoch":459,"log_start":"415'1298","ondisk_log_start":"415'1298","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6072,"log_dups_size":0,"ondisk_log_size":6072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33882112,"num_objects":517,"num_object_clones":0,"num_object_copies":1551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4687,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1998,"num_bytes_recovered":130842624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,5","objects":132},{"shards":"1,6","objects":385}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":180,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.495628+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:09:01.495628+0000","last_peered":"2023-09-23T23:09:01.495628+0000","last_clean":"2023-09-23T23:09:01.495628+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:09:01.495628+0000","last_undegraded":"2023-09-23T23:09:01.495628+0000","last_fullsized":"2023-09-23T23:09:01.495628+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":477,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506951+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:09:01.506951+0000","last_peered":"2023-09-23T23:09:01.506951+0000","last_clean":"2023-09-23T23:09:01.506951+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:09:01.506951+0000","last_undegraded":"2023-09-23T23:09:01.506951+0000","last_fullsized":"2023-09-23T23:09:01.506951+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"475'7306","reported_seq":7403,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.748325+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:09:16.748325+0000","last_peered":"2023-09-23T23:09:16.748325+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:09:16.748325+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7306,"log_dups_size":0,"ondisk_log_size":7306,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32604160,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4642,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":466},{"shards":"2,3","objects":-1898}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506451+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:09:01.506451+0000","last_peered":"2023-09-23T23:09:01.506451+0000","last_clean":"2023-09-23T23:09:01.506451+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:09:01.506451+0000","last_undegraded":"2023-09-23T23:09:01.506451+0000","last_fullsized":"2023-09-23T23:09:01.506451+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":485,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.509060+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:09:01.509060+0000","last_peered":"2023-09-23T23:09:01.509060+0000","last_clean":"2023-09-23T23:09:01.509060+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:09:01.509060+0000","last_undegraded":"2023-09-23T23:09:01.509060+0000","last_fullsized":"2023-09-23T23:09:01.509060+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"475'7387","reported_seq":11653,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:18.228304+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:09:18.228304+0000","last_peered":"2023-09-23T23:09:18.228304+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:09:18.228304+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7387,"log_dups_size":0,"ondisk_log_size":7387,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25985024,"num_objects":396,"num_object_clones":0,"num_object_copies":1188,"num_objects_missing_on_primary":236,"num_objects_missing":236,"num_objects_degraded":667,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4733,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2081,"num_bytes_recovered":136249344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,2,3","objects":149},{"shards":"0,2,3,6","objects":1},{"shards":"0,3,6","objects":300},{"shards":"2,6","objects":-54}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":498,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.505902+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:09:01.505902+0000","last_peered":"2023-09-23T23:09:01.505902+0000","last_clean":"2023-09-23T23:09:01.505902+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:09:01.505902+0000","last_undegraded":"2023-09-23T23:09:01.505902+0000","last_fullsized":"2023-09-23T23:09:01.505902+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":503,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.507079+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:09:01.507079+0000","last_peered":"2023-09-23T23:09:01.507079+0000","last_clean":"2023-09-23T23:09:01.507079+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:09:01.507079+0000","last_undegraded":"2023-09-23T23:09:01.507079+0000","last_fullsized":"2023-09-23T23:09:01.507079+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"475'7226","reported_seq":11150,"reported_epoch":475,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:14.175864+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:09:14.175864+0000","last_peered":"2023-09-23T23:09:14.175864+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:09:14.175864+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:09:14.175864+0000","mapping_epoch":429,"log_start":"414'66","ondisk_log_start":"414'66","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7160,"log_dups_size":0,"ondisk_log_size":7160,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4570,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1914,"num_bytes_recovered":125435904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":173},{"shards":"1,3,6","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":148,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493629+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:09:01.493629+0000","last_peered":"2023-09-23T23:09:01.493629+0000","last_clean":"2023-09-23T23:09:01.493629+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:09:01.493629+0000","last_undegraded":"2023-09-23T23:09:01.493629+0000","last_fullsized":"2023-09-23T23:09:01.493629+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":381,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.489819+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:09:01.489819+0000","last_peered":"2023-09-23T23:09:01.489819+0000","last_clean":"2023-09-23T23:09:01.489819+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:09:01.489819+0000","last_undegraded":"2023-09-23T23:09:01.489819+0000","last_fullsized":"2023-09-23T23:09:01.489819+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"475'9483","reported_seq":10952,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.759265+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:09:16.759265+0000","last_peered":"2023-09-23T23:09:16.759265+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:09:16.759265+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"418'3674","ondisk_log_start":"418'3674","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63701016,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9483,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":297},{"shards":"2,4","objects":-2893}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":395,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.487640+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:09:01.487640+0000","last_peered":"2023-09-23T23:09:01.487640+0000","last_clean":"2023-09-23T23:09:01.487640+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:09:01.487640+0000","last_undegraded":"2023-09-23T23:09:01.487640+0000","last_fullsized":"2023-09-23T23:09:01.487640+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":313,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493157+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:09:01.493157+0000","last_peered":"2023-09-23T23:09:01.493157+0000","last_clean":"2023-09-23T23:09:01.493157+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:09:01.493157+0000","last_undegraded":"2023-09-23T23:09:01.493157+0000","last_fullsized":"2023-09-23T23:09:01.493157+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":216,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506310+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:09:01.506310+0000","last_peered":"2023-09-23T23:09:01.506310+0000","last_clean":"2023-09-23T23:09:01.506310+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:09:01.506310+0000","last_undegraded":"2023-09-23T23:09:01.506310+0000","last_fullsized":"2023-09-23T23:09:01.506310+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"475'9241","reported_seq":9360,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:14.323358+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:09:14.323358+0000","last_peered":"2023-09-23T23:09:14.323358+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:09:14.323358+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"425'4667","ondisk_log_start":"425'4667","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4574,"log_dups_size":0,"ondisk_log_size":4574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9241,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1035},{"shards":"1,3","objects":-3790}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":211,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.486454+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:09:01.486454+0000","last_peered":"2023-09-23T23:09:01.486454+0000","last_clean":"2023-09-23T23:09:01.486454+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:09:01.486454+0000","last_undegraded":"2023-09-23T23:09:01.486454+0000","last_fullsized":"2023-09-23T23:09:01.486454+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":513,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506709+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:09:01.506709+0000","last_peered":"2023-09-23T23:09:01.506709+0000","last_clean":"2023-09-23T23:09:01.506709+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:09:01.506709+0000","last_undegraded":"2023-09-23T23:09:01.506709+0000","last_fullsized":"2023-09-23T23:09:01.506709+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":380,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.487563+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:09:01.487563+0000","last_peered":"2023-09-23T23:09:01.487563+0000","last_clean":"2023-09-23T23:09:01.487563+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:09:01.487563+0000","last_undegraded":"2023-09-23T23:09:01.487563+0000","last_fullsized":"2023-09-23T23:09:01.487563+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"475'7393","reported_seq":13376,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:14.357700+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:09:14.357700+0000","last_peered":"2023-09-23T23:09:14.357700+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:09:14.357700+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"414'491","ondisk_log_start":"414'491","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30900224,"num_objects":471,"num_object_clones":0,"num_object_copies":1413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1050,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4702,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2014,"num_bytes_recovered":131923968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":182},{"shards":"3,5,6","objects":315},{"shards":"3,6","objects":-26}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":455,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.486115+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:09:01.486115+0000","last_peered":"2023-09-23T23:09:01.486115+0000","last_clean":"2023-09-23T23:09:01.486115+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:09:01.486115+0000","last_undegraded":"2023-09-23T23:09:01.486115+0000","last_fullsized":"2023-09-23T23:09:01.486115+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":307,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.485881+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:09:01.485881+0000","last_peered":"2023-09-23T23:09:01.485881+0000","last_clean":"2023-09-23T23:09:01.485881+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:09:01.485881+0000","last_undegraded":"2023-09-23T23:09:01.485881+0000","last_fullsized":"2023-09-23T23:09:01.485881+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":412,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492740+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:09:01.492740+0000","last_peered":"2023-09-23T23:09:01.492740+0000","last_clean":"2023-09-23T23:09:01.492740+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:09:01.492740+0000","last_undegraded":"2023-09-23T23:09:01.492740+0000","last_fullsized":"2023-09-23T23:09:01.492740+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"475'9266","reported_seq":17649,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:14.344034+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:09:14.344034+0000","last_peered":"2023-09-23T23:09:14.344034+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:09:14.344034+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"425'4734","ondisk_log_start":"425'4734","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4532,"log_dups_size":0,"ondisk_log_size":4532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64094208,"num_objects":978,"num_object_clones":0,"num_object_copies":2934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9266,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4144,"num_bytes_recovered":271581184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":358},{"shards":"3,5","objects":1},{"shards":"3,5,6","objects":619}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":386,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.489482+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:09:01.489482+0000","last_peered":"2023-09-23T23:09:01.489482+0000","last_clean":"2023-09-23T23:09:01.489482+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:09:01.489482+0000","last_undegraded":"2023-09-23T23:09:01.489482+0000","last_fullsized":"2023-09-23T23:09:01.489482+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":587,"reported_epoch":474,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:01.486161+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:09:01.486161+0000","last_peered":"2023-09-23T23:09:01.486161+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:09:01.486161+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:09:01.486161+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":166,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.492204+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:09:01.492204+0000","last_peered":"2023-09-23T23:09:01.492204+0000","last_clean":"2023-09-23T23:09:01.492204+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:09:01.492204+0000","last_undegraded":"2023-09-23T23:09:01.492204+0000","last_fullsized":"2023-09-23T23:09:01.492204+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":393,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.509208+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:09:01.509208+0000","last_peered":"2023-09-23T23:09:01.509208+0000","last_clean":"2023-09-23T23:09:01.509208+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:09:01.509208+0000","last_undegraded":"2023-09-23T23:09:01.509208+0000","last_fullsized":"2023-09-23T23:09:01.509208+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"475'7355","reported_seq":7582,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:14.326496+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:09:14.326496+0000","last_peered":"2023-09-23T23:09:14.326496+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:09:14.326496+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"414'644","ondisk_log_start":"414'644","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25460736,"num_objects":388,"num_object_clones":0,"num_object_copies":1164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2754,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4030464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,1,3","objects":183},{"shards":"0,3,6","objects":321},{"shards":"1,3","objects":-2110}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":399,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.491923+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:09:01.491923+0000","last_peered":"2023-09-23T23:09:01.491923+0000","last_clean":"2023-09-23T23:09:01.491923+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:09:01.491923+0000","last_undegraded":"2023-09-23T23:09:01.491923+0000","last_fullsized":"2023-09-23T23:09:01.491923+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":369,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.505923+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:09:01.505923+0000","last_peered":"2023-09-23T23:09:01.505923+0000","last_clean":"2023-09-23T23:09:01.505923+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:09:01.505923+0000","last_undegraded":"2023-09-23T23:09:01.505923+0000","last_fullsized":"2023-09-23T23:09:01.505923+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":335,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.486684+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:09:01.486684+0000","last_peered":"2023-09-23T23:09:01.486684+0000","last_clean":"2023-09-23T23:09:01.486684+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:09:01.486684+0000","last_undegraded":"2023-09-23T23:09:01.486684+0000","last_fullsized":"2023-09-23T23:09:01.486684+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"475'9320","reported_seq":9396,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:16.749587+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:09:16.749587+0000","last_peered":"2023-09-23T23:09:16.749587+0000","last_clean":"2023-09-23T23:09:16.749587+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:09:16.749587+0000","last_undegraded":"2023-09-23T23:09:16.749587+0000","last_fullsized":"2023-09-23T23:09:16.749587+0000","mapping_epoch":410,"log_start":"418'3511","ondisk_log_start":"418'3511","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59768832,"num_objects":912,"num_object_clones":0,"num_object_copies":2736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9320,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":334,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.508968+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:09:01.508968+0000","last_peered":"2023-09-23T23:09:01.508968+0000","last_clean":"2023-09-23T23:09:01.508968+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:09:01.508968+0000","last_undegraded":"2023-09-23T23:09:01.508968+0000","last_fullsized":"2023-09-23T23:09:01.508968+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":332,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.487943+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:09:01.487943+0000","last_peered":"2023-09-23T23:09:01.487943+0000","last_clean":"2023-09-23T23:09:01.487943+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:09:01.487943+0000","last_undegraded":"2023-09-23T23:09:01.487943+0000","last_fullsized":"2023-09-23T23:09:01.487943+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":297,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.488919+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:09:01.488919+0000","last_peered":"2023-09-23T23:09:01.488919+0000","last_clean":"2023-09-23T23:09:01.488919+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:09:01.488919+0000","last_undegraded":"2023-09-23T23:09:01.488919+0000","last_fullsized":"2023-09-23T23:09:01.488919+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"475'9230","reported_seq":9314,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.783017+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:09:16.783017+0000","last_peered":"2023-09-23T23:09:16.783017+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:09:16.783017+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"418'3383","ondisk_log_start":"418'3383","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5847,"log_dups_size":0,"ondisk_log_size":5847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58982400,"num_objects":900,"num_object_clones":0,"num_object_copies":2700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9230,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":900},{"shards":"1,2","objects":-3869}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":198,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.493074+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:09:01.493074+0000","last_peered":"2023-09-23T23:09:01.493074+0000","last_clean":"2023-09-23T23:09:01.493074+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:09:01.493074+0000","last_undegraded":"2023-09-23T23:09:01.493074+0000","last_fullsized":"2023-09-23T23:09:01.493074+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":173,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.507005+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:09:01.507005+0000","last_peered":"2023-09-23T23:09:01.507005+0000","last_clean":"2023-09-23T23:09:01.507005+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:09:01.507005+0000","last_undegraded":"2023-09-23T23:09:01.507005+0000","last_fullsized":"2023-09-23T23:09:01.507005+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":181,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.494307+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:09:01.494307+0000","last_peered":"2023-09-23T23:09:01.494307+0000","last_clean":"2023-09-23T23:09:01.494307+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:09:01.494307+0000","last_undegraded":"2023-09-23T23:09:01.494307+0000","last_fullsized":"2023-09-23T23:09:01.494307+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"475'9355","reported_seq":18465,"reported_epoch":475,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:15.493354+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:09:15.493354+0000","last_peered":"2023-09-23T23:09:15.493354+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:09:15.493354+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"418'3620","ondisk_log_start":"418'3620","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64028672,"num_objects":977,"num_object_clones":0,"num_object_copies":2931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":183,"num_objects_unfound":0,"num_objects_dirty":977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9355,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4514,"num_bytes_recovered":295829504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":183},{"shards":"1,4","objects":610}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":424,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.553980+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:09:11.553980+0000","last_peered":"2023-09-23T23:09:11.553980+0000","last_clean":"2023-09-23T23:09:11.553980+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:09:11.553980+0000","last_undegraded":"2023-09-23T23:09:11.553980+0000","last_fullsized":"2023-09-23T23:09:11.553980+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":96,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.507199+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:09:01.507199+0000","last_peered":"2023-09-23T23:09:01.507199+0000","last_clean":"2023-09-23T23:09:01.507199+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:09:01.507199+0000","last_undegraded":"2023-09-23T23:09:01.507199+0000","last_fullsized":"2023-09-23T23:09:01.507199+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":467,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506097+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:09:01.506097+0000","last_peered":"2023-09-23T23:09:01.506097+0000","last_clean":"2023-09-23T23:09:01.506097+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:09:01.506097+0000","last_undegraded":"2023-09-23T23:09:01.506097+0000","last_fullsized":"2023-09-23T23:09:01.506097+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"475'7448","reported_seq":9537,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:17.741939+0000","last_change":"2023-09-23T23:07:56.517407+0000","last_active":"2023-09-23T23:09:17.741939+0000","last_peered":"2023-09-23T23:09:17.741939+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.507318+0000","last_became_peered":"2023-09-23T23:07:56.507318+0000","last_unstale":"2023-09-23T23:09:17.741939+0000","last_undegraded":"2023-09-23T23:07:56.466720+0000","last_fullsized":"2023-09-23T23:07:56.443187+0000","mapping_epoch":459,"log_start":"415'1569","ondisk_log_start":"415'1569","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28770304,"num_objects":439,"num_object_clones":0,"num_object_copies":1317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4765,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":64454656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2","objects":-792},{"shards":"1,2,5","objects":136}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":66,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506378+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:09:01.506378+0000","last_peered":"2023-09-23T23:09:01.506378+0000","last_clean":"2023-09-23T23:09:01.506378+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:09:01.506378+0000","last_undegraded":"2023-09-23T23:09:01.506378+0000","last_fullsized":"2023-09-23T23:09:01.506378+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":226,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.488008+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:09:01.488008+0000","last_peered":"2023-09-23T23:09:01.488008+0000","last_clean":"2023-09-23T23:09:01.488008+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:09:01.488008+0000","last_undegraded":"2023-09-23T23:09:01.488008+0000","last_fullsized":"2023-09-23T23:09:01.488008+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"475'7423","reported_seq":7848,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.812038+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:09:16.812038+0000","last_peered":"2023-09-23T23:09:16.812038+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:09:16.812038+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7423,"log_dups_size":0,"ondisk_log_size":7423,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33783808,"num_objects":516,"num_object_clones":0,"num_object_copies":1548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4708,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1459},{"shards":"1,2,3","objects":146}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":297,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506873+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:09:01.506873+0000","last_peered":"2023-09-23T23:09:01.506873+0000","last_clean":"2023-09-23T23:09:01.506873+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:09:01.506873+0000","last_undegraded":"2023-09-23T23:09:01.506873+0000","last_fullsized":"2023-09-23T23:09:01.506873+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":498,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506754+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:09:01.506754+0000","last_peered":"2023-09-23T23:09:01.506754+0000","last_clean":"2023-09-23T23:09:01.506754+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:09:01.506754+0000","last_undegraded":"2023-09-23T23:09:01.506754+0000","last_fullsized":"2023-09-23T23:09:01.506754+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"475'7272","reported_seq":13186,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:18.240524+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:09:18.240524+0000","last_peered":"2023-09-23T23:09:18.240524+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:09:18.240524+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7272,"log_dups_size":0,"ondisk_log_size":7272,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30310400,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":234,"num_objects_missing":234,"num_objects_degraded":879,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4624,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1974,"num_bytes_recovered":129302528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":118},{"shards":"3,5,6","objects":298},{"shards":"3,6","objects":47}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":324,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.509140+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:09:01.509140+0000","last_peered":"2023-09-23T23:09:01.509140+0000","last_clean":"2023-09-23T23:09:01.509140+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:09:01.509140+0000","last_undegraded":"2023-09-23T23:09:01.509140+0000","last_fullsized":"2023-09-23T23:09:01.509140+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":394,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.506388+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:09:01.506388+0000","last_peered":"2023-09-23T23:09:01.506388+0000","last_clean":"2023-09-23T23:09:01.506388+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:09:01.506388+0000","last_undegraded":"2023-09-23T23:09:01.506388+0000","last_fullsized":"2023-09-23T23:09:01.506388+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"475'7278","reported_seq":8293,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:16.774303+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:09:16.774303+0000","last_peered":"2023-09-23T23:09:16.774303+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:09:16.774303+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7278,"log_dups_size":0,"ondisk_log_size":7278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27295744,"num_objects":417,"num_object_clones":0,"num_object_copies":1251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4647,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":495},{"shards":"1,2","objects":-1570}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":50,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.494175+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:09:01.494175+0000","last_peered":"2023-09-23T23:09:01.494175+0000","last_clean":"2023-09-23T23:09:01.494175+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:09:01.494175+0000","last_undegraded":"2023-09-23T23:09:01.494175+0000","last_fullsized":"2023-09-23T23:09:01.494175+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.494362+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:09:01.494362+0000","last_peered":"2023-09-23T23:09:01.494362+0000","last_clean":"2023-09-23T23:09:01.494362+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:09:01.494362+0000","last_undegraded":"2023-09-23T23:09:01.494362+0000","last_fullsized":"2023-09-23T23:09:01.494362+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"475'7223","reported_seq":7313,"reported_epoch":475,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:15.455793+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:09:15.455793+0000","last_peered":"2023-09-23T23:09:15.455793+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:09:15.455793+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7223,"log_dups_size":0,"ondisk_log_size":7223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34308096,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4573,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1732},{"shards":"1,4,5","objects":482}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":243,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.489886+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:09:01.489886+0000","last_peered":"2023-09-23T23:09:01.489886+0000","last_clean":"2023-09-23T23:09:01.489886+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:09:01.489886+0000","last_undegraded":"2023-09-23T23:09:01.489886+0000","last_fullsized":"2023-09-23T23:09:01.489886+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":469,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T23:09:01.505934+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:09:01.505934+0000","last_peered":"2023-09-23T23:09:01.505934+0000","last_clean":"2023-09-23T23:09:01.505934+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:09:01.505934+0000","last_undegraded":"2023-09-23T23:09:01.505934+0000","last_fullsized":"2023-09-23T23:09:01.505934+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1004732440,"num_objects":15332,"num_object_clones":0,"num_object_copies":45996,"num_objects_missing_on_primary":685,"num_objects_missing":685,"num_objects_degraded":42640,"num_objects_misplaced":183,"num_objects_unfound":0,"num_objects_dirty":15332,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148972,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33959,"num_bytes_recovered":2199191552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7441494016,"data_stored":7441481800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173788,"ondisk_log_size":173788,"up":78,"acting":58,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430715,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626940,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275948,"kb_used_data":381412,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92469204,"statfs":{"total":95995035648,"available":94688464896,"internally_reserved":0,"allocated":390565888,"data_stored":389499817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3,1,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.304,"15min":4.9790000000000001},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":6.4320000000000004,"5min":10.853,"15min":115.32899999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.371,"5min":1.1919999999999999,"15min":4.9989999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":115.348},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1559999999999997,"5min":3.2839999999999998,"15min":3.1099999999999999},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":38.259999999999998,"5min":149.947,"15min":149.947},"last":1.4319999999999999},{"interface":"front","average":{"1min":3.5950000000000002,"5min":3.4940000000000002,"15min":3.1030000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":38.530000000000001,"5min":150.22499999999999,"15min":150.22499999999999},"last":2.0270000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:08:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9420000000000002,"5min":5.5709999999999997,"15min":2.9260000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":45.051000000000002,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":4.8579999999999997,"5min":5.25,"15min":2.9969999999999999},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32500000000000001},"max":{"1min":45.258000000000003,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.314}]},{"osd":3,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.831,"5min":4.5940000000000003,"15min":3.2719999999999998},"min":{"1min":0.39300000000000002,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":47.734000000000002,"5min":142.24700000000001,"15min":142.24700000000001},"last":1.149},{"interface":"front","average":{"1min":3.1549999999999998,"5min":4.617,"15min":3.3210000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":47.162999999999997,"5min":143.86699999999999,"15min":143.86699999999999},"last":1.9950000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.1970000000000001,"15min":1.925},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.23999999999999999},"max":{"1min":7.3869999999999996,"5min":12.407999999999999,"15min":119.804},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.117,"15min":1.95},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.24099999999999999},"max":{"1min":8.8940000000000001,"5min":12.388,"15min":122.31999999999999},"last":0.375}]},{"osd":5,"last update":"Sat Sep 23 23:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.863,"15min":2.5600000000000001},"min":{"1min":0.42599999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":1.665,"5min":44.781999999999996,"15min":186.27600000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.9930000000000001,"15min":2.6840000000000002},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":8.7720000000000002,"5min":44.601999999999997,"15min":191.28700000000001},"last":0.40300000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.224,"15min":1.5880000000000001},"min":{"1min":0.46700000000000003,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":8.8670000000000009,"5min":8.8670000000000009,"15min":115.61},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.698,"5min":0.97499999999999998,"15min":1.7070000000000001},"min":{"1min":0.39600000000000002,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":8.2609999999999992,"5min":8.2609999999999992,"15min":115.578},"last":0.45400000000000001}]}]},{"osd":1,"up_from":84,"seq":360777253253,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1889720,"kb_used_data":918448,"kb_used_omap":9,"kb_used_meta":971254,"kb_avail":91855432,"statfs":{"total":95995035648,"available":94059962368,"internally_reserved":0,"allocated":940490752,"data_stored":938553024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":994565097},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":5.5030000000000001,"15min":2.544},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.389,"5min":279.30500000000001,"15min":279.30500000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.81100000000000005,"5min":5.6820000000000004,"15min":2.8799999999999999},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.22600000000000001},"max":{"1min":3.4500000000000002,"5min":279.39800000000002,"15min":279.39800000000002},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":7.96,"5min":6.2439999999999998,"15min":2.4449999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.527,"5min":161.61699999999999,"15min":161.61699999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":7.5199999999999996,"5min":6.4059999999999997,"15min":2.6059999999999999},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":118.29900000000001,"5min":162.61699999999999,"15min":162.61699999999999},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.4510000000000001,"15min":1.389},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":3.8039999999999998,"5min":59.424999999999997,"15min":59.424999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.5680000000000001,"15min":1.454},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":7.1580000000000004,"5min":55.174999999999997,"15min":55.174999999999997},"last":0.24199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2320000000000002,"5min":3.0569999999999999,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":67.813999999999993,"5min":129.69,"15min":151.036},"last":0.379},{"interface":"front","average":{"1min":5.4039999999999999,"5min":3.427,"15min":2.9180000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":67.796999999999997,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.318}]},{"osd":5,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.4419999999999999,"15min":2.2370000000000001},"min":{"1min":0.40300000000000002,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":1.522,"5min":18.228999999999999,"15min":116.098},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.0920000000000001,"15min":2.0419999999999998},"min":{"1min":0.38900000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.5800000000000001,"5min":6.585,"15min":116.08499999999999},"last":5.7779999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2759999999999998,"5min":2.1850000000000001,"15min":2.056},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":64.058000000000007,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.57999999999999996},{"interface":"front","average":{"1min":4.7190000000000003,"5min":2.3690000000000002,"15min":2.0110000000000001},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.33300000000000002},"max":{"1min":64.239999999999995,"5min":64.239999999999995,"15min":67.366},"last":3.7669999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":1.337,"15min":1.1910000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":5.7549999999999999,"5min":7.0960000000000001,"15min":8.1400000000000006},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.472,"5min":1.7290000000000001,"15min":1.478},"min":{"1min":0.40699999999999997,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":8.8719999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.44400000000000001}]}]},{"osd":0,"up_from":0,"seq":1516123455623,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259396,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":787848,"kb_used_data":489984,"kb_used_omap":83,"kb_used_meta":297772,"kb_avail":92957304,"statfs":{"total":95995035648,"available":95188279296,"internally_reserved":0,"allocated":501743616,"data_stored":499676480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":304918981},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.82999999999999996,"15min":1.212},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":4.1390000000000002,"5min":10.199999999999999,"15min":74.241},"last":0.27800000000000002},{"interface":"front","average":{"1min":1.5,"5min":1.29,"15min":1.5309999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":14.944000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":4.1109999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0670000000000002,"5min":2.5049999999999999,"15min":1.3009999999999999},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":83.897000000000006,"5min":83.897000000000006,"15min":83.897000000000006},"last":4.2290000000000001},{"interface":"front","average":{"1min":4.5289999999999999,"5min":2.1669999999999998,"15min":1.462},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":83.088999999999999,"5min":83.088999999999999,"15min":83.088999999999999},"last":4.343}]},{"osd":3,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7130000000000001,"5min":3.98,"15min":1.8380000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.23000000000000001},"max":{"1min":79.655000000000001,"5min":110.31699999999999,"15min":110.31699999999999},"last":4.1600000000000001},{"interface":"front","average":{"1min":7.2039999999999997,"5min":3.8650000000000002,"15min":1.851},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26100000000000001},"max":{"1min":79.677999999999997,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.73599999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":1.9279999999999999,"15min":1.1379999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":68.325000000000003,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":4.9020000000000001,"5min":2.2610000000000001,"15min":1.351},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":70.608000000000004,"5min":70.608000000000004,"15min":70.608000000000004},"last":4.2599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":3.105,"15min":1.6839999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.6600000000000001,"5min":139.405,"15min":139.405},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.002,"5min":2.9129999999999998,"15min":1.5089999999999999},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":4.0499999999999998,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.65100000000000002}]},{"osd":6,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":3.4180000000000001,"15min":1.8600000000000001},"min":{"1min":0.377,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.6989999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":4.431},{"interface":"front","average":{"1min":1.1439999999999999,"5min":3.1749999999999998,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34599999999999997,"15min":0.32200000000000001},"max":{"1min":4.0830000000000002,"5min":108.496,"15min":108.496},"last":0.67600000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.421,"15min":1.1890000000000001},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.1449999999999996,"5min":10.164,"15min":10.164},"last":0.623},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.393,"15min":1.2050000000000001},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":15.037000000000001,"5min":15.037000000000001,"15min":15.037000000000001},"last":4.1920000000000002}]}]},{"osd":3,"up_from":94,"seq":403726926200,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1570988,"kb_used_data":868708,"kb_used_omap":7,"kb_used_meta":702264,"kb_avail":92174164,"statfs":{"total":95995035648,"available":94386343936,"internally_reserved":0,"allocated":889556992,"data_stored":887510189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":719118952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.534,"15min":1.4099999999999999},"min":{"1min":0.22600000000000001,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":8.1660000000000004,"5min":17.015000000000001,"15min":122.289},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.004,"15min":1.1419999999999999},"min":{"1min":0.25600000000000001,"5min":0.218,"15min":0.17799999999999999},"max":{"1min":4.1470000000000002,"5min":11.382999999999999,"15min":122.041},"last":1.1779999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":2.1720000000000002,"15min":1.4710000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":10.935,"5min":23.969000000000001,"15min":23.969000000000001},"last":3.984},{"interface":"front","average":{"1min":1.109,"5min":1.9119999999999999,"15min":1.373},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.20699999999999999},"max":{"1min":4.0119999999999996,"5min":16.986000000000001,"15min":16.986000000000001},"last":2.6589999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:08:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8479999999999999,"5min":2.3370000000000002,"15min":1.232},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":55.311999999999998,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.437},{"interface":"front","average":{"1min":5.532,"5min":2.7480000000000002,"15min":1.5389999999999999},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.215},"max":{"1min":57.308,"5min":57.308,"15min":57.308},"last":4.0289999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.6629999999999998,"15min":2.1110000000000002},"min":{"1min":0.374,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":11.01,"5min":115.69,"15min":115.69},"last":1.9910000000000001},{"interface":"front","average":{"1min":1.351,"5min":2.4009999999999998,"15min":2.0419999999999998},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":5.9169999999999998,"5min":115.617,"15min":115.617},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.401},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":4.8810000000000002,"5min":16.959,"15min":16.959},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.359,"5min":1.476,"15min":1.369},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":6.6130000000000004,"5min":10.545,"15min":10.545},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":2.589,"15min":1.8999999999999999},"min":{"1min":0.41699999999999998,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":77.707999999999998,"5min":77.707999999999998,"15min":77.707999999999998},"last":1.363},{"interface":"front","average":{"1min":5.4080000000000004,"5min":2.1930000000000001,"15min":1.6890000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":77.721999999999994,"5min":77.721999999999994,"15min":77.721999999999994},"last":1.3819999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.4630000000000001,"15min":1.272},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":4.5220000000000002,"5min":8.1620000000000008,"15min":8.1620000000000008},"last":1.1240000000000001},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.0499999999999998,"15min":1.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":1.9530000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019397,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1197932,"kb_used_data":286372,"kb_used_omap":0,"kb_used_meta":911552,"kb_avail":92547220,"statfs":{"total":95995035648,"available":94768353280,"internally_reserved":0,"allocated":293244928,"data_stored":292178335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933429248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.0409999999999999,"15min":2.2360000000000002},"min":{"1min":0.27400000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.778,"5min":99.132999999999996,"15min":156.375},"last":1.925},{"interface":"front","average":{"1min":0.95399999999999996,"5min":2.1829999999999998,"15min":2.371},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":2.9910000000000001,"5min":101.062,"15min":157.02199999999999},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":3.4900000000000002,"15min":3.1970000000000001},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":6.915,"5min":95.591999999999999,"15min":98.846999999999994},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":3.4039999999999999,"15min":3.157},"min":{"1min":0.33800000000000002,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":15.728,"5min":97.929000000000002,"15min":98.938999999999993},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":11.628,"5min":4.2610000000000001,"15min":2.1459999999999999},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":124.819,"5min":124.819,"15min":124.819},"last":2.8290000000000002},{"interface":"front","average":{"1min":10.571999999999999,"5min":4.077,"15min":1.9850000000000001},"min":{"1min":0.36599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":122.086,"5min":122.086,"15min":122.086},"last":1.2}]},{"osd":3,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":2.5379999999999998,"15min":1.841},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":55.466999999999999,"5min":60.274000000000001,"15min":60.274000000000001},"last":1.8109999999999999},{"interface":"front","average":{"1min":3.3820000000000001,"5min":2.677,"15min":1.9259999999999999},"min":{"1min":0.38500000000000001,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":55.625999999999998,"5min":60.677999999999997,"15min":60.677999999999997},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.177,"15min":1.764},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.214},"max":{"1min":3.036,"5min":10.130000000000001,"15min":148.61600000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.522,"5min":1.157,"15min":1.861},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":8.5489999999999995,"5min":8.5489999999999995,"15min":151.905},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.3899999999999999,"15min":1.4359999999999999},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.2590000000000003,"5min":35.896000000000001,"15min":35.896000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.5629999999999999,"15min":1.403},"min":{"1min":0.35499999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":7.008,"5min":35.908000000000001,"15min":35.908000000000001},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.161,"15min":1.288},"min":{"1min":0.373,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":3.1080000000000001,"5min":7.0789999999999997,"15min":12.84},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.28,"15min":1.232},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":15.808,"5min":15.808,"15min":15.808},"last":0.56000000000000005}]}]},{"osd":5,"up_from":0,"seq":154618823171,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2934833152,"data_stored":2934833152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":933953536,"data_stored":933953536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":491458560,"data_stored":491454488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":876806144,"data_stored":876806144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":288034816,"data_stored":288030744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":384372736,"data_stored":384368664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:09:21.521 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:21.551 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:09:21.594 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:21.774 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:21.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:21.890 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:21.935 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:21.982 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:22.106 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:22.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:22.219 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:22.259 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:22.317 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:22 smithi052 ceph-mon[180979]: pgmap v1499: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 884 MiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 517 op/s; 41930/42495 objects degraded (98.670%); 76/42495 objects misplaced (0.179%); 12 MiB/s, 186 objects/s recovering 2023-09-23T23:09:22.325 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:22 smithi052 ceph-mon[187743]: pgmap v1499: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 884 MiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 517 op/s; 41930/42495 objects degraded (98.670%); 76/42495 objects misplaced (0.179%); 12 MiB/s, 186 objects/s recovering 2023-09-23T23:09:22.346 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:22.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:22 smithi151 ceph-mon[159210]: pgmap v1499: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 884 MiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 517 op/s; 41930/42495 objects degraded (98.670%); 76/42495 objects misplaced (0.179%); 12 MiB/s, 186 objects/s recovering 2023-09-23T23:09:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:22.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:22.456 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:22.527 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:22.731 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:22.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:22.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:22.838 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:22.881 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:22.944 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:22.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:23.037 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:09:23.047 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:23.073 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:23.115 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:23.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:23.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:23.229 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:23.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:23 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4251322820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:09:23.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:23 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4251322820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:09:23.269 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:23.340 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:23.354 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:23 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4251322820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:09:23.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:23.455 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:23.501 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:23.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:09:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:09:23] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T23:09:24.157 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:24.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:24.260 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:24.274 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[187743]: pgmap v1500: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 812 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 558 op/s; 41850/39027 objects degraded (107.233%); 76/39027 objects misplaced (0.195%); 11 MiB/s, 171 objects/s recovering 2023-09-23T23:09:24.274 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:24.274 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/4251322820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:09:24.274 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[187743]: osdmap e476: 8 total, 8 up, 5 in 2023-09-23T23:09:24.274 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[187743]: failed to encode map e476 with expected crc 2023-09-23T23:09:24.274 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[187743]: failed to encode map e476 with expected crc 2023-09-23T23:09:24.275 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[180979]: pgmap v1500: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 812 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 558 op/s; 41850/39027 objects degraded (107.233%); 76/39027 objects misplaced (0.195%); 11 MiB/s, 171 objects/s recovering 2023-09-23T23:09:24.275 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:24.275 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/4251322820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:09:24.275 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[180979]: osdmap e476: 8 total, 8 up, 5 in 2023-09-23T23:09:24.275 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[180979]: failed to encode map e476 with expected crc 2023-09-23T23:09:24.275 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:24 smithi052 ceph-mon[180979]: failed to encode map e476 with expected crc 2023-09-23T23:09:24.316 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:24.319 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:24.320 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:24 smithi151 ceph-mon[159210]: pgmap v1500: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 812 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 558 op/s; 41850/39027 objects degraded (107.233%); 76/39027 objects misplaced (0.195%); 11 MiB/s, 171 objects/s recovering 2023-09-23T23:09:24.320 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:24 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:24.320 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:24 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/4251322820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:09:24.320 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:24 smithi151 ceph-mon[159210]: osdmap e476: 8 total, 8 up, 5 in 2023-09-23T23:09:24.320 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:24 smithi151 ceph-mon[159210]: failed to encode map e476 with expected crc 2023-09-23T23:09:24.320 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:24 smithi151 ceph-mon[159210]: failed to encode map e476 with expected crc 2023-09-23T23:09:24.415 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:24.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:24.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:24.521 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:09:24.523 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:24.559 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:24.628 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:24.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:24.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:24.736 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:24.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:24.814 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:09:24.841 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:24.848 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:09:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:24.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:24.886 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:09:24.937 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:09:24.954 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:24.979 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:09:25.020 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:09:25.049 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:25.063 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:09:25.098 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.118 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:09:25.169 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:25.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:25.179 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:25 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/496429502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:25.179 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:25 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/496429502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:25.279 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:25.313 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:25.358 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:25.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:25.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:25 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/496429502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:25.464 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:25.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:25.611 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:25.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:25.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:25.717 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:25.747 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:26.277 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:09:26.418 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:26 smithi052 ceph-mon[187743]: pgmap v1502: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 746 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 516 op/s; 40794/35835 objects degraded (113.838%); 76/35835 objects misplaced (0.212%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:09:26.418 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:26 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 40794/35835 objects degraded (113.838%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:26.419 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:26 smithi052 ceph-mon[180979]: pgmap v1502: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 746 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 516 op/s; 40794/35835 objects degraded (113.838%); 76/35835 objects misplaced (0.212%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:09:26.419 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:26 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 40794/35835 objects degraded (113.838%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:26 smithi151 ceph-mon[159210]: pgmap v1502: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 746 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 516 op/s; 40794/35835 objects degraded (113.838%); 76/35835 objects misplaced (0.212%); 12 MiB/s, 192 objects/s recovering 2023-09-23T23:09:26.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:26 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 40794/35835 objects degraded (113.838%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:26.628 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:26.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:26.750 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:26.780 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:26.846 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:26.851 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:26.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:26.962 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:26.999 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:27.084 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:27.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:27.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:27.195 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:27.240 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:27.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:27 smithi052 ceph-mon[187743]: pgmap v1503: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 727 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 511 op/s; 40794/34941 objects degraded (116.751%); 12 MiB/s, 191 objects/s recovering 2023-09-23T23:09:27.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:27 smithi052 ceph-mon[180979]: pgmap v1503: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 727 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 511 op/s; 40794/34941 objects degraded (116.751%); 12 MiB/s, 191 objects/s recovering 2023-09-23T23:09:27.305 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:27.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:27.411 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:27.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:27 smithi151 ceph-mon[159210]: pgmap v1503: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 727 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 511 op/s; 40794/34941 objects degraded (116.751%); 12 MiB/s, 191 objects/s recovering 2023-09-23T23:09:27.455 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:27.525 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "size": 20, 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 600, 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [ 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138140 10.8 10:10bb8653:::benchmark_data_smithi052_461816_object41339:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 539.90895843299995, 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.272336093, 2023-09-23T23:09:27.526 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138140 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.527 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612733+0000", 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612731+0000", 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.528 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612741+0000", 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.4800000000000007e-06 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612762+0000", 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1761000000000001e-05 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.529 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612772+0000", 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6229999999999992e-06 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.612837+0000", 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4789999999999995e-05 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613071+0000", 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00023441400000000001 2023-09-23T23:09:27.530 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.613293+0000", 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00022140800000000001 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.616187+0000", 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028945080000000001 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.531 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.618246+0000", 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020593949999999999 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883729+0000", 2023-09-23T23:09:27.532 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.265482459 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.883802+0000", 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.3008000000000003e-05 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.885069+0000", 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012670579999999999 2023-09-23T23:09:27.533 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:138142 10.8 10:10a11927:::benchmark_data_smithi052_461816_object41341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 539.89486696799997, 2023-09-23T23:09:27.534 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.26049065999999998, 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 138142 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:09:27.535 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626824+0000", 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.536 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626822+0000", 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999971 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626834+0000", 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2293000000000001e-05 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.537 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626866+0000", 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.1934999999999998e-05 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626881+0000", 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.5491000000000001e-05 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.538 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.626950+0000", 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.8701000000000005e-05 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627110+0000", 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000160347 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.539 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 2,5", 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.627220+0000", 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000110009 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.629339+0000", 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0021183280000000001 2023-09-23T23:09:27.540 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.635341+0000", 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0060026009999999998 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886154+0000", 2023-09-23T23:09:27.541 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25081289200000001 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.886210+0000", 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5321000000000001e-05 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.542 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:27.887315+0000", 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001105396 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.45953.0:140723 10.6 10:68c22ebc:::benchmark_data_smithi052_461816_object43922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e385)", 2023-09-23T23:09:27.543 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 528.30666802099995, 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.195061394, 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.45953", 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/1601405544", 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 140723 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.544 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215023+0000", 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215022+0000", 2023-09-23T23:09:27.545 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215028+0000", 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.8850000000000004e-06 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215039+0000", 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0998e-05 2023-09-23T23:09:27.546 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215044+0000", 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.6380000000000001e-06 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215089+0000", 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.4929000000000003e-05 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.547 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215188+0000", 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8741999999999998e-05 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 4,5", 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.215268+0000", 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0049000000000006e-05 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217087+0000", 2023-09-23T23:09:27.548 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0018184539999999999 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.217880+0000", 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00079313599999999995 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410012+0000", 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19213219200000001 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.549 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410050+0000", 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.8262000000000001e-05 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:00:39.410085+0000", 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.4588000000000002e-05 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.550 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_repop(client.45953.0:188923 10.d e403/354 10:ba1021e3:::benchmark_data_smithi052_461816_object92122:head v 403'11913, mlcod=344'7206)", 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 376.76724969499998, 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.205832343, 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.551 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754442+0000", 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754440+0000", 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.552 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754450+0000", 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3270000000000007e-06 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754462+0000", 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.2313999999999999e-05 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.553 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754470+0000", 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2209999999999997e-06 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754510+0000", 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9437999999999997e-05 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.554 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.754531+0000", 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1557999999999999e-05 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.960234+0000", 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20570290699999999 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.555 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:03:10.960274+0000", 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9780999999999997e-05 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:44802 11.6 11:6ab43c8b:::benchmark_data_smithi052_606809_object44801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e417)", 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 299.60376598699997, 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22081410400000001, 2023-09-23T23:09:27.556 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 44802 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:09:27.557 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917925+0000", 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917924+0000", 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.558 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917950+0000", 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5999e-05 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917964+0000", 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4412e-05 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.559 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.917972+0000", 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.2800000000000003e-06 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918102+0000", 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012992200000000001 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918229+0000", 2023-09-23T23:09:27.560 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012657599999999999 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.918357+0000", 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012783599999999999 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.920436+0000", 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0020790959999999999 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.561 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:27.921317+0000", 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00088133199999999997 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.562 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138645+0000", 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.21732747999999999 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138699+0000", 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4261000000000001e-05 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:28.138739+0000", 2023-09-23T23:09:27.563 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.0448000000000003e-05 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:48637 11.6 11:6a40531f:::benchmark_data_smithi052_606809_object48636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e418)", 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 295.24718081600003, 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18496931799999999, 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.564 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 48637 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.565 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274510+0000", 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.999999 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.566 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274531+0000", 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1537000000000001e-05 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274548+0000", 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.6458000000000001e-05 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.567 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.274555+0000", 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.0389999999999999e-06 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.278791+0000", 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0042367860000000002 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.568 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.278955+0000", 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00016336700000000001 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.279146+0000", 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019095900000000001 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.569 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283306+0000", 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041598900000000003 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.283525+0000", 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00021912200000000001 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.570 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459382+0000", 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.17585730099999999 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459433+0000", 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1341999999999997e-05 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.571 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:32.459480+0000", 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6394999999999999e-05 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.572 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:63348 11.6 11:69e6f9e0:::benchmark_data_smithi052_606809_object63347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e419)", 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:50.778092+0000", 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 276.74359955400001, 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19583917300000001, 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.573 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 63348 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778092+0000", 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.574 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778092+0000", 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.575 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778090+0000", 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778116+0000", 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5840999999999999e-05 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.576 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778134+0000", 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8145e-05 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778143+0000", 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7329999999999992e-06 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778197+0000", 2023-09-23T23:09:27.577 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4280999999999997e-05 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778334+0000", 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013696300000000001 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.778463+0000", 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00012883999999999999 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.578 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790115+0000", 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.011652199 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791359+0000", 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0012437889999999999 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.579 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.973800+0000", 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18244153799999999 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.973876+0000", 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.5632999999999999e-05 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.580 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.973931+0000", 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.4935999999999997e-05 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:63357 11.6 11:631bdb38:::benchmark_data_smithi052_606809_object63356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e419)", 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:50.790896+0000", 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 276.730794855, 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.186466243, 2023-09-23T23:09:27.581 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 63357 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790896+0000", 2023-09-23T23:09:27.582 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790896+0000", 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790895+0000", 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.583 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790919+0000", 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.3969e-05 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790934+0000", 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.539e-05 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.584 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.790942+0000", 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.8120000000000008e-06 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791391+0000", 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00044866900000000001 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791559+0000", 2023-09-23T23:09:27.585 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000168643 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.791665+0000", 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000105408 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.793549+0000", 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00188408 2023-09-23T23:09:27.586 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.793803+0000", 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00025422800000000001 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.977275+0000", 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.18347145100000001 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.587 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.977326+0000", 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.1465000000000003e-05 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:50.977363+0000", 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6607999999999999e-05 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.588 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:66508 11.6 11:6896caef:::benchmark_data_smithi052_606809_object66507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:04:54.685256+0000", 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 272.83643488899997, 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.233501501, 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 66508 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.589 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685256+0000", 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685256+0000", 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685255+0000", 2023-09-23T23:09:27.590 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685281+0000", 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.6123e-05 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685295+0000", 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.4019000000000001e-05 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.591 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.685303+0000", 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0209999999999999e-06 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.687147+0000", 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001843988 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.687286+0000", 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000139083 2023-09-23T23:09:27.592 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.687419+0000", 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00013247500000000001 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.690234+0000", 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0028158459999999999 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.593 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.693191+0000", 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0029565989999999999 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918666+0000", 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.22547516100000001 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.594 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918721+0000", 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5090999999999998e-05 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:04:54.918758+0000", 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.6711000000000001e-05 2023-09-23T23:09:27.595 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:72298 11.6 11:6e6f37f0:::benchmark_data_smithi052_606809_object72297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e424)", 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:05.400540+0000", 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 262.121151107, 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.20392650300000001, 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.596 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 72298 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.597 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400540+0000", 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400540+0000", 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400539+0000", 2023-09-23T23:09:27.598 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400561+0000", 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2129000000000001e-05 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.599 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400574+0000", 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.3552e-05 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400582+0000", 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 7.4749999999999996e-06 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400637+0000", 2023-09-23T23:09:27.600 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.5077000000000001e-05 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400792+0000", 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00015541700000000001 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.601 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.400939+0000", 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.000146766 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.404192+0000", 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.003252376 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.602 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.406041+0000", 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.001849313 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.604379+0000", 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.19833788699999999 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.603 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.604427+0000", 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.8018e-05 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:05.604467+0000", 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 3.9919000000000002e-05 2023-09-23T23:09:27.604 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "osd_op(client.46892.0:85554 11.6 11:6191145a:::benchmark_data_smithi052_606809_object3400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e431)", 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:05:34.193299+0000", 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 233.32839206899999, 2023-09-23T23:09:27.605 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.260360021, 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "client_info": { 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "client": "client.46892", 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "tid": 85554 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.606 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193299+0000", 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193299+0000", 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.607 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193297+0000", 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999976 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193319+0000", 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2223000000000001e-05 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.608 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193330+0000", 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0623e-05 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193336+0000", 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.9410000000000004e-06 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193385+0000", 2023-09-23T23:09:27.609 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9261999999999997e-05 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "started", 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193470+0000", 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.5123000000000005e-05 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "waiting for subops from 1,2", 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.193534+0000", 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.4336000000000002e-05 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.610 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "op_commit", 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.194964+0000", 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0014301019999999999 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.195157+0000", 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00019253299999999999 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:09:27.611 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.453544+0000", 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.25838723800000002 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "commit_sent", 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.453609+0000", 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.5129999999999995e-05 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:05:34.453659+0000", 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.9976999999999999e-05 2023-09-23T23:09:27.612 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.15 e462/440 cost 1049576)", 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:08:11.154816+0000", 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 76.366875495000002, 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0053698610000000001, 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.613 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154816+0000", 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154816+0000", 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.614 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154815+0000", 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154825+0000", 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3579999999999992e-06 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154835+0000", 2023-09-23T23:09:27.615 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.9769999999999995e-06 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154844+0000", 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.1430000000000004e-06 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.154890+0000", 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4.6167e-05 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.616 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:11.160186+0000", 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0052957630000000002 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.5 e464/441 cost 1049576)", 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:08:20.968582+0000", 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 66.553108843000004, 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0046665589999999998, 2023-09-23T23:09:27.617 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.968582+0000", 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.968582+0000", 2023-09-23T23:09:27.618 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.968581+0000", 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.968598+0000", 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7563000000000001e-05 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.619 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.968623+0000", 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.4375e-05 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.968632+0000", 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.6579999999999997e-06 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.620 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.972984+0000", 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0043518860000000001 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:20.973249+0000", 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00026466700000000001 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.621 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.15 e466/440 cost 1049576)", 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:08:32.068846+0000", 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 55.452845705000001, 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0068443510000000003, 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.068846+0000", 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.622 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.068846+0000", 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.068844+0000", 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.623 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.068861+0000", 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7e-05 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.068884+0000", 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2429000000000002e-05 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.624 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.068893+0000", 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.3810000000000005e-06 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.072414+0000", 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0035206500000000002 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.625 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:32.075690+0000", 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0032761829999999998 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.1 e472/440 cost 1049576)", 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:08:51.819020+0000", 2023-09-23T23:09:27.626 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 35.702670961999999, 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0041604679999999996, 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819020+0000", 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.627 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819020+0000", 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819019+0000", 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.628 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819041+0000", 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1951999999999999e-05 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819071+0000", 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.9601e-05 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.629 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819080+0000", 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.7890000000000006e-06 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.819161+0000", 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.0266999999999996e-05 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:51.823181+0000", 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0040201940000000004 2023-09-23T23:09:27.630 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.5 e473/441 cost 1049576)", 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:08:53.134830+0000", 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 34.386861312999997, 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0048422359999999998, 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.631 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134830+0000", 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134830+0000", 2023-09-23T23:09:27.632 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134829+0000", 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999995 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134839+0000", 2023-09-23T23:09:27.633 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 9.8030000000000008e-06 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134860+0000", 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.0696999999999999e-05 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134866+0000", 2023-09-23T23:09:27.634 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.19e-06 2023-09-23T23:09:27.635 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.635 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.635 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.636 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.134924+0000", 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.7799e-05 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:53.139672+0000", 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0047483050000000004 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.637 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.5 e473/441 cost 1049576)", 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:08:55.752416+0000", 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 31.769275771, 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0081184369999999992, 2023-09-23T23:09:27.638 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.752416+0000", 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.639 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.752416+0000", 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.752414+0000", 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999981 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.752432+0000", 2023-09-23T23:09:27.640 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.8703e-05 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.752455+0000", 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2932999999999999e-05 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.752465+0000", 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0081000000000001e-05 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.641 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.754113+0000", 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0016476939999999999 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:08:55.760534+0000", 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0064210059999999999 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.642 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.15 e474/440 cost 1049576)", 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:09:10.099428+0000", 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.422263151999999, 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056534669999999997, 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099428+0000", 2023-09-23T23:09:27.643 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099428+0000", 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099427+0000", 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.644 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099445+0000", 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7611000000000001e-05 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099467+0000", 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.2198e-05 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.645 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099478+0000", 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.0794e-05 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.099539+0000", 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 6.1755999999999996e-05 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.105082+0000", 2023-09-23T23:09:27.646 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0055423449999999997 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.1 e474/440 cost 1049576)", 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:09:10.503520+0000", 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 17.018170864999998, 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0050454829999999999, 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.647 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503520+0000", 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503520+0000", 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.648 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503519+0000", 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503537+0000", 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.7805e-05 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.649 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503559+0000", 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.1911000000000001e-05 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503568+0000", 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.7900000000000005e-06 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.650 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.503620+0000", 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 5.198e-05 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:10.508566+0000", 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0049462869999999997 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.651 INFO:teuthology.orchestra.run.smithi052.stdout: "description": "MOSDPGPull(11.1 e475/440 cost 1049576)", 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "initiated_at": "2023-09-23T23:09:13.304072+0000", 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "age": 14.217619343000001, 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0056987449999999999, 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "type_data": { 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "flag_point": "reached pg", 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "events": [ 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "initiated", 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.304072+0000", 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.652 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "throttled", 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.304072+0000", 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "header_read", 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.304071+0000", 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 4294967295.9999986 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.653 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "all_read", 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.304091+0000", 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 1.9874e-05 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "dispatched", 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.304116+0000", 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 2.5843999999999999e-05 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "queued_for_pg", 2023-09-23T23:09:27.654 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.304125+0000", 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 8.9269999999999994e-06 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "reached_pg", 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.309486+0000", 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.0053608220000000003 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: }, 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: { 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "event": "done", 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "time": "2023-09-23T23:09:13.309771+0000", 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: "duration": 0.00028449500000000001 2023-09-23T23:09:27.655 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.656 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.656 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.656 INFO:teuthology.orchestra.run.smithi052.stdout: } 2023-09-23T23:09:27.656 INFO:teuthology.orchestra.run.smithi052.stdout: ] 2023-09-23T23:09:27.656 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:09:27.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:27.657 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:09:27.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:09:27.691 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:27.758 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:27.863 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:28.730 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:09:29.495 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:29.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:29.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:29.527 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:09:29.600 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:29.704 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:29.918 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:29.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:30.024 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:30.076 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:29 smithi052 ceph-mon[180979]: pgmap v1504: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 641 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 507 op/s; 40439/30834 objects degraded (131.151%); 12 MiB/s, 187 objects/s recovering 2023-09-23T23:09:30.078 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:29 smithi052 ceph-mon[187743]: pgmap v1504: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 641 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 507 op/s; 40439/30834 objects degraded (131.151%); 12 MiB/s, 187 objects/s recovering 2023-09-23T23:09:30.129 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:29 smithi151 ceph-mon[159210]: pgmap v1504: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 641 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 507 op/s; 40439/30834 objects degraded (131.151%); 12 MiB/s, 187 objects/s recovering 2023-09-23T23:09:30.211 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:30.447 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:30.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:30.554 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:30.652 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:30.857 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:09:30.884 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:09:31.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:30 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 40439/30834 objects degraded (131.151%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:31.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:30 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 40439/30834 objects degraded (131.151%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:31.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:30 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 40439/30834 objects degraded (131.151%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:31.572 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:31.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:31.687 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:31.746 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:31.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:31 smithi052 ceph-mon[187743]: pgmap v1505: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 551 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 40076/26496 objects degraded (151.253%); 9.4 MiB/s, 165 objects/s recovering 2023-09-23T23:09:31.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:31 smithi052 ceph-mon[187743]: from='client.47405 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:31.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:31 smithi052 ceph-mon[180979]: pgmap v1505: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 551 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 40076/26496 objects degraded (151.253%); 9.4 MiB/s, 165 objects/s recovering 2023-09-23T23:09:31.998 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:31 smithi052 ceph-mon[180979]: from='client.47405 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:31 smithi151 ceph-mon[159210]: pgmap v1505: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 551 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 40076/26496 objects degraded (151.253%); 9.4 MiB/s, 165 objects/s recovering 2023-09-23T23:09:32.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:31 smithi151 ceph-mon[159210]: from='client.47405 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:32.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:32.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:32.339 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:32.367 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:32.394 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1505,"stamp":"2023-09-23T23:09:30.812439+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":577771448,"num_objects":8832,"num_object_clones":0,"num_object_copies":26496,"num_objects_missing_on_primary":300,"num_objects_missing":300,"num_objects_degraded":40076,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8832,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":155756,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36331,"num_bytes_recovered":2340180416,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":174036,"ondisk_log_size":174036,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5914444,"kb_used_data":2104676,"kb_used_omap":99,"kb_used_meta":3809372,"kb_avail":462811316,"statfs":{"total":479975178240,"available":473918787584,"internally_reserved":0,"allocated":2155188224,"data_stored":2146973770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3900796950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,2,9,0,0,1,0,7],"upper_bound":1024},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-349503488,"num_objects":-5333,"num_object_clones":0,"num_object_copies":-15999,"num_objects_missing_on_primary":-387,"num_objects_missing":-387,"num_objects_degraded":-1854,"num_objects_misplaced":-76,"num_objects_unfound":0,"num_objects_dirty":-5333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5333,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":98369536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004279"},"pg_stats":[{"pgid":"11.12","version":"476'7521","reported_seq":7618,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.883562+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:09:26.883562+0000","last_peered":"2023-09-23T23:09:26.883562+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:09:26.883562+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18513920,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4858,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":314},{"shards":"2,3","objects":-2174}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":488,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523345+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:09:11.523345+0000","last_peered":"2023-09-23T23:09:11.523345+0000","last_clean":"2023-09-23T23:09:11.523345+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:09:11.523345+0000","last_undegraded":"2023-09-23T23:09:11.523345+0000","last_fullsized":"2023-09-23T23:09:11.523345+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":148,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543793+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:09:11.543793+0000","last_peered":"2023-09-23T23:09:11.543793+0000","last_clean":"2023-09-23T23:09:11.543793+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:09:11.543793+0000","last_undegraded":"2023-09-23T23:09:11.543793+0000","last_fullsized":"2023-09-23T23:09:11.543793+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"476'7506","reported_seq":11764,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:29.336031+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:09:29.336031+0000","last_peered":"2023-09-23T23:09:29.336031+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:09:29.336031+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:09:29.336031+0000","mapping_epoch":441,"log_start":"414'639","ondisk_log_start":"414'639","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18186240,"num_objects":278,"num_object_clones":0,"num_object_copies":834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4853,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2077,"num_bytes_recovered":136052736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3,6","objects":223},{"shards":"2,3,6","objects":55}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543748+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:09:11.543748+0000","last_peered":"2023-09-23T23:09:11.543748+0000","last_clean":"2023-09-23T23:09:11.543748+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:09:11.543748+0000","last_undegraded":"2023-09-23T23:09:11.543748+0000","last_fullsized":"2023-09-23T23:09:11.543748+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":494,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543046+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:09:11.543046+0000","last_peered":"2023-09-23T23:09:11.543046+0000","last_clean":"2023-09-23T23:09:11.543046+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:09:11.543046+0000","last_undegraded":"2023-09-23T23:09:11.543046+0000","last_fullsized":"2023-09-23T23:09:11.543046+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"476'7520","reported_seq":11905,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:29.251537+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:09:29.251537+0000","last_peered":"2023-09-23T23:09:29.251537+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:09:29.251537+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:09:29.251537+0000","mapping_epoch":428,"log_start":"415'860","ondisk_log_start":"415'860","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6660,"log_dups_size":0,"ondisk_log_size":6660,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15925248,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4864,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2145,"num_bytes_recovered":140574720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"1,3,6","objects":243}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":55,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.478993+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:09:11.478993+0000","last_peered":"2023-09-23T23:09:11.478993+0000","last_clean":"2023-09-23T23:09:11.478993+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:09:11.478993+0000","last_undegraded":"2023-09-23T23:09:11.478993+0000","last_fullsized":"2023-09-23T23:09:11.478993+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.542719+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:09:11.542719+0000","last_peered":"2023-09-23T23:09:11.542719+0000","last_clean":"2023-09-23T23:09:11.542719+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:09:11.542719+0000","last_undegraded":"2023-09-23T23:09:11.542719+0000","last_fullsized":"2023-09-23T23:09:11.542719+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":420,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543032+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:09:11.543032+0000","last_peered":"2023-09-23T23:09:11.543032+0000","last_clean":"2023-09-23T23:09:11.543032+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:09:11.543032+0000","last_undegraded":"2023-09-23T23:09:11.543032+0000","last_fullsized":"2023-09-23T23:09:11.543032+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":167,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543915+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:09:11.543915+0000","last_peered":"2023-09-23T23:09:11.543915+0000","last_clean":"2023-09-23T23:09:11.543915+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:09:11.543915+0000","last_undegraded":"2023-09-23T23:09:11.543915+0000","last_fullsized":"2023-09-23T23:09:11.543915+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":282,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543210+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:09:11.543210+0000","last_peered":"2023-09-23T23:09:11.543210+0000","last_clean":"2023-09-23T23:09:11.543210+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:09:11.543210+0000","last_undegraded":"2023-09-23T23:09:11.543210+0000","last_fullsized":"2023-09-23T23:09:11.543210+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":181,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.542742+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:09:11.542742+0000","last_peered":"2023-09-23T23:09:11.542742+0000","last_clean":"2023-09-23T23:09:11.542742+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:09:11.542742+0000","last_undegraded":"2023-09-23T23:09:11.542742+0000","last_fullsized":"2023-09-23T23:09:11.542742+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":213,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543724+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:09:11.543724+0000","last_peered":"2023-09-23T23:09:11.543724+0000","last_clean":"2023-09-23T23:09:11.543724+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:09:11.543724+0000","last_undegraded":"2023-09-23T23:09:11.543724+0000","last_fullsized":"2023-09-23T23:09:11.543724+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":231,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.551729+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:09:11.551729+0000","last_peered":"2023-09-23T23:09:11.551729+0000","last_clean":"2023-09-23T23:09:11.551729+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:09:11.551729+0000","last_undegraded":"2023-09-23T23:09:11.551729+0000","last_fullsized":"2023-09-23T23:09:11.551729+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":566,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523411+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:09:11.523411+0000","last_peered":"2023-09-23T23:09:11.523411+0000","last_clean":"2023-09-23T23:09:11.523411+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:09:11.523411+0000","last_undegraded":"2023-09-23T23:09:11.523411+0000","last_fullsized":"2023-09-23T23:09:11.523411+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":459,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.524854+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:09:11.524854+0000","last_peered":"2023-09-23T23:09:11.524854+0000","last_clean":"2023-09-23T23:09:11.524854+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:09:11.524854+0000","last_undegraded":"2023-09-23T23:09:11.524854+0000","last_fullsized":"2023-09-23T23:09:11.524854+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":391,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.524925+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:09:11.524925+0000","last_peered":"2023-09-23T23:09:11.524925+0000","last_clean":"2023-09-23T23:09:11.524925+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:09:11.524925+0000","last_undegraded":"2023-09-23T23:09:11.524925+0000","last_fullsized":"2023-09-23T23:09:11.524925+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":160,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.550881+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:09:11.550881+0000","last_peered":"2023-09-23T23:09:11.550881+0000","last_clean":"2023-09-23T23:09:11.550881+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:09:11.550881+0000","last_undegraded":"2023-09-23T23:09:11.550881+0000","last_fullsized":"2023-09-23T23:09:11.550881+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":478,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:19.362985+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:09:19.362985+0000","last_peered":"2023-09-23T23:09:19.362985+0000","last_clean":"2023-09-23T23:09:19.362985+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:09:19.362985+0000","last_undegraded":"2023-09-23T23:09:19.362985+0000","last_fullsized":"2023-09-23T23:09:19.362985+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":392,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543648+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:09:11.543648+0000","last_peered":"2023-09-23T23:09:11.543648+0000","last_clean":"2023-09-23T23:09:11.543648+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:09:11.543648+0000","last_undegraded":"2023-09-23T23:09:11.543648+0000","last_fullsized":"2023-09-23T23:09:11.543648+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"476'7375","reported_seq":12546,"reported_epoch":476,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:09:29.284352+0000","last_change":"2023-09-23T23:06:32.889539+0000","last_active":"2023-09-23T23:09:29.284352+0000","last_peered":"2023-09-23T23:09:29.284352+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:06:32.875369+0000","last_became_peered":"2023-09-23T23:06:32.875369+0000","last_unstale":"2023-09-23T23:09:29.284352+0000","last_undegraded":"2023-09-23T23:09:29.284352+0000","last_fullsized":"2023-09-23T23:09:29.284352+0000","mapping_epoch":445,"log_start":"414'293","ondisk_log_start":"414'293","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7082,"log_dups_size":0,"ondisk_log_size":7082,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24150016,"num_objects":369,"num_object_clones":0,"num_object_copies":1107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4726,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":369}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":379,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523354+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:09:11.523354+0000","last_peered":"2023-09-23T23:09:11.523354+0000","last_clean":"2023-09-23T23:09:11.523354+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:09:11.523354+0000","last_undegraded":"2023-09-23T23:09:11.523354+0000","last_fullsized":"2023-09-23T23:09:11.523354+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":501,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.542961+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:09:11.542961+0000","last_peered":"2023-09-23T23:09:11.542961+0000","last_clean":"2023-09-23T23:09:11.542961+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:09:11.542961+0000","last_undegraded":"2023-09-23T23:09:11.542961+0000","last_fullsized":"2023-09-23T23:09:11.542961+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"476'7584","reported_seq":13970,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:29.337348+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:09:29.337348+0000","last_peered":"2023-09-23T23:09:29.337348+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:09:29.337348+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"414'796","ondisk_log_start":"414'796","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6788,"log_dups_size":0,"ondisk_log_size":6788,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18382848,"num_objects":281,"num_object_clones":0,"num_object_copies":843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":649,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4894,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2143,"num_bytes_recovered":140247040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"3,5,6","objects":254},{"shards":"3,6","objects":-36}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":90,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.479102+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:09:11.479102+0000","last_peered":"2023-09-23T23:09:11.479102+0000","last_clean":"2023-09-23T23:09:11.479102+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:09:11.479102+0000","last_undegraded":"2023-09-23T23:09:11.479102+0000","last_fullsized":"2023-09-23T23:09:11.479102+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":172,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543178+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:09:11.543178+0000","last_peered":"2023-09-23T23:09:11.543178+0000","last_clean":"2023-09-23T23:09:11.543178+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:09:11.543178+0000","last_undegraded":"2023-09-23T23:09:11.543178+0000","last_fullsized":"2023-09-23T23:09:11.543178+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"476'7563","reported_seq":7664,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:30.415754+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:09:30.415754+0000","last_peered":"2023-09-23T23:09:30.415754+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:09:30.415754+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7563,"log_dups_size":0,"ondisk_log_size":7563,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12025856,"num_objects":183,"num_object_clones":0,"num_object_copies":549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4912,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-2182},{"shards":"1,4,5","objects":225}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":386,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523771+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:09:11.523771+0000","last_peered":"2023-09-23T23:09:11.523771+0000","last_clean":"2023-09-23T23:09:11.523771+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:09:11.523771+0000","last_undegraded":"2023-09-23T23:09:11.523771+0000","last_fullsized":"2023-09-23T23:09:11.523771+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":342,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523036+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:09:11.523036+0000","last_peered":"2023-09-23T23:09:11.523036+0000","last_clean":"2023-09-23T23:09:11.523036+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:09:11.523036+0000","last_undegraded":"2023-09-23T23:09:11.523036+0000","last_fullsized":"2023-09-23T23:09:11.523036+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"476'7311","reported_seq":12321,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:26.784299+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:09:26.784299+0000","last_peered":"2023-09-23T23:09:26.784299+0000","last_clean":"2023-09-23T23:09:26.784299+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:09:26.784299+0000","last_undegraded":"2023-09-23T23:09:26.784299+0000","last_fullsized":"2023-09-23T23:09:26.784299+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7311,"log_dups_size":0,"ondisk_log_size":7311,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25133056,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4679,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":105,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.555277+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:09:11.555277+0000","last_peered":"2023-09-23T23:09:11.555277+0000","last_clean":"2023-09-23T23:09:11.555277+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:09:11.555277+0000","last_undegraded":"2023-09-23T23:09:11.555277+0000","last_fullsized":"2023-09-23T23:09:11.555277+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":483,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523265+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:09:11.523265+0000","last_peered":"2023-09-23T23:09:11.523265+0000","last_clean":"2023-09-23T23:09:11.523265+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:09:11.523265+0000","last_undegraded":"2023-09-23T23:09:11.523265+0000","last_fullsized":"2023-09-23T23:09:11.523265+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"476'7509","reported_seq":14090,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:28.232084+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:09:28.232084+0000","last_peered":"2023-09-23T23:09:28.232084+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:09:28.232084+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14778368,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":95,"num_objects_missing":95,"num_objects_degraded":590,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4860,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2212,"num_bytes_recovered":144965632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"1,6","objects":-47},{"shards":"3,5,6","objects":272}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543575+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:09:11.543575+0000","last_peered":"2023-09-23T23:09:11.543575+0000","last_clean":"2023-09-23T23:09:11.543575+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:09:11.543575+0000","last_undegraded":"2023-09-23T23:09:11.543575+0000","last_fullsized":"2023-09-23T23:09:11.543575+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":73,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.479651+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:09:11.479651+0000","last_peered":"2023-09-23T23:09:11.479651+0000","last_clean":"2023-09-23T23:09:11.479651+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:09:11.479651+0000","last_undegraded":"2023-09-23T23:09:11.479651+0000","last_fullsized":"2023-09-23T23:09:11.479651+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"476'7663","reported_seq":8104,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.937956+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:09:26.937956+0000","last_peered":"2023-09-23T23:09:26.937956+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:09:26.937956+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7663,"log_dups_size":0,"ondisk_log_size":7663,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18055168,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4947,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1769}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.550798+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:09:11.550798+0000","last_peered":"2023-09-23T23:09:11.550798+0000","last_clean":"2023-09-23T23:09:11.550798+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:09:11.550798+0000","last_undegraded":"2023-09-23T23:09:11.550798+0000","last_fullsized":"2023-09-23T23:09:11.550798+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":298,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.524795+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:09:11.524795+0000","last_peered":"2023-09-23T23:09:11.524795+0000","last_clean":"2023-09-23T23:09:11.524795+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:09:11.524795+0000","last_undegraded":"2023-09-23T23:09:11.524795+0000","last_fullsized":"2023-09-23T23:09:11.524795+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"476'7541","reported_seq":11923,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:27.734282+0000","last_change":"2023-09-23T23:07:56.507184+0000","last_active":"2023-09-23T23:09:27.734282+0000","last_peered":"2023-09-23T23:09:27.734282+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.488393+0000","last_became_peered":"2023-09-23T23:07:56.488393+0000","last_unstale":"2023-09-23T23:09:27.734282+0000","last_undegraded":"2023-09-23T23:07:56.465702+0000","last_fullsized":"2023-09-23T23:07:56.443434+0000","mapping_epoch":459,"log_start":"416'1670","ondisk_log_start":"416'1670","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5871,"log_dups_size":0,"ondisk_log_size":5871,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22675456,"num_objects":346,"num_object_clones":0,"num_object_copies":1038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":194,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4858,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2130,"num_bytes_recovered":139493376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,6","objects":346}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":181,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.480012+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:09:11.480012+0000","last_peered":"2023-09-23T23:09:11.480012+0000","last_clean":"2023-09-23T23:09:11.480012+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:09:11.480012+0000","last_undegraded":"2023-09-23T23:09:11.480012+0000","last_fullsized":"2023-09-23T23:09:11.480012+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":478,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523488+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:09:11.523488+0000","last_peered":"2023-09-23T23:09:11.523488+0000","last_clean":"2023-09-23T23:09:11.523488+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:09:11.523488+0000","last_undegraded":"2023-09-23T23:09:11.523488+0000","last_fullsized":"2023-09-23T23:09:11.523488+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"476'7463","reported_seq":7561,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.923298+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:09:26.923298+0000","last_peered":"2023-09-23T23:09:26.923298+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:09:26.923298+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7463,"log_dups_size":0,"ondisk_log_size":7463,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22315008,"num_objects":340,"num_object_clones":0,"num_object_copies":1020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4799,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":309},{"shards":"2,3","objects":-2055}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":249,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523437+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:09:11.523437+0000","last_peered":"2023-09-23T23:09:11.523437+0000","last_clean":"2023-09-23T23:09:11.523437+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:09:11.523437+0000","last_undegraded":"2023-09-23T23:09:11.523437+0000","last_fullsized":"2023-09-23T23:09:11.523437+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":486,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523650+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:09:11.523650+0000","last_peered":"2023-09-23T23:09:11.523650+0000","last_clean":"2023-09-23T23:09:11.523650+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:09:11.523650+0000","last_undegraded":"2023-09-23T23:09:11.523650+0000","last_fullsized":"2023-09-23T23:09:11.523650+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"476'7586","reported_seq":12248,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:28.258710+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:09:28.258710+0000","last_peered":"2023-09-23T23:09:28.258710+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:09:28.258710+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12943360,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":87,"num_objects_missing":87,"num_objects_degraded":470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4932,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2279,"num_bytes_recovered":149225472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3,6","objects":252},{"shards":"2,6","objects":-55}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":499,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.525399+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:09:11.525399+0000","last_peered":"2023-09-23T23:09:11.525399+0000","last_clean":"2023-09-23T23:09:11.525399+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:09:11.525399+0000","last_undegraded":"2023-09-23T23:09:11.525399+0000","last_fullsized":"2023-09-23T23:09:11.525399+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":504,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523646+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:09:11.523646+0000","last_peered":"2023-09-23T23:09:11.523646+0000","last_clean":"2023-09-23T23:09:11.523646+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:09:11.523646+0000","last_undegraded":"2023-09-23T23:09:11.523646+0000","last_fullsized":"2023-09-23T23:09:11.523646+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"476'7472","reported_seq":11743,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:29.296339+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:09:29.296339+0000","last_peered":"2023-09-23T23:09:29.296339+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:09:29.296339+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:09:29.296339+0000","mapping_epoch":429,"log_start":"414'573","ondisk_log_start":"414'573","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19070976,"num_objects":291,"num_object_clones":0,"num_object_copies":873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":90,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4816,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2087,"num_bytes_recovered":136773632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"1,3,6","objects":291}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":149,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.544045+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:09:11.544045+0000","last_peered":"2023-09-23T23:09:11.544045+0000","last_clean":"2023-09-23T23:09:11.544045+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:09:11.544045+0000","last_undegraded":"2023-09-23T23:09:11.544045+0000","last_fullsized":"2023-09-23T23:09:11.544045+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":382,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.554132+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:09:11.554132+0000","last_peered":"2023-09-23T23:09:11.554132+0000","last_clean":"2023-09-23T23:09:11.554132+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:09:11.554132+0000","last_undegraded":"2023-09-23T23:09:11.554132+0000","last_fullsized":"2023-09-23T23:09:11.554132+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"476'9802","reported_seq":11272,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.941429+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:09:26.941429+0000","last_peered":"2023-09-23T23:09:26.941429+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:09:26.941429+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"419'3974","ondisk_log_start":"419'3974","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42795032,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9802,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"2,4","objects":-3212}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":396,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.555142+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:09:11.555142+0000","last_peered":"2023-09-23T23:09:11.555142+0000","last_clean":"2023-09-23T23:09:11.555142+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:09:11.555142+0000","last_undegraded":"2023-09-23T23:09:11.555142+0000","last_fullsized":"2023-09-23T23:09:11.555142+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.480032+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:09:11.480032+0000","last_peered":"2023-09-23T23:09:11.480032+0000","last_clean":"2023-09-23T23:09:11.480032+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:09:11.480032+0000","last_undegraded":"2023-09-23T23:09:11.480032+0000","last_fullsized":"2023-09-23T23:09:11.480032+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":217,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523248+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:09:11.523248+0000","last_peered":"2023-09-23T23:09:11.523248+0000","last_clean":"2023-09-23T23:09:11.523248+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:09:11.523248+0000","last_undegraded":"2023-09-23T23:09:11.523248+0000","last_fullsized":"2023-09-23T23:09:11.523248+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"476'9768","reported_seq":9888,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:29.355243+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:09:29.355243+0000","last_peered":"2023-09-23T23:09:29.355243+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:09:29.355243+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"427'5267","ondisk_log_start":"427'5267","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4501,"log_dups_size":0,"ondisk_log_size":4501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33292288,"num_objects":508,"num_object_clones":0,"num_object_copies":1524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9768,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":508},{"shards":"1,3","objects":-4317}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":212,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523483+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:09:11.523483+0000","last_peered":"2023-09-23T23:09:11.523483+0000","last_clean":"2023-09-23T23:09:11.523483+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:09:11.523483+0000","last_undegraded":"2023-09-23T23:09:11.523483+0000","last_fullsized":"2023-09-23T23:09:11.523483+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":514,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.525048+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:09:11.525048+0000","last_peered":"2023-09-23T23:09:11.525048+0000","last_clean":"2023-09-23T23:09:11.525048+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:09:11.525048+0000","last_undegraded":"2023-09-23T23:09:11.525048+0000","last_fullsized":"2023-09-23T23:09:11.525048+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":381,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.555065+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:09:11.555065+0000","last_peered":"2023-09-23T23:09:11.555065+0000","last_clean":"2023-09-23T23:09:11.555065+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:09:11.555065+0000","last_undegraded":"2023-09-23T23:09:11.555065+0000","last_fullsized":"2023-09-23T23:09:11.555065+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"476'7644","reported_seq":14174,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:29.241350+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:09:29.241350+0000","last_peered":"2023-09-23T23:09:29.241350+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:09:29.241350+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"415'983","ondisk_log_start":"415'983","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6661,"log_dups_size":0,"ondisk_log_size":6661,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14450688,"num_objects":220,"num_object_clones":0,"num_object_copies":660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4953,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2196,"num_bytes_recovered":143851520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"3,5,6","objects":247},{"shards":"3,6","objects":-95}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":456,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.557394+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:09:11.557394+0000","last_peered":"2023-09-23T23:09:11.557394+0000","last_clean":"2023-09-23T23:09:11.557394+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:09:11.557394+0000","last_undegraded":"2023-09-23T23:09:11.557394+0000","last_fullsized":"2023-09-23T23:09:11.557394+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":308,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.551398+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:09:11.551398+0000","last_peered":"2023-09-23T23:09:11.551398+0000","last_clean":"2023-09-23T23:09:11.551398+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:09:11.551398+0000","last_undegraded":"2023-09-23T23:09:11.551398+0000","last_fullsized":"2023-09-23T23:09:11.551398+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":413,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.542920+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:09:11.542920+0000","last_peered":"2023-09-23T23:09:11.542920+0000","last_clean":"2023-09-23T23:09:11.542920+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:09:11.542920+0000","last_undegraded":"2023-09-23T23:09:11.542920+0000","last_fullsized":"2023-09-23T23:09:11.542920+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"476'9763","reported_seq":18866,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:29.370948+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:09:29.370948+0000","last_peered":"2023-09-23T23:09:29.370948+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:09:29.370948+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"426'5234","ondisk_log_start":"426'5234","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4529,"log_dups_size":0,"ondisk_log_size":4529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31522816,"num_objects":481,"num_object_clones":0,"num_object_copies":1443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1076,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9763,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4503,"num_bytes_recovered":295108608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":481},{"shards":"3,6","objects":-138}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":387,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.552931+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:09:11.552931+0000","last_peered":"2023-09-23T23:09:11.552931+0000","last_clean":"2023-09-23T23:09:11.552931+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:09:11.552931+0000","last_undegraded":"2023-09-23T23:09:11.552931+0000","last_fullsized":"2023-09-23T23:09:11.552931+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":588,"reported_epoch":475,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:11.552226+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:09:11.552226+0000","last_peered":"2023-09-23T23:09:11.552226+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:09:11.552226+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:09:11.552226+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":167,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.479753+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:09:11.479753+0000","last_peered":"2023-09-23T23:09:11.479753+0000","last_clean":"2023-09-23T23:09:11.479753+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:09:11.479753+0000","last_undegraded":"2023-09-23T23:09:11.479753+0000","last_fullsized":"2023-09-23T23:09:11.479753+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":394,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523726+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:09:11.523726+0000","last_peered":"2023-09-23T23:09:11.523726+0000","last_clean":"2023-09-23T23:09:11.523726+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:09:11.523726+0000","last_undegraded":"2023-09-23T23:09:11.523726+0000","last_fullsized":"2023-09-23T23:09:11.523726+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"476'7604","reported_seq":7964,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:29.340256+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:09:29.340256+0000","last_peered":"2023-09-23T23:09:29.340256+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:09:29.340256+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"415'1112","ondisk_log_start":"415'1112","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6492,"log_dups_size":0,"ondisk_log_size":6492,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9142272,"num_objects":139,"num_object_clones":0,"num_object_copies":417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2688,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4954,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":128,"num_bytes_recovered":8355840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3,6","objects":255},{"shards":"1,3","objects":-2293}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":400,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.479357+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:09:11.479357+0000","last_peered":"2023-09-23T23:09:11.479357+0000","last_clean":"2023-09-23T23:09:11.479357+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:09:11.479357+0000","last_undegraded":"2023-09-23T23:09:11.479357+0000","last_fullsized":"2023-09-23T23:09:11.479357+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":370,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523034+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:09:11.523034+0000","last_peered":"2023-09-23T23:09:11.523034+0000","last_clean":"2023-09-23T23:09:11.523034+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:09:11.523034+0000","last_undegraded":"2023-09-23T23:09:11.523034+0000","last_fullsized":"2023-09-23T23:09:11.523034+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":336,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.553197+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:09:11.553197+0000","last_peered":"2023-09-23T23:09:11.553197+0000","last_clean":"2023-09-23T23:09:11.553197+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:09:11.553197+0000","last_undegraded":"2023-09-23T23:09:11.553197+0000","last_fullsized":"2023-09-23T23:09:11.553197+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"476'9625","reported_seq":9702,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:26.937299+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:09:26.937299+0000","last_peered":"2023-09-23T23:09:26.937299+0000","last_clean":"2023-09-23T23:09:26.937299+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:09:26.937299+0000","last_undegraded":"2023-09-23T23:09:26.937299+0000","last_fullsized":"2023-09-23T23:09:26.937299+0000","mapping_epoch":410,"log_start":"419'3811","ondisk_log_start":"419'3811","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39780352,"num_objects":607,"num_object_clones":0,"num_object_copies":1821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9625,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":335,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523594+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:09:11.523594+0000","last_peered":"2023-09-23T23:09:11.523594+0000","last_clean":"2023-09-23T23:09:11.523594+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:09:11.523594+0000","last_undegraded":"2023-09-23T23:09:11.523594+0000","last_fullsized":"2023-09-23T23:09:11.523594+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":333,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.555466+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:09:11.555466+0000","last_peered":"2023-09-23T23:09:11.555466+0000","last_clean":"2023-09-23T23:09:11.555466+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:09:11.555466+0000","last_undegraded":"2023-09-23T23:09:11.555466+0000","last_fullsized":"2023-09-23T23:09:11.555466+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":298,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.552718+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:09:11.552718+0000","last_peered":"2023-09-23T23:09:11.552718+0000","last_clean":"2023-09-23T23:09:11.552718+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:09:11.552718+0000","last_undegraded":"2023-09-23T23:09:11.552718+0000","last_fullsized":"2023-09-23T23:09:11.552718+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"476'9525","reported_seq":9610,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.892599+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:09:26.892599+0000","last_peered":"2023-09-23T23:09:26.892599+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:09:26.892599+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"419'3683","ondisk_log_start":"419'3683","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39649280,"num_objects":605,"num_object_clones":0,"num_object_copies":1815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9525,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":605},{"shards":"1,2","objects":-4164}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":199,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543346+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:09:11.543346+0000","last_peered":"2023-09-23T23:09:11.543346+0000","last_clean":"2023-09-23T23:09:11.543346+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:09:11.543346+0000","last_undegraded":"2023-09-23T23:09:11.543346+0000","last_fullsized":"2023-09-23T23:09:11.543346+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":174,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523558+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:09:11.523558+0000","last_peered":"2023-09-23T23:09:11.523558+0000","last_clean":"2023-09-23T23:09:11.523558+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:09:11.523558+0000","last_undegraded":"2023-09-23T23:09:11.523558+0000","last_fullsized":"2023-09-23T23:09:11.523558+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":183,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109822+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:09:23.109822+0000","last_peered":"2023-09-23T23:09:23.109822+0000","last_clean":"2023-09-23T23:09:23.109822+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:09:23.109822+0000","last_undegraded":"2023-09-23T23:09:23.109822+0000","last_fullsized":"2023-09-23T23:09:23.109822+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"476'9855","reported_seq":19634,"reported_epoch":476,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:30.538583+0000","last_change":"2023-09-23T23:05:53.677657+0000","last_active":"2023-09-23T23:09:30.538583+0000","last_peered":"2023-09-23T23:09:30.538583+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:09:30.538583+0000","last_undegraded":"2023-09-23T23:05:40.908093+0000","last_fullsized":"2023-09-23T23:05:40.876204+0000","mapping_epoch":433,"log_start":"420'4120","ondisk_log_start":"420'4120","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31260672,"num_objects":477,"num_object_clones":0,"num_object_copies":1431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9855,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4848,"num_bytes_recovered":307822592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"1,4","objects":293}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":425,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:16.937974+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:09:16.937974+0000","last_peered":"2023-09-23T23:09:16.937974+0000","last_clean":"2023-09-23T23:09:16.937974+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:09:16.937974+0000","last_undegraded":"2023-09-23T23:09:16.937974+0000","last_fullsized":"2023-09-23T23:09:16.937974+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":97,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523745+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:09:11.523745+0000","last_peered":"2023-09-23T23:09:11.523745+0000","last_clean":"2023-09-23T23:09:11.523745+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:09:11.523745+0000","last_undegraded":"2023-09-23T23:09:11.523745+0000","last_fullsized":"2023-09-23T23:09:11.523745+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":468,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523128+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:09:11.523128+0000","last_peered":"2023-09-23T23:09:11.523128+0000","last_clean":"2023-09-23T23:09:11.523128+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:09:11.523128+0000","last_undegraded":"2023-09-23T23:09:11.523128+0000","last_fullsized":"2023-09-23T23:09:11.523128+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"476'7611","reported_seq":9701,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:27.714657+0000","last_change":"2023-09-23T23:07:56.517407+0000","last_active":"2023-09-23T23:09:27.714657+0000","last_peered":"2023-09-23T23:09:27.714657+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.507318+0000","last_became_peered":"2023-09-23T23:07:56.507318+0000","last_unstale":"2023-09-23T23:09:27.714657+0000","last_undegraded":"2023-09-23T23:07:56.466720+0000","last_fullsized":"2023-09-23T23:07:56.443187+0000","mapping_epoch":459,"log_start":"416'1923","ondisk_log_start":"416'1923","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4928,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":64454656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2","objects":-955}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":67,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.524678+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:09:11.524678+0000","last_peered":"2023-09-23T23:09:11.524678+0000","last_clean":"2023-09-23T23:09:11.524678+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:09:11.524678+0000","last_undegraded":"2023-09-23T23:09:11.524678+0000","last_fullsized":"2023-09-23T23:09:11.524678+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":227,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.555534+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:09:11.555534+0000","last_peered":"2023-09-23T23:09:11.555534+0000","last_clean":"2023-09-23T23:09:11.555534+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:09:11.555534+0000","last_undegraded":"2023-09-23T23:09:11.555534+0000","last_fullsized":"2023-09-23T23:09:11.555534+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"476'7580","reported_seq":8006,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.821413+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:09:26.821413+0000","last_peered":"2023-09-23T23:09:26.821413+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:09:26.821413+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23494656,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4865,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1616}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":298,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523299+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:09:11.523299+0000","last_peered":"2023-09-23T23:09:11.523299+0000","last_clean":"2023-09-23T23:09:11.523299+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:09:11.523299+0000","last_undegraded":"2023-09-23T23:09:11.523299+0000","last_fullsized":"2023-09-23T23:09:11.523299+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":499,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.525115+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:09:11.525115+0000","last_peered":"2023-09-23T23:09:11.525115+0000","last_clean":"2023-09-23T23:09:11.525115+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:09:11.525115+0000","last_undegraded":"2023-09-23T23:09:11.525115+0000","last_fullsized":"2023-09-23T23:09:11.525115+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"476'7432","reported_seq":13699,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:28.252922+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:09:28.252922+0000","last_peered":"2023-09-23T23:09:28.252922+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:09:28.252922+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19824640,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":116,"num_objects_missing":116,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4784,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2092,"num_bytes_recovered":137035776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":256},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":325,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523684+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:09:11.523684+0000","last_peered":"2023-09-23T23:09:11.523684+0000","last_clean":"2023-09-23T23:09:11.523684+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:09:11.523684+0000","last_undegraded":"2023-09-23T23:09:11.523684+0000","last_fullsized":"2023-09-23T23:09:11.523684+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":395,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523908+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:09:11.523908+0000","last_peered":"2023-09-23T23:09:11.523908+0000","last_clean":"2023-09-23T23:09:11.523908+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:09:11.523908+0000","last_undegraded":"2023-09-23T23:09:11.523908+0000","last_fullsized":"2023-09-23T23:09:11.523908+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"476'7470","reported_seq":8486,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:26.932253+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:09:26.932253+0000","last_peered":"2023-09-23T23:09:26.932253+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:09:26.932253+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7470,"log_dups_size":0,"ondisk_log_size":7470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14712832,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4839,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":303},{"shards":"1,2","objects":-1762}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":51,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543304+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:09:11.543304+0000","last_peered":"2023-09-23T23:09:11.543304+0000","last_clean":"2023-09-23T23:09:11.543304+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:09:11.543304+0000","last_undegraded":"2023-09-23T23:09:11.543304+0000","last_fullsized":"2023-09-23T23:09:11.543304+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.543895+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:09:11.543895+0000","last_peered":"2023-09-23T23:09:11.543895+0000","last_clean":"2023-09-23T23:09:11.543895+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:09:11.543895+0000","last_undegraded":"2023-09-23T23:09:11.543895+0000","last_fullsized":"2023-09-23T23:09:11.543895+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"476'7487","reported_seq":7578,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:30.513159+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:09:30.513159+0000","last_peered":"2023-09-23T23:09:30.513159+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:09:30.513159+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7487,"log_dups_size":0,"ondisk_log_size":7487,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17006592,"num_objects":260,"num_object_clones":0,"num_object_copies":780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4837,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-1996},{"shards":"1,4,5","objects":218}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":244,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.554223+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:09:11.554223+0000","last_peered":"2023-09-23T23:09:11.554223+0000","last_clean":"2023-09-23T23:09:11.554223+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:09:11.554223+0000","last_undegraded":"2023-09-23T23:09:11.554223+0000","last_fullsized":"2023-09-23T23:09:11.554223+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":470,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T23:09:11.523091+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:09:11.523091+0000","last_peered":"2023-09-23T23:09:11.523091+0000","last_clean":"2023-09-23T23:09:11.523091+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:09:11.523091+0000","last_undegraded":"2023-09-23T23:09:11.523091+0000","last_fullsized":"2023-09-23T23:09:11.523091+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":577175576,"num_objects":8808,"num_object_clones":0,"num_object_copies":26424,"num_objects_missing_on_primary":298,"num_objects_missing":298,"num_objects_degraded":40074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8808,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155496,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36261,"num_bytes_recovered":2340159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6550532096,"data_stored":6550519880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173842,"ondisk_log_size":173842,"up":78,"acting":58,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430718,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626942,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240888,"kb_used_data":346352,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92504264,"statfs":{"total":95995035648,"available":94724366336,"internally_reserved":0,"allocated":354664448,"data_stored":353594594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1,0,7],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.3109999999999999,"15min":4.6340000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.923,"5min":10.853,"15min":115.32899999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.304,"15min":4.6719999999999997},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":4.383,"5min":8.7010000000000005,"15min":115.348},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":3.629,"15min":3.056},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":9.4060000000000006,"5min":149.947,"15min":149.947},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.718,"15min":2.9940000000000002},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":6.3689999999999998,"5min":150.22499999999999,"15min":150.22499999999999},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:08:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9420000000000002,"5min":5.5709999999999997,"15min":2.9260000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":45.051000000000002,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":4.8579999999999997,"5min":5.25,"15min":2.9969999999999999},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32500000000000001},"max":{"1min":45.258000000000003,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":4.9630000000000001,"15min":3.3210000000000002},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":9.4329999999999998,"5min":142.24700000000001,"15min":142.24700000000001},"last":14.426},{"interface":"front","average":{"1min":1.986,"5min":4.8369999999999997,"15min":3.2349999999999999},"min":{"1min":0.41799999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.4100000000000001,"5min":143.86699999999999,"15min":143.86699999999999},"last":14.955}]},{"osd":4,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.2490000000000001,"15min":1.857},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.23999999999999999},"max":{"1min":6.3470000000000004,"5min":12.407999999999999,"15min":119.804},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.3109999999999999,"15min":1.925},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.24099999999999999},"max":{"1min":6.2140000000000004,"5min":8.8940000000000001,"15min":122.31999999999999},"last":0.30499999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.369,"15min":2.52},"min":{"1min":0.40400000000000003,"5min":0.33000000000000002,"15min":0.27900000000000003},"max":{"1min":6.4530000000000003,"5min":11.862,"15min":186.27600000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":2.238,"5min":1.506,"15min":2.6629999999999998},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.29599999999999999},"max":{"1min":6.2729999999999997,"5min":10.254,"15min":191.28700000000001},"last":0.436}]},{"osd":7,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.224,"15min":1.5880000000000001},"min":{"1min":0.46700000000000003,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":8.8670000000000009,"5min":8.8670000000000009,"15min":115.61},"last":0.375},{"interface":"front","average":{"1min":1.698,"5min":0.97499999999999998,"15min":1.7070000000000001},"min":{"1min":0.39600000000000002,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":8.2609999999999992,"5min":8.2609999999999992,"15min":115.578},"last":0.42699999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253255,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1684552,"kb_used_data":695104,"kb_used_omap":9,"kb_used_meta":989302,"kb_avail":92060600,"statfs":{"total":95995035648,"available":94270054400,"internally_reserved":0,"allocated":711786496,"data_stored":709841249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1013046249},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.5629999999999999,"15min":2.5649999999999999},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.2189999999999999,"5min":59.475000000000001,"15min":279.30500000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7490000000000001,"15min":2.8799999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":4.8559999999999999,"5min":59.393999999999998,"15min":279.39800000000002},"last":2.476}]},{"osd":2,"last update":"Sat Sep 23 23:08:37 2023","interfaces":[{"interface":"back","average":{"1min":7.96,"5min":6.2439999999999998,"15min":2.4449999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.527,"5min":161.61699999999999,"15min":161.61699999999999},"last":0.379},{"interface":"front","average":{"1min":7.5199999999999996,"5min":6.4059999999999997,"15min":2.6059999999999999},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":118.29900000000001,"5min":162.61699999999999,"15min":162.61699999999999},"last":2.5139999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.4510000000000001,"15min":1.389},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":3.8039999999999998,"5min":59.424999999999997,"15min":59.424999999999997},"last":5.8280000000000003},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.5680000000000001,"15min":1.454},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":7.1580000000000004,"5min":55.174999999999997,"15min":55.174999999999997},"last":2.5539999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2320000000000002,"5min":3.0569999999999999,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":67.813999999999993,"5min":129.69,"15min":151.036},"last":2.585},{"interface":"front","average":{"1min":5.4039999999999999,"5min":3.427,"15min":2.9180000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":67.796999999999997,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.92400000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.4419999999999999,"15min":2.2370000000000001},"min":{"1min":0.40300000000000002,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":1.522,"5min":18.228999999999999,"15min":116.098},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.0920000000000001,"15min":2.0419999999999998},"min":{"1min":0.38900000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.5800000000000001,"5min":6.585,"15min":116.08499999999999},"last":2.9849999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2759999999999998,"5min":2.1850000000000001,"15min":2.056},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":64.058000000000007,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.84199999999999997},{"interface":"front","average":{"1min":4.7190000000000003,"5min":2.3690000000000002,"15min":2.0110000000000001},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.33300000000000002},"max":{"1min":64.239999999999995,"5min":64.239999999999995,"15min":67.366},"last":1.3080000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":1.337,"15min":1.1910000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":5.7549999999999999,"5min":7.0960000000000001,"15min":8.1400000000000006},"last":3.0259999999999998},{"interface":"front","average":{"1min":1.472,"5min":1.7290000000000001,"15min":1.478},"min":{"1min":0.40699999999999997,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":8.8719999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.502}]}]},{"osd":0,"up_from":0,"seq":1516123455626,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259398,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":639512,"kb_used_data":327696,"kb_used_omap":83,"kb_used_meta":311724,"kb_avail":93105640,"statfs":{"total":95995035648,"available":95340175360,"internally_reserved":0,"allocated":335560704,"data_stored":333486049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":319205829},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.82999999999999996,"15min":1.212},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":4.1390000000000002,"5min":10.199999999999999,"15min":74.241},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.5,"5min":1.29,"15min":1.5309999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":14.944000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0670000000000002,"5min":2.5049999999999999,"15min":1.3009999999999999},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":83.897000000000006,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.43099999999999999},{"interface":"front","average":{"1min":4.5289999999999999,"5min":2.1669999999999998,"15min":1.462},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":83.088999999999999,"5min":83.088999999999999,"15min":83.088999999999999},"last":7.1689999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7130000000000001,"5min":3.98,"15min":1.8380000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.23000000000000001},"max":{"1min":79.655000000000001,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":7.2039999999999997,"5min":3.8650000000000002,"15min":1.851},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26100000000000001},"max":{"1min":79.677999999999997,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":1.9279999999999999,"15min":1.1379999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":68.325000000000003,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":4.9020000000000001,"5min":2.2610000000000001,"15min":1.351},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":70.608000000000004,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":3.105,"15min":1.6839999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.6600000000000001,"5min":139.405,"15min":139.405},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.002,"5min":2.9129999999999998,"15min":1.5089999999999999},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":4.0499999999999998,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":3.4180000000000001,"15min":1.8600000000000001},"min":{"1min":0.377,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.6989999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":3.1749999999999998,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34599999999999997,"15min":0.32200000000000001},"max":{"1min":4.0830000000000002,"5min":108.496,"15min":108.496},"last":0.70799999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.421,"15min":1.1890000000000001},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.1449999999999996,"5min":10.164,"15min":10.164},"last":0.621},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.393,"15min":1.2050000000000001},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":15.037000000000001,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.747}]}]},{"osd":3,"up_from":94,"seq":403726926203,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1300028,"kb_used_data":593524,"kb_used_omap":7,"kb_used_meta":706360,"kb_avail":92445124,"statfs":{"total":95995035648,"available":94663806976,"internally_reserved":0,"allocated":607768576,"data_stored":605714254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":723313256},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,7],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.486,"15min":1.4490000000000001},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":3.8439999999999999,"5min":17.015000000000001,"15min":122.289},"last":5.0439999999999996},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.115,"15min":1.1870000000000001},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":4.9909999999999997,"5min":11.382999999999999,"15min":122.041},"last":1.762}]},{"osd":1,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":2.1720000000000002,"15min":1.4710000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":10.935,"5min":23.969000000000001,"15min":23.969000000000001},"last":5.1120000000000001},{"interface":"front","average":{"1min":1.109,"5min":1.9119999999999999,"15min":1.373},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.20699999999999999},"max":{"1min":4.0119999999999996,"5min":16.986000000000001,"15min":16.986000000000001},"last":0.26900000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:08:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8479999999999999,"5min":2.3370000000000002,"15min":1.232},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":55.311999999999998,"5min":55.311999999999998,"15min":55.311999999999998},"last":2.1619999999999999},{"interface":"front","average":{"1min":5.532,"5min":2.7480000000000002,"15min":1.5389999999999999},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.215},"max":{"1min":57.308,"5min":57.308,"15min":57.308},"last":6.109}]},{"osd":4,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.6629999999999998,"15min":2.1110000000000002},"min":{"1min":0.374,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":11.01,"5min":115.69,"15min":115.69},"last":5.1470000000000002},{"interface":"front","average":{"1min":1.351,"5min":2.4009999999999998,"15min":2.0419999999999998},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":5.9169999999999998,"5min":115.617,"15min":115.617},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.401},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":4.8810000000000002,"5min":16.959,"15min":16.959},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.359,"5min":1.476,"15min":1.369},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":6.6130000000000004,"5min":10.545,"15min":10.545},"last":1.847}]},{"osd":6,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":2.589,"15min":1.8999999999999999},"min":{"1min":0.41699999999999998,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":77.707999999999998,"5min":77.707999999999998,"15min":77.707999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":5.4080000000000004,"5min":2.1930000000000001,"15min":1.6890000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":77.721999999999994,"5min":77.721999999999994,"15min":77.721999999999994},"last":1.806}]},{"osd":7,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.4630000000000001,"15min":1.272},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":4.5220000000000002,"5min":8.1620000000000008,"15min":8.1620000000000008},"last":1.883},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.0499999999999998,"15min":1.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":5.1769999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019400,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049464,"kb_used_data":142000,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92695688,"statfs":{"total":95995035648,"available":94920384512,"internally_reserved":0,"allocated":145408000,"data_stored":144337624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.4039999999999999,"15min":2.3580000000000001},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":9.8870000000000005,"5min":99.132999999999996,"15min":156.375},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.3370000000000002,"15min":2.4279999999999999},"min":{"1min":0.36799999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":4.4509999999999996,"5min":101.062,"15min":157.02199999999999},"last":2.964}]},{"osd":1,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.4340000000000002,"15min":2.7989999999999999},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":4.3840000000000003,"5min":95.591999999999999,"15min":98.846999999999994},"last":0.88600000000000001},{"interface":"front","average":{"1min":1.954,"5min":3.407,"15min":2.843},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":10.311,"5min":97.929000000000002,"15min":98.938999999999993},"last":0.67300000000000004}]},{"osd":2,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":11.628,"5min":4.2610000000000001,"15min":2.1459999999999999},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":124.819,"5min":124.819,"15min":124.819},"last":0.56299999999999994},{"interface":"front","average":{"1min":10.571999999999999,"5min":4.077,"15min":1.9850000000000001},"min":{"1min":0.36599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":122.086,"5min":122.086,"15min":122.086},"last":0.97799999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":11.974,"5min":4.7699999999999996,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":147.82499999999999,"5min":147.82499999999999,"15min":147.82499999999999},"last":3.153},{"interface":"front","average":{"1min":11.071,"5min":4.7309999999999999,"15min":2.4889999999999999},"min":{"1min":0.44,"5min":0.36599999999999999,"15min":0.32500000000000001},"max":{"1min":147.84100000000001,"5min":147.84100000000001,"15min":147.84100000000001},"last":3.0470000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:08:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.177,"15min":1.764},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.214},"max":{"1min":3.036,"5min":10.130000000000001,"15min":148.61600000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.522,"5min":1.157,"15min":1.861},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":8.5489999999999995,"5min":8.5489999999999995,"15min":151.905},"last":3.758}]},{"osd":6,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.4650000000000001,"15min":1.419},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.9740000000000002,"5min":35.896000000000001,"15min":35.896000000000001},"last":3.7850000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.778,"15min":1.3899999999999999},"min":{"1min":0.34899999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":6.5129999999999999,"5min":35.908000000000001,"15min":35.908000000000001},"last":0.753}]},{"osd":7,"last update":"Sat Sep 23 23:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.161,"15min":1.288},"min":{"1min":0.373,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":3.1080000000000001,"5min":7.0789999999999997,"15min":12.84},"last":0.629},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.28,"15min":1.232},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":15.808,"5min":15.808,"15min":15.808},"last":0.64000000000000001}]}]},{"osd":5,"up_from":0,"seq":154618823174,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2934833152,"data_stored":2934833152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":705298432,"data_stored":705298432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":321654784,"data_stored":321650712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":595656704,"data_stored":595656704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":135991296,"data_stored":135987224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":325062656,"data_stored":325058584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:09:32.401 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T23:09:32.438 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:32.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:32.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:32.544 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:32.610 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:32.721 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:32.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:32.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:32.830 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:32.918 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:33.384 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:33.397 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:33.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:33.499 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:33.578 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:09:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:09:33] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.33.4" 2023-09-23T23:09:33.581 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:33.758 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:33.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:33.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:33.865 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:33.902 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:33.985 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:33.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:34.049 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:34.093 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:34.128 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:34.177 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:09:34.196 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:34.200 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:33 smithi052 ceph-mon[187743]: pgmap v1506: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 451 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 40076/21678 objects degraded (184.869%); 8.9 MiB/s, 157 objects/s recovering 2023-09-23T23:09:34.200 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:33 smithi052 ceph-mon[180979]: pgmap v1506: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 451 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 40076/21678 objects degraded (184.869%); 8.9 MiB/s, 157 objects/s recovering 2023-09-23T23:09:34.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:33 smithi151 ceph-mon[159210]: pgmap v1506: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 451 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 40076/21678 objects degraded (184.869%); 8.9 MiB/s, 157 objects/s recovering 2023-09-23T23:09:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:34.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:34.225 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:09:34.272 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:09:34.303 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:34.317 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:09:34.372 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:09:34.422 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:09:34.432 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:34.470 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:09:34.506 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:09:34.516 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.517 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:09:34.727 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:34.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:34.839 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:34.949 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:34.976 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:09:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1255872462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3702258807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:34.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:34.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1255872462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3702258807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:34.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:35.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1255872462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:35.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:34 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3702258807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:35.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:34 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:09:35.202 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: pgmap v1507: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 362 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 611 op/s; 39698/17421 objects degraded (227.874%); 9.5 MiB/s, 165 objects/s recovering 2023-09-23T23:09:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: osdmap e477: 8 total, 8 up, 5 in 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:35 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 39698/17421 objects degraded (227.874%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: pgmap v1507: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 362 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 611 op/s; 39698/17421 objects degraded (227.874%); 9.5 MiB/s, 165 objects/s recovering 2023-09-23T23:09:36.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: osdmap e477: 8 total, 8 up, 5 in 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 39698/17421 objects degraded (227.874%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:36.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: pgmap v1507: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 362 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 611 op/s; 39698/17421 objects degraded (227.874%); 9.5 MiB/s, 165 objects/s recovering 2023-09-23T23:09:36.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:09:36.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:09:36.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: osdmap e477: 8 total, 8 up, 5 in 2023-09-23T23:09:36.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:35 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 39698/17421 objects degraded (227.874%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:36.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:36 smithi052 ceph-mon[187743]: failed to encode map e477 with expected crc 2023-09-23T23:09:36.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:36 smithi052 ceph-mon[180979]: failed to encode map e477 with expected crc 2023-09-23T23:09:37.143 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:37.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:37.252 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:37.281 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:37.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:36 smithi151 ceph-mon[159210]: failed to encode map e477 with expected crc 2023-09-23T23:09:37.700 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:37.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:37.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:37.846 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:37.901 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:37.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:38.008 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:38.091 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:38.201 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:37 smithi151 ceph-mon[159210]: pgmap v1509: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 336 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 39698/16158 objects degraded (245.686%); 6.0 MiB/s, 119 objects/s recovering 2023-09-23T23:09:38.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:37 smithi052 ceph-mon[187743]: pgmap v1509: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 336 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 39698/16158 objects degraded (245.686%); 6.0 MiB/s, 119 objects/s recovering 2023-09-23T23:09:38.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:37 smithi052 ceph-mon[180979]: pgmap v1509: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 336 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 39698/16158 objects degraded (245.686%); 6.0 MiB/s, 119 objects/s recovering 2023-09-23T23:09:38.268 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:38.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:38.374 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:38.465 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:38.673 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:38.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:38.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:38.786 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:38.803 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:09:38.804 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:09:38.813 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:38.893 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:38.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:39.016 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:39.049 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:39.063 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:09:39.107 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:39.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:39.214 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:39.228 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:09:39.228 INFO:teuthology.orchestra.run.smithi052.stdout:{"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 39479/10890 objects degraded (362.525%), 24 pgs degraded, 20 pgs undersized","count":44},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2111,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":477,"num_osds":8,"num_up_osds":8,"osd_up_since":1695509902,"num_in_osds":5,"osd_in_since":1695510352,"num_remapped_pgs":20},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":74},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":19},{"state_name":"active+recovery_wait+degraded","count":4},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":3630,"data_bytes":236853176,"bytes_used":5160308736,"bytes_avail":474814869504,"bytes_total":479975178240,"degraded_objects":39479,"degraded_total":10890,"degraded_ratio":3.6252525252525256,"recovering_objects_per_sec":113,"recovering_bytes_per_sec":5921917,"recovering_keys_per_sec":0,"num_objects_recovered":1134,"num_bytes_recovered":59244544,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":664},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi052.thpzxu","status":"up:active","gid":24518}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.52:8443/","prometheus":"http://172.21.15.52:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T23:09:04.798859+0000","services":{}},"progress_events":{"8c4c2550-2af0-42f2-9388-4b306baed78d":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true},"fadc4ea3-ec6a-4d25-82eb-03b384d527e2":{"message":"Global Recovery Event (4m)\n [====================........] (remaining: 86s)","progress":0.74747473001480103,"add_to_ceph_s":true}}} 2023-09-23T23:09:39.249 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:39.312 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:39.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:39.417 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:39.444 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:39.486 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:39.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:39.592 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:39.649 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:09:39.658 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:40.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:39 smithi052 ceph-mon[187743]: pgmap v1510: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 226 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 664 op/s; 39479/10890 objects degraded (362.525%); 5.6 MiB/s, 113 objects/s recovering 2023-09-23T23:09:40.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3471365011' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:09:40.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:39 smithi052 ceph-mon[180979]: pgmap v1510: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 226 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 664 op/s; 39479/10890 objects degraded (362.525%); 5.6 MiB/s, 113 objects/s recovering 2023-09-23T23:09:40.246 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3471365011' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:09:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:39 smithi151 ceph-mon[159210]: pgmap v1510: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 226 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 664 op/s; 39479/10890 objects degraded (362.525%); 5.6 MiB/s, 113 objects/s recovering 2023-09-23T23:09:40.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3471365011' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T23:09:40.609 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:40.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:40.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:40.658 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:09:40.688 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T23:09:40.688 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:09:40.721 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:40.806 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:40.891 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:40.990 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:40 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 39479/10890 objects degraded (362.525%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:41.027 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:41.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:41.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:41.070 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:41.150 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:41.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:41 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 39479/10890 objects degraded (362.525%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:41.247 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:41.366 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:41.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:41.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:41.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:40 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 39479/10890 objects degraded (362.525%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:41.472 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:41.501 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:41.572 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:41.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:41.679 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:41.751 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:41.755 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Removed 82151 objects 2023-09-23T23:09:41.755 INFO:tasks.radosbench.radosbench.0.smithi052.stdout:Clean up completed and total clean up time :257.932 2023-09-23T23:09:41.855 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:41.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:41.958 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:41.988 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:42.236 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:42.239 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:41 smithi052 ceph-mon[187743]: pgmap v1511: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 136 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 39207/6594 objects degraded (594.586%); 4.9 MiB/s, 100 objects/s recovering 2023-09-23T23:09:42.240 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:41 smithi052 ceph-mon[180979]: pgmap v1511: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 136 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 39207/6594 objects degraded (594.586%); 4.9 MiB/s, 100 objects/s recovering 2023-09-23T23:09:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:42.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:42.341 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:42.369 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:42.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:41 smithi151 ceph-mon[159210]: pgmap v1511: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 136 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 39207/6594 objects degraded (594.586%); 4.9 MiB/s, 100 objects/s recovering 2023-09-23T23:09:42.768 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:42.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:42.871 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:42.899 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:43.068 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:43.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:43.171 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:43.197 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:43.217 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:43.239 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:43.239 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:43.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:43.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:43.344 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:43.346 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:09:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:09:43] "GET /metrics HTTP/1.1" 200 36639 "" "Prometheus/2.33.4" 2023-09-23T23:09:43.369 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:43.412 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:43.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:43.517 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:43.543 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:43.585 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:43.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:43.690 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:43.715 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:43.757 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:43.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:43.864 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:43.890 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:43.937 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:43.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:43.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:43.997 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:09:44.018 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:09:44.038 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:09:44.043 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:44.061 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:09:44.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:43 smithi052 ceph-mon[180979]: pgmap v1512: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 67 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 642 op/s; 39207/3264 objects degraded (1201.195%); 4.9 MiB/s, 100 objects/s recovering 2023-09-23T23:09:44.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:43 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:09:44.063 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/609295742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:44.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:43 smithi052 ceph-mon[187743]: pgmap v1512: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 67 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 642 op/s; 39207/3264 objects degraded (1201.195%); 4.9 MiB/s, 100 objects/s recovering 2023-09-23T23:09:44.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:43 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:09:44.063 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/609295742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:44.068 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:44.082 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:09:44.119 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:09:44.120 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:44.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:44.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:44.148 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:09:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:43 smithi151 ceph-mon[159210]: pgmap v1512: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 67 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 642 op/s; 39207/3264 objects degraded (1201.195%); 4.9 MiB/s, 100 objects/s recovering 2023-09-23T23:09:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:43 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:09:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/609295742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:44.224 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.245 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:09:44.256 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:44.657 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:44.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:44.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:44.761 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:44.789 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:45.006 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:09:45.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3587309481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:09:45.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3587309481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:09:45.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3587309481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:09:45.639 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:45.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:45.743 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:45.828 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:45.922 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:45.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:45.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:46.028 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:46.057 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:46.157 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:46.159 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: pgmap v1513: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 39 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 567 op/s; 38986/1899 objects degraded (2052.975%); 3.9 MiB/s, 84 objects/s recovering 2023-09-23T23:09:46.159 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:46.159 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3587309481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:09:46.159 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: osdmap e478: 8 total, 8 up, 5 in 2023-09-23T23:09:46.159 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.159 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.160 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.160 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:45 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 38986/1899 objects degraded (2052.975%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:46.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:46.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: pgmap v1513: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 39 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 567 op/s; 38986/1899 objects degraded (2052.975%); 3.9 MiB/s, 84 objects/s recovering 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3587309481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: osdmap e478: 8 total, 8 up, 5 in 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 38986/1899 objects degraded (2052.975%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:46.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: pgmap v1513: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 39 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 567 op/s; 38986/1899 objects degraded (2052.975%); 3.9 MiB/s, 84 objects/s recovering 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3587309481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: osdmap e478: 8 total, 8 up, 5 in 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: failed to encode map e478 with expected crc 2023-09-23T23:09:46.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:45 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 38986/1899 objects degraded (2052.975%), 24 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:46.261 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:46.288 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:46.383 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:46.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:46.488 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:46.523 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:46.650 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T23:09:46.651 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T23:09:46.731 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:46.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:46.835 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:46.861 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:46.905 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:46.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:46.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:46.933 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:09:46.933 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:09:46.933 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:09:46.933 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:09:46.934 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:09:46.976 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:09:46.978 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:09:47.011 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:47.044 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: failed to encode map e478 with expected crc 2023-09-23T23:09:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: failed to encode map e478 with expected crc 2023-09-23T23:09:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: osdmap e479: 8 total, 8 up, 5 in 2023-09-23T23:09:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:47.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:46 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: failed to encode map e478 with expected crc 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: failed to encode map e478 with expected crc 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: osdmap e479: 8 total, 8 up, 5 in 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:47.215 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: failed to encode map e478 with expected crc 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: failed to encode map e478 with expected crc 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: osdmap e479: 8 total, 8 up, 5 in 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: failed to encode map e479 with expected crc 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:47.216 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:46 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:47.429 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:09:48.058 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:48.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:48.089 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:48 smithi052 ceph-mon[180979]: failed to encode map e479 with expected crc 2023-09-23T23:09:48.089 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:48 smithi052 ceph-mon[180979]: pgmap v1516: 99 pgs: 1 peering, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 450 op/s; 38890/1653 objects degraded (2352.692%); 3.2 MiB/s, 75 objects/s recovering 2023-09-23T23:09:48.090 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:48 smithi052 ceph-mon[180979]: from='client.57208 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:48.090 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:48 smithi052 ceph-mon[187743]: failed to encode map e479 with expected crc 2023-09-23T23:09:48.090 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:48 smithi052 ceph-mon[187743]: pgmap v1516: 99 pgs: 1 peering, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 450 op/s; 38890/1653 objects degraded (2352.692%); 3.2 MiB/s, 75 objects/s recovering 2023-09-23T23:09:48.090 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:48 smithi052 ceph-mon[187743]: from='client.57208 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:48.091 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:09:48.164 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:48.190 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:48.234 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:48.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:48.305 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1516,"stamp":"2023-09-23T23:09:46.821670+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":35002272,"num_objects":551,"num_object_clones":0,"num_object_copies":1653,"num_objects_missing_on_primary":300,"num_objects_missing":300,"num_objects_degraded":38890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":551,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":164564,"num_write_kb":5260110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37651,"num_bytes_recovered":2405847488,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173818,"ondisk_log_size":173818,"up":297,"acting":278,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4490520,"kb_used_data":684976,"kb_used_omap":99,"kb_used_meta":3805276,"kb_avail":464235240,"statfs":{"total":479975178240,"available":475376885760,"internally_reserved":0,"allocated":701415424,"data_stored":693175179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3896602646},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-201850904,"num_objects":-3079,"num_object_clones":0,"num_object_copies":-9237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3606,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":608,"num_bytes_recovered":26542080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005832"},"pg_stats":[{"pgid":"11.12","version":"477'7835","reported_seq":7933,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.728247+0000","last_change":"2023-09-23T23:05:41.135551+0000","last_active":"2023-09-23T23:09:41.728247+0000","last_peered":"2023-09-23T23:09:41.728247+0000","last_clean":"2023-09-23T23:05:38.489927+0000","last_became_active":"2023-09-23T23:05:41.119181+0000","last_became_peered":"2023-09-23T23:05:41.119181+0000","last_unstale":"2023-09-23T23:09:41.728247+0000","last_undegraded":"2023-09-23T23:05:40.932576+0000","last_fullsized":"2023-09-23T23:05:40.868231+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7835,"log_dups_size":0,"ondisk_log_size":7835,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"2,3","objects":-2457}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":490,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968141+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:09:34.968141+0000","last_peered":"2023-09-23T23:09:34.968141+0000","last_clean":"2023-09-23T23:09:34.968141+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:09:34.968141+0000","last_undegraded":"2023-09-23T23:09:34.968141+0000","last_fullsized":"2023-09-23T23:09:34.968141+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":149,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109061+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:09:23.109061+0000","last_peered":"2023-09-23T23:09:23.109061+0000","last_clean":"2023-09-23T23:09:23.109061+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:09:23.109061+0000","last_undegraded":"2023-09-23T23:09:23.109061+0000","last_fullsized":"2023-09-23T23:09:23.109061+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"477'7729","reported_seq":12434,"reported_epoch":477,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:41.755543+0000","last_change":"2023-09-23T23:06:11.871775+0000","last_active":"2023-09-23T23:09:41.755543+0000","last_peered":"2023-09-23T23:09:41.755543+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.867199+0000","last_became_peered":"2023-09-23T23:06:11.867199+0000","last_unstale":"2023-09-23T23:09:41.755543+0000","last_undegraded":"2023-09-23T23:06:10.848340+0000","last_fullsized":"2023-09-23T23:09:41.755543+0000","mapping_epoch":441,"log_start":"415'1110","ondisk_log_start":"415'1110","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3571712,"num_objects":55,"num_object_clones":0,"num_object_copies":165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":343,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5076,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2300,"num_bytes_recovered":150667264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"2,3,6","objects":55}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":149,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.969763+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:09:34.969763+0000","last_peered":"2023-09-23T23:09:34.969763+0000","last_clean":"2023-09-23T23:09:34.969763+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:09:34.969763+0000","last_undegraded":"2023-09-23T23:09:34.969763+0000","last_fullsized":"2023-09-23T23:09:34.969763+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":496,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.961477+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:09:34.961477+0000","last_peered":"2023-09-23T23:09:34.961477+0000","last_clean":"2023-09-23T23:09:34.961477+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:09:34.961477+0000","last_undegraded":"2023-09-23T23:09:34.961477+0000","last_fullsized":"2023-09-23T23:09:34.961477+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"477'7800","reported_seq":12186,"reported_epoch":477,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:41.670536+0000","last_change":"2023-09-23T23:05:53.577594+0000","last_active":"2023-09-23T23:09:41.670536+0000","last_peered":"2023-09-23T23:09:41.670536+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:29.956349+0000","last_became_peered":"2023-09-23T23:05:29.956349+0000","last_unstale":"2023-09-23T23:09:41.670536+0000","last_undegraded":"2023-09-23T23:05:53.577243+0000","last_fullsized":"2023-09-23T23:09:41.670536+0000","mapping_epoch":428,"log_start":"415'1370","ondisk_log_start":"415'1370","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5144,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2145,"num_bytes_recovered":140574720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.107436+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:09:23.107436+0000","last_peered":"2023-09-23T23:09:23.107436+0000","last_clean":"2023-09-23T23:09:23.107436+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:09:23.107436+0000","last_undegraded":"2023-09-23T23:09:23.107436+0000","last_fullsized":"2023-09-23T23:09:23.107436+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.107811+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:09:23.107811+0000","last_peered":"2023-09-23T23:09:23.107811+0000","last_clean":"2023-09-23T23:09:23.107811+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:09:23.107811+0000","last_undegraded":"2023-09-23T23:09:23.107811+0000","last_fullsized":"2023-09-23T23:09:23.107811+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":421,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.108637+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:09:23.108637+0000","last_peered":"2023-09-23T23:09:23.108637+0000","last_clean":"2023-09-23T23:09:23.108637+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:09:23.108637+0000","last_undegraded":"2023-09-23T23:09:23.108637+0000","last_fullsized":"2023-09-23T23:09:23.108637+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":168,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109789+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:09:23.109789+0000","last_peered":"2023-09-23T23:09:23.109789+0000","last_clean":"2023-09-23T23:09:23.109789+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:09:23.109789+0000","last_undegraded":"2023-09-23T23:09:23.109789+0000","last_fullsized":"2023-09-23T23:09:23.109789+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":283,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109246+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:09:23.109246+0000","last_peered":"2023-09-23T23:09:23.109246+0000","last_clean":"2023-09-23T23:09:23.109246+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:09:23.109246+0000","last_undegraded":"2023-09-23T23:09:23.109246+0000","last_fullsized":"2023-09-23T23:09:23.109246+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":182,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.112867+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:09:23.112867+0000","last_peered":"2023-09-23T23:09:23.112867+0000","last_clean":"2023-09-23T23:09:23.112867+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:09:23.112867+0000","last_undegraded":"2023-09-23T23:09:23.112867+0000","last_fullsized":"2023-09-23T23:09:23.112867+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":215,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.377578+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:09:34.377578+0000","last_peered":"2023-09-23T23:09:34.377578+0000","last_clean":"2023-09-23T23:09:34.377578+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:09:34.377578+0000","last_undegraded":"2023-09-23T23:09:34.377578+0000","last_fullsized":"2023-09-23T23:09:34.377578+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":232,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.123792+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:09:23.123792+0000","last_peered":"2023-09-23T23:09:23.123792+0000","last_clean":"2023-09-23T23:09:23.123792+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:09:23.123792+0000","last_undegraded":"2023-09-23T23:09:23.123792+0000","last_fullsized":"2023-09-23T23:09:23.123792+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":568,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967624+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:09:34.967624+0000","last_peered":"2023-09-23T23:09:34.967624+0000","last_clean":"2023-09-23T23:09:34.967624+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:09:34.967624+0000","last_undegraded":"2023-09-23T23:09:34.967624+0000","last_fullsized":"2023-09-23T23:09:34.967624+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":461,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.966890+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:09:34.966890+0000","last_peered":"2023-09-23T23:09:34.966890+0000","last_clean":"2023-09-23T23:09:34.966890+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:09:34.966890+0000","last_undegraded":"2023-09-23T23:09:34.966890+0000","last_fullsized":"2023-09-23T23:09:34.966890+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":393,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.966987+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:09:34.966987+0000","last_peered":"2023-09-23T23:09:34.966987+0000","last_clean":"2023-09-23T23:09:34.966987+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:09:34.966987+0000","last_undegraded":"2023-09-23T23:09:34.966987+0000","last_fullsized":"2023-09-23T23:09:34.966987+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":161,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109136+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:09:23.109136+0000","last_peered":"2023-09-23T23:09:23.109136+0000","last_clean":"2023-09-23T23:09:23.109136+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:09:23.109136+0000","last_undegraded":"2023-09-23T23:09:23.109136+0000","last_fullsized":"2023-09-23T23:09:23.109136+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":479,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109274+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:09:23.109274+0000","last_peered":"2023-09-23T23:09:23.109274+0000","last_clean":"2023-09-23T23:09:23.109274+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:09:23.109274+0000","last_undegraded":"2023-09-23T23:09:23.109274+0000","last_fullsized":"2023-09-23T23:09:23.109274+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":393,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.108903+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:09:23.108903+0000","last_peered":"2023-09-23T23:09:23.108903+0000","last_clean":"2023-09-23T23:09:23.108903+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:09:23.108903+0000","last_undegraded":"2023-09-23T23:09:23.108903+0000","last_fullsized":"2023-09-23T23:09:23.108903+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"477'7628","reported_seq":12800,"reported_epoch":477,"state":"active+recovery_wait","last_fresh":"2023-09-23T23:09:41.629983+0000","last_change":"2023-09-23T23:06:32.889539+0000","last_active":"2023-09-23T23:09:41.629983+0000","last_peered":"2023-09-23T23:09:41.629983+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:06:32.875369+0000","last_became_peered":"2023-09-23T23:06:32.875369+0000","last_unstale":"2023-09-23T23:09:41.629983+0000","last_undegraded":"2023-09-23T23:09:41.629983+0000","last_fullsized":"2023-09-23T23:09:41.629983+0000","mapping_epoch":445,"log_start":"414'813","ondisk_log_start":"414'813","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6815,"log_dups_size":0,"ondisk_log_size":6815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7569408,"num_objects":116,"num_object_clones":0,"num_object_copies":348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4979,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":152666112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":116}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":381,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967419+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:09:34.967419+0000","last_peered":"2023-09-23T23:09:34.967419+0000","last_clean":"2023-09-23T23:09:34.967419+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:09:34.967419+0000","last_undegraded":"2023-09-23T23:09:34.967419+0000","last_fullsized":"2023-09-23T23:09:34.967419+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.961428+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:09:34.961428+0000","last_peered":"2023-09-23T23:09:34.961428+0000","last_clean":"2023-09-23T23:09:34.961428+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:09:34.961428+0000","last_undegraded":"2023-09-23T23:09:34.961428+0000","last_fullsized":"2023-09-23T23:09:34.961428+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"477'7838","reported_seq":14225,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.720696+0000","last_change":"2023-09-23T23:05:54.758020+0000","last_active":"2023-09-23T23:09:41.720696+0000","last_peered":"2023-09-23T23:09:41.720696+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.744936+0000","last_became_peered":"2023-09-23T23:05:54.744936+0000","last_unstale":"2023-09-23T23:09:41.720696+0000","last_undegraded":"2023-09-23T23:05:54.632836+0000","last_fullsized":"2023-09-23T23:05:54.554468+0000","mapping_epoch":437,"log_start":"415'1313","ondisk_log_start":"415'1313","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1736704,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":649,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5148,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2143,"num_bytes_recovered":140247040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"3,6","objects":-290}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":91,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.107515+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:09:23.107515+0000","last_peered":"2023-09-23T23:09:23.107515+0000","last_clean":"2023-09-23T23:09:23.107515+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:09:23.107515+0000","last_undegraded":"2023-09-23T23:09:23.107515+0000","last_fullsized":"2023-09-23T23:09:23.107515+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":174,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.961612+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:09:34.961612+0000","last_peered":"2023-09-23T23:09:34.961612+0000","last_clean":"2023-09-23T23:09:34.961612+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:09:34.961612+0000","last_undegraded":"2023-09-23T23:09:34.961612+0000","last_fullsized":"2023-09-23T23:09:34.961612+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"477'7788","reported_seq":7891,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.709445+0000","last_change":"2023-09-23T23:06:12.267145+0000","last_active":"2023-09-23T23:09:41.709445+0000","last_peered":"2023-09-23T23:09:41.709445+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702243+0000","last_became_peered":"2023-09-23T23:05:54.702243+0000","last_unstale":"2023-09-23T23:09:41.709445+0000","last_undegraded":"2023-09-23T23:05:54.583979+0000","last_fullsized":"2023-09-23T23:05:54.563563+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7788,"log_dups_size":0,"ondisk_log_size":7788,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-2365}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":388,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.973650+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:09:34.973650+0000","last_peered":"2023-09-23T23:09:34.973650+0000","last_clean":"2023-09-23T23:09:34.973650+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:09:34.973650+0000","last_undegraded":"2023-09-23T23:09:34.973650+0000","last_fullsized":"2023-09-23T23:09:34.973650+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":344,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.966581+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:09:34.966581+0000","last_peered":"2023-09-23T23:09:34.966581+0000","last_clean":"2023-09-23T23:09:34.966581+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:09:34.966581+0000","last_undegraded":"2023-09-23T23:09:34.966581+0000","last_fullsized":"2023-09-23T23:09:34.966581+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"477'7616","reported_seq":12627,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:41.738144+0000","last_change":"2023-09-23T23:06:09.613618+0000","last_active":"2023-09-23T23:09:41.738144+0000","last_peered":"2023-09-23T23:09:41.738144+0000","last_clean":"2023-09-23T23:09:41.738144+0000","last_became_active":"2023-09-23T23:06:09.609837+0000","last_became_peered":"2023-09-23T23:06:09.609837+0000","last_unstale":"2023-09-23T23:09:41.738144+0000","last_undegraded":"2023-09-23T23:09:41.738144+0000","last_fullsized":"2023-09-23T23:09:41.738144+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7616,"log_dups_size":0,"ondisk_log_size":7616,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5144576,"num_objects":78,"num_object_clones":0,"num_object_copies":234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4984,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2453,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":106,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.123573+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:09:23.123573+0000","last_peered":"2023-09-23T23:09:23.123573+0000","last_clean":"2023-09-23T23:09:23.123573+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:09:23.123573+0000","last_undegraded":"2023-09-23T23:09:23.123573+0000","last_fullsized":"2023-09-23T23:09:23.123573+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":485,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967340+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:09:34.967340+0000","last_peered":"2023-09-23T23:09:34.967340+0000","last_clean":"2023-09-23T23:09:34.967340+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:09:34.967340+0000","last_undegraded":"2023-09-23T23:09:34.967340+0000","last_fullsized":"2023-09-23T23:09:34.967340+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"477'7781","reported_seq":14907,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.702497+0000","last_change":"2023-09-23T23:06:16.393155+0000","last_active":"2023-09-23T23:09:41.702497+0000","last_peered":"2023-09-23T23:09:41.702497+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:16.293083+0000","last_became_peered":"2023-09-23T23:06:16.293083+0000","last_unstale":"2023-09-23T23:09:41.702497+0000","last_undegraded":"2023-09-23T23:06:14.163673+0000","last_fullsized":"2023-09-23T23:06:12.174890+0000","mapping_epoch":441,"log_start":"414'334","ondisk_log_start":"414'334","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7447,"log_dups_size":0,"ondisk_log_size":7447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":95,"num_objects_missing":95,"num_objects_degraded":318,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5132,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2484,"num_bytes_recovered":162791424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,1],"avail_no_missing":["0","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":135,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.108827+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:09:23.108827+0000","last_peered":"2023-09-23T23:09:23.108827+0000","last_clean":"2023-09-23T23:09:23.108827+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:09:23.108827+0000","last_undegraded":"2023-09-23T23:09:23.108827+0000","last_fullsized":"2023-09-23T23:09:23.108827+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":74,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.107570+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:09:23.107570+0000","last_peered":"2023-09-23T23:09:23.107570+0000","last_clean":"2023-09-23T23:09:23.107570+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:09:23.107570+0000","last_undegraded":"2023-09-23T23:09:23.107570+0000","last_fullsized":"2023-09-23T23:09:23.107570+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.4","version":"477'7985","reported_seq":8427,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.668107+0000","last_change":"2023-09-23T23:06:09.618502+0000","last_active":"2023-09-23T23:09:41.668107+0000","last_peered":"2023-09-23T23:09:41.668107+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.610763+0000","last_became_peered":"2023-09-23T23:06:09.610763+0000","last_unstale":"2023-09-23T23:09:41.668107+0000","last_undegraded":"2023-09-23T23:06:09.594475+0000","last_fullsized":"2023-09-23T23:06:09.589009+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7985,"log_dups_size":0,"ondisk_log_size":7985,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5269,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11304960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-2044}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109065+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:09:23.109065+0000","last_peered":"2023-09-23T23:09:23.109065+0000","last_clean":"2023-09-23T23:09:23.109065+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:09:23.109065+0000","last_undegraded":"2023-09-23T23:09:23.109065+0000","last_fullsized":"2023-09-23T23:09:23.109065+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":300,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.966815+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:09:34.966815+0000","last_peered":"2023-09-23T23:09:34.966815+0000","last_clean":"2023-09-23T23:09:34.966815+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:09:34.966815+0000","last_undegraded":"2023-09-23T23:09:34.966815+0000","last_fullsized":"2023-09-23T23:09:34.966815+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.3","version":"477'7840","reported_seq":12223,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.477299+0000","last_change":"2023-09-23T23:07:56.507184+0000","last_active":"2023-09-23T23:09:41.477299+0000","last_peered":"2023-09-23T23:09:41.477299+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.488393+0000","last_became_peered":"2023-09-23T23:07:56.488393+0000","last_unstale":"2023-09-23T23:09:41.477299+0000","last_undegraded":"2023-09-23T23:07:56.465702+0000","last_fullsized":"2023-09-23T23:07:56.443434+0000","mapping_epoch":459,"log_start":"416'2232","ondisk_log_start":"416'2232","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3080192,"num_objects":47,"num_object_clones":0,"num_object_copies":141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":194,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5157,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2130,"num_bytes_recovered":139493376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,6","objects":47}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":182,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.110000+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:09:23.110000+0000","last_peered":"2023-09-23T23:09:23.110000+0000","last_clean":"2023-09-23T23:09:23.110000+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:09:23.110000+0000","last_undegraded":"2023-09-23T23:09:23.110000+0000","last_fullsized":"2023-09-23T23:09:23.110000+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":480,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967510+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:09:34.967510+0000","last_peered":"2023-09-23T23:09:34.967510+0000","last_clean":"2023-09-23T23:09:34.967510+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:09:34.967510+0000","last_undegraded":"2023-09-23T23:09:34.967510+0000","last_fullsized":"2023-09-23T23:09:34.967510+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"477'7772","reported_seq":7871,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.630882+0000","last_change":"2023-09-23T23:05:41.139261+0000","last_active":"2023-09-23T23:09:41.630882+0000","last_peered":"2023-09-23T23:09:41.630882+0000","last_clean":"2023-09-23T23:05:38.841794+0000","last_became_active":"2023-09-23T23:05:41.121014+0000","last_became_peered":"2023-09-23T23:05:41.121014+0000","last_unstale":"2023-09-23T23:09:41.630882+0000","last_undegraded":"2023-09-23T23:05:40.918064+0000","last_fullsized":"2023-09-23T23:05:40.867911+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":410,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7772,"log_dups_size":0,"ondisk_log_size":7772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2064384,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5108,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"2,3","objects":-2364}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968294+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:09:34.968294+0000","last_peered":"2023-09-23T23:09:34.968294+0000","last_clean":"2023-09-23T23:09:34.968294+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:09:34.968294+0000","last_undegraded":"2023-09-23T23:09:34.968294+0000","last_fullsized":"2023-09-23T23:09:34.968294+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":488,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968095+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:09:34.968095+0000","last_peered":"2023-09-23T23:09:34.968095+0000","last_clean":"2023-09-23T23:09:34.968095+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:09:34.968095+0000","last_undegraded":"2023-09-23T23:09:34.968095+0000","last_fullsized":"2023-09-23T23:09:34.968095+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"477'7838","reported_seq":13005,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.730253+0000","last_change":"2023-09-23T23:06:11.823176+0000","last_active":"2023-09-23T23:09:41.730253+0000","last_peered":"2023-09-23T23:09:41.730253+0000","last_clean":"2023-09-23T23:05:02.364362+0000","last_became_active":"2023-09-23T23:06:11.725948+0000","last_became_peered":"2023-09-23T23:06:11.725948+0000","last_unstale":"2023-09-23T23:09:41.730253+0000","last_undegraded":"2023-09-23T23:06:10.730907+0000","last_fullsized":"2023-09-23T23:06:08.586237+0000","mapping_epoch":440,"log_start":"414'474","ondisk_log_start":"414'474","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7364,"log_dups_size":0,"ondisk_log_size":7364,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":87,"num_objects_missing":87,"num_objects_degraded":218,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5184,"num_write_kb":164160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2531,"num_bytes_recovered":165740544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[6,2],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":501,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967124+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:09:34.967124+0000","last_peered":"2023-09-23T23:09:34.967124+0000","last_clean":"2023-09-23T23:09:34.967124+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:09:34.967124+0000","last_undegraded":"2023-09-23T23:09:34.967124+0000","last_fullsized":"2023-09-23T23:09:34.967124+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":506,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967694+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:09:34.967694+0000","last_peered":"2023-09-23T23:09:34.967694+0000","last_clean":"2023-09-23T23:09:34.967694+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:09:34.967694+0000","last_undegraded":"2023-09-23T23:09:34.967694+0000","last_fullsized":"2023-09-23T23:09:34.967694+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"477'7726","reported_seq":11998,"reported_epoch":477,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:41.674362+0000","last_change":"2023-09-23T23:05:53.809651+0000","last_active":"2023-09-23T23:09:41.674362+0000","last_peered":"2023-09-23T23:09:41.674362+0000","last_clean":"2023-09-23T23:05:02.419445+0000","last_became_active":"2023-09-23T23:05:30.887450+0000","last_became_peered":"2023-09-23T23:05:30.887450+0000","last_unstale":"2023-09-23T23:09:41.674362+0000","last_undegraded":"2023-09-23T23:05:53.808745+0000","last_fullsized":"2023-09-23T23:09:41.674362+0000","mapping_epoch":429,"log_start":"415'1132","ondisk_log_start":"415'1132","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":90,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5070,"num_write_kb":163424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2087,"num_bytes_recovered":136773632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"1,3,6","objects":37}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":150,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109912+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:09:23.109912+0000","last_peered":"2023-09-23T23:09:23.109912+0000","last_clean":"2023-09-23T23:09:23.109912+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:09:23.109912+0000","last_undegraded":"2023-09-23T23:09:23.109912+0000","last_fullsized":"2023-09-23T23:09:23.109912+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":384,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.958904+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:09:34.958904+0000","last_peered":"2023-09-23T23:09:34.958904+0000","last_clean":"2023-09-23T23:09:34.958904+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:09:34.958904+0000","last_undegraded":"2023-09-23T23:09:34.958904+0000","last_fullsized":"2023-09-23T23:09:34.958904+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"477'10456","reported_seq":11927,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.651642+0000","last_change":"2023-09-23T23:05:53.630525+0000","last_active":"2023-09-23T23:09:41.651642+0000","last_peered":"2023-09-23T23:09:41.651642+0000","last_clean":"2023-09-23T23:05:02.409992+0000","last_became_active":"2023-09-23T23:05:04.830351+0000","last_became_peered":"2023-09-23T23:05:04.830351+0000","last_unstale":"2023-09-23T23:09:41.651642+0000","last_undegraded":"2023-09-23T23:05:04.710915+0000","last_fullsized":"2023-09-23T23:05:04.430622+0000","mapping_epoch":422,"log_start":"425'4674","ondisk_log_start":"425'4674","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:25.049953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3866,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10456,"num_write_kb":334529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"2,4","objects":-3866}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":397,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.122996+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:09:23.122996+0000","last_peered":"2023-09-23T23:09:23.122996+0000","last_clean":"2023-09-23T23:09:23.122996+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:09:23.122996+0000","last_undegraded":"2023-09-23T23:09:23.122996+0000","last_fullsized":"2023-09-23T23:09:23.122996+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.108664+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:09:23.108664+0000","last_peered":"2023-09-23T23:09:23.108664+0000","last_clean":"2023-09-23T23:09:23.108664+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:09:23.108664+0000","last_undegraded":"2023-09-23T23:09:23.108664+0000","last_fullsized":"2023-09-23T23:09:23.108664+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":219,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968059+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:09:34.968059+0000","last_peered":"2023-09-23T23:09:34.968059+0000","last_clean":"2023-09-23T23:09:34.968059+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:09:34.968059+0000","last_undegraded":"2023-09-23T23:09:34.968059+0000","last_fullsized":"2023-09-23T23:09:34.968059+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"477'10276","reported_seq":10397,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.715272+0000","last_change":"2023-09-23T23:06:08.610656+0000","last_active":"2023-09-23T23:09:41.715272+0000","last_peered":"2023-09-23T23:09:41.715272+0000","last_clean":"2023-09-23T23:05:38.825441+0000","last_became_active":"2023-09-23T23:05:41.120820+0000","last_became_peered":"2023-09-23T23:05:41.120820+0000","last_unstale":"2023-09-23T23:09:41.715272+0000","last_undegraded":"2023-09-23T23:05:40.964086+0000","last_fullsized":"2023-09-23T23:05:40.871181+0000","mapping_epoch":433,"log_start":"443'5767","ondisk_log_start":"443'5767","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4509,"log_dups_size":0,"ondisk_log_size":4509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:11.062560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10276,"num_write_kb":328832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"1,3","objects":-4825}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":213,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.110436+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:09:23.110436+0000","last_peered":"2023-09-23T23:09:23.110436+0000","last_clean":"2023-09-23T23:09:23.110436+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:09:23.110436+0000","last_undegraded":"2023-09-23T23:09:23.110436+0000","last_fullsized":"2023-09-23T23:09:23.110436+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":516,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967151+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:09:34.967151+0000","last_peered":"2023-09-23T23:09:34.967151+0000","last_clean":"2023-09-23T23:09:34.967151+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:09:34.967151+0000","last_undegraded":"2023-09-23T23:09:34.967151+0000","last_fullsized":"2023-09-23T23:09:34.967151+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":382,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.122943+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:09:23.122943+0000","last_peered":"2023-09-23T23:09:23.122943+0000","last_clean":"2023-09-23T23:09:23.122943+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:09:23.122943+0000","last_undegraded":"2023-09-23T23:09:23.122943+0000","last_fullsized":"2023-09-23T23:09:23.122943+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"477'7891","reported_seq":14422,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.734697+0000","last_change":"2023-09-23T23:05:54.766995+0000","last_active":"2023-09-23T23:09:41.734697+0000","last_peered":"2023-09-23T23:09:41.734697+0000","last_clean":"2023-09-23T23:05:02.392363+0000","last_became_active":"2023-09-23T23:05:54.759585+0000","last_became_peered":"2023-09-23T23:05:54.759585+0000","last_unstale":"2023-09-23T23:09:41.734697+0000","last_undegraded":"2023-09-23T23:05:54.653133+0000","last_fullsized":"2023-09-23T23:05:54.557285+0000","mapping_epoch":437,"log_start":"415'1479","ondisk_log_start":"415'1479","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6412,"log_dups_size":0,"ondisk_log_size":6412,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5200,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2196,"num_bytes_recovered":143851520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"3,6","objects":-315}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":457,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.121884+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:09:23.121884+0000","last_peered":"2023-09-23T23:09:23.121884+0000","last_clean":"2023-09-23T23:09:23.121884+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:09:23.121884+0000","last_undegraded":"2023-09-23T23:09:23.121884+0000","last_fullsized":"2023-09-23T23:09:23.121884+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":309,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.121393+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:09:23.121393+0000","last_peered":"2023-09-23T23:09:23.121393+0000","last_clean":"2023-09-23T23:09:23.121393+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:09:23.121393+0000","last_undegraded":"2023-09-23T23:09:23.121393+0000","last_fullsized":"2023-09-23T23:09:23.121393+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":414,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.107958+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:09:23.107958+0000","last_peered":"2023-09-23T23:09:23.107958+0000","last_clean":"2023-09-23T23:09:23.107958+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:09:23.107958+0000","last_undegraded":"2023-09-23T23:09:23.107958+0000","last_fullsized":"2023-09-23T23:09:23.107958+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"477'10244","reported_seq":19348,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.734471+0000","last_change":"2023-09-23T23:05:54.764537+0000","last_active":"2023-09-23T23:09:41.734471+0000","last_peered":"2023-09-23T23:09:41.734471+0000","last_clean":"2023-09-23T23:05:02.401371+0000","last_became_active":"2023-09-23T23:05:54.716057+0000","last_became_peered":"2023-09-23T23:05:54.716057+0000","last_unstale":"2023-09-23T23:09:41.734471+0000","last_undegraded":"2023-09-23T23:05:54.698944+0000","last_fullsized":"2023-09-23T23:05:54.560652+0000","mapping_epoch":437,"log_start":"443'5734","ondisk_log_start":"443'5734","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:04.488082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1076,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10244,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4503,"num_bytes_recovered":295108608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,6","objects":-619}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":388,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.122081+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:09:23.122081+0000","last_peered":"2023-09-23T23:09:23.122081+0000","last_clean":"2023-09-23T23:09:23.122081+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:09:23.122081+0000","last_undegraded":"2023-09-23T23:09:23.122081+0000","last_fullsized":"2023-09-23T23:09:23.122081+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":589,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T23:09:23.121643+0000","last_change":"2023-09-23T23:06:09.610149+0000","last_active":"2023-09-23T23:09:23.121643+0000","last_peered":"2023-09-23T23:09:23.121643+0000","last_clean":"2023-09-23T22:32:49.505889+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:09:23.121643+0000","last_undegraded":"2023-09-23T23:06:09.588664+0000","last_fullsized":"2023-09-23T23:09:23.121643+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":168,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.107659+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:09:23.107659+0000","last_peered":"2023-09-23T23:09:23.107659+0000","last_clean":"2023-09-23T23:09:23.107659+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:09:23.107659+0000","last_undegraded":"2023-09-23T23:09:23.107659+0000","last_fullsized":"2023-09-23T23:09:23.107659+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":396,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968268+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:09:34.968268+0000","last_peered":"2023-09-23T23:09:34.968268+0000","last_clean":"2023-09-23T23:09:34.968268+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:09:34.968268+0000","last_undegraded":"2023-09-23T23:09:34.968268+0000","last_fullsized":"2023-09-23T23:09:34.968268+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"477'7859","reported_seq":8730,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.703610+0000","last_change":"2023-09-23T23:05:44.048640+0000","last_active":"2023-09-23T23:09:41.703610+0000","last_peered":"2023-09-23T23:09:41.703610+0000","last_clean":"2023-09-23T23:05:02.431507+0000","last_became_active":"2023-09-23T23:05:44.042037+0000","last_became_peered":"2023-09-23T23:05:44.042037+0000","last_unstale":"2023-09-23T23:09:41.703610+0000","last_undegraded":"2023-09-23T23:05:41.035728+0000","last_fullsized":"2023-09-23T23:05:40.879456+0000","mapping_epoch":433,"log_start":"416'1633","ondisk_log_start":"416'1633","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":6226,"log_dups_size":0,"ondisk_log_size":6226,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5209,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":383,"num_bytes_recovered":25067520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,1],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"1,3","objects":-2177}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":402,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.954637+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:09:34.954637+0000","last_peered":"2023-09-23T23:09:34.954637+0000","last_clean":"2023-09-23T23:09:34.954637+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:09:34.954637+0000","last_undegraded":"2023-09-23T23:09:34.954637+0000","last_fullsized":"2023-09-23T23:09:34.954637+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":372,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967028+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:09:34.967028+0000","last_peered":"2023-09-23T23:09:34.967028+0000","last_clean":"2023-09-23T23:09:34.967028+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:09:34.967028+0000","last_undegraded":"2023-09-23T23:09:34.967028+0000","last_fullsized":"2023-09-23T23:09:34.967028+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":337,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.122353+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:09:23.122353+0000","last_peered":"2023-09-23T23:09:23.122353+0000","last_clean":"2023-09-23T23:09:23.122353+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:09:23.122353+0000","last_undegraded":"2023-09-23T23:09:23.122353+0000","last_fullsized":"2023-09-23T23:09:23.122353+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.c","version":"477'10232","reported_seq":10310,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:41.742985+0000","last_change":"2023-09-23T23:03:43.136140+0000","last_active":"2023-09-23T23:09:41.742985+0000","last_peered":"2023-09-23T23:09:41.742985+0000","last_clean":"2023-09-23T23:09:41.742985+0000","last_became_active":"2023-09-23T23:03:43.135856+0000","last_became_peered":"2023-09-23T23:03:43.135856+0000","last_unstale":"2023-09-23T23:09:41.742985+0000","last_undegraded":"2023-09-23T23:09:41.742985+0000","last_fullsized":"2023-09-23T23:09:41.742985+0000","mapping_epoch":410,"log_start":"420'4411","ondisk_log_start":"420'4411","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5821,"log_dups_size":0,"ondisk_log_size":5821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:24.598873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10232,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":337,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968022+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:09:34.968022+0000","last_peered":"2023-09-23T23:09:34.968022+0000","last_clean":"2023-09-23T23:09:34.968022+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:09:34.968022+0000","last_undegraded":"2023-09-23T23:09:34.968022+0000","last_fullsized":"2023-09-23T23:09:34.968022+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":334,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.123190+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:09:23.123190+0000","last_peered":"2023-09-23T23:09:23.123190+0000","last_clean":"2023-09-23T23:09:23.123190+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:09:23.123190+0000","last_undegraded":"2023-09-23T23:09:23.123190+0000","last_fullsized":"2023-09-23T23:09:23.123190+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":299,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.121929+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:09:23.121929+0000","last_peered":"2023-09-23T23:09:23.121929+0000","last_clean":"2023-09-23T23:09:23.121929+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:09:23.121929+0000","last_undegraded":"2023-09-23T23:09:23.121929+0000","last_fullsized":"2023-09-23T23:09:23.121929+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.f","version":"477'10130","reported_seq":10216,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.728354+0000","last_change":"2023-09-23T23:05:41.146881+0000","last_active":"2023-09-23T23:09:41.728354+0000","last_peered":"2023-09-23T23:09:41.728354+0000","last_clean":"2023-09-23T23:05:38.792301+0000","last_became_active":"2023-09-23T23:05:41.135545+0000","last_became_peered":"2023-09-23T23:05:41.135545+0000","last_unstale":"2023-09-23T23:09:41.728354+0000","last_undegraded":"2023-09-23T23:05:40.953266+0000","last_fullsized":"2023-09-23T23:05:40.869206+0000","mapping_epoch":433,"log_start":"420'4283","ondisk_log_start":"420'4283","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5847,"log_dups_size":0,"ondisk_log_size":5847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:33.717722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10130,"num_write_kb":324160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"1,2","objects":-4769}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":200,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.108777+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:09:23.108777+0000","last_peered":"2023-09-23T23:09:23.108777+0000","last_clean":"2023-09-23T23:09:23.108777+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:09:23.108777+0000","last_undegraded":"2023-09-23T23:09:23.108777+0000","last_fullsized":"2023-09-23T23:09:23.108777+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":176,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967622+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:09:34.967622+0000","last_peered":"2023-09-23T23:09:34.967622+0000","last_clean":"2023-09-23T23:09:34.967622+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:09:34.967622+0000","last_undegraded":"2023-09-23T23:09:34.967622+0000","last_fullsized":"2023-09-23T23:09:34.967622+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":184,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:29.372986+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:09:29.372986+0000","last_peered":"2023-09-23T23:09:29.372986+0000","last_clean":"2023-09-23T23:09:29.372986+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:09:29.372986+0000","last_undegraded":"2023-09-23T23:09:29.372986+0000","last_fullsized":"2023-09-23T23:09:29.372986+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"477'10332","reported_seq":20751,"reported_epoch":478,"state":"peering","last_fresh":"2023-09-23T23:09:45.001373+0000","last_change":"2023-09-23T23:09:45.004110+0000","last_active":"2023-09-23T23:09:45.001373+0000","last_peered":"2023-09-23T23:09:44.704551+0000","last_clean":"2023-09-23T23:05:38.850063+0000","last_became_active":"2023-09-23T23:05:41.214184+0000","last_became_peered":"2023-09-23T23:05:41.214184+0000","last_unstale":"2023-09-23T23:09:45.001373+0000","last_undegraded":"2023-09-23T23:09:45.001373+0000","last_fullsized":"2023-09-23T23:09:45.001373+0000","mapping_epoch":478,"log_start":"425'4620","ondisk_log_start":"425'4620","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10332,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5166,"num_bytes_recovered":307822592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,3],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":426,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.125357+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:09:23.125357+0000","last_peered":"2023-09-23T23:09:23.125357+0000","last_clean":"2023-09-23T23:09:23.125357+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:09:23.125357+0000","last_undegraded":"2023-09-23T23:09:23.125357+0000","last_fullsized":"2023-09-23T23:09:23.125357+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":99,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967781+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:09:34.967781+0000","last_peered":"2023-09-23T23:09:34.967781+0000","last_clean":"2023-09-23T23:09:34.967781+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:09:34.967781+0000","last_undegraded":"2023-09-23T23:09:34.967781+0000","last_fullsized":"2023-09-23T23:09:34.967781+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":470,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967207+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:09:34.967207+0000","last_peered":"2023-09-23T23:09:34.967207+0000","last_clean":"2023-09-23T23:09:34.967207+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:09:34.967207+0000","last_undegraded":"2023-09-23T23:09:34.967207+0000","last_fullsized":"2023-09-23T23:09:34.967207+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"477'7934","reported_seq":10025,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.686472+0000","last_change":"2023-09-23T23:07:56.517407+0000","last_active":"2023-09-23T23:09:41.686472+0000","last_peered":"2023-09-23T23:09:41.686472+0000","last_clean":"2023-09-23T23:05:02.422124+0000","last_became_active":"2023-09-23T23:07:56.507318+0000","last_became_peered":"2023-09-23T23:07:56.507318+0000","last_unstale":"2023-09-23T23:09:41.686472+0000","last_undegraded":"2023-09-23T23:07:56.466720+0000","last_fullsized":"2023-09-23T23:07:56.443187+0000","mapping_epoch":459,"log_start":"417'2566","ondisk_log_start":"417'2566","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":5368,"log_dups_size":0,"ondisk_log_size":5368,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5251,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":64454656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2","objects":-1231}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":69,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.966681+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:09:34.966681+0000","last_peered":"2023-09-23T23:09:34.966681+0000","last_clean":"2023-09-23T23:09:34.966681+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:09:34.966681+0000","last_undegraded":"2023-09-23T23:09:34.966681+0000","last_fullsized":"2023-09-23T23:09:34.966681+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":228,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.123184+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:09:23.123184+0000","last_peered":"2023-09-23T23:09:23.123184+0000","last_clean":"2023-09-23T23:09:23.123184+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:09:23.123184+0000","last_undegraded":"2023-09-23T23:09:23.123184+0000","last_fullsized":"2023-09-23T23:09:23.123184+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"477'7892","reported_seq":8319,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.696576+0000","last_change":"2023-09-23T23:06:09.618456+0000","last_active":"2023-09-23T23:09:41.696576+0000","last_peered":"2023-09-23T23:09:41.696576+0000","last_clean":"2023-09-23T23:05:02.407690+0000","last_became_active":"2023-09-23T23:06:09.608079+0000","last_became_peered":"2023-09-23T23:06:09.608079+0000","last_unstale":"2023-09-23T23:09:41.696576+0000","last_undegraded":"2023-09-23T23:06:09.594306+0000","last_fullsized":"2023-09-23T23:06:09.588959+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7892,"log_dups_size":0,"ondisk_log_size":7892,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3047424,"num_objects":47,"num_object_clones":0,"num_object_copies":141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5177,"num_write_kb":167136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10715136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-1928}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":300,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967424+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:09:34.967424+0000","last_peered":"2023-09-23T23:09:34.967424+0000","last_clean":"2023-09-23T23:09:34.967424+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:09:34.967424+0000","last_undegraded":"2023-09-23T23:09:34.967424+0000","last_fullsized":"2023-09-23T23:09:34.967424+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":501,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.967234+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:09:34.967234+0000","last_peered":"2023-09-23T23:09:34.967234+0000","last_clean":"2023-09-23T23:09:34.967234+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:09:34.967234+0000","last_undegraded":"2023-09-23T23:09:34.967234+0000","last_fullsized":"2023-09-23T23:09:34.967234+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"477'7688","reported_seq":13958,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.615564+0000","last_change":"2023-09-23T23:06:12.270173+0000","last_active":"2023-09-23T23:09:41.615564+0000","last_peered":"2023-09-23T23:09:41.615564+0000","last_clean":"2023-09-23T23:05:02.421154+0000","last_became_active":"2023-09-23T23:06:12.181762+0000","last_became_peered":"2023-09-23T23:06:12.181762+0000","last_unstale":"2023-09-23T23:09:41.615564+0000","last_undegraded":"2023-09-23T23:06:10.254454+0000","last_fullsized":"2023-09-23T23:06:08.588756+0000","mapping_epoch":440,"log_start":"414'19","ondisk_log_start":"414'19","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7669,"log_dups_size":0,"ondisk_log_size":7669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3047424,"num_objects":47,"num_object_clones":0,"num_object_copies":141,"num_objects_missing_on_primary":116,"num_objects_missing":116,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5040,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2092,"num_bytes_recovered":137035776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[6,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"3,6","objects":-251}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":327,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968181+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:09:34.968181+0000","last_peered":"2023-09-23T23:09:34.968181+0000","last_clean":"2023-09-23T23:09:34.968181+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:09:34.968181+0000","last_undegraded":"2023-09-23T23:09:34.968181+0000","last_fullsized":"2023-09-23T23:09:34.968181+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":397,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.968219+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:09:34.968219+0000","last_peered":"2023-09-23T23:09:34.968219+0000","last_clean":"2023-09-23T23:09:34.968219+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:09:34.968219+0000","last_undegraded":"2023-09-23T23:09:34.968219+0000","last_fullsized":"2023-09-23T23:09:34.968219+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"477'7773","reported_seq":8790,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.622450+0000","last_change":"2023-09-23T23:06:12.175060+0000","last_active":"2023-09-23T23:09:41.622450+0000","last_peered":"2023-09-23T23:09:41.622450+0000","last_clean":"2023-09-23T23:05:38.747777+0000","last_became_active":"2023-09-23T23:05:41.068602+0000","last_became_peered":"2023-09-23T23:05:41.068602+0000","last_unstale":"2023-09-23T23:09:41.622450+0000","last_undegraded":"2023-09-23T23:05:40.922998+0000","last_fullsized":"2023-09-23T23:05:40.868625+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7773,"log_dups_size":0,"ondisk_log_size":7773,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5142,"num_write_kb":162016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"1,2","objects":-1987}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":52,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.109332+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:09:23.109332+0000","last_peered":"2023-09-23T23:09:23.109332+0000","last_clean":"2023-09-23T23:09:23.109332+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:09:23.109332+0000","last_undegraded":"2023-09-23T23:09:23.109332+0000","last_fullsized":"2023-09-23T23:09:23.109332+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":195,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.375537+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:09:34.375537+0000","last_peered":"2023-09-23T23:09:34.375537+0000","last_clean":"2023-09-23T23:09:34.375537+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:09:34.375537+0000","last_undegraded":"2023-09-23T23:09:34.375537+0000","last_fullsized":"2023-09-23T23:09:34.375537+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"477'7705","reported_seq":7798,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T23:09:41.736515+0000","last_change":"2023-09-23T23:06:08.617232+0000","last_active":"2023-09-23T23:09:41.736515+0000","last_peered":"2023-09-23T23:09:41.736515+0000","last_clean":"2023-09-23T23:05:28.792850+0000","last_became_active":"2023-09-23T23:05:54.702020+0000","last_became_peered":"2023-09-23T23:05:54.702020+0000","last_unstale":"2023-09-23T23:09:41.736515+0000","last_undegraded":"2023-09-23T23:05:54.583698+0000","last_fullsized":"2023-09-23T23:05:54.563744+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:03:41.870679+0000","last_clean_scrub_stamp":"2023-09-23T23:03:41.870679+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2719744,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2514,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5055,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":-2214}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":245,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T23:09:23.125579+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:09:23.125579+0000","last_peered":"2023-09-23T23:09:23.125579+0000","last_clean":"2023-09-23T23:09:23.125579+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:09:23.125579+0000","last_undegraded":"2023-09-23T23:09:23.125579+0000","last_fullsized":"2023-09-23T23:09:23.125579+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":472,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T23:09:34.966663+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:09:34.966663+0000","last_peered":"2023-09-23T23:09:34.966663+0000","last_clean":"2023-09-23T23:09:34.966663+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:09:34.966663+0000","last_undegraded":"2023-09-23T23:09:34.966663+0000","last_fullsized":"2023-09-23T23:09:34.966663+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":34406400,"num_objects":527,"num_object_clones":0,"num_object_copies":1581,"num_objects_missing_on_primary":298,"num_objects_missing":298,"num_objects_degraded":38888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":527,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":164304,"num_write_kb":5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37581,"num_bytes_recovered":2405826560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5139005440,"data_stored":5139005440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173624,"ondisk_log_size":173624,"up":78,"acting":59,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430721,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626945,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903808,"kb_used_data":5176,"kb_used_omap":0,"kb_used_meta":898624,"kb_avail":92841344,"statfs":{"total":95995035648,"available":95069536256,"internally_reserved":0,"allocated":5300224,"data_stored":4230659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920190976},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.3109999999999999,"15min":4.6340000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.923,"5min":10.853,"15min":115.32899999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.304,"15min":4.6719999999999997},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":4.383,"5min":8.7010000000000005,"15min":115.348},"last":0.51100000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":3.629,"15min":3.056},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":9.4060000000000006,"5min":149.947,"15min":149.947},"last":0.62},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.718,"15min":2.9940000000000002},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":6.3689999999999998,"5min":150.22499999999999,"15min":150.22499999999999},"last":0.47399999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:08:52 2023","interfaces":[{"interface":"back","average":{"1min":4.9420000000000002,"5min":5.5709999999999997,"15min":2.9260000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":45.051000000000002,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":4.8579999999999997,"5min":5.25,"15min":2.9969999999999999},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32500000000000001},"max":{"1min":45.258000000000003,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":4.9630000000000001,"15min":3.3210000000000002},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":9.4329999999999998,"5min":142.24700000000001,"15min":142.24700000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.986,"5min":4.8369999999999997,"15min":3.2349999999999999},"min":{"1min":0.41799999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.4100000000000001,"5min":143.86699999999999,"15min":143.86699999999999},"last":0.502}]},{"osd":4,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.2490000000000001,"15min":1.857},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.23999999999999999},"max":{"1min":6.3470000000000004,"5min":12.407999999999999,"15min":119.804},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.3109999999999999,"15min":1.925},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.24099999999999999},"max":{"1min":6.2140000000000004,"5min":8.8940000000000001,"15min":122.31999999999999},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.369,"15min":2.52},"min":{"1min":0.40400000000000003,"5min":0.33000000000000002,"15min":0.27900000000000003},"max":{"1min":6.4530000000000003,"5min":11.862,"15min":186.27600000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.238,"5min":1.506,"15min":2.6629999999999998},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.29599999999999999},"max":{"1min":6.2729999999999997,"5min":10.254,"15min":191.28700000000001},"last":0.34399999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.4159999999999999,"15min":1.5569999999999999},"min":{"1min":0.375,"5min":0.372,"15min":0.29999999999999999},"max":{"1min":5.3959999999999999,"5min":8.8670000000000009,"15min":115.61},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.117,"5min":1.3,"15min":1.7350000000000001},"min":{"1min":0.42699999999999999,"5min":0.38300000000000001,"15min":0.29999999999999999},"max":{"1min":7.1920000000000002,"5min":8.2609999999999992,"15min":115.578},"last":0.42899999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253258,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1317976,"kb_used_data":336848,"kb_used_omap":9,"kb_used_meta":981110,"kb_avail":92427176,"statfs":{"total":95995035648,"available":94645428224,"internally_reserved":0,"allocated":344932352,"data_stored":342979586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1004657641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.5629999999999999,"15min":2.5649999999999999},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.2189999999999999,"5min":59.475000000000001,"15min":279.30500000000001},"last":0.218},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7490000000000001,"15min":2.8799999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":4.8559999999999999,"5min":59.393999999999998,"15min":279.39800000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:09:38 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.7429999999999999,"15min":2.7229999999999999},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":62.177,"5min":121.527,"15min":161.61699999999999},"last":72.251999999999995},{"interface":"front","average":{"1min":5.2619999999999996,"5min":4.1280000000000001,"15min":2.9079999999999999},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.460999999999999,"5min":118.29900000000001,"15min":162.61699999999999},"last":75.563000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.4510000000000001,"15min":1.389},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":3.8039999999999998,"5min":59.424999999999997,"15min":59.424999999999997},"last":60.838999999999999},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.5680000000000001,"15min":1.454},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.19500000000000001},"max":{"1min":7.1580000000000004,"5min":55.174999999999997,"15min":55.174999999999997},"last":60.518999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2320000000000002,"5min":3.0569999999999999,"15min":2.677},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":67.813999999999993,"5min":129.69,"15min":151.036},"last":0.47799999999999998},{"interface":"front","average":{"1min":5.4039999999999999,"5min":3.427,"15min":2.9180000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":67.796999999999997,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.4419999999999999,"15min":2.2370000000000001},"min":{"1min":0.40300000000000002,"5min":0.32800000000000001,"15min":0.29899999999999999},"max":{"1min":1.522,"5min":18.228999999999999,"15min":116.098},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.0920000000000001,"15min":2.0419999999999998},"min":{"1min":0.38900000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.5800000000000001,"5min":6.585,"15min":116.08499999999999},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":5.2759999999999998,"5min":2.1850000000000001,"15min":2.056},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":64.058000000000007,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.37},{"interface":"front","average":{"1min":4.7190000000000003,"5min":2.3690000000000002,"15min":2.0110000000000001},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.33300000000000002},"max":{"1min":64.239999999999995,"5min":64.239999999999995,"15min":67.366},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":1.337,"15min":1.1910000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":5.7549999999999999,"5min":7.0960000000000001,"15min":8.1400000000000006},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.472,"5min":1.7290000000000001,"15min":1.478},"min":{"1min":0.40699999999999997,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":8.8719999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.46700000000000003}]}]},{"osd":0,"up_from":0,"seq":1516123455629,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259401,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":356648,"kb_used_data":40608,"kb_used_omap":83,"kb_used_meta":315820,"kb_avail":93388504,"statfs":{"total":95995035648,"available":95629828096,"internally_reserved":0,"allocated":41582592,"data_stored":39500418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":323400133},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.82999999999999996,"15min":1.212},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":4.1390000000000002,"5min":10.199999999999999,"15min":74.241},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.5,"5min":1.29,"15min":1.5309999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":14.944000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.71899999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0670000000000002,"5min":2.5049999999999999,"15min":1.3009999999999999},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":83.897000000000006,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.74199999999999999},{"interface":"front","average":{"1min":4.5289999999999999,"5min":2.1669999999999998,"15min":1.462},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":83.088999999999999,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.66000000000000003}]},{"osd":3,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7130000000000001,"5min":3.98,"15min":1.8380000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.23000000000000001},"max":{"1min":79.655000000000001,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.77000000000000002},{"interface":"front","average":{"1min":7.2039999999999997,"5min":3.8650000000000002,"15min":1.851},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26100000000000001},"max":{"1min":79.677999999999997,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.59899999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":1.9279999999999999,"15min":1.1379999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":68.325000000000003,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.77900000000000003},{"interface":"front","average":{"1min":4.9020000000000001,"5min":2.2610000000000001,"15min":1.351},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":70.608000000000004,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.79200000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":3.105,"15min":1.6839999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.6600000000000001,"5min":139.405,"15min":139.405},"last":0.628},{"interface":"front","average":{"1min":1.002,"5min":2.9129999999999998,"15min":1.5089999999999999},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":4.0499999999999998,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.83499999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":3.4180000000000001,"15min":1.8600000000000001},"min":{"1min":0.377,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.6989999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.1439999999999999,"5min":3.1749999999999998,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34599999999999997,"15min":0.32200000000000001},"max":{"1min":4.0830000000000002,"5min":108.496,"15min":108.496},"last":0.90600000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.421,"15min":1.1890000000000001},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.1449999999999996,"5min":10.164,"15min":10.164},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.393,"15min":1.2050000000000001},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":15.037000000000001,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.98699999999999999}]}]},{"osd":3,"up_from":94,"seq":403726926206,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004108,"kb_used_data":297732,"kb_used_omap":7,"kb_used_meta":706360,"kb_avail":92741044,"statfs":{"total":95995035648,"available":94966829056,"internally_reserved":0,"allocated":304877568,"data_stored":302815727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":723313256},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.486,"15min":1.4490000000000001},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":3.8439999999999999,"5min":17.015000000000001,"15min":122.289},"last":6.1449999999999996},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.115,"15min":1.1870000000000001},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":4.9909999999999997,"5min":11.382999999999999,"15min":122.041},"last":8.1519999999999992}]},{"osd":1,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":2.1720000000000002,"15min":1.4710000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":10.935,"5min":23.969000000000001,"15min":23.969000000000001},"last":8.093},{"interface":"front","average":{"1min":1.109,"5min":1.9119999999999999,"15min":1.373},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.20699999999999999},"max":{"1min":4.0119999999999996,"5min":16.986000000000001,"15min":16.986000000000001},"last":3.4580000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:08:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8479999999999999,"5min":2.3370000000000002,"15min":1.232},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":55.311999999999998,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.27700000000000002},{"interface":"front","average":{"1min":5.532,"5min":2.7480000000000002,"15min":1.5389999999999999},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.215},"max":{"1min":57.308,"5min":57.308,"15min":57.308},"last":5.6539999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.6629999999999998,"15min":2.1110000000000002},"min":{"1min":0.374,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":11.01,"5min":115.69,"15min":115.69},"last":5.5759999999999996},{"interface":"front","average":{"1min":1.351,"5min":2.4009999999999998,"15min":2.0419999999999998},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":5.9169999999999998,"5min":115.617,"15min":115.617},"last":3.5259999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.401},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":4.8810000000000002,"5min":16.959,"15min":16.959},"last":3.5489999999999999},{"interface":"front","average":{"1min":1.359,"5min":1.476,"15min":1.369},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":6.6130000000000004,"5min":10.545,"15min":10.545},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":2.589,"15min":1.8999999999999999},"min":{"1min":0.41699999999999998,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":77.707999999999998,"5min":77.707999999999998,"15min":77.707999999999998},"last":3.4910000000000001},{"interface":"front","average":{"1min":5.4080000000000004,"5min":2.1930000000000001,"15min":1.6890000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":77.721999999999994,"5min":77.721999999999994,"15min":77.721999999999994},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.4630000000000001,"15min":1.272},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":4.5220000000000002,"5min":8.1620000000000008,"15min":8.1620000000000008},"last":2.754},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.0499999999999998,"15min":1.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":5.6189999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019403,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907980,"kb_used_data":4612,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92837172,"statfs":{"total":95995035648,"available":95065264128,"internally_reserved":0,"allocated":4722688,"data_stored":3648789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.4039999999999999,"15min":2.3580000000000001},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":9.8870000000000005,"5min":99.132999999999996,"15min":156.375},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.3370000000000002,"15min":2.4279999999999999},"min":{"1min":0.36799999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":4.4509999999999996,"5min":101.062,"15min":157.02199999999999},"last":0.53400000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.4340000000000002,"15min":2.7989999999999999},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":4.3840000000000003,"5min":95.591999999999999,"15min":98.846999999999994},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.954,"5min":3.407,"15min":2.843},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":10.311,"5min":97.929000000000002,"15min":98.938999999999993},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:44 2023","interfaces":[{"interface":"back","average":{"1min":9.6500000000000004,"5min":5.8819999999999997,"15min":2.7210000000000001},"min":{"1min":0.41599999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":151.20400000000001,"5min":151.20400000000001,"15min":151.20400000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":9.3279999999999994,"5min":5.7590000000000003,"15min":2.5590000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":148.852,"5min":148.852,"15min":148.852},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":11.974,"5min":4.7699999999999996,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":147.82499999999999,"5min":147.82499999999999,"15min":147.82499999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":11.071,"5min":4.7309999999999999,"15min":2.4889999999999999},"min":{"1min":0.44,"5min":0.36599999999999999,"15min":0.32500000000000001},"max":{"1min":147.84100000000001,"5min":147.84100000000001,"15min":147.84100000000001},"last":0.93500000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.4159999999999999,"15min":1.7769999999999999},"min":{"1min":0.37,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":10.282,"5min":10.282,"15min":148.61600000000001},"last":0.495},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.601,"15min":1.9430000000000001},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.247},"max":{"1min":17.02,"5min":17.02,"15min":151.905},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.4650000000000001,"15min":1.419},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.9740000000000002,"5min":35.896000000000001,"15min":35.896000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.778,"15min":1.3899999999999999},"min":{"1min":0.34899999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":6.5129999999999999,"5min":35.908000000000001,"15min":35.908000000000001},"last":0.55400000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:09:34 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":1.5680000000000001,"15min":1.3660000000000001},"min":{"1min":0.432,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":11.096,"5min":11.096,"15min":12.84},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.448,"5min":1.4670000000000001,"15min":1.25},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":7.3179999999999996,"5min":15.808,"15min":15.808},"last":0.66400000000000003}]}]},{"osd":5,"up_from":0,"seq":154618823177,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2934833152,"data_stored":2934833152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":338624512,"data_stored":338624512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":34471936,"data_stored":34471936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":299040768,"data_stored":299040768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1532035072,"data_stored":1532035072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:09:48.340 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:48.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:48 smithi151 ceph-mon[159210]: failed to encode map e479 with expected crc 2023-09-23T23:09:48.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:48 smithi151 ceph-mon[159210]: pgmap v1516: 99 pgs: 1 peering, 1 active+recovery_wait, 19 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 450 op/s; 38890/1653 objects degraded (2352.692%); 3.2 MiB/s, 75 objects/s recovering 2023-09-23T23:09:48.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:48 smithi151 ceph-mon[159210]: from='client.57208 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:09:48.366 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:48.366 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:09:48.389 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:48.394 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:48.413 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:48.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:48.518 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:48.543 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:48.584 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:48.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:48.692 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:48.717 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:48.760 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:48.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:48.867 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:48.905 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:48.951 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:48.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:49.055 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:49.084 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:49.114 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:49.135 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:49.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:49.240 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:49.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:49 smithi052 ceph-mon[187743]: failed to encode map e479 with expected crc 2023-09-23T23:09:49.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:49 smithi052 ceph-mon[180979]: failed to encode map e479 with expected crc 2023-09-23T23:09:49.268 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:49.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:49 smithi151 ceph-mon[159210]: failed to encode map e479 with expected crc 2023-09-23T23:09:50.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:50 smithi052 ceph-mon[187743]: pgmap v1517: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 18 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 260 op/s; 38672/1653 objects degraded (2339.504%); 1.7 MiB/s, 49 objects/s recovering 2023-09-23T23:09:50.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:50 smithi052 ceph-mon[180979]: pgmap v1517: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 18 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 260 op/s; 38672/1653 objects degraded (2339.504%); 1.7 MiB/s, 49 objects/s recovering 2023-09-23T23:09:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:50.367 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:50.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:50 smithi151 ceph-mon[159210]: pgmap v1517: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 18 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 74 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 260 op/s; 38672/1653 objects degraded (2339.504%); 1.7 MiB/s, 49 objects/s recovering 2023-09-23T23:09:50.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:09:50.471 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:50.523 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:50.690 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:50.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:50.730 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:50.794 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:50.840 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:50.919 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:50.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:50.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:51.024 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:51.058 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:51.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:51 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 38672/1653 objects degraded (2339.504%), 22 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:51.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:51 smithi052 ceph-mon[187743]: osdmap e480: 8 total, 8 up, 5 in 2023-09-23T23:09:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:51 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 38672/1653 objects degraded (2339.504%), 22 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:51.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:51 smithi052 ceph-mon[180979]: osdmap e480: 8 total, 8 up, 5 in 2023-09-23T23:09:51.312 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:09:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:51 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 38672/1653 objects degraded (2339.504%), 22 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:51.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:51 smithi151 ceph-mon[159210]: osdmap e480: 8 total, 8 up, 5 in 2023-09-23T23:09:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:52 smithi151 ceph-mon[159210]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:52 smithi151 ceph-mon[159210]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:52 smithi151 ceph-mon[159210]: pgmap v1519: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 75 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 16 op/s; 38672/1653 objects degraded (2339.504%); 0 B/s, 29 objects/s recovering 2023-09-23T23:09:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:52 smithi151 ceph-mon[159210]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:52 smithi151 ceph-mon[159210]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:52 smithi151 ceph-mon[159210]: osdmap e481: 8 total, 8 up, 5 in 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[180979]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[180979]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[180979]: pgmap v1519: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 75 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 16 op/s; 38672/1653 objects degraded (2339.504%); 0 B/s, 29 objects/s recovering 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[180979]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[180979]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[180979]: osdmap e481: 8 total, 8 up, 5 in 2023-09-23T23:09:52.494 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[187743]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[187743]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[187743]: pgmap v1519: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 75 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 16 op/s; 38672/1653 objects degraded (2339.504%); 0 B/s, 29 objects/s recovering 2023-09-23T23:09:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[187743]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[187743]: failed to encode map e480 with expected crc 2023-09-23T23:09:52.495 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:52 smithi052 ceph-mon[187743]: osdmap e481: 8 total, 8 up, 5 in 2023-09-23T23:09:52.987 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:52.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:53.092 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:53.131 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:53.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[187743]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[187743]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[187743]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[187743]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[187743]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[187743]: osdmap e482: 8 total, 8 up, 5 in 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[180979]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[180979]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[180979]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[180979]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[180979]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:53 smithi052 ceph-mon[180979]: osdmap e482: 8 total, 8 up, 5 in 2023-09-23T23:09:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:53 smithi151 ceph-mon[159210]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:53 smithi151 ceph-mon[159210]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:53 smithi151 ceph-mon[159210]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:53 smithi151 ceph-mon[159210]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:53 smithi151 ceph-mon[159210]: failed to encode map e481 with expected crc 2023-09-23T23:09:53.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:53 smithi151 ceph-mon[159210]: osdmap e482: 8 total, 8 up, 5 in 2023-09-23T23:09:53.514 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:09:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:09:53] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-09-23T23:09:54.080 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[187743]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.080 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[187743]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[187743]: pgmap v1522: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 76 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 37641/1653 objects degraded (2277.132%); 0 B/s, 185 objects/s recovering 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[187743]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[187743]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[187743]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[180979]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[180979]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[180979]: pgmap v1522: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 76 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 37641/1653 objects degraded (2277.132%); 0 B/s, 185 objects/s recovering 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[180979]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.081 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[180979]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.082 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:54 smithi052 ceph-mon[180979]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.082 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:09:54.113 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:09:54.138 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:09:54.161 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:09:54.185 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:09:54.208 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:09:54.232 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:09:54.259 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.261 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:09:54.302 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:54.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:54.409 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:54.424 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:09:54.424 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:09:54.432 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:54 smithi151 ceph-mon[159210]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.433 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:54 smithi151 ceph-mon[159210]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.433 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:54 smithi151 ceph-mon[159210]: pgmap v1522: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 76 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 37641/1653 objects degraded (2277.132%); 0 B/s, 185 objects/s recovering 2023-09-23T23:09:54.433 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:54 smithi151 ceph-mon[159210]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.433 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:54 smithi151 ceph-mon[159210]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.433 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:54 smithi151 ceph-mon[159210]: failed to encode map e482 with expected crc 2023-09-23T23:09:54.435 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:54.478 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:54.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:54.584 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:54.618 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:55.048 INFO:teuthology.orchestra.run.smithi052.stderr:pool 'unique_pool_5' removed 2023-09-23T23:09:55.232 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:55.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:55.338 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:09:55.375 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3741168674' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:55.375 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/84972811' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:09:55.375 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3741168674' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:55.375 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/84972811' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:09:55.378 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3741168674' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:09:55.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/84972811' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T23:09:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:56 smithi151 ceph-mon[159210]: pgmap v1523: 99 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 77 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 37641/1653 objects degraded (2277.132%); 0 B/s, 157 objects/s recovering 2023-09-23T23:09:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/84972811' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:09:56.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:56 smithi151 ceph-mon[159210]: osdmap e483: 8 total, 8 up, 5 in 2023-09-23T23:09:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:56 smithi151 ceph-mon[159210]: failed to encode map e483 with expected crc 2023-09-23T23:09:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:56 smithi151 ceph-mon[159210]: failed to encode map e483 with expected crc 2023-09-23T23:09:56.450 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:56 smithi151 ceph-mon[159210]: Health check update: Degraded data redundancy: 37641/1653 objects degraded (2277.132%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:56.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[180979]: pgmap v1523: 99 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 77 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 37641/1653 objects degraded (2277.132%); 0 B/s, 157 objects/s recovering 2023-09-23T23:09:56.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/84972811' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:09:56.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[180979]: osdmap e483: 8 total, 8 up, 5 in 2023-09-23T23:09:56.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[180979]: failed to encode map e483 with expected crc 2023-09-23T23:09:56.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[180979]: failed to encode map e483 with expected crc 2023-09-23T23:09:56.495 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[180979]: Health check update: Degraded data redundancy: 37641/1653 objects degraded (2277.132%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:56.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[187743]: pgmap v1523: 99 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 77 active+clean; 33 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 37641/1653 objects degraded (2277.132%); 0 B/s, 157 objects/s recovering 2023-09-23T23:09:56.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/84972811' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T23:09:56.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[187743]: osdmap e483: 8 total, 8 up, 5 in 2023-09-23T23:09:56.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[187743]: failed to encode map e483 with expected crc 2023-09-23T23:09:56.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[187743]: failed to encode map e483 with expected crc 2023-09-23T23:09:56.496 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:56 smithi052 ceph-mon[187743]: Health check update: Degraded data redundancy: 37641/1653 objects degraded (2277.132%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T23:09:57.037 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:09:57.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:57.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:09:57.142 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:57.176 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:57.225 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:57.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:57.332 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:09:57.363 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:57.407 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:09:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:57.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:09:57.512 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:09:57.537 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:57.579 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:09:57.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:57.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:09:57.685 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:09:57.715 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:58.225 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:58.290 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:58 smithi052 ceph-mon[187743]: pgmap v1525: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T23:09:58.291 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:58 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:58.291 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:58 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:58.292 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:58 smithi052 ceph-mon[180979]: pgmap v1525: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T23:09:58.292 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:58 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:58.292 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:58 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:58.292 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T23:09:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:58 smithi151 ceph-mon[159210]: pgmap v1525: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T23:09:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:58 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:58.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:58 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:58.465 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:09:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:58.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:09:58.535 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:09:58.535 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T23:09:58.535 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T23:09:58.536 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T23:09:58.536 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T23:09:58.536 DEBUG:teuthology.orchestra.run.smithi052:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:09:58.569 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:09:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:09:58.580 INFO:teuthology.orchestra.run.smithi052.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T23:09:58.580 DEBUG:teuthology.orchestra.run.smithi052:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:09:58.600 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:58.648 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:09:58.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:58.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:09:58.659 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T23:09:58.659 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T23:09:58.717 INFO:tasks.workunit:timeout=3h 2023-09-23T23:09:58.717 INFO:tasks.workunit:cleanup=True 2023-09-23T23:09:58.717 DEBUG:teuthology.orchestra.run.smithi052:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:09:58.754 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:58.787 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:58.854 INFO:tasks.workunit.client.0.smithi052.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T23:09:59.185 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:59.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:59.289 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:09:59.315 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:09:59.315 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:09:59.315 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:09:59 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:59.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:09:59.315 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:09:59.316 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:09:59 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:59.318 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:09:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:09:59.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:09:59 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:09:59.629 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:09:59.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:59.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:09:59.733 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:09:59.766 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:09:59.770 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:09:59.846 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:09:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:09:59.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:09:59.950 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:09:59.981 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:00.030 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:00.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:00.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:00.117 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:10:00.138 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:00.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[187743]: pgmap v1526: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:00.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T23:10:00.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1558221589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:00.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[187743]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T23:10:00.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[187743]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T23:10:00.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[180979]: pgmap v1526: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:00.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T23:10:00.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1558221589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:00.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[180979]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T23:10:00.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:00 smithi052 ceph-mon[180979]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T23:10:00.196 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:00.246 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:10:00.265 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:00.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:00.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:00.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:00 smithi151 ceph-mon[159210]: pgmap v1526: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:00.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:00 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T23:10:00.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1558221589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:00.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:00 smithi151 ceph-mon[159210]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T23:10:00.400 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:00 smithi151 ceph-mon[159210]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T23:10:00.402 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:00.445 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:10:00.446 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:10:00.452 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:00.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:00.558 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:00.587 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:01.190 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:01.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:01 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:01.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1558221589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:01.193 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:01 smithi052 ceph-mon[187743]: osdmap e484: 8 total, 8 up, 5 in 2023-09-23T23:10:01.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:01 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:01.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1558221589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:01.194 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:01 smithi052 ceph-mon[180979]: osdmap e484: 8 total, 8 up, 5 in 2023-09-23T23:10:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:01.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:01.295 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:01.325 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:01 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:01.325 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1558221589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:01.325 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:01 smithi151 ceph-mon[159210]: osdmap e484: 8 total, 8 up, 5 in 2023-09-23T23:10:01.328 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:01.378 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:01.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:01.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:01.484 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:01.511 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:01.555 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:01.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:01.661 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:01.691 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:01.917 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1527,"stamp":"2023-09-23T23:10:00.233267+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":611296,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":252,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4473272,"kb_used_data":623504,"kb_used_omap":99,"kb_used_meta":3849628,"kb_avail":464252488,"statfs":{"total":479975178240,"available":475394547712,"internally_reserved":0,"allocated":638468096,"data_stored":630052251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3942019094},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":45,"apply_latency_ms":45,"commit_latency_ns":45000000,"apply_latency_ns":45000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-34406400,"num_objects":-527,"num_object_clones":0,"num_object_copies":-1581,"num_objects_missing_on_primary":-216,"num_objects_missing":-216,"num_objects_degraded":-38307,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-527,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-164304,"num_write_kb":-5257665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-38027,"num_bytes_recovered":-2405236192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.743146"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":495,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854375+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:09:52.854375+0000","last_peered":"2023-09-23T23:09:52.854375+0000","last_clean":"2023-09-23T23:09:52.854375+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:09:52.854375+0000","last_undegraded":"2023-09-23T23:09:52.854375+0000","last_fullsized":"2023-09-23T23:09:52.854375+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145977+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:09:55.145977+0000","last_peered":"2023-09-23T23:09:55.145977+0000","last_clean":"2023-09-23T23:09:55.145977+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:09:55.145977+0000","last_undegraded":"2023-09-23T23:09:55.145977+0000","last_fullsized":"2023-09-23T23:09:55.145977+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":155,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494295+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:09:52.494295+0000","last_peered":"2023-09-23T23:09:52.494295+0000","last_clean":"2023-09-23T23:09:52.494295+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:09:52.494295+0000","last_undegraded":"2023-09-23T23:09:52.494295+0000","last_fullsized":"2023-09-23T23:09:52.494295+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":502,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493912+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:09:52.493912+0000","last_peered":"2023-09-23T23:09:52.493912+0000","last_clean":"2023-09-23T23:09:52.493912+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:09:52.493912+0000","last_undegraded":"2023-09-23T23:09:52.493912+0000","last_fullsized":"2023-09-23T23:09:52.493912+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":62,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.491359+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:09:52.491359+0000","last_peered":"2023-09-23T23:09:52.491359+0000","last_clean":"2023-09-23T23:09:52.491359+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:09:52.491359+0000","last_undegraded":"2023-09-23T23:09:52.491359+0000","last_fullsized":"2023-09-23T23:09:52.491359+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494235+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:09:52.494235+0000","last_peered":"2023-09-23T23:09:52.494235+0000","last_clean":"2023-09-23T23:09:52.494235+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:09:52.494235+0000","last_undegraded":"2023-09-23T23:09:52.494235+0000","last_fullsized":"2023-09-23T23:09:52.494235+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":427,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494498+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:09:52.494498+0000","last_peered":"2023-09-23T23:09:52.494498+0000","last_clean":"2023-09-23T23:09:52.494498+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:09:52.494498+0000","last_undegraded":"2023-09-23T23:09:52.494498+0000","last_fullsized":"2023-09-23T23:09:52.494498+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":174,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492222+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:09:52.492222+0000","last_peered":"2023-09-23T23:09:52.492222+0000","last_clean":"2023-09-23T23:09:52.492222+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:09:52.492222+0000","last_undegraded":"2023-09-23T23:09:52.492222+0000","last_fullsized":"2023-09-23T23:09:52.492222+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494216+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:09:52.494216+0000","last_peered":"2023-09-23T23:09:52.494216+0000","last_clean":"2023-09-23T23:09:52.494216+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:09:52.494216+0000","last_undegraded":"2023-09-23T23:09:52.494216+0000","last_fullsized":"2023-09-23T23:09:52.494216+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493928+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:09:52.493928+0000","last_peered":"2023-09-23T23:09:52.493928+0000","last_clean":"2023-09-23T23:09:52.493928+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:09:52.493928+0000","last_undegraded":"2023-09-23T23:09:52.493928+0000","last_fullsized":"2023-09-23T23:09:52.493928+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":221,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493103+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:09:52.493103+0000","last_peered":"2023-09-23T23:09:52.493103+0000","last_clean":"2023-09-23T23:09:52.493103+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:09:52.493103+0000","last_undegraded":"2023-09-23T23:09:52.493103+0000","last_fullsized":"2023-09-23T23:09:52.493103+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":238,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.499389+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:09:52.499389+0000","last_peered":"2023-09-23T23:09:52.499389+0000","last_clean":"2023-09-23T23:09:52.499389+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:09:52.499389+0000","last_undegraded":"2023-09-23T23:09:52.499389+0000","last_fullsized":"2023-09-23T23:09:52.499389+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":573,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854767+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:09:52.854767+0000","last_peered":"2023-09-23T23:09:52.854767+0000","last_clean":"2023-09-23T23:09:52.854767+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:09:52.854767+0000","last_undegraded":"2023-09-23T23:09:52.854767+0000","last_fullsized":"2023-09-23T23:09:52.854767+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":466,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854489+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:09:52.854489+0000","last_peered":"2023-09-23T23:09:52.854489+0000","last_clean":"2023-09-23T23:09:52.854489+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:09:52.854489+0000","last_undegraded":"2023-09-23T23:09:52.854489+0000","last_fullsized":"2023-09-23T23:09:52.854489+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":398,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854618+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:09:52.854618+0000","last_peered":"2023-09-23T23:09:52.854618+0000","last_clean":"2023-09-23T23:09:52.854618+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:09:52.854618+0000","last_undegraded":"2023-09-23T23:09:52.854618+0000","last_fullsized":"2023-09-23T23:09:52.854618+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":167,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493670+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:09:52.493670+0000","last_peered":"2023-09-23T23:09:52.493670+0000","last_clean":"2023-09-23T23:09:52.493670+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:09:52.493670+0000","last_undegraded":"2023-09-23T23:09:52.493670+0000","last_fullsized":"2023-09-23T23:09:52.493670+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":485,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492451+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:09:52.492451+0000","last_peered":"2023-09-23T23:09:52.492451+0000","last_clean":"2023-09-23T23:09:52.492451+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:09:52.492451+0000","last_undegraded":"2023-09-23T23:09:52.492451+0000","last_fullsized":"2023-09-23T23:09:52.492451+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":399,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493013+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:09:52.493013+0000","last_peered":"2023-09-23T23:09:52.493013+0000","last_clean":"2023-09-23T23:09:52.493013+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:09:52.493013+0000","last_undegraded":"2023-09-23T23:09:52.493013+0000","last_fullsized":"2023-09-23T23:09:52.493013+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":386,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854329+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:09:52.854329+0000","last_peered":"2023-09-23T23:09:52.854329+0000","last_clean":"2023-09-23T23:09:52.854329+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:09:52.854329+0000","last_undegraded":"2023-09-23T23:09:52.854329+0000","last_fullsized":"2023-09-23T23:09:52.854329+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":509,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493833+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:09:52.493833+0000","last_peered":"2023-09-23T23:09:52.493833+0000","last_clean":"2023-09-23T23:09:52.493833+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:09:52.493833+0000","last_undegraded":"2023-09-23T23:09:52.493833+0000","last_fullsized":"2023-09-23T23:09:52.493833+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":97,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.491666+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:09:52.491666+0000","last_peered":"2023-09-23T23:09:52.491666+0000","last_clean":"2023-09-23T23:09:52.491666+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:09:52.491666+0000","last_undegraded":"2023-09-23T23:09:52.491666+0000","last_fullsized":"2023-09-23T23:09:52.491666+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494577+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:09:52.494577+0000","last_peered":"2023-09-23T23:09:52.494577+0000","last_clean":"2023-09-23T23:09:52.494577+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:09:52.494577+0000","last_undegraded":"2023-09-23T23:09:52.494577+0000","last_fullsized":"2023-09-23T23:09:52.494577+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":393,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854832+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:09:52.854832+0000","last_peered":"2023-09-23T23:09:52.854832+0000","last_clean":"2023-09-23T23:09:52.854832+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:09:52.854832+0000","last_undegraded":"2023-09-23T23:09:52.854832+0000","last_fullsized":"2023-09-23T23:09:52.854832+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":349,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.853952+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:09:52.853952+0000","last_peered":"2023-09-23T23:09:52.853952+0000","last_clean":"2023-09-23T23:09:52.853952+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:09:52.853952+0000","last_undegraded":"2023-09-23T23:09:52.853952+0000","last_fullsized":"2023-09-23T23:09:52.853952+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":112,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.500976+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:09:52.500976+0000","last_peered":"2023-09-23T23:09:52.500976+0000","last_clean":"2023-09-23T23:09:52.500976+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:09:52.500976+0000","last_undegraded":"2023-09-23T23:09:52.500976+0000","last_fullsized":"2023-09-23T23:09:52.500976+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":490,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854215+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:09:52.854215+0000","last_peered":"2023-09-23T23:09:52.854215+0000","last_clean":"2023-09-23T23:09:52.854215+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:09:52.854215+0000","last_undegraded":"2023-09-23T23:09:52.854215+0000","last_fullsized":"2023-09-23T23:09:52.854215+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":141,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492935+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:09:52.492935+0000","last_peered":"2023-09-23T23:09:52.492935+0000","last_clean":"2023-09-23T23:09:52.492935+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:09:52.492935+0000","last_undegraded":"2023-09-23T23:09:52.492935+0000","last_fullsized":"2023-09-23T23:09:52.492935+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.491469+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:09:52.491469+0000","last_peered":"2023-09-23T23:09:52.491469+0000","last_clean":"2023-09-23T23:09:52.491469+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:09:52.491469+0000","last_undegraded":"2023-09-23T23:09:52.491469+0000","last_fullsized":"2023-09-23T23:09:52.491469+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":174,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493596+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:09:52.493596+0000","last_peered":"2023-09-23T23:09:52.493596+0000","last_clean":"2023-09-23T23:09:52.493596+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:09:52.493596+0000","last_undegraded":"2023-09-23T23:09:52.493596+0000","last_fullsized":"2023-09-23T23:09:52.493596+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":305,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854518+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:09:52.854518+0000","last_peered":"2023-09-23T23:09:52.854518+0000","last_clean":"2023-09-23T23:09:52.854518+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:09:52.854518+0000","last_undegraded":"2023-09-23T23:09:52.854518+0000","last_fullsized":"2023-09-23T23:09:52.854518+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":188,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.493991+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:09:52.493991+0000","last_peered":"2023-09-23T23:09:52.493991+0000","last_clean":"2023-09-23T23:09:52.493991+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:09:52.493991+0000","last_undegraded":"2023-09-23T23:09:52.493991+0000","last_fullsized":"2023-09-23T23:09:52.493991+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":485,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854479+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:09:52.854479+0000","last_peered":"2023-09-23T23:09:52.854479+0000","last_clean":"2023-09-23T23:09:52.854479+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:09:52.854479+0000","last_undegraded":"2023-09-23T23:09:52.854479+0000","last_fullsized":"2023-09-23T23:09:52.854479+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":256,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.853958+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:09:52.853958+0000","last_peered":"2023-09-23T23:09:52.853958+0000","last_clean":"2023-09-23T23:09:52.853958+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:09:52.853958+0000","last_undegraded":"2023-09-23T23:09:52.853958+0000","last_fullsized":"2023-09-23T23:09:52.853958+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":493,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854599+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:09:52.854599+0000","last_peered":"2023-09-23T23:09:52.854599+0000","last_clean":"2023-09-23T23:09:52.854599+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:09:52.854599+0000","last_undegraded":"2023-09-23T23:09:52.854599+0000","last_fullsized":"2023-09-23T23:09:52.854599+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":506,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854087+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:09:52.854087+0000","last_peered":"2023-09-23T23:09:52.854087+0000","last_clean":"2023-09-23T23:09:52.854087+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:09:52.854087+0000","last_undegraded":"2023-09-23T23:09:52.854087+0000","last_fullsized":"2023-09-23T23:09:52.854087+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":511,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854414+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:09:52.854414+0000","last_peered":"2023-09-23T23:09:52.854414+0000","last_clean":"2023-09-23T23:09:52.854414+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:09:52.854414+0000","last_undegraded":"2023-09-23T23:09:52.854414+0000","last_fullsized":"2023-09-23T23:09:52.854414+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":156,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492512+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:09:52.492512+0000","last_peered":"2023-09-23T23:09:52.492512+0000","last_clean":"2023-09-23T23:09:52.492512+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:09:52.492512+0000","last_undegraded":"2023-09-23T23:09:52.492512+0000","last_fullsized":"2023-09-23T23:09:52.492512+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":390,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.502231+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:09:52.502231+0000","last_peered":"2023-09-23T23:09:52.502231+0000","last_clean":"2023-09-23T23:09:52.502231+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:09:52.502231+0000","last_undegraded":"2023-09-23T23:09:52.502231+0000","last_fullsized":"2023-09-23T23:09:52.502231+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":403,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.500149+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:09:52.500149+0000","last_peered":"2023-09-23T23:09:52.500149+0000","last_clean":"2023-09-23T23:09:52.500149+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:09:52.500149+0000","last_undegraded":"2023-09-23T23:09:52.500149+0000","last_fullsized":"2023-09-23T23:09:52.500149+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":321,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492782+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:09:52.492782+0000","last_peered":"2023-09-23T23:09:52.492782+0000","last_clean":"2023-09-23T23:09:52.492782+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:09:52.492782+0000","last_undegraded":"2023-09-23T23:09:52.492782+0000","last_fullsized":"2023-09-23T23:09:52.492782+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":224,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854283+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:09:52.854283+0000","last_peered":"2023-09-23T23:09:52.854283+0000","last_clean":"2023-09-23T23:09:52.854283+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:09:52.854283+0000","last_undegraded":"2023-09-23T23:09:52.854283+0000","last_fullsized":"2023-09-23T23:09:52.854283+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":219,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.497542+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:09:52.497542+0000","last_peered":"2023-09-23T23:09:52.497542+0000","last_clean":"2023-09-23T23:09:52.497542+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:09:52.497542+0000","last_undegraded":"2023-09-23T23:09:52.497542+0000","last_fullsized":"2023-09-23T23:09:52.497542+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":521,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854695+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:09:52.854695+0000","last_peered":"2023-09-23T23:09:52.854695+0000","last_clean":"2023-09-23T23:09:52.854695+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:09:52.854695+0000","last_undegraded":"2023-09-23T23:09:52.854695+0000","last_fullsized":"2023-09-23T23:09:52.854695+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":388,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.500087+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:09:52.500087+0000","last_peered":"2023-09-23T23:09:52.500087+0000","last_clean":"2023-09-23T23:09:52.500087+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:09:52.500087+0000","last_undegraded":"2023-09-23T23:09:52.500087+0000","last_fullsized":"2023-09-23T23:09:52.500087+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":463,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.498937+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:09:52.498937+0000","last_peered":"2023-09-23T23:09:52.498937+0000","last_clean":"2023-09-23T23:09:52.498937+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:09:52.498937+0000","last_undegraded":"2023-09-23T23:09:52.498937+0000","last_fullsized":"2023-09-23T23:09:52.498937+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":316,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.018099+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:09:55.018099+0000","last_peered":"2023-09-23T23:09:55.018099+0000","last_clean":"2023-09-23T23:09:55.018099+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:09:55.018099+0000","last_undegraded":"2023-09-23T23:09:55.018099+0000","last_fullsized":"2023-09-23T23:09:55.018099+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":420,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494404+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:09:52.494404+0000","last_peered":"2023-09-23T23:09:52.494404+0000","last_clean":"2023-09-23T23:09:52.494404+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:09:52.494404+0000","last_undegraded":"2023-09-23T23:09:52.494404+0000","last_fullsized":"2023-09-23T23:09:52.494404+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":394,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.499041+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:09:52.499041+0000","last_peered":"2023-09-23T23:09:52.499041+0000","last_clean":"2023-09-23T23:09:52.499041+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:09:52.499041+0000","last_undegraded":"2023-09-23T23:09:52.499041+0000","last_fullsized":"2023-09-23T23:09:52.499041+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":603,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.052855+0000","last_change":"2023-09-23T23:09:55.052855+0000","last_active":"2023-09-23T23:09:55.052855+0000","last_peered":"2023-09-23T23:09:55.052855+0000","last_clean":"2023-09-23T23:09:55.052855+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:09:55.052855+0000","last_undegraded":"2023-09-23T23:09:55.052855+0000","last_fullsized":"2023-09-23T23:09:55.052855+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:54.527823+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:09:54.527823+0000","last_peered":"2023-09-23T23:09:54.527823+0000","last_clean":"2023-09-23T23:09:54.527823+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:09:54.527823+0000","last_undegraded":"2023-09-23T23:09:54.527823+0000","last_fullsized":"2023-09-23T23:09:54.527823+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":401,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854719+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:09:52.854719+0000","last_peered":"2023-09-23T23:09:52.854719+0000","last_clean":"2023-09-23T23:09:52.854719+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:09:52.854719+0000","last_undegraded":"2023-09-23T23:09:52.854719+0000","last_fullsized":"2023-09-23T23:09:52.854719+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":408,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492153+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:09:52.492153+0000","last_peered":"2023-09-23T23:09:52.492153+0000","last_clean":"2023-09-23T23:09:52.492153+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:09:52.492153+0000","last_undegraded":"2023-09-23T23:09:52.492153+0000","last_fullsized":"2023-09-23T23:09:52.492153+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":377,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854050+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:09:52.854050+0000","last_peered":"2023-09-23T23:09:52.854050+0000","last_clean":"2023-09-23T23:09:52.854050+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:09:52.854050+0000","last_undegraded":"2023-09-23T23:09:52.854050+0000","last_fullsized":"2023-09-23T23:09:52.854050+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":343,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.500805+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:09:52.500805+0000","last_peered":"2023-09-23T23:09:52.500805+0000","last_clean":"2023-09-23T23:09:52.500805+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:09:52.500805+0000","last_undegraded":"2023-09-23T23:09:52.500805+0000","last_fullsized":"2023-09-23T23:09:52.500805+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":342,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854621+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:09:52.854621+0000","last_peered":"2023-09-23T23:09:52.854621+0000","last_clean":"2023-09-23T23:09:52.854621+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:09:52.854621+0000","last_undegraded":"2023-09-23T23:09:52.854621+0000","last_fullsized":"2023-09-23T23:09:52.854621+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":340,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.500324+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:09:52.500324+0000","last_peered":"2023-09-23T23:09:52.500324+0000","last_clean":"2023-09-23T23:09:52.500324+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:09:52.500324+0000","last_undegraded":"2023-09-23T23:09:52.500324+0000","last_fullsized":"2023-09-23T23:09:52.500324+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":305,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.498848+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:09:52.498848+0000","last_peered":"2023-09-23T23:09:52.498848+0000","last_clean":"2023-09-23T23:09:52.498848+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:09:52.498848+0000","last_undegraded":"2023-09-23T23:09:52.498848+0000","last_fullsized":"2023-09-23T23:09:52.498848+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":206,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494788+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:09:52.494788+0000","last_peered":"2023-09-23T23:09:52.494788+0000","last_clean":"2023-09-23T23:09:52.494788+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:09:52.494788+0000","last_undegraded":"2023-09-23T23:09:52.494788+0000","last_fullsized":"2023-09-23T23:09:52.494788+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":181,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.853977+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:09:52.853977+0000","last_peered":"2023-09-23T23:09:52.853977+0000","last_clean":"2023-09-23T23:09:52.853977+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:09:52.853977+0000","last_undegraded":"2023-09-23T23:09:52.853977+0000","last_fullsized":"2023-09-23T23:09:52.853977+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":190,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.492290+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:09:52.492290+0000","last_peered":"2023-09-23T23:09:52.492290+0000","last_clean":"2023-09-23T23:09:52.492290+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:09:52.492290+0000","last_undegraded":"2023-09-23T23:09:52.492290+0000","last_fullsized":"2023-09-23T23:09:52.492290+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":432,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.502154+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:09:52.502154+0000","last_peered":"2023-09-23T23:09:52.502154+0000","last_clean":"2023-09-23T23:09:52.502154+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:09:52.502154+0000","last_undegraded":"2023-09-23T23:09:52.502154+0000","last_fullsized":"2023-09-23T23:09:52.502154+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":104,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854451+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:09:52.854451+0000","last_peered":"2023-09-23T23:09:52.854451+0000","last_clean":"2023-09-23T23:09:52.854451+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:09:52.854451+0000","last_undegraded":"2023-09-23T23:09:52.854451+0000","last_fullsized":"2023-09-23T23:09:52.854451+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":475,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854134+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:09:52.854134+0000","last_peered":"2023-09-23T23:09:52.854134+0000","last_clean":"2023-09-23T23:09:52.854134+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:09:52.854134+0000","last_undegraded":"2023-09-23T23:09:52.854134+0000","last_fullsized":"2023-09-23T23:09:52.854134+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":74,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854194+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:09:52.854194+0000","last_peered":"2023-09-23T23:09:52.854194+0000","last_clean":"2023-09-23T23:09:52.854194+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:09:52.854194+0000","last_undegraded":"2023-09-23T23:09:52.854194+0000","last_fullsized":"2023-09-23T23:09:52.854194+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":234,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.500366+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:09:52.500366+0000","last_peered":"2023-09-23T23:09:52.500366+0000","last_clean":"2023-09-23T23:09:52.500366+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:09:52.500366+0000","last_undegraded":"2023-09-23T23:09:52.500366+0000","last_fullsized":"2023-09-23T23:09:52.500366+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":305,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854364+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:09:52.854364+0000","last_peered":"2023-09-23T23:09:52.854364+0000","last_clean":"2023-09-23T23:09:52.854364+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:09:52.854364+0000","last_undegraded":"2023-09-23T23:09:52.854364+0000","last_fullsized":"2023-09-23T23:09:52.854364+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":506,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854676+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:09:52.854676+0000","last_peered":"2023-09-23T23:09:52.854676+0000","last_clean":"2023-09-23T23:09:52.854676+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:09:52.854676+0000","last_undegraded":"2023-09-23T23:09:52.854676+0000","last_fullsized":"2023-09-23T23:09:52.854676+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":332,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854682+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:09:52.854682+0000","last_peered":"2023-09-23T23:09:52.854682+0000","last_clean":"2023-09-23T23:09:52.854682+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:09:52.854682+0000","last_undegraded":"2023-09-23T23:09:52.854682+0000","last_fullsized":"2023-09-23T23:09:52.854682+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":402,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854484+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:09:52.854484+0000","last_peered":"2023-09-23T23:09:52.854484+0000","last_clean":"2023-09-23T23:09:52.854484+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:09:52.854484+0000","last_undegraded":"2023-09-23T23:09:52.854484+0000","last_fullsized":"2023-09-23T23:09:52.854484+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":59,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.144457+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:09:55.144457+0000","last_peered":"2023-09-23T23:09:55.144457+0000","last_clean":"2023-09-23T23:09:55.144457+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:09:55.144457+0000","last_undegraded":"2023-09-23T23:09:55.144457+0000","last_fullsized":"2023-09-23T23:09:55.144457+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":201,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.494428+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:09:52.494428+0000","last_peered":"2023-09-23T23:09:52.494428+0000","last_clean":"2023-09-23T23:09:52.494428+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:09:52.494428+0000","last_undegraded":"2023-09-23T23:09:52.494428+0000","last_fullsized":"2023-09-23T23:09:52.494428+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":252,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.502333+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:09:52.502333+0000","last_peered":"2023-09-23T23:09:52.502333+0000","last_clean":"2023-09-23T23:09:52.502333+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:09:52.502333+0000","last_undegraded":"2023-09-23T23:09:52.502333+0000","last_fullsized":"2023-09-23T23:09:52.502333+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":477,"reported_epoch":482,"state":"active+clean","last_fresh":"2023-09-23T23:09:52.854237+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:09:52.854237+0000","last_peered":"2023-09-23T23:09:52.854237+0000","last_clean":"2023-09-23T23:09:52.854237+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:09:52.854237+0000","last_undegraded":"2023-09-23T23:09:52.854237+0000","last_fullsized":"2023-09-23T23:09:52.854237+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430723,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626948,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899784,"kb_used_data":5248,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92845368,"statfs":{"total":95995035648,"available":95073656832,"internally_reserved":0,"allocated":5373952,"data_stored":4280819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.3109999999999999,"15min":4.6340000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.923,"5min":10.853,"15min":115.32899999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.304,"15min":4.6719999999999997},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":4.383,"5min":8.7010000000000005,"15min":115.348},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":3.629,"15min":3.056},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":9.4060000000000006,"5min":149.947,"15min":149.947},"last":0.439},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.718,"15min":2.9940000000000002},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":6.3689999999999998,"5min":150.22499999999999,"15min":150.22499999999999},"last":0.65100000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":5.508,"15min":2.9369999999999998},"min":{"1min":0.29499999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.2949999999999999,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.423,"5min":5.4909999999999997,"15min":3.081},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.3629999999999995,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":4.9630000000000001,"15min":3.3210000000000002},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":9.4329999999999998,"5min":142.24700000000001,"15min":142.24700000000001},"last":0.499},{"interface":"front","average":{"1min":1.986,"5min":4.8369999999999997,"15min":3.2349999999999999},"min":{"1min":0.41799999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.4100000000000001,"5min":143.86699999999999,"15min":143.86699999999999},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.2490000000000001,"15min":1.857},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.23999999999999999},"max":{"1min":6.3470000000000004,"5min":12.407999999999999,"15min":119.804},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.3109999999999999,"15min":1.925},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.24099999999999999},"max":{"1min":6.2140000000000004,"5min":8.8940000000000001,"15min":122.31999999999999},"last":0.63}]},{"osd":5,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.369,"15min":2.52},"min":{"1min":0.40400000000000003,"5min":0.33000000000000002,"15min":0.27900000000000003},"max":{"1min":6.4530000000000003,"5min":11.862,"15min":186.27600000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.238,"5min":1.506,"15min":2.6629999999999998},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.29599999999999999},"max":{"1min":6.2729999999999997,"5min":10.254,"15min":191.28700000000001},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.4159999999999999,"15min":1.5569999999999999},"min":{"1min":0.375,"5min":0.372,"15min":0.29999999999999999},"max":{"1min":5.3959999999999999,"5min":8.8670000000000009,"15min":115.61},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.117,"5min":1.3,"15min":1.7350000000000001},"min":{"1min":0.42699999999999999,"5min":0.38300000000000001,"15min":0.29999999999999999},"max":{"1min":7.1920000000000002,"5min":8.2609999999999992,"15min":115.578},"last":0.66300000000000003}]}]},{"osd":1,"up_from":84,"seq":360777253261,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1329652,"kb_used_data":319852,"kb_used_omap":9,"kb_used_meta":1009782,"kb_avail":92415500,"statfs":{"total":95995035648,"available":94633472000,"internally_reserved":0,"allocated":327528448,"data_stored":325533902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1034017769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.5629999999999999,"15min":2.5649999999999999},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.2189999999999999,"5min":59.475000000000001,"15min":279.30500000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7490000000000001,"15min":2.8799999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":4.8559999999999999,"5min":59.393999999999998,"15min":279.39800000000002},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:38 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.7429999999999999,"15min":2.7229999999999999},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":62.177,"5min":121.527,"15min":161.61699999999999},"last":0.375},{"interface":"front","average":{"1min":5.2619999999999996,"5min":4.1280000000000001,"15min":2.9079999999999999},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.460999999999999,"5min":118.29900000000001,"15min":162.61699999999999},"last":0.28499999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":7.2039999999999997,"5min":2.2789999999999999,"15min":1.7270000000000001},"min":{"1min":0.29699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":67.646000000000001,"5min":67.646000000000001,"15min":67.646000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":7.1609999999999996,"5min":2.5099999999999998,"15min":1.7809999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19500000000000001},"max":{"1min":65.269999999999996,"5min":65.269999999999996,"15min":65.269999999999996},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":3.0979999999999999,"15min":2.5670000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.274,"5min":129.69,"15min":151.036},"last":0.433},{"interface":"front","average":{"1min":1.5129999999999999,"5min":3.3799999999999999,"15min":2.859},"min":{"1min":0.318,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":6.1230000000000002,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.2869999999999999,"15min":2.153},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":3.2719999999999998,"5min":18.228999999999999,"15min":116.098},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.353,"5min":1.1890000000000001,"15min":2.008},"min":{"1min":0.375,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":5.7779999999999996,"5min":6.585,"15min":116.08499999999999},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.194,"15min":2.0369999999999999},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":6.1740000000000004,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.177,"5min":2.282,"15min":1.9159999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":4.0759999999999996,"5min":64.239999999999995,"15min":67.366},"last":0.497}]},{"osd":7,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.3779999999999999,"15min":1.117},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":4.0890000000000004,"5min":7.0960000000000001,"15min":7.0960000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.595,"5min":1.6819999999999999,"15min":1.4339999999999999},"min":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.33100000000000002},"max":{"1min":6.2389999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.54200000000000004}]}]},{"osd":0,"up_from":0,"seq":1516123455631,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259404,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328776,"kb_used_data":11136,"kb_used_omap":83,"kb_used_meta":317548,"kb_avail":93416376,"statfs":{"total":95995035648,"available":95658369024,"internally_reserved":0,"allocated":11403264,"data_stored":9276102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":325169605},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":18,"apply_latency_ms":18,"commit_latency_ns":18000000,"apply_latency_ns":18000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.82999999999999996,"15min":1.212},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":4.1390000000000002,"5min":10.199999999999999,"15min":74.241},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.5,"5min":1.29,"15min":1.5309999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":14.944000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0670000000000002,"5min":2.5049999999999999,"15min":1.3009999999999999},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":83.897000000000006,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.31},{"interface":"front","average":{"1min":4.5289999999999999,"5min":2.1669999999999998,"15min":1.462},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":83.088999999999999,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.97199999999999998}]},{"osd":3,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":7.7130000000000001,"5min":3.98,"15min":1.8380000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.23000000000000001},"max":{"1min":79.655000000000001,"5min":110.31699999999999,"15min":110.31699999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":7.2039999999999997,"5min":3.8650000000000002,"15min":1.851},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26100000000000001},"max":{"1min":79.677999999999997,"5min":110.56999999999999,"15min":110.56999999999999},"last":0.28599999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":1.9279999999999999,"15min":1.1379999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":68.325000000000003,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.502},{"interface":"front","average":{"1min":4.9020000000000001,"5min":2.2610000000000001,"15min":1.351},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":70.608000000000004,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":3.105,"15min":1.6839999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.6600000000000001,"5min":139.405,"15min":139.405},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.002,"5min":2.9129999999999998,"15min":1.5089999999999999},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":4.0499999999999998,"5min":139.42099999999999,"15min":139.42099999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":3.4180000000000001,"15min":1.8600000000000001},"min":{"1min":0.377,"5min":0.36799999999999999,"15min":0.34999999999999998},"max":{"1min":3.6989999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":3.1749999999999998,"15min":1.5740000000000001},"min":{"1min":0.44,"5min":0.34599999999999997,"15min":0.32200000000000001},"max":{"1min":4.0830000000000002,"5min":108.496,"15min":108.496},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:08:53 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.421,"15min":1.1890000000000001},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.1449999999999996,"5min":10.164,"15min":10.164},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.393,"15min":1.2050000000000001},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":15.037000000000001,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.48099999999999998}]}]},{"osd":3,"up_from":94,"seq":403726926209,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1007020,"kb_used_data":282596,"kb_used_omap":7,"kb_used_meta":724408,"kb_avail":92738132,"statfs":{"total":95995035648,"available":94963847168,"internally_reserved":0,"allocated":289378304,"data_stored":287270955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":741794408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.486,"15min":1.4490000000000001},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":3.8439999999999999,"5min":17.015000000000001,"15min":122.289},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.115,"15min":1.1870000000000001},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":4.9909999999999997,"5min":11.382999999999999,"15min":122.041},"last":0.373}]},{"osd":1,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":2.1720000000000002,"15min":1.4710000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":10.935,"5min":23.969000000000001,"15min":23.969000000000001},"last":0.314},{"interface":"front","average":{"1min":1.109,"5min":1.9119999999999999,"15min":1.373},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.20699999999999999},"max":{"1min":4.0119999999999996,"5min":16.986000000000001,"15min":16.986000000000001},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.5030000000000001,"15min":1.3340000000000001},"min":{"1min":0.27700000000000002,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":16.149000000000001,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.496},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.875,"15min":1.6799999999999999},"min":{"1min":0.41299999999999998,"5min":0.251,"15min":0.215},"max":{"1min":11.375999999999999,"5min":57.308,"15min":57.308},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.6629999999999998,"15min":2.1110000000000002},"min":{"1min":0.374,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":11.01,"5min":115.69,"15min":115.69},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.351,"5min":2.4009999999999998,"15min":2.0419999999999998},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":5.9169999999999998,"5min":115.617,"15min":115.617},"last":0.70299999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.401},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":4.8810000000000002,"5min":16.959,"15min":16.959},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.359,"5min":1.476,"15min":1.369},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":6.6130000000000004,"5min":10.545,"15min":10.545},"last":0.629}]},{"osd":6,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":2.589,"15min":1.8999999999999999},"min":{"1min":0.41699999999999998,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":77.707999999999998,"5min":77.707999999999998,"15min":77.707999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":5.4080000000000004,"5min":2.1930000000000001,"15min":1.6890000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":77.721999999999994,"5min":77.721999999999994,"15min":77.721999999999994},"last":0.71699999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.4630000000000001,"15min":1.272},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":4.5220000000000002,"5min":8.1620000000000008,"15min":8.1620000000000008},"last":0.67600000000000005},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.0499999999999998,"15min":1.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":0.621}]}]},{"osd":4,"up_from":30,"seq":128849019405,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908040,"kb_used_data":4672,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92837112,"statfs":{"total":95995035648,"available":95065202688,"internally_reserved":0,"allocated":4784128,"data_stored":3690473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.4039999999999999,"15min":2.3580000000000001},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":9.8870000000000005,"5min":99.132999999999996,"15min":156.375},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.3370000000000002,"15min":2.4279999999999999},"min":{"1min":0.36799999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":4.4509999999999996,"5min":101.062,"15min":157.02199999999999},"last":13.504}]},{"osd":1,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.4340000000000002,"15min":2.7989999999999999},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":4.3840000000000003,"5min":95.591999999999999,"15min":98.846999999999994},"last":1.161},{"interface":"front","average":{"1min":1.954,"5min":3.407,"15min":2.843},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":10.311,"5min":97.929000000000002,"15min":98.938999999999993},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:09:44 2023","interfaces":[{"interface":"back","average":{"1min":9.6500000000000004,"5min":5.8819999999999997,"15min":2.7210000000000001},"min":{"1min":0.41599999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":151.20400000000001,"5min":151.20400000000001,"15min":151.20400000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":9.3279999999999994,"5min":5.7590000000000003,"15min":2.5590000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":148.852,"5min":148.852,"15min":148.852},"last":0.70199999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":11.974,"5min":4.7699999999999996,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":147.82499999999999,"5min":147.82499999999999,"15min":147.82499999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":11.071,"5min":4.7309999999999999,"15min":2.4889999999999999},"min":{"1min":0.44,"5min":0.36599999999999999,"15min":0.32500000000000001},"max":{"1min":147.84100000000001,"5min":147.84100000000001,"15min":147.84100000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.4159999999999999,"15min":1.7769999999999999},"min":{"1min":0.37,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":10.282,"5min":10.282,"15min":148.61600000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.601,"15min":1.9430000000000001},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.247},"max":{"1min":17.02,"5min":17.02,"15min":151.905},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.4650000000000001,"15min":1.419},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.9740000000000002,"5min":35.896000000000001,"15min":35.896000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.611,"5min":1.778,"15min":1.3899999999999999},"min":{"1min":0.34899999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":6.5129999999999999,"5min":35.908000000000001,"15min":35.908000000000001},"last":1.4870000000000001}]},{"osd":7,"last update":"Sat Sep 23 23:09:34 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":1.5680000000000001,"15min":1.3660000000000001},"min":{"1min":0.432,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":11.096,"5min":11.096,"15min":12.84},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.448,"5min":1.4670000000000001,"15min":1.25},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":7.3179999999999996,"5min":15.808,"15min":15.808},"last":0.68400000000000005}]}]},{"osd":5,"up_from":0,"seq":154618823179,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:10:01.922 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:10:01.922 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:10:01.922 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-09-23T23:10:01.923 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:10:02.002 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:02.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:02.047 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:02.112 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:02.145 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:02.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[180979]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[180979]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[180979]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[180979]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[180979]: from='client.37335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:02.195 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[180979]: pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:02.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[187743]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[187743]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[187743]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[187743]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[187743]: from='client.37335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:02.196 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:02 smithi052 ceph-mon[187743]: pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:02.200 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:02.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:02.306 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:02.336 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:02.443 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:02.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:02 smithi151 ceph-mon[159210]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:02 smithi151 ceph-mon[159210]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:02 smithi151 ceph-mon[159210]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:02 smithi151 ceph-mon[159210]: failed to encode map e484 with expected crc 2023-09-23T23:10:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:02 smithi151 ceph-mon[159210]: from='client.37335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:02.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:02 smithi151 ceph-mon[159210]: pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:02.457 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:10:02.457 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:10:02.548 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:02.581 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:03.150 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:10:03.422 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:10:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:10:03] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.33.4" 2023-09-23T23:10:03.439 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:03.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:03.544 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:03.572 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:04.077 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T23:10:04.102 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:04.123 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:04.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:04.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:04.188 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:04.230 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:04.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:04.312 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:04.314 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:04 smithi052 ceph-mon[180979]: pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:04.315 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:04 smithi052 ceph-mon[187743]: pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:04.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:04.416 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:04.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:04 smithi151 ceph-mon[159210]: pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:04.447 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:04.495 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:04.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:04.602 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:04.630 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:04.658 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:10:04.674 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:04.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:04.679 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:10:04.699 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:10:04.719 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:10:04.737 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:10:04.756 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:10:04.780 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:04.781 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:10:04.805 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:04.807 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:10:04.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.810 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.811 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.811 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:10:04.849 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:04.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:04.955 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:04.983 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:04.986 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T23:10:05.025 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:05.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:05.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:05.033 INFO:tasks.workunit.client.0.smithi052.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T23:10:05.035 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:10:05.035 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T23:10:05.129 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:05.142 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T23:10:05.155 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:05.191 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T23:10:05.197 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:05.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:05.231 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:10:05.233 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:10:05.233 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T23:10:05.303 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:05.333 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:05.380 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:10:05.519 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T23:10:05.519 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T23:10:05.591 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T23:10:05.592 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T23:10:05.592 DEBUG:teuthology.orchestra.run.smithi052:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T23:10:05.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3707237401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[187743]: pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3707237401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[180979]: pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-09-23T23:10:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:05.780 INFO:tasks.workunit.client.0.smithi052.stdout:Running main() from gmock_main.cc 2023-09-23T23:10:05.780 INFO:tasks.workunit.client.0.smithi052.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T23:10:05.780 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] Global test environment set-up. 2023-09-23T23:10:05.780 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T23:10:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3707237401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:05 smithi151 ceph-mon[159210]: pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-09-23T23:10:05.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:05.984 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:05.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:06.088 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:06.116 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:06.159 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:06.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:06.264 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:06.290 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:06.332 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:06.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:06.439 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:06.465 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:06.507 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:06.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:06.612 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:06.644 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:06.695 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:06.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:06.798 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:06.824 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:06.866 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:06.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:06.973 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:06.999 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:07.044 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:07.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:07.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:07.149 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:07.183 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:07.235 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:07.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:07.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:07.339 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:07.368 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:07.418 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:07.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:07.521 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:07.542 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T23:10:07.551 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:07.593 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:07.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:07.701 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:07.730 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: osdmap e485: 8 total, 8 up, 5 in 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3213926960' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:07 smithi151 ceph-mon[159210]: pgmap v1532: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: osdmap e485: 8 total, 8 up, 5 in 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3213926960' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[180979]: pgmap v1532: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: osdmap e485: 8 total, 8 up, 5 in 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3213926960' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: failed to encode map e485 with expected crc 2023-09-23T23:10:07.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:07 smithi052 ceph-mon[187743]: pgmap v1532: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:08.012 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_all_features (470 ms) 2023-09-23T23:10:08.012 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T23:10:08.204 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.copyup (192 ms) 2023-09-23T23:10:08.204 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T23:10:08.222 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.sparse_copyup (18 ms) 2023-09-23T23:10:08.222 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T23:10:08.230 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2023-09-23T23:10:08.230 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T23:10:08.253 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.add_remove_child (23 ms) 2023-09-23T23:10:08.253 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T23:10:08.339 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.directory_methods (86 ms) 2023-09-23T23:10:08.340 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.create 2023-09-23T23:10:08.368 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.create (29 ms) 2023-09-23T23:10:08.368 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T23:10:08.369 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-09-23T23:10:08.369 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T23:10:08.372 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_object_prefix (3 ms) 2023-09-23T23:10:08.372 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T23:10:08.373 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-09-23T23:10:08.373 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T23:10:08.374 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T23:10:08.374 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T23:10:08.376 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T23:10:08.376 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T23:10:08.379 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T23:10:08.379 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T23:10:08.382 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-09-23T23:10:08.383 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T23:10:08.388 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-23T23:10:08.388 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T23:10:08.407 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-09-23T23:10:08.407 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T23:10:08.415 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-09-23T23:10:08.415 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T23:10:08.457 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.parents_v1 (42 ms) 2023-09-23T23:10:08.457 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T23:10:08.488 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2023-09-23T23:10:08.488 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T23:10:08.510 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-09-23T23:10:08.511 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T23:10:08.514 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2023-09-23T23:10:08.514 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T23:10:08.524 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-23T23:10:08.524 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T23:10:08.525 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2023-09-23T23:10:08.525 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T23:10:08.530 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-09-23T23:10:08.530 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T23:10:08.536 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2023-09-23T23:10:08.536 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T23:10:08.536 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T23:10:08.537 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T23:10:08.541 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-09-23T23:10:08.541 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T23:10:08.551 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.object_map_snap_remove (11 ms) 2023-09-23T23:10:08.551 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T23:10:08.557 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.flags (6 ms) 2023-09-23T23:10:08.557 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T23:10:08.587 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.metadata (30 ms) 2023-09-23T23:10:08.587 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T23:10:08.594 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-09-23T23:10:08.594 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T23:10:08.629 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2023-09-23T23:10:08.629 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T23:10:08.645 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-09-23T23:10:08.645 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T23:10:08.667 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:08.683 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:08.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:08.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:08.788 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:08.812 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:08 smithi151 ceph-mon[159210]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.812 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:08 smithi151 ceph-mon[159210]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.812 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:08 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T23:10:08.812 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:08 smithi151 ceph-mon[159210]: osdmap e486: 8 total, 8 up, 5 in 2023-09-23T23:10:08.813 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:08 smithi151 ceph-mon[159210]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.814 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:08.858 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:08.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:08.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[187743]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[187743]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[187743]: osdmap e486: 8 total, 8 up, 5 in 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[187743]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[180979]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[180979]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi052-696532-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T23:10:08.932 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[180979]: osdmap e486: 8 total, 8 up, 5 in 2023-09-23T23:10:08.933 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:08 smithi052 ceph-mon[180979]: failed to encode map e486 with expected crc 2023-09-23T23:10:08.963 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:08.989 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:09.034 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:09.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:09.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:09.143 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:09.169 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:09.213 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:09.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:09.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:09.319 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:09.360 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[187743]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[187743]: osdmap e487: 8 total, 8 up, 5 in 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[187743]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[187743]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[187743]: pgmap v1535: 105 pgs: 13 creating+peering, 19 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[180979]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[180979]: osdmap e487: 8 total, 8 up, 5 in 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[180979]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[180979]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.899 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:09 smithi052 ceph-mon[180979]: pgmap v1535: 105 pgs: 13 creating+peering, 19 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:09.934 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:09.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:09.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:09 smithi151 ceph-mon[159210]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:09 smithi151 ceph-mon[159210]: osdmap e487: 8 total, 8 up, 5 in 2023-09-23T23:10:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:09 smithi151 ceph-mon[159210]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:09 smithi151 ceph-mon[159210]: failed to encode map e487 with expected crc 2023-09-23T23:10:09.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:09 smithi151 ceph-mon[159210]: pgmap v1535: 105 pgs: 13 creating+peering, 19 unknown, 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:10.039 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:10.066 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:10.070 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:10:10.071 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:10:10.109 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:10.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:10.214 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:10.252 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:10.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2820791133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:10.551 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:10.562 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:10:10.703 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:10:10.776 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:10.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:10.880 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:10.904 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2820791133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:10.904 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:10.906 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:10.949 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:10.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2820791133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:10.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:11.055 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:11.087 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:11.479 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:11.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:11.545 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_image_status (2900 ms) 2023-09-23T23:10:11.545 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T23:10:11.584 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:11.615 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:11.638 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_image_map (93 ms) 2023-09-23T23:10:11.638 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T23:10:11.647 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2023-09-23T23:10:11.647 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T23:10:11.659 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2023-09-23T23:10:11.659 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T23:10:11.661 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T23:10:11.661 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T23:10:11.663 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-09-23T23:10:11.664 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T23:10:11.667 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T23:10:11.667 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T23:10:11.673 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-09-23T23:10:11.673 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T23:10:11.676 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T23:10:11.676 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T23:10:11.681 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2023-09-23T23:10:11.681 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T23:10:11.683 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T23:10:11.683 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T23:10:11.686 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T23:10:11.686 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T23:10:11.689 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-23T23:10:11.689 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T23:10:11.692 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_image_clean (3 ms) 2023-09-23T23:10:11.692 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T23:10:11.730 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.image_group_add (38 ms) 2023-09-23T23:10:11.731 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T23:10:11.733 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-09-23T23:10:11.734 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T23:10:11.736 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T23:10:11.736 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T23:10:11.737 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2023-09-23T23:10:11.737 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T23:10:11.739 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T23:10:11.739 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T23:10:11.741 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2023-09-23T23:10:11.741 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T23:10:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: from='client.37359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:11.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: failed to encode map e488 with expected crc 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: osdmap e488: 8 total, 8 up, 5 in 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: failed to encode map e488 with expected crc 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[180979]: pgmap v1537: 105 pgs: 13 creating+peering, 92 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 489 KiB/s rd, 292 KiB/s wr, 794 op/s 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: from='client.37359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: failed to encode map e488 with expected crc 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:10:11.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: osdmap e488: 8 total, 8 up, 5 in 2023-09-23T23:10:11.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: failed to encode map e488 with expected crc 2023-09-23T23:10:11.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:11 smithi052 ceph-mon[187743]: pgmap v1537: 105 pgs: 13 creating+peering, 92 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 489 KiB/s rd, 292 KiB/s wr, 794 op/s 2023-09-23T23:10:11.745 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T23:10:11.745 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T23:10:11.853 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_set (109 ms) 2023-09-23T23:10:11.853 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T23:10:11.856 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T23:10:11.856 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T23:10:11.872 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2023-09-23T23:10:11.872 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: from='client.37359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: failed to encode map e488 with expected crc 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: osdmap e488: 8 total, 8 up, 5 in 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: failed to encode map e488 with expected crc 2023-09-23T23:10:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:11 smithi151 ceph-mon[159210]: pgmap v1537: 105 pgs: 13 creating+peering, 92 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 489 KiB/s rd, 292 KiB/s wr, 794 op/s 2023-09-23T23:10:12.115 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1535,"stamp":"2023-09-23T23:10:08.828281+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":260,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":611296,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194,"ondisk_log_size":194,"up":258,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":223,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4406604,"kb_used_data":556836,"kb_used_omap":99,"kb_used_meta":3849628,"kb_avail":464319156,"statfs":{"total":479975178240,"available":475462815744,"internally_reserved":0,"allocated":570200064,"data_stored":561762403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3942019094},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001635"},"pg_stats":[{"pgid":"12.15","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.541768+0000","last_change":"2023-09-23T23:10:06.541768+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.541768+0000","last_undegraded":"2023-09-23T23:10:06.541768+0000","last_fullsized":"2023-09-23T23:10:06.541768+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:00.280895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":496,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054852+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:09:55.054852+0000","last_peered":"2023-09-23T23:09:55.054852+0000","last_clean":"2023-09-23T23:09:55.054852+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:09:55.054852+0000","last_undegraded":"2023-09-23T23:09:55.054852+0000","last_fullsized":"2023-09-23T23:09:55.054852+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:59.528840+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:09:59.528840+0000","last_peered":"2023-09-23T23:09:59.528840+0000","last_clean":"2023-09-23T23:09:59.528840+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:09:59.528840+0000","last_undegraded":"2023-09-23T23:09:59.528840+0000","last_fullsized":"2023-09-23T23:09:59.528840+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":156,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.147982+0000","last_change":"2023-09-23T22:57:27.528484+0000","last_active":"2023-09-23T23:09:55.147982+0000","last_peered":"2023-09-23T23:09:55.147982+0000","last_clean":"2023-09-23T23:09:55.147982+0000","last_became_active":"2023-09-23T22:57:27.528121+0000","last_became_peered":"2023-09-23T22:57:27.528121+0000","last_unstale":"2023-09-23T23:09:55.147982+0000","last_undegraded":"2023-09-23T23:09:55.147982+0000","last_fullsized":"2023-09-23T23:09:55.147982+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:20.146529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":503,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.143805+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:09:55.143805+0000","last_peered":"2023-09-23T23:09:55.143805+0000","last_clean":"2023-09-23T23:09:55.143805+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:09:55.143805+0000","last_undegraded":"2023-09-23T23:09:55.143805+0000","last_fullsized":"2023-09-23T23:09:55.143805+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":63,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.148123+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:09:55.148123+0000","last_peered":"2023-09-23T23:09:55.148123+0000","last_clean":"2023-09-23T23:09:55.148123+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:09:55.148123+0000","last_undegraded":"2023-09-23T23:09:55.148123+0000","last_fullsized":"2023-09-23T23:09:55.148123+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":73,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145011+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:09:55.145011+0000","last_peered":"2023-09-23T23:09:55.145011+0000","last_clean":"2023-09-23T23:09:55.145011+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:09:55.145011+0000","last_undegraded":"2023-09-23T23:09:55.145011+0000","last_fullsized":"2023-09-23T23:09:55.145011+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":428,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145285+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:09:55.145285+0000","last_peered":"2023-09-23T23:09:55.145285+0000","last_clean":"2023-09-23T23:09:55.145285+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:09:55.145285+0000","last_undegraded":"2023-09-23T23:09:55.145285+0000","last_fullsized":"2023-09-23T23:09:55.145285+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":175,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.154945+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:09:55.154945+0000","last_peered":"2023-09-23T23:09:55.154945+0000","last_clean":"2023-09-23T23:09:55.154945+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:09:55.154945+0000","last_undegraded":"2023-09-23T23:09:55.154945+0000","last_fullsized":"2023-09-23T23:09:55.154945+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.542772+0000","last_change":"2023-09-23T23:10:06.542772+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.542772+0000","last_undegraded":"2023-09-23T23:10:06.542772+0000","last_fullsized":"2023-09-23T23:10:06.542772+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:45.154282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":290,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.144362+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:09:55.144362+0000","last_peered":"2023-09-23T23:09:55.144362+0000","last_clean":"2023-09-23T23:09:55.144362+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:09:55.144362+0000","last_undegraded":"2023-09-23T23:09:55.144362+0000","last_fullsized":"2023-09-23T23:09:55.144362+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":189,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145202+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:09:55.145202+0000","last_peered":"2023-09-23T23:09:55.145202+0000","last_clean":"2023-09-23T23:09:55.145202+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:09:55.145202+0000","last_undegraded":"2023-09-23T23:09:55.145202+0000","last_fullsized":"2023-09-23T23:09:55.145202+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":222,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145908+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:09:55.145908+0000","last_peered":"2023-09-23T23:09:55.145908+0000","last_clean":"2023-09-23T23:09:55.145908+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:09:55.145908+0000","last_undegraded":"2023-09-23T23:09:55.145908+0000","last_fullsized":"2023-09-23T23:09:55.145908+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":240,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267466+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:10:00.267466+0000","last_peered":"2023-09-23T23:10:00.267466+0000","last_clean":"2023-09-23T23:10:00.267466+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:10:00.267466+0000","last_undegraded":"2023-09-23T23:10:00.267466+0000","last_fullsized":"2023-09-23T23:10:00.267466+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":574,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056198+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:09:55.056198+0000","last_peered":"2023-09-23T23:09:55.056198+0000","last_clean":"2023-09-23T23:09:55.056198+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:09:55.056198+0000","last_undegraded":"2023-09-23T23:09:55.056198+0000","last_fullsized":"2023-09-23T23:09:55.056198+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":467,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054895+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:09:55.054895+0000","last_peered":"2023-09-23T23:09:55.054895+0000","last_clean":"2023-09-23T23:09:55.054895+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:09:55.054895+0000","last_undegraded":"2023-09-23T23:09:55.054895+0000","last_fullsized":"2023-09-23T23:09:55.054895+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1c","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.542157+0000","last_change":"2023-09-23T23:10:06.542157+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.542157+0000","last_undegraded":"2023-09-23T23:10:06.542157+0000","last_fullsized":"2023-09-23T23:10:06.542157+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:09.305549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":399,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054995+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:09:55.054995+0000","last_peered":"2023-09-23T23:09:55.054995+0000","last_clean":"2023-09-23T23:09:55.054995+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:09:55.054995+0000","last_undegraded":"2023-09-23T23:09:55.054995+0000","last_fullsized":"2023-09-23T23:09:55.054995+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":168,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.144770+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:09:55.144770+0000","last_peered":"2023-09-23T23:09:55.144770+0000","last_clean":"2023-09-23T23:09:55.144770+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:09:55.144770+0000","last_undegraded":"2023-09-23T23:09:55.144770+0000","last_fullsized":"2023-09-23T23:09:55.144770+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1d","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.543515+0000","last_change":"2023-09-23T23:10:06.543515+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.543515+0000","last_undegraded":"2023-09-23T23:10:06.543515+0000","last_fullsized":"2023-09-23T23:10:06.543515+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:31.938380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":486,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.140190+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:09:55.140190+0000","last_peered":"2023-09-23T23:09:55.140190+0000","last_clean":"2023-09-23T23:09:55.140190+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:09:55.140190+0000","last_undegraded":"2023-09-23T23:09:55.140190+0000","last_fullsized":"2023-09-23T23:09:55.140190+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":400,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145812+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:09:55.145812+0000","last_peered":"2023-09-23T23:09:55.145812+0000","last_clean":"2023-09-23T23:09:55.145812+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:09:55.145812+0000","last_undegraded":"2023-09-23T23:09:55.145812+0000","last_fullsized":"2023-09-23T23:09:55.145812+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":387,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056028+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:09:55.056028+0000","last_peered":"2023-09-23T23:09:55.056028+0000","last_clean":"2023-09-23T23:09:55.056028+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:09:55.056028+0000","last_undegraded":"2023-09-23T23:09:55.056028+0000","last_fullsized":"2023-09-23T23:09:55.056028+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":510,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.143723+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:09:55.143723+0000","last_peered":"2023-09-23T23:09:55.143723+0000","last_clean":"2023-09-23T23:09:55.143723+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:09:55.143723+0000","last_undegraded":"2023-09-23T23:09:55.143723+0000","last_fullsized":"2023-09-23T23:09:55.143723+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1f","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.543117+0000","last_change":"2023-09-23T23:10:06.543117+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.543117+0000","last_undegraded":"2023-09-23T23:10:06.543117+0000","last_fullsized":"2023-09-23T23:10:06.543117+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:58.199608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":98,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.151550+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:09:55.151550+0000","last_peered":"2023-09-23T23:09:55.151550+0000","last_clean":"2023-09-23T23:09:55.151550+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:09:55.151550+0000","last_undegraded":"2023-09-23T23:09:55.151550+0000","last_fullsized":"2023-09-23T23:09:55.151550+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":181,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145368+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:09:55.145368+0000","last_peered":"2023-09-23T23:09:55.145368+0000","last_clean":"2023-09-23T23:09:55.145368+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:09:55.145368+0000","last_undegraded":"2023-09-23T23:09:55.145368+0000","last_fullsized":"2023-09-23T23:09:55.145368+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":394,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.057348+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:09:55.057348+0000","last_peered":"2023-09-23T23:09:55.057348+0000","last_clean":"2023-09-23T23:09:55.057348+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:09:55.057348+0000","last_undegraded":"2023-09-23T23:09:55.057348+0000","last_fullsized":"2023-09-23T23:09:55.057348+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":350,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054479+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:09:55.054479+0000","last_peered":"2023-09-23T23:09:55.054479+0000","last_clean":"2023-09-23T23:09:55.054479+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:09:55.054479+0000","last_undegraded":"2023-09-23T23:09:55.054479+0000","last_fullsized":"2023-09-23T23:09:55.054479+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":114,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.268179+0000","last_change":"2023-09-23T23:05:53.631535+0000","last_active":"2023-09-23T23:10:00.268179+0000","last_peered":"2023-09-23T23:10:00.268179+0000","last_clean":"2023-09-23T23:10:00.268179+0000","last_became_active":"2023-09-23T23:05:53.574828+0000","last_became_peered":"2023-09-23T23:05:53.574828+0000","last_unstale":"2023-09-23T23:10:00.268179+0000","last_undegraded":"2023-09-23T23:10:00.268179+0000","last_fullsized":"2023-09-23T23:10:00.268179+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:55.413525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":491,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055958+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:09:55.055958+0000","last_peered":"2023-09-23T23:09:55.055958+0000","last_clean":"2023-09-23T23:09:55.055958+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:09:55.055958+0000","last_undegraded":"2023-09-23T23:09:55.055958+0000","last_fullsized":"2023-09-23T23:09:55.055958+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":142,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145697+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:09:55.145697+0000","last_peered":"2023-09-23T23:09:55.145697+0000","last_clean":"2023-09-23T23:09:55.145697+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:09:55.145697+0000","last_undegraded":"2023-09-23T23:09:55.145697+0000","last_fullsized":"2023-09-23T23:09:55.145697+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":81,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.134219+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:09:55.134219+0000","last_peered":"2023-09-23T23:09:55.134219+0000","last_clean":"2023-09-23T23:09:55.134219+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:09:55.134219+0000","last_undegraded":"2023-09-23T23:09:55.134219+0000","last_fullsized":"2023-09-23T23:09:55.134219+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":175,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.144687+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:09:55.144687+0000","last_peered":"2023-09-23T23:09:55.144687+0000","last_clean":"2023-09-23T23:09:55.144687+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:09:55.144687+0000","last_undegraded":"2023-09-23T23:09:55.144687+0000","last_fullsized":"2023-09-23T23:09:55.144687+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":306,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054814+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:09:55.054814+0000","last_peered":"2023-09-23T23:09:55.054814+0000","last_clean":"2023-09-23T23:09:55.054814+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:09:55.054814+0000","last_undegraded":"2023-09-23T23:09:55.054814+0000","last_fullsized":"2023-09-23T23:09:55.054814+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":189,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.146800+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:09:55.146800+0000","last_peered":"2023-09-23T23:09:55.146800+0000","last_clean":"2023-09-23T23:09:55.146800+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:09:55.146800+0000","last_undegraded":"2023-09-23T23:09:55.146800+0000","last_fullsized":"2023-09-23T23:09:55.146800+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":486,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056112+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:09:55.056112+0000","last_peered":"2023-09-23T23:09:55.056112+0000","last_clean":"2023-09-23T23:09:55.056112+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:09:55.056112+0000","last_undegraded":"2023-09-23T23:09:55.056112+0000","last_fullsized":"2023-09-23T23:09:55.056112+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.543085+0000","last_change":"2023-09-23T23:10:06.543085+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.543085+0000","last_undegraded":"2023-09-23T23:10:06.543085+0000","last_fullsized":"2023-09-23T23:10:06.543085+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:41.869411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":257,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055153+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:09:55.055153+0000","last_peered":"2023-09-23T23:09:55.055153+0000","last_clean":"2023-09-23T23:09:55.055153+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:09:55.055153+0000","last_undegraded":"2023-09-23T23:09:55.055153+0000","last_fullsized":"2023-09-23T23:09:55.055153+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":494,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056733+0000","last_change":"2023-09-23T22:58:23.267136+0000","last_active":"2023-09-23T23:09:55.056733+0000","last_peered":"2023-09-23T23:09:55.056733+0000","last_clean":"2023-09-23T23:09:55.056733+0000","last_became_active":"2023-09-23T22:58:23.266832+0000","last_became_peered":"2023-09-23T22:58:23.266832+0000","last_unstale":"2023-09-23T23:09:55.056733+0000","last_undegraded":"2023-09-23T23:09:55.056733+0000","last_fullsized":"2023-09-23T23:09:55.056733+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":507,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055728+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:09:55.055728+0000","last_peered":"2023-09-23T23:09:55.055728+0000","last_clean":"2023-09-23T23:09:55.055728+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:09:55.055728+0000","last_undegraded":"2023-09-23T23:09:55.055728+0000","last_fullsized":"2023-09-23T23:09:55.055728+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":512,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056364+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:09:55.056364+0000","last_peered":"2023-09-23T23:09:55.056364+0000","last_clean":"2023-09-23T23:09:55.056364+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:09:55.056364+0000","last_undegraded":"2023-09-23T23:09:55.056364+0000","last_fullsized":"2023-09-23T23:09:55.056364+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.541839+0000","last_change":"2023-09-23T23:10:06.541839+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.541839+0000","last_undegraded":"2023-09-23T23:10:06.541839+0000","last_fullsized":"2023-09-23T23:10:06.541839+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:12.062431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":157,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.156724+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:09:55.156724+0000","last_peered":"2023-09-23T23:09:55.156724+0000","last_clean":"2023-09-23T23:09:55.156724+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:09:55.156724+0000","last_undegraded":"2023-09-23T23:09:55.156724+0000","last_fullsized":"2023-09-23T23:09:55.156724+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":392,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.268388+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:10:00.268388+0000","last_peered":"2023-09-23T23:10:00.268388+0000","last_clean":"2023-09-23T23:10:00.268388+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:10:00.268388+0000","last_undegraded":"2023-09-23T23:10:00.268388+0000","last_fullsized":"2023-09-23T23:10:00.268388+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.541518+0000","last_change":"2023-09-23T23:10:06.541518+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.541518+0000","last_undegraded":"2023-09-23T23:10:06.541518+0000","last_fullsized":"2023-09-23T23:10:06.541518+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:14.836833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":405,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267677+0000","last_change":"2023-09-23T23:05:39.874927+0000","last_active":"2023-09-23T23:10:00.267677+0000","last_peered":"2023-09-23T23:10:00.267677+0000","last_clean":"2023-09-23T23:10:00.267677+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:10:00.267677+0000","last_undegraded":"2023-09-23T23:10:00.267677+0000","last_fullsized":"2023-09-23T23:10:00.267677+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":322,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145160+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:09:55.145160+0000","last_peered":"2023-09-23T23:09:55.145160+0000","last_clean":"2023-09-23T23:09:55.145160+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:09:55.145160+0000","last_undegraded":"2023-09-23T23:09:55.145160+0000","last_fullsized":"2023-09-23T23:09:55.145160+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":225,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054763+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:09:55.054763+0000","last_peered":"2023-09-23T23:09:55.054763+0000","last_clean":"2023-09-23T23:09:55.054763+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:09:55.054763+0000","last_undegraded":"2023-09-23T23:09:55.054763+0000","last_fullsized":"2023-09-23T23:09:55.054763+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.542956+0000","last_change":"2023-09-23T23:10:06.542956+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.542956+0000","last_undegraded":"2023-09-23T23:10:06.542956+0000","last_fullsized":"2023-09-23T23:10:06.542956+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:58.441725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":222,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:02.786946+0000","last_change":"2023-09-23T23:07:55.442378+0000","last_active":"2023-09-23T23:10:02.786946+0000","last_peered":"2023-09-23T23:10:02.786946+0000","last_clean":"2023-09-23T23:10:02.786946+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:10:02.786946+0000","last_undegraded":"2023-09-23T23:10:02.786946+0000","last_fullsized":"2023-09-23T23:10:02.786946+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":522,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055596+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:09:55.055596+0000","last_peered":"2023-09-23T23:09:55.055596+0000","last_clean":"2023-09-23T23:09:55.055596+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:09:55.055596+0000","last_undegraded":"2023-09-23T23:09:55.055596+0000","last_fullsized":"2023-09-23T23:09:55.055596+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":390,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267634+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:10:00.267634+0000","last_peered":"2023-09-23T23:10:00.267634+0000","last_clean":"2023-09-23T23:10:00.267634+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:10:00.267634+0000","last_undegraded":"2023-09-23T23:10:00.267634+0000","last_fullsized":"2023-09-23T23:10:00.267634+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.543626+0000","last_change":"2023-09-23T23:10:06.543626+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.543626+0000","last_undegraded":"2023-09-23T23:10:06.543626+0000","last_fullsized":"2023-09-23T23:10:06.543626+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:16.794818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":465,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267350+0000","last_change":"2023-09-23T23:05:39.880854+0000","last_active":"2023-09-23T23:10:00.267350+0000","last_peered":"2023-09-23T23:10:00.267350+0000","last_clean":"2023-09-23T23:10:00.267350+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:10:00.267350+0000","last_undegraded":"2023-09-23T23:10:00.267350+0000","last_fullsized":"2023-09-23T23:10:00.267350+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":318,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267165+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:10:00.267165+0000","last_peered":"2023-09-23T23:10:00.267165+0000","last_clean":"2023-09-23T23:10:00.267165+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:10:00.267165+0000","last_undegraded":"2023-09-23T23:10:00.267165+0000","last_fullsized":"2023-09-23T23:10:00.267165+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":421,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145196+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:09:55.145196+0000","last_peered":"2023-09-23T23:09:55.145196+0000","last_clean":"2023-09-23T23:09:55.145196+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:09:55.145196+0000","last_undegraded":"2023-09-23T23:09:55.145196+0000","last_fullsized":"2023-09-23T23:09:55.145196+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.542653+0000","last_change":"2023-09-23T23:10:06.542653+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.542653+0000","last_undegraded":"2023-09-23T23:10:06.542653+0000","last_fullsized":"2023-09-23T23:10:06.542653+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:15:53.428559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":397,"reported_epoch":485,"state":"active+clean","last_fresh":"2023-09-23T23:10:06.544743+0000","last_change":"2023-09-23T23:05:53.571833+0000","last_active":"2023-09-23T23:10:06.544743+0000","last_peered":"2023-09-23T23:10:06.544743+0000","last_clean":"2023-09-23T23:10:06.544743+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:10:06.544743+0000","last_undegraded":"2023-09-23T23:10:06.544743+0000","last_fullsized":"2023-09-23T23:10:06.544743+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":605,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267293+0000","last_change":"2023-09-23T23:09:55.052855+0000","last_active":"2023-09-23T23:10:00.267293+0000","last_peered":"2023-09-23T23:10:00.267293+0000","last_clean":"2023-09-23T23:10:00.267293+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:10:00.267293+0000","last_undegraded":"2023-09-23T23:10:00.267293+0000","last_fullsized":"2023-09-23T23:10:00.267293+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.134306+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:09:55.134306+0000","last_peered":"2023-09-23T23:09:55.134306+0000","last_clean":"2023-09-23T23:09:55.134306+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:09:55.134306+0000","last_undegraded":"2023-09-23T23:09:55.134306+0000","last_fullsized":"2023-09-23T23:09:55.134306+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":402,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056939+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:09:55.056939+0000","last_peered":"2023-09-23T23:09:55.056939+0000","last_clean":"2023-09-23T23:09:55.056939+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:09:55.056939+0000","last_undegraded":"2023-09-23T23:09:55.056939+0000","last_fullsized":"2023-09-23T23:09:55.056939+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":409,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.153452+0000","last_change":"2023-09-23T23:05:53.608808+0000","last_active":"2023-09-23T23:09:55.153452+0000","last_peered":"2023-09-23T23:09:55.153452+0000","last_clean":"2023-09-23T23:09:55.153452+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:09:55.153452+0000","last_undegraded":"2023-09-23T23:09:55.153452+0000","last_fullsized":"2023-09-23T23:09:55.153452+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":378,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055647+0000","last_change":"2023-09-23T23:05:53.572522+0000","last_active":"2023-09-23T23:09:55.055647+0000","last_peered":"2023-09-23T23:09:55.055647+0000","last_clean":"2023-09-23T23:09:55.055647+0000","last_became_active":"2023-09-23T23:05:53.572181+0000","last_became_peered":"2023-09-23T23:05:53.572181+0000","last_unstale":"2023-09-23T23:09:55.055647+0000","last_undegraded":"2023-09-23T23:09:55.055647+0000","last_fullsized":"2023-09-23T23:09:55.055647+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":346,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:01.975957+0000","last_change":"2023-09-23T23:05:53.574765+0000","last_active":"2023-09-23T23:10:01.975957+0000","last_peered":"2023-09-23T23:10:01.975957+0000","last_clean":"2023-09-23T23:10:01.975957+0000","last_became_active":"2023-09-23T23:05:53.574414+0000","last_became_peered":"2023-09-23T23:05:53.574414+0000","last_unstale":"2023-09-23T23:10:01.975957+0000","last_undegraded":"2023-09-23T23:10:01.975957+0000","last_fullsized":"2023-09-23T23:10:01.975957+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:00.534814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":343,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056656+0000","last_change":"2023-09-23T23:05:39.898872+0000","last_active":"2023-09-23T23:09:55.056656+0000","last_peered":"2023-09-23T23:09:55.056656+0000","last_clean":"2023-09-23T23:09:55.056656+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:09:55.056656+0000","last_undegraded":"2023-09-23T23:09:55.056656+0000","last_fullsized":"2023-09-23T23:09:55.056656+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":342,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267863+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:10:00.267863+0000","last_peered":"2023-09-23T23:10:00.267863+0000","last_clean":"2023-09-23T23:10:00.267863+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:10:00.267863+0000","last_undegraded":"2023-09-23T23:10:00.267863+0000","last_fullsized":"2023-09-23T23:10:00.267863+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":308,"reported_epoch":485,"state":"active+clean","last_fresh":"2023-09-23T23:10:06.544609+0000","last_change":"2023-09-23T23:05:53.573664+0000","last_active":"2023-09-23T23:10:06.544609+0000","last_peered":"2023-09-23T23:10:06.544609+0000","last_clean":"2023-09-23T23:10:06.544609+0000","last_became_active":"2023-09-23T23:05:53.573348+0000","last_became_peered":"2023-09-23T23:05:53.573348+0000","last_unstale":"2023-09-23T23:10:06.544609+0000","last_undegraded":"2023-09-23T23:10:06.544609+0000","last_fullsized":"2023-09-23T23:10:06.544609+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:15.419143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":207,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.145516+0000","last_change":"2023-09-23T23:05:53.629673+0000","last_active":"2023-09-23T23:09:55.145516+0000","last_peered":"2023-09-23T23:09:55.145516+0000","last_clean":"2023-09-23T23:09:55.145516+0000","last_became_active":"2023-09-23T23:05:53.573737+0000","last_became_peered":"2023-09-23T23:05:53.573737+0000","last_unstale":"2023-09-23T23:09:55.145516+0000","last_undegraded":"2023-09-23T23:09:55.145516+0000","last_fullsized":"2023-09-23T23:09:55.145516+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:05.832274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056322+0000","last_change":"2023-09-23T23:06:08.602502+0000","last_active":"2023-09-23T23:09:55.056322+0000","last_peered":"2023-09-23T23:09:55.056322+0000","last_clean":"2023-09-23T23:09:55.056322+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:09:55.056322+0000","last_undegraded":"2023-09-23T23:09:55.056322+0000","last_fullsized":"2023-09-23T23:09:55.056322+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.155021+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:09:55.155021+0000","last_peered":"2023-09-23T23:09:55.155021+0000","last_clean":"2023-09-23T23:09:55.155021+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:09:55.155021+0000","last_undegraded":"2023-09-23T23:09:55.155021+0000","last_fullsized":"2023-09-23T23:09:55.155021+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.541381+0000","last_change":"2023-09-23T23:10:06.541381+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.541381+0000","last_undegraded":"2023-09-23T23:10:06.541381+0000","last_fullsized":"2023-09-23T23:10:06.541381+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:17.626367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":434,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.268235+0000","last_change":"2023-09-23T23:06:08.613829+0000","last_active":"2023-09-23T23:10:00.268235+0000","last_peered":"2023-09-23T23:10:00.268235+0000","last_clean":"2023-09-23T23:10:00.268235+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:10:00.268235+0000","last_undegraded":"2023-09-23T23:10:00.268235+0000","last_fullsized":"2023-09-23T23:10:00.268235+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":105,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056414+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:09:55.056414+0000","last_peered":"2023-09-23T23:09:55.056414+0000","last_clean":"2023-09-23T23:09:55.056414+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:09:55.056414+0000","last_undegraded":"2023-09-23T23:09:55.056414+0000","last_fullsized":"2023-09-23T23:09:55.056414+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":476,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055835+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:09:55.055835+0000","last_peered":"2023-09-23T23:09:55.055835+0000","last_clean":"2023-09-23T23:09:55.055835+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:09:55.055835+0000","last_undegraded":"2023-09-23T23:09:55.055835+0000","last_fullsized":"2023-09-23T23:09:55.055835+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":75,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054664+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:09:55.054664+0000","last_peered":"2023-09-23T23:09:55.054664+0000","last_clean":"2023-09-23T23:09:55.054664+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:09:55.054664+0000","last_undegraded":"2023-09-23T23:09:55.054664+0000","last_fullsized":"2023-09-23T23:09:55.054664+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":236,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.267926+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:10:00.267926+0000","last_peered":"2023-09-23T23:10:00.267926+0000","last_clean":"2023-09-23T23:10:00.267926+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:10:00.267926+0000","last_undegraded":"2023-09-23T23:10:00.267926+0000","last_fullsized":"2023-09-23T23:10:00.267926+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"0'0","reported_seq":1,"reported_epoch":485,"state":"creating+peering","last_fresh":"2023-09-23T23:10:06.542483+0000","last_change":"2023-09-23T23:10:06.542483+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.542483+0000","last_undegraded":"2023-09-23T23:10:06.542483+0000","last_fullsized":"2023-09-23T23:10:06.542483+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":485,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:09.406749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":306,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055544+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:09:55.055544+0000","last_peered":"2023-09-23T23:09:55.055544+0000","last_clean":"2023-09-23T23:09:55.055544+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:09:55.055544+0000","last_undegraded":"2023-09-23T23:09:55.055544+0000","last_fullsized":"2023-09-23T23:09:55.055544+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":507,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055685+0000","last_change":"2023-09-23T22:57:26.514894+0000","last_active":"2023-09-23T23:09:55.055685+0000","last_peered":"2023-09-23T23:09:55.055685+0000","last_clean":"2023-09-23T23:09:55.055685+0000","last_became_active":"2023-09-23T22:57:26.514556+0000","last_became_peered":"2023-09-23T22:57:26.514556+0000","last_unstale":"2023-09-23T23:09:55.055685+0000","last_undegraded":"2023-09-23T23:09:55.055685+0000","last_fullsized":"2023-09-23T23:09:55.055685+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":333,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.056857+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:09:55.056857+0000","last_peered":"2023-09-23T23:09:55.056857+0000","last_clean":"2023-09-23T23:09:55.056857+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:09:55.056857+0000","last_undegraded":"2023-09-23T23:09:55.056857+0000","last_fullsized":"2023-09-23T23:09:55.056857+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":403,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.055103+0000","last_change":"2023-09-23T23:05:39.887461+0000","last_active":"2023-09-23T23:09:55.055103+0000","last_peered":"2023-09-23T23:09:55.055103+0000","last_clean":"2023-09-23T23:09:55.055103+0000","last_became_active":"2023-09-23T23:05:39.887154+0000","last_became_peered":"2023-09-23T23:05:39.887154+0000","last_unstale":"2023-09-23T23:09:55.055103+0000","last_undegraded":"2023-09-23T23:09:55.055103+0000","last_fullsized":"2023-09-23T23:09:55.055103+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":60,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:59.528907+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:09:59.528907+0000","last_peered":"2023-09-23T23:09:59.528907+0000","last_clean":"2023-09-23T23:09:59.528907+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:09:59.528907+0000","last_undegraded":"2023-09-23T23:09:59.528907+0000","last_fullsized":"2023-09-23T23:09:59.528907+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.148121+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:09:55.148121+0000","last_peered":"2023-09-23T23:09:55.148121+0000","last_clean":"2023-09-23T23:09:55.148121+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:09:55.148121+0000","last_undegraded":"2023-09-23T23:09:55.148121+0000","last_fullsized":"2023-09-23T23:09:55.148121+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T23:10:06.531616+0000","last_change":"2023-09-23T23:10:06.531616+0000","last_active":"2023-09-23T23:10:06.531616+0000","last_peered":"2023-09-23T23:10:06.531616+0000","last_clean":"2023-09-23T23:10:06.531616+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T23:10:06.531616+0000","last_undegraded":"2023-09-23T23:10:06.531616+0000","last_fullsized":"2023-09-23T23:10:06.531616+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T23:10:06.531616+0000","last_clean_scrub_stamp":"2023-09-23T23:10:06.531616+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":254,"reported_epoch":484,"state":"active+clean","last_fresh":"2023-09-23T23:10:00.268514+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:10:00.268514+0000","last_peered":"2023-09-23T23:10:00.268514+0000","last_clean":"2023-09-23T23:10:00.268514+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:10:00.268514+0000","last_undegraded":"2023-09-23T23:10:00.268514+0000","last_fullsized":"2023-09-23T23:10:00.268514+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":478,"reported_epoch":483,"state":"active+clean","last_fresh":"2023-09-23T23:09:55.054544+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:09:55.054544+0000","last_peered":"2023-09-23T23:09:55.054544+0000","last_clean":"2023-09-23T23:09:55.054544+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:09:55.054544+0000","last_undegraded":"2023-09-23T23:09:55.054544+0000","last_fullsized":"2023-09-23T23:09:55.054544+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":39,"acting":39,"num_store_stats":2},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430725,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626950,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899800,"kb_used_data":5264,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92845352,"statfs":{"total":95995035648,"available":95073640448,"internally_reserved":0,"allocated":5390336,"data_stored":4296331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.3109999999999999,"15min":4.6340000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.923,"5min":10.853,"15min":115.32899999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.304,"15min":4.6719999999999997},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":4.383,"5min":8.7010000000000005,"15min":115.348},"last":0.63900000000000001}]},{"osd":1,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":3.629,"15min":3.056},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":9.4060000000000006,"5min":149.947,"15min":149.947},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.718,"15min":2.9940000000000002},"min":{"1min":0.33800000000000002,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":6.3689999999999998,"5min":150.22499999999999,"15min":150.22499999999999},"last":0.629}]},{"osd":2,"last update":"Sat Sep 23 23:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":5.508,"15min":2.9369999999999998},"min":{"1min":0.29499999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.2949999999999999,"5min":138.43299999999999,"15min":138.43299999999999},"last":1.2789999999999999},{"interface":"front","average":{"1min":2.423,"5min":5.4909999999999997,"15min":3.081},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.3629999999999995,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":4.9630000000000001,"15min":3.3210000000000002},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.26800000000000002},"max":{"1min":9.4329999999999998,"5min":142.24700000000001,"15min":142.24700000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.986,"5min":4.8369999999999997,"15min":3.2349999999999999},"min":{"1min":0.41799999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":6.4100000000000001,"5min":143.86699999999999,"15min":143.86699999999999},"last":3.375}]},{"osd":4,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.2490000000000001,"15min":1.857},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.23999999999999999},"max":{"1min":6.3470000000000004,"5min":12.407999999999999,"15min":119.804},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.3109999999999999,"15min":1.925},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.24099999999999999},"max":{"1min":6.2140000000000004,"5min":8.8940000000000001,"15min":122.31999999999999},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.369,"15min":2.52},"min":{"1min":0.40400000000000003,"5min":0.33000000000000002,"15min":0.27900000000000003},"max":{"1min":6.4530000000000003,"5min":11.862,"15min":186.27600000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.238,"5min":1.506,"15min":2.6629999999999998},"min":{"1min":0.40300000000000002,"5min":0.35399999999999998,"15min":0.29599999999999999},"max":{"1min":6.2729999999999997,"5min":10.254,"15min":191.28700000000001},"last":0.501}]},{"osd":7,"last update":"Sat Sep 23 23:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.4159999999999999,"15min":1.5569999999999999},"min":{"1min":0.375,"5min":0.372,"15min":0.29999999999999999},"max":{"1min":5.3959999999999999,"5min":8.8670000000000009,"15min":115.61},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.117,"5min":1.3,"15min":1.7350000000000001},"min":{"1min":0.42699999999999999,"5min":0.38300000000000001,"15min":0.29999999999999999},"max":{"1min":7.1920000000000002,"5min":8.2609999999999992,"15min":115.578},"last":0.61199999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253263,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1286860,"kb_used_data":277060,"kb_used_omap":9,"kb_used_meta":1009782,"kb_avail":92458292,"statfs":{"total":95995035648,"available":94677291008,"internally_reserved":0,"allocated":283709440,"data_stored":281706922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1034017769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.5629999999999999,"15min":2.5649999999999999},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.2189999999999999,"5min":59.475000000000001,"15min":279.30500000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.7490000000000001,"15min":2.8799999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":4.8559999999999999,"5min":59.393999999999998,"15min":279.39800000000002},"last":0.496}]},{"osd":2,"last update":"Sat Sep 23 23:09:38 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.7429999999999999,"15min":2.7229999999999999},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":62.177,"5min":121.527,"15min":161.61699999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":5.2619999999999996,"5min":4.1280000000000001,"15min":2.9079999999999999},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":62.460999999999999,"5min":118.29900000000001,"15min":162.61699999999999},"last":0.55500000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":7.2039999999999997,"5min":2.2789999999999999,"15min":1.7270000000000001},"min":{"1min":0.29699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":67.646000000000001,"5min":67.646000000000001,"15min":67.646000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":7.1609999999999996,"5min":2.5099999999999998,"15min":1.7809999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19500000000000001},"max":{"1min":65.269999999999996,"5min":65.269999999999996,"15min":65.269999999999996},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":3.0979999999999999,"15min":2.5670000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.274,"5min":129.69,"15min":151.036},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.5129999999999999,"5min":3.3799999999999999,"15min":2.859},"min":{"1min":0.318,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":6.1230000000000002,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.2869999999999999,"15min":2.153},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":3.2719999999999998,"5min":18.228999999999999,"15min":116.098},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.353,"5min":1.1890000000000001,"15min":2.008},"min":{"1min":0.375,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":5.7779999999999996,"5min":6.585,"15min":116.08499999999999},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.194,"15min":2.0369999999999999},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":6.1740000000000004,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.177,"5min":2.282,"15min":1.9159999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":4.0759999999999996,"5min":64.239999999999995,"15min":67.366},"last":0.59099999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.3779999999999999,"15min":1.117},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":4.0890000000000004,"5min":7.0960000000000001,"15min":7.0960000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.595,"5min":1.6819999999999999,"15min":1.4339999999999999},"min":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.33100000000000002},"max":{"1min":6.2389999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.54000000000000004}]}]},{"osd":0,"up_from":0,"seq":1516123455633,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259406,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324064,"kb_used_data":6424,"kb_used_omap":83,"kb_used_meta":317548,"kb_avail":93421088,"statfs":{"total":95995035648,"available":95663194112,"internally_reserved":0,"allocated":6578176,"data_stored":4443042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":325169605},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.82399999999999995,"15min":1.238},"min":{"1min":0.22800000000000001,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":3.8039999999999998,"5min":10.199999999999999,"15min":74.241},"last":0.251},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.3720000000000001,"15min":1.609},"min":{"1min":0.25700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":10.973000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":2.7589999999999999,"15min":1.4390000000000001},"min":{"1min":0.308,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":16.995999999999999,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.376},{"interface":"front","average":{"1min":1.964,"5min":2.2240000000000002,"15min":1.516},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.3730000000000002,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.29799999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":5.7439999999999998,"5min":3.7029999999999998,"15min":2.1739999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.23000000000000001},"max":{"1min":55.255000000000003,"5min":79.655000000000001,"15min":110.31699999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":5.383,"5min":3.427,"15min":2.1549999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":61.197000000000003,"5min":79.677999999999997,"15min":110.56999999999999},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":1.5820000000000001,"15min":1.1519999999999999},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":2.7290000000000001,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.0710000000000002,"15min":1.4119999999999999},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":8.1099999999999994,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.4079999999999999,"15min":1.7110000000000001},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.0460000000000003,"5min":9.8019999999999996,"15min":139.405},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.105,"5min":1.22,"15min":1.538},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":5.5880000000000001,"5min":10.109999999999999,"15min":139.42099999999999},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":3.5539999999999998,"15min":1.917},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.0179999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.623},{"interface":"front","average":{"1min":1.1639999999999999,"5min":3.2269999999999999,"15min":1.611},"min":{"1min":0.434,"5min":0.42399999999999999,"15min":0.32200000000000001},"max":{"1min":3.3239999999999998,"5min":108.496,"15min":108.496},"last":0.56599999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.548,"15min":1.2270000000000001},"min":{"1min":0.48699999999999999,"5min":0.40300000000000002,"15min":0.36699999999999999},"max":{"1min":6.0899999999999999,"5min":10.164,"15min":10.164},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.4330000000000001,"15min":1.214},"min":{"1min":0.42999999999999999,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":8.3179999999999996,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.66100000000000003}]}]},{"osd":3,"up_from":94,"seq":403726926210,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":987832,"kb_used_data":263408,"kb_used_omap":7,"kb_used_meta":724408,"kb_avail":92757320,"statfs":{"total":95995035648,"available":94983495680,"internally_reserved":0,"allocated":269729792,"data_stored":267618075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":741794408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.486,"15min":1.4490000000000001},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":3.8439999999999999,"5min":17.015000000000001,"15min":122.289},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.115,"15min":1.1870000000000001},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":4.9909999999999997,"5min":11.382999999999999,"15min":122.041},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":2.1720000000000002,"15min":1.4710000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":10.935,"5min":23.969000000000001,"15min":23.969000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.109,"5min":1.9119999999999999,"15min":1.373},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.20699999999999999},"max":{"1min":4.0119999999999996,"5min":16.986000000000001,"15min":16.986000000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.5030000000000001,"15min":1.3340000000000001},"min":{"1min":0.27700000000000002,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":16.149000000000001,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.875,"15min":1.6799999999999999},"min":{"1min":0.41299999999999998,"5min":0.251,"15min":0.215},"max":{"1min":11.375999999999999,"5min":57.308,"15min":57.308},"last":0.441}]},{"osd":4,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.6629999999999998,"15min":2.1110000000000002},"min":{"1min":0.374,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":11.01,"5min":115.69,"15min":115.69},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.351,"5min":2.4009999999999998,"15min":2.0419999999999998},"min":{"1min":0.32900000000000001,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":5.9169999999999998,"5min":115.617,"15min":115.617},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.401},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.28499999999999998},"max":{"1min":4.8810000000000002,"5min":16.959,"15min":16.959},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.359,"5min":1.476,"15min":1.369},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":6.6130000000000004,"5min":10.545,"15min":10.545},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":5.6639999999999997,"5min":2.589,"15min":1.8999999999999999},"min":{"1min":0.41699999999999998,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":77.707999999999998,"5min":77.707999999999998,"15min":77.707999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":5.4080000000000004,"5min":2.1930000000000001,"15min":1.6890000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":77.721999999999994,"5min":77.721999999999994,"15min":77.721999999999994},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.4630000000000001,"15min":1.272},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":4.5220000000000002,"5min":8.1620000000000008,"15min":8.1620000000000008},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.0499999999999998,"15min":1.5249999999999999},"min":{"1min":0.42099999999999999,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":0.75}]}]},{"osd":4,"up_from":30,"seq":128849019407,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908048,"kb_used_data":4680,"kb_used_omap":0,"kb_used_meta":903360,"kb_avail":92837104,"statfs":{"total":95995035648,"available":95065194496,"internally_reserved":0,"allocated":4792320,"data_stored":3698033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925040640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.4039999999999999,"15min":2.3580000000000001},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":9.8870000000000005,"5min":99.132999999999996,"15min":156.375},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.3370000000000002,"15min":2.4279999999999999},"min":{"1min":0.36799999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":4.4509999999999996,"5min":101.062,"15min":157.02199999999999},"last":0.61499999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.4340000000000002,"15min":2.7989999999999999},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":4.3840000000000003,"5min":95.591999999999999,"15min":98.846999999999994},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.954,"5min":3.407,"15min":2.843},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.23699999999999999},"max":{"1min":10.311,"5min":97.929000000000002,"15min":98.938999999999993},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:44 2023","interfaces":[{"interface":"back","average":{"1min":9.6500000000000004,"5min":5.8819999999999997,"15min":2.7210000000000001},"min":{"1min":0.41599999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":151.20400000000001,"5min":151.20400000000001,"15min":151.20400000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":9.3279999999999994,"5min":5.7590000000000003,"15min":2.5590000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":148.852,"5min":148.852,"15min":148.852},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":11.974,"5min":4.7699999999999996,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":147.82499999999999,"5min":147.82499999999999,"15min":147.82499999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":11.071,"5min":4.7309999999999999,"15min":2.4889999999999999},"min":{"1min":0.44,"5min":0.36599999999999999,"15min":0.32500000000000001},"max":{"1min":147.84100000000001,"5min":147.84100000000001,"15min":147.84100000000001},"last":0.47599999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.4159999999999999,"15min":1.7769999999999999},"min":{"1min":0.37,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":10.282,"5min":10.282,"15min":148.61600000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.601,"15min":1.9430000000000001},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.247},"max":{"1min":17.02,"5min":17.02,"15min":151.905},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 23:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.4650000000000001,"15min":1.419},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":6.9740000000000002,"5min":35.896000000000001,"15min":35.896000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.611,"5min":1.778,"15min":1.3899999999999999},"min":{"1min":0.34899999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":6.5129999999999999,"5min":35.908000000000001,"15min":35.908000000000001},"last":0.71199999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:09:34 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":1.5680000000000001,"15min":1.3660000000000001},"min":{"1min":0.432,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":11.096,"5min":11.096,"15min":12.84},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.448,"5min":1.4670000000000001,"15min":1.25},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":7.3179999999999996,"5min":15.808,"15min":15.808},"last":0.67400000000000004}]}]},{"osd":5,"up_from":0,"seq":154618823181,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:10:12.122 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (246 ms) 2023-09-23T23:10:12.122 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T23:10:12.122 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T23:10:12.123 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T23:10:12.124 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T23:10:12.124 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T23:10:12.131 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-09-23T23:10:12.131 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T23:10:12.139 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T23:10:12.139 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T23:10:12.159 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-09-23T23:10:12.159 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T23:10:12.168 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-09-23T23:10:12.168 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T23:10:12.180 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-09-23T23:10:12.180 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T23:10:12.186 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2023-09-23T23:10:12.186 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T23:10:12.198 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.migration (12 ms) 2023-09-23T23:10:12.198 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T23:10:12.207 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2023-09-23T23:10:12.207 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T23:10:12.400 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:12.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:12.505 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:12.531 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:12.575 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:12.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:12.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:12.591 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:12.627 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:10:12.628 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:10:12.628 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:10:12.628 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:10:12.628 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:10:12.680 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:12.714 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:12.914 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:12.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:13.019 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:13.047 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:13.528 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:13.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:13.572 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1364 ms) 2023-09-23T23:10:13.572 INFO:tasks.workunit.client.0.smithi052.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T23:10:13.594 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T23:10:13.604 INFO:tasks.workunit.client.0.smithi052.stdout:[ OK ] TestClsRbd.sparsify (33 ms) 2023-09-23T23:10:13.628 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:10:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:10:13] "GET /metrics HTTP/1.1" 200 34191 "" "Prometheus/2.33.4" 2023-09-23T23:10:13.628 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:13 smithi052 ceph-mon[180979]: failed to encode map e489 with expected crc 2023-09-23T23:10:13.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:13 smithi052 ceph-mon[180979]: osdmap e489: 8 total, 8 up, 5 in 2023-09-23T23:10:13.629 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:13 smithi052 ceph-mon[180979]: pgmap v1539: 105 pgs: 105 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 2.0 MiB/s rd, 1.0 MiB/s wr, 795 op/s 2023-09-23T23:10:13.629 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:13 smithi052 ceph-mon[187743]: failed to encode map e489 with expected crc 2023-09-23T23:10:13.629 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:13 smithi052 ceph-mon[187743]: osdmap e489: 8 total, 8 up, 5 in 2023-09-23T23:10:13.629 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:13 smithi052 ceph-mon[187743]: pgmap v1539: 105 pgs: 105 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 2.0 MiB/s rd, 1.0 MiB/s wr, 795 op/s 2023-09-23T23:10:13.631 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:13.662 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:13.709 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:13.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:13.814 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:13.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:13 smithi151 ceph-mon[159210]: failed to encode map e489 with expected crc 2023-09-23T23:10:13.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:13 smithi151 ceph-mon[159210]: osdmap e489: 8 total, 8 up, 5 in 2023-09-23T23:10:13.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:13 smithi151 ceph-mon[159210]: pgmap v1539: 105 pgs: 105 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 2.0 MiB/s rd, 1.0 MiB/s wr, 795 op/s 2023-09-23T23:10:13.842 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:13.885 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:13.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:13.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:13.989 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:14.019 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:14.310 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:14.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:14.414 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:14.448 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:14.451 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:14.455 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:10:14.503 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:14.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:14.565 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[180979]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.575 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] 70 tests from TestClsRbd (6062 ms total) 2023-09-23T23:10:14.575 INFO:tasks.workunit.client.0.smithi052.stdout: 2023-09-23T23:10:14.575 INFO:tasks.workunit.client.0.smithi052.stdout:[----------] Global test environment tear-down 2023-09-23T23:10:14.576 INFO:tasks.workunit.client.0.smithi052.stdout:[==========] 70 tests from 1 test suite ran. (8796 ms total) 2023-09-23T23:10:14.576 INFO:tasks.workunit.client.0.smithi052.stdout:[ PASSED ] 70 tests. 2023-09-23T23:10:14.578 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T23:10:14.578 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T23:10:14.607 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:14.608 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T23:10:14.608 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:10:14.636 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:14.679 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:14.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:14.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:14.785 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:14.811 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:14.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[187743]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[187743]: osdmap e490: 8 total, 8 up, 5 in 2023-09-23T23:10:14.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[187743]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[187743]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.847 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[187743]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[180979]: osdmap e490: 8 total, 8 up, 5 in 2023-09-23T23:10:14.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[180979]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[180979]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.848 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:14 smithi052 ceph-mon[180979]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.853 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:14.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:14 smithi151 ceph-mon[159210]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:14 smithi151 ceph-mon[159210]: osdmap e490: 8 total, 8 up, 5 in 2023-09-23T23:10:14.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:14 smithi151 ceph-mon[159210]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:14 smithi151 ceph-mon[159210]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:14 smithi151 ceph-mon[159210]: failed to encode map e490 with expected crc 2023-09-23T23:10:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:14.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:14.896 DEBUG:teuthology.parallel:result is None 2023-09-23T23:10:14.896 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:10:14.926 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:10:14.926 DEBUG:teuthology.orchestra.run.smithi052:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:14.960 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:14.984 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:10:14.984 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:10:14.985 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T23:10:14.985 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T23:10:14.985 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T23:10:14.985 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T23:10:14.985 DEBUG:teuthology.orchestra.run.smithi052:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:14.996 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:15.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:10:15.016 INFO:teuthology.orchestra.run.smithi052.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T23:10:15.016 DEBUG:teuthology.orchestra.run.smithi052:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:15.032 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:10:15.050 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:15.050 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T23:10:15.058 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:10:15.079 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:10:15.082 INFO:tasks.workunit:timeout=3h 2023-09-23T23:10:15.082 INFO:tasks.workunit:cleanup=True 2023-09-23T23:10:15.082 DEBUG:teuthology.orchestra.run.smithi052:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:10:15.097 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:10:15.103 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:15.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:15.119 INFO:tasks.workunit.client.0.smithi052.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T23:10:15.121 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:10:15.144 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:10:15.162 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:10:15.181 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:10:15.182 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.183 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:10:15.209 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:15.235 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:15.277 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:15.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:15.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:15.382 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:15.408 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:15.449 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:15.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:15.556 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:15.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:15 smithi151 ceph-mon[159210]: failed to encode map e491 with expected crc 2023-09-23T23:10:15.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:15 smithi151 ceph-mon[159210]: osdmap e491: 8 total, 8 up, 5 in 2023-09-23T23:10:15.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:15 smithi151 ceph-mon[159210]: pgmap v1542: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:15.574 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3446631502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:15.583 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:15.624 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:15.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:15.729 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:15.758 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:15.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[180979]: failed to encode map e491 with expected crc 2023-09-23T23:10:15.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[180979]: osdmap e491: 8 total, 8 up, 5 in 2023-09-23T23:10:15.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[180979]: pgmap v1542: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:15.971 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3446631502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:15.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[187743]: failed to encode map e491 with expected crc 2023-09-23T23:10:15.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[187743]: osdmap e491: 8 total, 8 up, 5 in 2023-09-23T23:10:15.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[187743]: pgmap v1542: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:15.971 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3446631502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:15.990 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:15.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:16.093 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:16.120 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:16.368 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:16.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:16.463 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:16.472 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:16.504 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:16.552 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:16.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:16.632 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:16 smithi052 ceph-mon[180979]: failed to encode map e491 with expected crc 2023-09-23T23:10:16.633 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:16 smithi052 ceph-mon[180979]: failed to encode map e491 with expected crc 2023-09-23T23:10:16.633 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:16 smithi052 ceph-mon[187743]: failed to encode map e491 with expected crc 2023-09-23T23:10:16.633 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:16 smithi052 ceph-mon[187743]: failed to encode map e491 with expected crc 2023-09-23T23:10:16.655 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:16.687 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:16.736 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:16.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:16.863 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:16.880 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:10:16.880 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:10:16.896 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:16 smithi151 ceph-mon[159210]: failed to encode map e491 with expected crc 2023-09-23T23:10:16.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:16 smithi151 ceph-mon[159210]: failed to encode map e491 with expected crc 2023-09-23T23:10:16.949 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:16.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:16.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:17.056 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:17.084 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:17.122 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:10:17.123 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:10:17.123 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T23:10:17.123 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T23:10:17.123 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T23:10:17.123 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T23:10:17.575 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:17.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:17.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T23:10:17.679 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:17.711 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:17.760 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:17.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:17.868 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:17.893 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:17 smithi052 ceph-mon[187743]: pgmap v1543: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:17.893 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:17 smithi052 ceph-mon[180979]: pgmap v1543: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:17.896 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:17 smithi151 ceph-mon[159210]: pgmap v1543: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:18.198 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:18.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:18.301 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:18.341 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:18.366 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:18.402 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:18.484 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:10:18.512 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:18.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:18.617 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:18.646 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:18.915 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:18.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:19.019 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:19.050 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:19.259 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:19.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:19.362 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:19.392 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:19.892 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:10:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:19 smithi052 ceph-mon[187743]: pgmap v1544: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:19.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/200029667' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:19.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:19 smithi052 ceph-mon[180979]: pgmap v1544: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/200029667' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:19.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:19 smithi151 ceph-mon[159210]: pgmap v1544: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/200029667' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:20.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:20.937 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:10:20.937 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:10:20.937 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[187743]: failed to encode map e492 with expected crc 2023-09-23T23:10:20.937 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[187743]: failed to encode map e492 with expected crc 2023-09-23T23:10:20.937 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/200029667' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:20.937 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[187743]: osdmap e492: 8 total, 8 up, 5 in 2023-09-23T23:10:20.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:10:20.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:10:20.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[180979]: failed to encode map e492 with expected crc 2023-09-23T23:10:20.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[180979]: failed to encode map e492 with expected crc 2023-09-23T23:10:20.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/200029667' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:20.938 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:20 smithi052 ceph-mon[180979]: osdmap e492: 8 total, 8 up, 5 in 2023-09-23T23:10:20.990 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:21.000 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:21.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:21.111 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:21.140 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:21.188 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:21.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:20 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:10:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:20 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:10:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:20 smithi151 ceph-mon[159210]: failed to encode map e492 with expected crc 2023-09-23T23:10:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:20 smithi151 ceph-mon[159210]: failed to encode map e492 with expected crc 2023-09-23T23:10:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:20 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/200029667' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:21.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:20 smithi151 ceph-mon[159210]: osdmap e492: 8 total, 8 up, 5 in 2023-09-23T23:10:21.292 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:21.323 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:21.372 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:21.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:21.479 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:21.510 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:21.718 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:10:21.719 INFO:teuthology.orchestra.run.smithi052.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T23:10:21.719 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:10:21.762 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:21.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:21.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:21.868 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:21.910 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:22.106 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T23:10:22.171 INFO:tasks.workunit.client.0.smithi052.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T23:10:22.174 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:10:22.174 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T23:10:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:21 smithi151 ceph-mon[159210]: failed to encode map e492 with expected crc 2023-09-23T23:10:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:21 smithi151 ceph-mon[159210]: failed to encode map e492 with expected crc 2023-09-23T23:10:22.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:21 smithi151 ceph-mon[159210]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:22.217 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:10:22.218 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:10:22.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:21 smithi052 ceph-mon[187743]: failed to encode map e492 with expected crc 2023-09-23T23:10:22.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:21 smithi052 ceph-mon[187743]: failed to encode map e492 with expected crc 2023-09-23T23:10:22.221 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:21 smithi052 ceph-mon[187743]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:22.221 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:21 smithi052 ceph-mon[180979]: failed to encode map e492 with expected crc 2023-09-23T23:10:22.221 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:21 smithi052 ceph-mon[180979]: failed to encode map e492 with expected crc 2023-09-23T23:10:22.222 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:21 smithi052 ceph-mon[180979]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:22.348 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T23:10:22.437 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T23:10:22.538 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T23:10:22.540 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:10:22.540 INFO:tasks.workunit.client.0.smithi052.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T23:10:22.673 INFO:tasks.workunit.client.0.smithi052.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T23:10:22.676 DEBUG:teuthology.orchestra.run.smithi052:> set -ex 2023-09-23T23:10:22.676 DEBUG:teuthology.orchestra.run.smithi052:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T23:10:22.738 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T23:10:22.739 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T23:10:22.739 DEBUG:teuthology.orchestra.run.smithi052:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T23:10:22.801 INFO:tasks.workunit.client.0.smithi052.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T23:10:22.801 INFO:tasks.workunit.client.0.smithi052.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T23:10:22.801 INFO:tasks.workunit.client.0.smithi052.stderr:+ TMPDIR=/tmp/rbd_import_export_701000 2023-09-23T23:10:22.801 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm -rf /tmp/rbd_import_export_701000 2023-09-23T23:10:22.801 INFO:tasks.workunit.client.0.smithi052.stderr:+ mkdir /tmp/rbd_import_export_701000 2023-09-23T23:10:22.802 INFO:tasks.workunit.client.0.smithi052.stderr:+ trap 'rm -rf /tmp/rbd_import_export_701000' INT TERM EXIT 2023-09-23T23:10:22.803 INFO:tasks.workunit.client.0.smithi052.stderr:+ mkdir foo.701000 2023-09-23T23:10:22.803 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import foo.701000 foo.dir 2023-09-23T23:10:22.940 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T23:10:23.069 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:23.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:23.149 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: cannot import a directory 2023-09-23T23:10:23.149 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 0% complete...failed. 2023-09-23T23:10:23.149 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: import failed: (21) Is a directory 2023-09-23T23:10:23.151 INFO:tasks.workunit.client.0.smithi052.stderr:+ true 2023-09-23T23:10:23.152 INFO:tasks.workunit.client.0.smithi052.stderr:+ rmdir foo.701000 2023-09-23T23:10:23.152 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_701000/img bs=1k count=1 seek=10 2023-09-23T23:10:23.153 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records in 2023-09-23T23:10:23.154 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records out 2023-09-23T23:10:23.154 INFO:tasks.workunit.client.0.smithi052.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.9614e-05 s, 20.6 MB/s 2023-09-23T23:10:23.154 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_701000/img bs=1k count=10 seek=100 2023-09-23T23:10:23.154 INFO:tasks.workunit.client.0.smithi052.stderr:10+0 records in 2023-09-23T23:10:23.155 INFO:tasks.workunit.client.0.smithi052.stderr:10+0 records out 2023-09-23T23:10:23.155 INFO:tasks.workunit.client.0.smithi052.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.9668e-05 s, 172 MB/s 2023-09-23T23:10:23.155 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_701000/img bs=1k count=100 seek=1000 2023-09-23T23:10:23.155 INFO:tasks.workunit.client.0.smithi052.stderr:70+1 records in 2023-09-23T23:10:23.155 INFO:tasks.workunit.client.0.smithi052.stderr:70+1 records out 2023-09-23T23:10:23.156 INFO:tasks.workunit.client.0.smithi052.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000202549 s, 356 MB/s 2023-09-23T23:10:23.156 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_701000/img bs=1k seek=10000 2023-09-23T23:10:23.156 INFO:tasks.workunit.client.0.smithi052.stderr:139+1 records in 2023-09-23T23:10:23.156 INFO:tasks.workunit.client.0.smithi052.stderr:139+1 records out 2023-09-23T23:10:23.156 INFO:tasks.workunit.client.0.smithi052.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000507318 s, 282 MB/s 2023-09-23T23:10:23.157 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_701000/img bs=1k seek=100000 2023-09-23T23:10:23.174 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:23.201 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:23.310 INFO:tasks.workunit.client.0.smithi052.stderr:70+1 records in 2023-09-23T23:10:23.310 INFO:tasks.workunit.client.0.smithi052.stderr:70+1 records out 2023-09-23T23:10:23.310 INFO:tasks.workunit.client.0.smithi052.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.152306 s, 474 kB/s 2023-09-23T23:10:23.310 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_701000/img bs=1k seek=1000000 2023-09-23T23:10:23.311 INFO:tasks.workunit.client.0.smithi052.stderr:189+1 records in 2023-09-23T23:10:23.311 INFO:tasks.workunit.client.0.smithi052.stderr:189+1 records out 2023-09-23T23:10:23.311 INFO:tasks.workunit.client.0.smithi052.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000544467 s, 356 MB/s 2023-09-23T23:10:23.311 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm testimg 2023-09-23T23:10:23.347 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 0% complete...failed. 2023-09-23T23:10:23.347 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T23:10:23.350 INFO:tasks.workunit.client.0.smithi052.stderr:+ true 2023-09-23T23:10:23.351 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format /tmp/rbd_import_export_701000/img testimg 2023-09-23T23:10:23.649 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:23.651 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:10:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:10:23] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.33.4" 2023-09-23T23:10:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:23.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:23.753 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:23.786 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:23.837 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:23.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:23.942 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:23.974 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:23 smithi052 ceph-mon[180979]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:23.974 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:23 smithi052 ceph-mon[187743]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:23.978 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:24.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:23 smithi151 ceph-mon[159210]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:24 smithi151 ceph-mon[159210]: failed to encode map e493 with expected crc 2023-09-23T23:10:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:24 smithi151 ceph-mon[159210]: osdmap e493: 8 total, 8 up, 5 in 2023-09-23T23:10:25.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:24 smithi151 ceph-mon[159210]: failed to encode map e493 with expected crc 2023-09-23T23:10:25.237 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:24 smithi052 ceph-mon[187743]: failed to encode map e493 with expected crc 2023-09-23T23:10:25.237 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:24 smithi052 ceph-mon[187743]: osdmap e493: 8 total, 8 up, 5 in 2023-09-23T23:10:25.237 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:24 smithi052 ceph-mon[187743]: failed to encode map e493 with expected crc 2023-09-23T23:10:25.237 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:24 smithi052 ceph-mon[180979]: failed to encode map e493 with expected crc 2023-09-23T23:10:25.238 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:24 smithi052 ceph-mon[180979]: osdmap e493: 8 total, 8 up, 5 in 2023-09-23T23:10:25.238 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:24 smithi052 ceph-mon[180979]: failed to encode map e493 with expected crc 2023-09-23T23:10:25.348 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:25.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:25.454 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:25.486 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:25.536 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:25.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:25.641 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:25.669 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:25.675 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T23:10:25.678 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:25.681 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg /tmp/rbd_import_export_701000/img2 2023-09-23T23:10:25.714 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:25.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:25.819 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:25.890 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:25.949 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T23:10:25.985 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:25.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:25.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:26.032 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:26.036 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg - 2023-09-23T23:10:26.065 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:26.093 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:26.129 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:25 smithi052 ceph-mon[187743]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:26.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:25 smithi052 ceph-mon[187743]: osdmap e494: 8 total, 8 up, 5 in 2023-09-23T23:10:26.130 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:25 smithi052 ceph-mon[187743]: failed to encode map e494 with expected crc 2023-09-23T23:10:26.131 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:25 smithi052 ceph-mon[180979]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:26.131 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:25 smithi052 ceph-mon[180979]: osdmap e494: 8 total, 8 up, 5 in 2023-09-23T23:10:26.131 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:25 smithi052 ceph-mon[180979]: failed to encode map e494 with expected crc 2023-09-23T23:10:26.131 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:25 smithi151 ceph-mon[159210]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:25 smithi151 ceph-mon[159210]: osdmap e494: 8 total, 8 up, 5 in 2023-09-23T23:10:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:25 smithi151 ceph-mon[159210]: failed to encode map e494 with expected crc 2023-09-23T23:10:26.206 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:26.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:26.310 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:26.346 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:26.524 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:26.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:26.632 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:26.660 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:26.704 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:26.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:26.734 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:10:26.754 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:10:26.778 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:10:26.800 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:10:26.810 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:26.821 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:10:26.843 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:26.845 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:10:26.880 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:10:26.889 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:26.895 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm testimg 2023-09-23T23:10:26.905 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:10:26.907 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:26.907 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.907 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.907 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.907 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.908 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.908 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.908 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.908 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:10:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:26.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:26.997 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T23:10:27.001 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img2 2023-09-23T23:10:27.020 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:27.046 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:27.088 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:27.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:27.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:27.193 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:27.198 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:26 smithi151 ceph-mon[159210]: failed to encode map e494 with expected crc 2023-09-23T23:10:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1180589499' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:27.219 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:27.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:26 smithi052 ceph-mon[187743]: failed to encode map e494 with expected crc 2023-09-23T23:10:27.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1180589499' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:27.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:26 smithi052 ceph-mon[180979]: failed to encode map e494 with expected crc 2023-09-23T23:10:27.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1180589499' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:27.261 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:27.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:27.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:27.368 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:27.400 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:27.506 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:27.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:27.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:27.610 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:27.639 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:27.780 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img3 2023-09-23T23:10:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:27 smithi151 ceph-mon[159210]: pgmap v1551: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:28.240 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:28.242 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:27 smithi052 ceph-mon[180979]: pgmap v1551: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:28.246 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:27 smithi052 ceph-mon[187743]: pgmap v1551: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:10:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:28.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:28.287 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img2 /tmp/rbd_import_export_701000/img3 2023-09-23T23:10:28.348 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:28.380 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:28.427 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:28.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:28.536 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:28.564 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:28.611 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:28.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:28.714 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:28.745 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:28.791 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:28.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:28.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:28.898 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:28.930 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:28.982 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:28.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:28.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:29.086 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:29.114 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:29.160 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:29.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:29.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:29.263 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:29.295 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:29.344 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:29.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:29.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:29.452 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:29.480 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:29.524 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:29.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:29.628 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:29.662 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:29.715 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:29.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:29.819 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:29.851 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:29.902 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:29.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:30.006 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:30.035 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:30.082 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:30.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:30.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:30.188 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:29 smithi151 ceph-mon[159210]: pgmap v1552: 73 pgs: 73 active+clean; 3.5 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 99 KiB/s rd, 40 KiB/s wr, 33 op/s 2023-09-23T23:10:30.218 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:29 smithi052 ceph-mon[187743]: pgmap v1552: 73 pgs: 73 active+clean; 3.5 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 99 KiB/s rd, 40 KiB/s wr, 33 op/s 2023-09-23T23:10:30.218 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:29 smithi052 ceph-mon[180979]: pgmap v1552: 73 pgs: 73 active+clean; 3.5 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 99 KiB/s rd, 40 KiB/s wr, 33 op/s 2023-09-23T23:10:30.221 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:30.274 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:30.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:30.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:30.378 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:30.408 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:30.455 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:30.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:30.559 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:30.587 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:30.631 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:30.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:30.735 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:30.766 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:30.814 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:30.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:30.919 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:30.947 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:30.984 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:10:30.985 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:10:30.985 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.807741613406335 2023-09-23T23:10:30.985 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 0.807741613406335 2023-09-23T23:10:30.990 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:30.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:31.096 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:31.124 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:31.176 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:31.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:31.280 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:31.309 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:31.350 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:31.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:31.457 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:31.486 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:31.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:31 smithi151 ceph-mon[159210]: pgmap v1553: 73 pgs: 73 active+clean; 3.5 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 146 KiB/s rd, 63 KiB/s wr, 34 op/s 2023-09-23T23:10:31.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:31 smithi052 ceph-mon[187743]: pgmap v1553: 73 pgs: 73 active+clean; 3.5 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 146 KiB/s rd, 63 KiB/s wr, 34 op/s 2023-09-23T23:10:31.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:31 smithi052 ceph-mon[180979]: pgmap v1553: 73 pgs: 73 active+clean; 3.5 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 146 KiB/s rd, 63 KiB/s wr, 34 op/s 2023-09-23T23:10:33.398 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format - testimg 2023-09-23T23:10:33.690 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:10:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:10:33] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.33.4" 2023-09-23T23:10:33.873 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:33.873 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:33.876 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg /tmp/rbd_import_export_701000/img2 2023-09-23T23:10:33.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:33 smithi052 ceph-mon[187743]: pgmap v1554: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 133 KiB/s rd, 57 KiB/s wr, 34 op/s 2023-09-23T23:10:33.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:33 smithi052 ceph-mon[180979]: pgmap v1554: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 133 KiB/s rd, 57 KiB/s wr, 34 op/s 2023-09-23T23:10:34.139 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:34.144 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg - 2023-09-23T23:10:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:33 smithi151 ceph-mon[159210]: pgmap v1554: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 133 KiB/s rd, 57 KiB/s wr, 34 op/s 2023-09-23T23:10:34.203 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:34.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:34.308 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:34.346 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:34.356 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:34.417 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:34.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:34.521 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:34.523 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:34.554 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:34.605 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:34.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:34.712 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:34.747 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:34.898 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:34.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:34.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:34.978 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:34.981 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm testimg 2023-09-23T23:10:35.003 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:35.034 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:35.080 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T23:10:35.084 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img2 2023-09-23T23:10:35.084 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:35.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:35.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:35.192 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:35.220 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:35.264 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:35.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:35.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:35.396 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:35.437 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:35.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:35.495 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is set 2023-09-23T23:10:35.544 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:35.570 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:35.611 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:35.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: pgmap v1555: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T23:10:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/740585156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:10:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:35.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/740585156' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:10:35.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:35 smithi151 ceph-mon[159210]: osdmap e495: 8 total, 8 up, 5 in 2023-09-23T23:10:35.721 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:35.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: pgmap v1555: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/740585156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/740585156' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[180979]: osdmap e495: 8 total, 8 up, 5 in 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:35.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: pgmap v1555: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T23:10:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/740585156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T23:10:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/740585156' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T23:10:35.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:35 smithi052 ceph-mon[187743]: osdmap e495: 8 total, 8 up, 5 in 2023-09-23T23:10:35.747 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:35.789 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:35.791 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img3 2023-09-23T23:10:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:35.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:35.895 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:35.933 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:36.329 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img2 /tmp/rbd_import_export_701000/img3 2023-09-23T23:10:36.391 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:10:38.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:37 smithi151 ceph-mon[159210]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:37 smithi151 ceph-mon[159210]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:37 smithi151 ceph-mon[159210]: pgmap v1557: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 163 KiB/s rd, 72 KiB/s wr, 42 op/s 2023-09-23T23:10:38.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:37 smithi151 ceph-mon[159210]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:37 smithi151 ceph-mon[159210]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[187743]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[187743]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[187743]: pgmap v1557: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 163 KiB/s rd, 72 KiB/s wr, 42 op/s 2023-09-23T23:10:38.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[187743]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[187743]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[180979]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[180979]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[180979]: pgmap v1557: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 163 KiB/s rd, 72 KiB/s wr, 42 op/s 2023-09-23T23:10:38.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[180979]: failed to encode map e495 with expected crc 2023-09-23T23:10:38.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:37 smithi052 ceph-mon[180979]: failed to encode map e495 with expected crc 2023-09-23T23:10:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:39 smithi151 ceph-mon[159210]: pgmap v1558: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 173 KiB/s rd, 72 KiB/s wr, 53 op/s 2023-09-23T23:10:40.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:39 smithi052 ceph-mon[187743]: pgmap v1558: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 173 KiB/s rd, 72 KiB/s wr, 53 op/s 2023-09-23T23:10:40.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:39 smithi052 ceph-mon[180979]: pgmap v1558: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 173 KiB/s rd, 72 KiB/s wr, 53 op/s 2023-09-23T23:10:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:41 smithi151 ceph-mon[159210]: pgmap v1559: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 135 KiB/s rd, 53 KiB/s wr, 52 op/s 2023-09-23T23:10:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:41 smithi052 ceph-mon[187743]: pgmap v1559: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 135 KiB/s rd, 53 KiB/s wr, 52 op/s 2023-09-23T23:10:41.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:41 smithi052 ceph-mon[180979]: pgmap v1559: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 135 KiB/s rd, 53 KiB/s wr, 52 op/s 2023-09-23T23:10:42.208 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd help export 2023-09-23T23:10:42.209 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep -q export-format 2023-09-23T23:10:42.228 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_701000/img bs=1k count=10 seek=100 2023-09-23T23:10:42.284 INFO:tasks.workunit.client.0.smithi052.stderr:10+0 records in 2023-09-23T23:10:42.284 INFO:tasks.workunit.client.0.smithi052.stderr:10+0 records out 2023-09-23T23:10:42.284 INFO:tasks.workunit.client.0.smithi052.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.2023e-05 s, 142 MB/s 2023-09-23T23:10:42.284 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format /tmp/rbd_import_export_701000/img testimg 2023-09-23T23:10:42.338 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:42.339 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:42.341 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap create testimg@snap 2023-09-23T23:10:42.587 INFO:tasks.workunit.client.0.smithi052.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T23:10:42.594 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T23:10:42.640 INFO:tasks.workunit.client.0.smithi052.stderr:++ rbd image-meta list testimg 2023-09-23T23:10:42.679 INFO:tasks.workunit.client.0.smithi052.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T23:10:42.679 INFO:tasks.workunit.client.0.smithi052.stderr: 2023-09-23T23:10:42.679 INFO:tasks.workunit.client.0.smithi052.stderr:Key Value 2023-09-23T23:10:42.679 INFO:tasks.workunit.client.0.smithi052.stderr:key1 value1' 2023-09-23T23:10:42.679 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:42.810 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:42.813 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_701000/img_v2 testimg_import 2023-09-23T23:10:42.817 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:42.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:42.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:42.921 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:42.953 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:43.001 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:43.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:43.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:43.109 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:43.138 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:43.183 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:43.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:43.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:43.287 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:43.314 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:43.359 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:43.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:43.464 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:43.493 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:43.497 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:43.501 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:10:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:10:43] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.33.4" 2023-09-23T23:10:43.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[180979]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[180979]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[180979]: osdmap e496: 8 total, 8 up, 5 in 2023-09-23T23:10:43.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[180979]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[180979]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.503 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[180979]: pgmap v1561: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 170 KiB/s rd, 66 KiB/s wr, 67 op/s 2023-09-23T23:10:43.533 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:10:43.537 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import 2023-09-23T23:10:43.575 INFO:tasks.workunit.client.0.smithi052.stdout:rbd image 'testimg_import': 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: size 110 KiB in 1 objects 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: order 22 (4 MiB objects) 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: snapshot_count: 1 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: id: e01a97ae0d0e 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: block_name_prefix: rbd_data.e01a97ae0d0e 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: format: 2 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: op_features: 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: flags: 2023-09-23T23:10:43.576 INFO:tasks.workunit.client.0.smithi052.stdout: create_timestamp: Sat Sep 23 23:10:42 2023 2023-09-23T23:10:43.577 INFO:tasks.workunit.client.0.smithi052.stdout: access_timestamp: Sat Sep 23 23:10:42 2023 2023-09-23T23:10:43.577 INFO:tasks.workunit.client.0.smithi052.stdout: modify_timestamp: Sat Sep 23 23:10:42 2023 2023-09-23T23:10:43.580 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import@snap 2023-09-23T23:10:43.617 INFO:tasks.workunit.client.0.smithi052.stdout:rbd image 'testimg_import': 2023-09-23T23:10:43.617 INFO:tasks.workunit.client.0.smithi052.stdout: size 110 KiB in 1 objects 2023-09-23T23:10:43.617 INFO:tasks.workunit.client.0.smithi052.stdout: order 22 (4 MiB objects) 2023-09-23T23:10:43.617 INFO:tasks.workunit.client.0.smithi052.stdout: snapshot_count: 1 2023-09-23T23:10:43.617 INFO:tasks.workunit.client.0.smithi052.stdout: id: e01a97ae0d0e 2023-09-23T23:10:43.617 INFO:tasks.workunit.client.0.smithi052.stdout: block_name_prefix: rbd_data.e01a97ae0d0e 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: format: 2 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: op_features: 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: flags: 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: create_timestamp: Sat Sep 23 23:10:42 2023 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: access_timestamp: Sat Sep 23 23:10:42 2023 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: modify_timestamp: Sat Sep 23 23:10:42 2023 2023-09-23T23:10:43.618 INFO:tasks.workunit.client.0.smithi052.stdout: protected: False 2023-09-23T23:10:43.622 INFO:tasks.workunit.client.0.smithi052.stderr:++ rbd image-meta list testimg_import 2023-09-23T23:10:43.662 INFO:tasks.workunit.client.0.smithi052.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T23:10:43.662 INFO:tasks.workunit.client.0.smithi052.stderr: 2023-09-23T23:10:43.662 INFO:tasks.workunit.client.0.smithi052.stderr:Key Value 2023-09-23T23:10:43.662 INFO:tasks.workunit.client.0.smithi052.stderr:key1 value1' 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr: 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr:Key Value 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr: 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr:Key Value 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr:key1 value1' ']' 2023-09-23T23:10:43.663 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg_import /tmp/rbd_import_export_701000/img_import 2023-09-23T23:10:43.698 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete...done. 2023-09-23T23:10:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:43 smithi151 ceph-mon[159210]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:43 smithi151 ceph-mon[159210]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:43 smithi151 ceph-mon[159210]: osdmap e496: 8 total, 8 up, 5 in 2023-09-23T23:10:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:43 smithi151 ceph-mon[159210]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:43 smithi151 ceph-mon[159210]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:43 smithi151 ceph-mon[159210]: pgmap v1561: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 170 KiB/s rd, 66 KiB/s wr, 67 op/s 2023-09-23T23:10:43.701 INFO:tasks.workunit.client.0.smithi052.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img_import 2023-09-23T23:10:43.701 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp -l /tmp/rbd_import_export_701000/img /tmp/rbd_import_export_701000/img_import 2023-09-23T23:10:43.702 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/img --format %b 2023-09-23T23:10:43.703 INFO:tasks.workunit.client.0.smithi052.stderr:+ origsize=24 2023-09-23T23:10:43.703 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/img_import --format %b 2023-09-23T23:10:43.704 INFO:tasks.workunit.client.0.smithi052.stderr:+ exportsize=224 2023-09-23T23:10:43.704 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=200 2023-09-23T23:10:43.704 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=200 2023-09-23T23:10:43.704 INFO:tasks.workunit.client.0.smithi052.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T23:10:43.705 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_701000/img_snap 2023-09-23T23:10:43.751 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete...done. 2023-09-23T23:10:43.754 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_701000/img_snap_import 2023-09-23T23:10:43.779 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:43.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:43.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[187743]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.783 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[187743]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.784 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[187743]: osdmap e496: 8 total, 8 up, 5 in 2023-09-23T23:10:43.784 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[187743]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.784 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[187743]: failed to encode map e496 with expected crc 2023-09-23T23:10:43.784 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:43 smithi052 ceph-mon[187743]: pgmap v1561: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 170 KiB/s rd, 66 KiB/s wr, 67 op/s 2023-09-23T23:10:43.790 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete...done. 2023-09-23T23:10:43.794 INFO:tasks.workunit.client.0.smithi052.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_701000/img_snap /tmp/rbd_import_export_701000/img_snap_import 2023-09-23T23:10:43.794 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp -l /tmp/rbd_import_export_701000/img_snap /tmp/rbd_import_export_701000/img_snap_import 2023-09-23T23:10:43.795 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/img_snap --format %b 2023-09-23T23:10:43.795 INFO:tasks.workunit.client.0.smithi052.stderr:+ origsize=224 2023-09-23T23:10:43.796 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/img_snap_import --format %b 2023-09-23T23:10:43.797 INFO:tasks.workunit.client.0.smithi052.stderr:+ exportsize=224 2023-09-23T23:10:43.797 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:43.797 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:43.797 INFO:tasks.workunit.client.0.smithi052.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:10:43.797 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:43.798 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_import 2023-09-23T23:10:43.798 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_snap 2023-09-23T23:10:43.799 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_snap_import 2023-09-23T23:10:43.799 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T23:10:43.884 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:43.910 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:43.952 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:43.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:44.058 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:44.083 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:44.124 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:44.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:44.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:44.129 INFO:teuthology.orchestra.run.smithi052.stdout:{ 2023-09-23T23:10:44.129 INFO:teuthology.orchestra.run.smithi052.stdout: "ops": [], 2023-09-23T23:10:44.129 INFO:teuthology.orchestra.run.smithi052.stdout: "complaint_time": 30, 2023-09-23T23:10:44.129 INFO:teuthology.orchestra.run.smithi052.stdout: "num_blocked_ops": 0 2023-09-23T23:10:44.130 INFO:teuthology.orchestra.run.smithi052.stdout:} 2023-09-23T23:10:44.229 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:44.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:44.524 INFO:tasks.workunit.client.0.smithi052.stderr: Removing snap: 100% complete...done. 2023-09-23T23:10:44.538 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg_import 2023-09-23T23:10:44.565 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 0.807742 (cec8) 2023-09-23T23:10:44.696 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:44.698 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap rm testimg@snap 2023-09-23T23:10:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:44 smithi151 ceph-mon[159210]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:44 smithi151 ceph-mon[159210]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:44 smithi151 ceph-mon[159210]: osdmap e497: 8 total, 8 up, 5 in 2023-09-23T23:10:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:44 smithi151 ceph-mon[159210]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3077423530' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]: dispatch 2023-09-23T23:10:44.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]: dispatch 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[187743]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[187743]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[187743]: osdmap e497: 8 total, 8 up, 5 in 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[187743]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3077423530' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]: dispatch 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]: dispatch 2023-09-23T23:10:44.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[180979]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[180979]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[180979]: osdmap e497: 8 total, 8 up, 5 in 2023-09-23T23:10:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[180979]: failed to encode map e497 with expected crc 2023-09-23T23:10:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3077423530' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]: dispatch 2023-09-23T23:10:44.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]: dispatch 2023-09-23T23:10:45.113 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:45.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:45.218 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:45.245 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:45.288 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:45.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:45.363 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T23:10:45.394 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:45.435 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:45.524 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.524 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.524 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]': finished 2023-09-23T23:10:45.524 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: osdmap e498: 8 total, 8 up, 5 in 2023-09-23T23:10:45.524 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:45 smithi151 ceph-mon[159210]: pgmap v1564: 73 pgs: 4 peering, 69 active+clean; 692 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 7.2 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2023-09-23T23:10:45.534 INFO:tasks.workunit.client.0.smithi052.stderr: Removing snap: 100% complete...done. 2023-09-23T23:10:45.543 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm testimg 2023-09-23T23:10:45.637 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:45.640 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --order 20 /tmp/rbd_import_export_701000/img testimg 2023-09-23T23:10:45.743 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]': finished 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: osdmap e498: 8 total, 8 up, 5 in 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[180979]: pgmap v1564: 73 pgs: 4 peering, 69 active+clean; 692 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 7.2 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2023-09-23T23:10:45.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.807741613406335}]': finished 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: osdmap e498: 8 total, 8 up, 5 in 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: failed to encode map e498 with expected crc 2023-09-23T23:10:45.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:45 smithi052 ceph-mon[187743]: pgmap v1564: 73 pgs: 4 peering, 69 active+clean; 692 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 7.2 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2023-09-23T23:10:45.748 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:45.748 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:10:45.751 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:45.787 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:45.790 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_701000/img_v2 testimg_import 2023-09-23T23:10:45.848 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:10:45.852 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import 2023-09-23T23:10:45.852 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep order 2023-09-23T23:10:45.852 INFO:tasks.workunit.client.0.smithi052.stderr:+ awk '{print $2}' 2023-09-23T23:10:45.852 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep 20 2023-09-23T23:10:45.886 INFO:tasks.workunit.client.0.smithi052.stdout:20 2023-09-23T23:10:45.887 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:45.887 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg_import 2023-09-23T23:10:45.993 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:45.996 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg 2023-09-23T23:10:46.010 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T23:10:46.089 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:46.092 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_701000/img testimg 2023-09-23T23:10:46.140 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:46.142 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:46.144 INFO:tasks.workunit.client.0.smithi052.stderr:++ rbd info testimg 2023-09-23T23:10:46.144 INFO:tasks.workunit.client.0.smithi052.stderr:++ grep features 2023-09-23T23:10:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:46.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:46.180 INFO:tasks.workunit.client.0.smithi052.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T23:10:46.181 INFO:tasks.workunit.client.0.smithi052.stderr: op_features: ' 2023-09-23T23:10:46.181 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:46.223 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:46.228 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_701000/img_v2 testimg_import 2023-09-23T23:10:46.246 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:46.272 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:46.300 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:10:46.305 INFO:tasks.workunit.client.0.smithi052.stderr:++ rbd info testimg_import 2023-09-23T23:10:46.305 INFO:tasks.workunit.client.0.smithi052.stderr:++ grep features 2023-09-23T23:10:46.315 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:46.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr: op_features: ' 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' ' features: layering 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr: op_features: ' '!=' ' features: layering 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr: op_features: ' ']' 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:46.355 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg_import 2023-09-23T23:10:46.363 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:46.422 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:46.436 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:46.440 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg 2023-09-23T23:10:46.453 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:46.528 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:46.531 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_701000/img testimg 2023-09-23T23:10:46.597 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:46.599 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:46.639 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:46.642 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_701000/img_v2 testimg_import 2023-09-23T23:10:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:46 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:10:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:46 smithi151 ceph-mon[159210]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:46 smithi151 ceph-mon[159210]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:46 smithi151 ceph-mon[159210]: osdmap e499: 8 total, 8 up, 5 in 2023-09-23T23:10:46.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:46 smithi151 ceph-mon[159210]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:46 smithi151 ceph-mon[159210]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.713 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:10:46.716 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import 2023-09-23T23:10:46.716 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep 'stripe unit' 2023-09-23T23:10:46.716 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T23:10:46.727 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:46.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:10:46.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[187743]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[187743]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[187743]: osdmap e499: 8 total, 8 up, 5 in 2023-09-23T23:10:46.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[187743]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[187743]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T23:10:46.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[180979]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[180979]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[180979]: osdmap e499: 8 total, 8 up, 5 in 2023-09-23T23:10:46.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[180979]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:46 smithi052 ceph-mon[180979]: failed to encode map e499 with expected crc 2023-09-23T23:10:46.758 INFO:tasks.workunit.client.0.smithi052.stdout: stripe unit: 4 KiB 2023-09-23T23:10:46.759 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import 2023-09-23T23:10:46.759 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep 'stripe count' 2023-09-23T23:10:46.760 INFO:tasks.workunit.client.0.smithi052.stderr:+ awk '{print $3}' 2023-09-23T23:10:46.761 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep 1000 2023-09-23T23:10:46.799 INFO:tasks.workunit.client.0.smithi052.stdout:1000 2023-09-23T23:10:46.799 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/img_v2 2023-09-23T23:10:46.800 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg_import 2023-09-23T23:10:46.810 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:46.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:46.824 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:10:46.854 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:10:46.876 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:10:46.901 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:10:46.904 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T23:10:46.907 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg 2023-09-23T23:10:46.916 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:46.921 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:10:46.941 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:46.942 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:10:46.966 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:10:46.985 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:46.990 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.992 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.993 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:10:46.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:46.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:47.005 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T23:10:47.009 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_701000/img testimg 2023-09-23T23:10:47.068 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:47.071 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap create testimg@snap1 2023-09-23T23:10:47.094 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:47.120 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:47.162 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:47.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:47.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:47.267 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:47.292 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:47.334 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:47.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:47.441 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:47.471 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:47.537 INFO:tasks.workunit.client.0.smithi052.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T23:10:47.543 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap create testimg@snap2 2023-09-23T23:10:47.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/284125660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:47.716 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:47 smithi151 ceph-mon[159210]: pgmap v1566: 73 pgs: 4 peering, 69 active+clean; 692 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 13 KiB/s rd, 3.9 KiB/s wr, 19 op/s 2023-09-23T23:10:47.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/284125660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:47.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:47 smithi052 ceph-mon[187743]: pgmap v1566: 73 pgs: 4 peering, 69 active+clean; 692 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 13 KiB/s rd, 3.9 KiB/s wr, 19 op/s 2023-09-23T23:10:47.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/284125660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:47.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:47 smithi052 ceph-mon[180979]: pgmap v1566: 73 pgs: 4 peering, 69 active+clean; 692 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 13 KiB/s rd, 3.9 KiB/s wr, 19 op/s 2023-09-23T23:10:48.374 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:48.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:48.478 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:48.509 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:48.540 INFO:tasks.workunit.client.0.smithi052.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T23:10:48.546 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap protect testimg@snap2 2023-09-23T23:10:48.561 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:48.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:48.594 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_701000/snap_protect 2023-09-23T23:10:48.634 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:48.637 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_701000/snap_protect testimg_import 2023-09-23T23:10:48.647 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:48 smithi052 ceph-mon[187743]: failed to encode map e500 with expected crc 2023-09-23T23:10:48.648 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:48 smithi052 ceph-mon[187743]: osdmap e500: 8 total, 8 up, 5 in 2023-09-23T23:10:48.648 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:48 smithi052 ceph-mon[187743]: failed to encode map e500 with expected crc 2023-09-23T23:10:48.648 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:48 smithi052 ceph-mon[180979]: failed to encode map e500 with expected crc 2023-09-23T23:10:48.648 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:48 smithi052 ceph-mon[180979]: osdmap e500: 8 total, 8 up, 5 in 2023-09-23T23:10:48.648 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:48 smithi052 ceph-mon[180979]: failed to encode map e500 with expected crc 2023-09-23T23:10:48.666 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:48.694 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:48.736 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:48.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:48.843 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:48.873 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:48 smithi151 ceph-mon[159210]: failed to encode map e500 with expected crc 2023-09-23T23:10:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:48 smithi151 ceph-mon[159210]: osdmap e500: 8 total, 8 up, 5 in 2023-09-23T23:10:48.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:48 smithi151 ceph-mon[159210]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.202 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:49.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:49.306 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:49.334 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:49.378 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:49.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:49.484 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:49.510 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:49.555 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:49.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:49.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:49.663 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: osdmap e501: 8 total, 8 up, 5 in 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.687 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:49 smithi151 ceph-mon[159210]: pgmap v1569: 73 pgs: 4 peering, 69 active+clean; 696 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 402 KiB/s rd, 43 KiB/s wr, 516 op/s 2023-09-23T23:10:49.689 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:49.731 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:49.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:49.800 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:10:49.800 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:10:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.820 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: osdmap e501: 8 total, 8 up, 5 in 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[180979]: pgmap v1569: 73 pgs: 4 peering, 69 active+clean; 696 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 402 KiB/s rd, 43 KiB/s wr, 516 op/s 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: failed to encode map e500 with expected crc 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.821 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.822 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: osdmap e501: 8 total, 8 up, 5 in 2023-09-23T23:10:49.822 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: failed to encode map e501 with expected crc 2023-09-23T23:10:49.822 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:49 smithi052 ceph-mon[187743]: pgmap v1569: 73 pgs: 4 peering, 69 active+clean; 696 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 402 KiB/s rd, 43 KiB/s wr, 516 op/s 2023-09-23T23:10:49.836 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:49.870 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:50.018 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:50.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:50.122 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:50.149 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:50.191 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:50.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:50.297 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:50.323 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:50.365 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:50.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:50.472 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:50.499 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:50.502 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is set 2023-09-23T23:10:50.517 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T23:10:50.519 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import@snap1 2023-09-23T23:10:50.519 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep 'protected: False' 2023-09-23T23:10:50.542 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:50.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:50.562 INFO:tasks.workunit.client.0.smithi052.stdout: protected: False 2023-09-23T23:10:50.562 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd info testimg_import@snap2 2023-09-23T23:10:50.562 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep 'protected: True' 2023-09-23T23:10:50.605 INFO:tasks.workunit.client.0.smithi052.stdout: protected: True 2023-09-23T23:10:50.605 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/snap_protect 2023-09-23T23:10:50.607 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T23:10:50.613 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T23:10:50.648 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:50.649 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T23:10:50.671 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T23:10:50.682 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:50.695 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap purge testimg 2023-09-23T23:10:50.727 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:50.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:50.834 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: osdmap e502: 8 total, 8 up, 5 in 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: failed to encode map e501 with expected crc 2023-09-23T23:10:50.859 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/122464181' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:50.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:50.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:50.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:50.860 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:50 smithi151 ceph-mon[159210]: osdmap e503: 8 total, 8 up, 5 in 2023-09-23T23:10:50.861 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:50.955 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:50.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:50.975 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:10:50.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: osdmap e502: 8 total, 8 up, 5 in 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: failed to encode map e501 with expected crc 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/122464181' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[187743]: osdmap e503: 8 total, 8 up, 5 in 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: osdmap e502: 8 total, 8 up, 5 in 2023-09-23T23:10:50.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: failed to encode map e501 with expected crc 2023-09-23T23:10:50.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: failed to encode map e502 with expected crc 2023-09-23T23:10:50.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/122464181' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:50.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:10:50.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:50.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T23:10:50.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:50 smithi052 ceph-mon[180979]: osdmap e503: 8 total, 8 up, 5 in 2023-09-23T23:10:51.017 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 0, 5] dead_osds: [] live_osds: [1, 3, 4, 5, 6, 7, 0, 2] 2023-09-23T23:10:51.017 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T23:10:51.018 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-09-23T23:10:51.018 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:10:51.061 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:51.089 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:51.257 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:51.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:51.341 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:51.361 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:51.389 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:51.481 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:51.634 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:51.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:51.738 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:51.773 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[187743]: failed to encode map e502 with expected crc 2023-09-23T23:10:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[187743]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[187743]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[187743]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-09-23T23:10:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[187743]: pgmap v1572: 73 pgs: 73 active+clean; 806 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 519 KiB/s rd, 57 KiB/s wr, 679 op/s 2023-09-23T23:10:51.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[180979]: failed to encode map e502 with expected crc 2023-09-23T23:10:51.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[180979]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[180979]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[180979]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-09-23T23:10:51.917 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:51 smithi052 ceph-mon[180979]: pgmap v1572: 73 pgs: 73 active+clean; 806 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 519 KiB/s rd, 57 KiB/s wr, 679 op/s 2023-09-23T23:10:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:51 smithi151 ceph-mon[159210]: failed to encode map e502 with expected crc 2023-09-23T23:10:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:51 smithi151 ceph-mon[159210]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:51 smithi151 ceph-mon[159210]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:51 smithi151 ceph-mon[159210]: failed to encode map e503 with expected crc 2023-09-23T23:10:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:51 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-09-23T23:10:51.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:51 smithi151 ceph-mon[159210]: pgmap v1572: 73 pgs: 73 active+clean; 806 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 519 KiB/s rd, 57 KiB/s wr, 679 op/s 2023-09-23T23:10:51.971 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:51.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:51.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:52.075 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:52.104 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:52.557 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T23:10:52.557 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.557 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.557 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-09-23T23:10:52.557 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: osdmap e504: 8 total, 8 up, 5 in 2023-09-23T23:10:52.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.558 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:52 smithi151 ceph-mon[159210]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.558 INFO:tasks.workunit.client.0.smithi052.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T23:10:52.565 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd snap purge testimg_import 2023-09-23T23:10:52.721 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:52.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:52.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:52.762 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:10:52.762 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:10:52.825 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:52.855 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:52.948 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:52.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: osdmap e504: 8 total, 8 up, 5 in 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[187743]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: osdmap e504: 8 total, 8 up, 5 in 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:52.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:52 smithi052 ceph-mon[180979]: failed to encode map e504 with expected crc 2023-09-23T23:10:53.052 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:53.078 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:53.120 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:53.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:53.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:53.223 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:53.249 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:53.271 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:10:53.414 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:53.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:53.468 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T23:10:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:10:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:10:53] "GET /metrics HTTP/1.1" 200 34244 "" "Prometheus/2.33.4" 2023-09-23T23:10:53.496 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:10:53.518 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:53.518 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:10:53.540 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:10:53.546 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:53.564 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:10:53.589 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:10:53.594 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:53.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:53.612 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:10:53.631 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:10:53.651 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:10:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.652 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.653 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T23:10:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:53 smithi151 ceph-mon[159210]: osdmap e505: 8 total, 8 up, 5 in 2023-09-23T23:10:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:53 smithi151 ceph-mon[159210]: failed to encode map e505 with expected crc 2023-09-23T23:10:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:53 smithi151 ceph-mon[159210]: failed to encode map e505 with expected crc 2023-09-23T23:10:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:53 smithi151 ceph-mon[159210]: pgmap v1575: 73 pgs: 1 peering, 72 active+clean; 802 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 170 KiB/s rd, 19 KiB/s wr, 233 op/s 2023-09-23T23:10:53.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:53 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3788620157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:53.700 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:53.728 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:53.831 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:10:53.832 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[187743]: osdmap e505: 8 total, 8 up, 5 in 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[187743]: failed to encode map e505 with expected crc 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[187743]: failed to encode map e505 with expected crc 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[187743]: pgmap v1575: 73 pgs: 1 peering, 72 active+clean; 802 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 170 KiB/s rd, 19 KiB/s wr, 233 op/s 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3788620157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[180979]: osdmap e505: 8 total, 8 up, 5 in 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[180979]: failed to encode map e505 with expected crc 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[180979]: failed to encode map e505 with expected crc 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[180979]: pgmap v1575: 73 pgs: 1 peering, 72 active+clean; 802 KiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 170 KiB/s rd, 19 KiB/s wr, 233 op/s 2023-09-23T23:10:53.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:53 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3788620157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:10:53.874 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:53.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:53.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:53.968 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:10:53.969 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:10:53.978 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:54.004 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:54.100 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:54.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:54.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:54.204 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:54.234 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:54.567 INFO:tasks.workunit.client.0.smithi052.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T23:10:54.575 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg 2023-09-23T23:10:54.663 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:54.666 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd remove testimg_import 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: failed to encode map e505 with expected crc 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: osdmap e506: 8 total, 8 up, 5 in 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:54 smithi151 ceph-mon[159210]: from='client.47882 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:54.752 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:54.753 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:54.756 INFO:tasks.workunit.client.0.smithi052.stderr:+ tiered=0 2023-09-23T23:10:54.756 INFO:tasks.workunit.client.0.smithi052.stderr:+ ceph osd dump 2023-09-23T23:10:54.758 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep '^pool' 2023-09-23T23:10:54.758 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep ''\''rbd'\''' 2023-09-23T23:10:54.758 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep tier 2023-09-23T23:10:54.964 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:54.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: failed to encode map e505 with expected crc 2023-09-23T23:10:54.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:10:54.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: osdmap e506: 8 total, 8 up, 5 in 2023-09-23T23:10:54.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[187743]: from='client.47882 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: failed to encode map e505 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: osdmap e506: 8 total, 8 up, 5 in 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.967 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: failed to encode map e506 with expected crc 2023-09-23T23:10:54.968 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:54 smithi052 ceph-mon[180979]: from='client.47882 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:10:54.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:54.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:55.069 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:55.097 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1576,"stamp":"2023-09-23T23:10:53.555344+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":821378,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":3818,"num_read_kb":5467,"num_write":753,"num_write_kb":3746,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":611296,"num_keys_recovered":40,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":759,"ondisk_log_size":759,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4152596,"kb_used_data":290536,"kb_used_omap":99,"kb_used_meta":3861916,"kb_avail":464573164,"statfs":{"total":479975178240,"available":475722919936,"internally_reserved":0,"allocated":297508864,"data_stored":288748423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102378,"internal_metadata":3954602006},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":108751,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":837,"num_read_kb":680,"num_write":99,"num_write_kb":75,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.017352"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":515,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246834+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:10:50.246834+0000","last_peered":"2023-09-23T23:10:50.246834+0000","last_clean":"2023-09-23T23:10:50.246834+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:10:50.246834+0000","last_undegraded":"2023-09-23T23:10:50.246834+0000","last_fullsized":"2023-09-23T23:10:50.246834+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":174,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529128+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:10:47.529128+0000","last_peered":"2023-09-23T23:10:47.529128+0000","last_clean":"2023-09-23T23:10:47.529128+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:10:47.529128+0000","last_undegraded":"2023-09-23T23:10:47.529128+0000","last_fullsized":"2023-09-23T23:10:47.529128+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":182,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528570+0000","last_change":"2023-09-23T23:10:45.539525+0000","last_active":"2023-09-23T23:10:47.528570+0000","last_peered":"2023-09-23T23:10:47.528570+0000","last_clean":"2023-09-23T23:10:47.528570+0000","last_became_active":"2023-09-23T23:10:45.539171+0000","last_became_peered":"2023-09-23T23:10:45.539171+0000","last_unstale":"2023-09-23T23:10:47.528570+0000","last_undegraded":"2023-09-23T23:10:47.528570+0000","last_fullsized":"2023-09-23T23:10:47.528570+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:38.240145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":520,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529086+0000","last_change":"2023-09-23T23:05:53.573960+0000","last_active":"2023-09-23T23:10:47.529086+0000","last_peered":"2023-09-23T23:10:47.529086+0000","last_clean":"2023-09-23T23:10:47.529086+0000","last_became_active":"2023-09-23T23:05:53.572999+0000","last_became_peered":"2023-09-23T23:05:53.572999+0000","last_unstale":"2023-09-23T23:10:47.529086+0000","last_undegraded":"2023-09-23T23:10:47.529086+0000","last_fullsized":"2023-09-23T23:10:47.529086+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:54.634648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528346+0000","last_change":"2023-09-23T23:05:39.883388+0000","last_active":"2023-09-23T23:10:47.528346+0000","last_peered":"2023-09-23T23:10:47.528346+0000","last_clean":"2023-09-23T23:10:47.528346+0000","last_became_active":"2023-09-23T23:05:39.883088+0000","last_became_peered":"2023-09-23T23:05:39.883088+0000","last_unstale":"2023-09-23T23:10:47.528346+0000","last_undegraded":"2023-09-23T23:10:47.528346+0000","last_fullsized":"2023-09-23T23:10:47.528346+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:16.799876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:10:48.534019+0000","last_change":"2023-09-23T23:08:43.480306+0000","last_active":"2023-09-23T23:10:48.534019+0000","last_peered":"2023-09-23T23:10:48.534019+0000","last_clean":"2023-09-23T23:10:48.534019+0000","last_became_active":"2023-09-23T23:08:43.479968+0000","last_became_peered":"2023-09-23T23:08:43.479968+0000","last_unstale":"2023-09-23T23:10:48.534019+0000","last_undegraded":"2023-09-23T23:10:48.534019+0000","last_fullsized":"2023-09-23T23:10:48.534019+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:13.349701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":446,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:10:48.534138+0000","last_change":"2023-09-23T23:05:53.581928+0000","last_active":"2023-09-23T23:10:48.534138+0000","last_peered":"2023-09-23T23:10:48.534138+0000","last_clean":"2023-09-23T23:10:48.534138+0000","last_became_active":"2023-09-23T23:05:53.581522+0000","last_became_peered":"2023-09-23T23:05:53.581522+0000","last_unstale":"2023-09-23T23:10:48.534138+0000","last_undegraded":"2023-09-23T23:10:48.534138+0000","last_fullsized":"2023-09-23T23:10:48.534138+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:00.485293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":192,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528512+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:10:47.528512+0000","last_peered":"2023-09-23T23:10:47.528512+0000","last_clean":"2023-09-23T23:10:47.528512+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:10:47.528512+0000","last_undegraded":"2023-09-23T23:10:47.528512+0000","last_fullsized":"2023-09-23T23:10:47.528512+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":308,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529119+0000","last_change":"2023-09-23T23:05:53.576352+0000","last_active":"2023-09-23T23:10:47.529119+0000","last_peered":"2023-09-23T23:10:47.529119+0000","last_clean":"2023-09-23T23:10:47.529119+0000","last_became_active":"2023-09-23T23:05:53.572953+0000","last_became_peered":"2023-09-23T23:05:53.572953+0000","last_unstale":"2023-09-23T23:10:47.529119+0000","last_undegraded":"2023-09-23T23:10:47.529119+0000","last_fullsized":"2023-09-23T23:10:47.529119+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:48.851652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":206,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528445+0000","last_change":"2023-09-23T23:05:53.599042+0000","last_active":"2023-09-23T23:10:47.528445+0000","last_peered":"2023-09-23T23:10:47.528445+0000","last_clean":"2023-09-23T23:10:47.528445+0000","last_became_active":"2023-09-23T23:05:53.597821+0000","last_became_peered":"2023-09-23T23:05:53.597821+0000","last_unstale":"2023-09-23T23:10:47.528445+0000","last_undegraded":"2023-09-23T23:10:47.528445+0000","last_fullsized":"2023-09-23T23:10:47.528445+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:33.628757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":239,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529035+0000","last_change":"2023-09-23T23:05:53.633278+0000","last_active":"2023-09-23T23:10:47.529035+0000","last_peered":"2023-09-23T23:10:47.529035+0000","last_clean":"2023-09-23T23:10:47.529035+0000","last_became_active":"2023-09-23T23:05:53.598456+0000","last_became_peered":"2023-09-23T23:05:53.598456+0000","last_unstale":"2023-09-23T23:10:47.529035+0000","last_undegraded":"2023-09-23T23:10:47.529035+0000","last_fullsized":"2023-09-23T23:10:47.529035+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:46.295675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":260,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.554954+0000","last_change":"2023-09-23T23:08:18.922767+0000","last_active":"2023-09-23T23:10:51.554954+0000","last_peered":"2023-09-23T23:10:51.554954+0000","last_clean":"2023-09-23T23:10:51.554954+0000","last_became_active":"2023-09-23T23:08:18.922443+0000","last_became_peered":"2023-09-23T23:08:18.922443+0000","last_unstale":"2023-09-23T23:10:51.554954+0000","last_undegraded":"2023-09-23T23:10:51.554954+0000","last_fullsized":"2023-09-23T23:10:51.554954+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:30.280172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":593,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246681+0000","last_change":"2023-09-23T23:05:04.019163+0000","last_active":"2023-09-23T23:10:50.246681+0000","last_peered":"2023-09-23T23:10:50.246681+0000","last_clean":"2023-09-23T23:10:50.246681+0000","last_became_active":"2023-09-23T23:05:03.451557+0000","last_became_peered":"2023-09-23T23:05:03.451557+0000","last_unstale":"2023-09-23T23:10:50.246681+0000","last_undegraded":"2023-09-23T23:10:50.246681+0000","last_fullsized":"2023-09-23T23:10:50.246681+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":486,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246747+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:10:50.246747+0000","last_peered":"2023-09-23T23:10:50.246747+0000","last_clean":"2023-09-23T23:10:50.246747+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:10:50.246747+0000","last_undegraded":"2023-09-23T23:10:50.246747+0000","last_fullsized":"2023-09-23T23:10:50.246747+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":418,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246776+0000","last_change":"2023-09-23T23:05:03.460914+0000","last_active":"2023-09-23T23:10:50.246776+0000","last_peered":"2023-09-23T23:10:50.246776+0000","last_clean":"2023-09-23T23:10:50.246776+0000","last_became_active":"2023-09-23T23:05:03.451534+0000","last_became_peered":"2023-09-23T23:05:03.451534+0000","last_unstale":"2023-09-23T23:10:50.246776+0000","last_undegraded":"2023-09-23T23:10:50.246776+0000","last_fullsized":"2023-09-23T23:10:50.246776+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":185,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529076+0000","last_change":"2023-09-23T22:58:24.244897+0000","last_active":"2023-09-23T23:10:47.529076+0000","last_peered":"2023-09-23T23:10:47.529076+0000","last_clean":"2023-09-23T23:10:47.529076+0000","last_became_active":"2023-09-23T22:58:24.244489+0000","last_became_peered":"2023-09-23T22:58:24.244489+0000","last_unstale":"2023-09-23T23:10:47.529076+0000","last_undegraded":"2023-09-23T23:10:47.529076+0000","last_fullsized":"2023-09-23T23:10:47.529076+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:53.388913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":503,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528797+0000","last_change":"2023-09-23T23:05:39.888759+0000","last_active":"2023-09-23T23:10:47.528797+0000","last_peered":"2023-09-23T23:10:47.528797+0000","last_clean":"2023-09-23T23:10:47.528797+0000","last_became_active":"2023-09-23T23:05:39.888259+0000","last_became_peered":"2023-09-23T23:05:39.888259+0000","last_unstale":"2023-09-23T23:10:47.528797+0000","last_undegraded":"2023-09-23T23:10:47.528797+0000","last_fullsized":"2023-09-23T23:10:47.528797+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:47.054708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":418,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528963+0000","last_change":"2023-09-23T23:08:18.954331+0000","last_active":"2023-09-23T23:10:47.528963+0000","last_peered":"2023-09-23T23:10:47.528963+0000","last_clean":"2023-09-23T23:10:47.528963+0000","last_became_active":"2023-09-23T23:08:18.953968+0000","last_became_peered":"2023-09-23T23:08:18.953968+0000","last_unstale":"2023-09-23T23:10:47.528963+0000","last_undegraded":"2023-09-23T23:10:47.528963+0000","last_fullsized":"2023-09-23T23:10:47.528963+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:47.487155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":406,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246801+0000","last_change":"2023-09-23T23:05:39.897133+0000","last_active":"2023-09-23T23:10:50.246801+0000","last_peered":"2023-09-23T23:10:50.246801+0000","last_clean":"2023-09-23T23:10:50.246801+0000","last_became_active":"2023-09-23T23:05:39.896814+0000","last_became_peered":"2023-09-23T23:05:39.896814+0000","last_unstale":"2023-09-23T23:10:50.246801+0000","last_undegraded":"2023-09-23T23:10:50.246801+0000","last_fullsized":"2023-09-23T23:10:50.246801+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":527,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528803+0000","last_change":"2023-09-23T23:05:39.904115+0000","last_active":"2023-09-23T23:10:47.528803+0000","last_peered":"2023-09-23T23:10:47.528803+0000","last_clean":"2023-09-23T23:10:47.528803+0000","last_became_active":"2023-09-23T23:05:39.903767+0000","last_became_peered":"2023-09-23T23:05:39.903767+0000","last_unstale":"2023-09-23T23:10:47.528803+0000","last_undegraded":"2023-09-23T23:10:47.528803+0000","last_fullsized":"2023-09-23T23:10:47.528803+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:42.788990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":116,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:10:48.533651+0000","last_change":"2023-09-23T23:06:08.627536+0000","last_active":"2023-09-23T23:10:48.533651+0000","last_peered":"2023-09-23T23:10:48.533651+0000","last_clean":"2023-09-23T23:10:48.533651+0000","last_became_active":"2023-09-23T23:06:08.610287+0000","last_became_peered":"2023-09-23T23:06:08.610287+0000","last_unstale":"2023-09-23T23:10:48.533651+0000","last_undegraded":"2023-09-23T23:10:48.533651+0000","last_fullsized":"2023-09-23T23:10:48.533651+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:24.626898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":198,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529363+0000","last_change":"2023-09-23T22:58:24.249172+0000","last_active":"2023-09-23T23:10:47.529363+0000","last_peered":"2023-09-23T23:10:47.529363+0000","last_clean":"2023-09-23T23:10:47.529363+0000","last_became_active":"2023-09-23T22:58:24.248822+0000","last_became_peered":"2023-09-23T22:58:24.248822+0000","last_unstale":"2023-09-23T23:10:47.529363+0000","last_undegraded":"2023-09-23T23:10:47.529363+0000","last_fullsized":"2023-09-23T23:10:47.529363+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:50.956152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":413,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246529+0000","last_change":"2023-09-23T23:05:53.626675+0000","last_active":"2023-09-23T23:10:50.246529+0000","last_peered":"2023-09-23T23:10:50.246529+0000","last_clean":"2023-09-23T23:10:50.246529+0000","last_became_active":"2023-09-23T23:05:53.573885+0000","last_became_peered":"2023-09-23T23:05:53.573885+0000","last_unstale":"2023-09-23T23:10:50.246529+0000","last_undegraded":"2023-09-23T23:10:50.246529+0000","last_fullsized":"2023-09-23T23:10:50.246529+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":369,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246554+0000","last_change":"2023-09-23T23:06:08.605813+0000","last_active":"2023-09-23T23:10:50.246554+0000","last_peered":"2023-09-23T23:10:50.246554+0000","last_clean":"2023-09-23T23:10:50.246554+0000","last_became_active":"2023-09-23T23:06:08.605450+0000","last_became_peered":"2023-09-23T23:06:08.605450+0000","last_unstale":"2023-09-23T23:10:50.246554+0000","last_undegraded":"2023-09-23T23:10:50.246554+0000","last_fullsized":"2023-09-23T23:10:50.246554+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":29,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.552238+0000","last_change":"2023-09-23T23:10:45.531660+0000","last_active":"2023-09-23T23:10:51.552238+0000","last_peered":"2023-09-23T23:10:51.552238+0000","last_clean":"2023-09-23T23:10:51.552238+0000","last_became_active":"2023-09-23T23:10:45.531292+0000","last_became_peered":"2023-09-23T23:10:45.531292+0000","last_unstale":"2023-09-23T23:10:51.552238+0000","last_undegraded":"2023-09-23T23:10:51.552238+0000","last_fullsized":"2023-09-23T23:10:51.552238+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:44.930770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":510,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246535+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:10:50.246535+0000","last_peered":"2023-09-23T23:10:50.246535+0000","last_clean":"2023-09-23T23:10:50.246535+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:10:50.246535+0000","last_undegraded":"2023-09-23T23:10:50.246535+0000","last_fullsized":"2023-09-23T23:10:50.246535+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":160,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528914+0000","last_change":"2023-09-23T23:06:12.280135+0000","last_active":"2023-09-23T23:10:47.528914+0000","last_peered":"2023-09-23T23:10:47.528914+0000","last_clean":"2023-09-23T23:10:47.528914+0000","last_became_active":"2023-09-23T23:06:12.172721+0000","last_became_peered":"2023-09-23T23:06:12.172721+0000","last_unstale":"2023-09-23T23:10:47.528914+0000","last_undegraded":"2023-09-23T23:10:47.528914+0000","last_fullsized":"2023-09-23T23:10:47.528914+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:24.448782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":99,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528501+0000","last_change":"2023-09-23T23:06:08.606906+0000","last_active":"2023-09-23T23:10:47.528501+0000","last_peered":"2023-09-23T23:10:47.528501+0000","last_clean":"2023-09-23T23:10:47.528501+0000","last_became_active":"2023-09-23T23:06:08.606288+0000","last_became_peered":"2023-09-23T23:06:08.606288+0000","last_unstale":"2023-09-23T23:10:47.528501+0000","last_undegraded":"2023-09-23T23:10:47.528501+0000","last_fullsized":"2023-09-23T23:10:47.528501+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:50.840492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":192,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528986+0000","last_change":"2023-09-23T23:05:53.581856+0000","last_active":"2023-09-23T23:10:47.528986+0000","last_peered":"2023-09-23T23:10:47.528986+0000","last_clean":"2023-09-23T23:10:47.528986+0000","last_became_active":"2023-09-23T23:05:53.581454+0000","last_became_peered":"2023-09-23T23:05:53.581454+0000","last_unstale":"2023-09-23T23:10:47.528986+0000","last_undegraded":"2023-09-23T23:10:47.528986+0000","last_fullsized":"2023-09-23T23:10:47.528986+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:13.659949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":325,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246644+0000","last_change":"2023-09-23T23:05:39.891864+0000","last_active":"2023-09-23T23:10:50.246644+0000","last_peered":"2023-09-23T23:10:50.246644+0000","last_clean":"2023-09-23T23:10:50.246644+0000","last_became_active":"2023-09-23T23:05:39.891546+0000","last_became_peered":"2023-09-23T23:05:39.891546+0000","last_unstale":"2023-09-23T23:10:50.246644+0000","last_undegraded":"2023-09-23T23:10:50.246644+0000","last_fullsized":"2023-09-23T23:10:50.246644+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":206,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528708+0000","last_change":"2023-09-23T23:05:39.933757+0000","last_active":"2023-09-23T23:10:47.528708+0000","last_peered":"2023-09-23T23:10:47.528708+0000","last_clean":"2023-09-23T23:10:47.528708+0000","last_became_active":"2023-09-23T23:05:39.887953+0000","last_became_peered":"2023-09-23T23:05:39.887953+0000","last_unstale":"2023-09-23T23:10:47.528708+0000","last_undegraded":"2023-09-23T23:10:47.528708+0000","last_fullsized":"2023-09-23T23:10:47.528708+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:27.421378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":505,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246581+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:10:50.246581+0000","last_peered":"2023-09-23T23:10:50.246581+0000","last_clean":"2023-09-23T23:10:50.246581+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:10:50.246581+0000","last_undegraded":"2023-09-23T23:10:50.246581+0000","last_fullsized":"2023-09-23T23:10:50.246581+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":276,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246728+0000","last_change":"2023-09-23T23:06:08.599828+0000","last_active":"2023-09-23T23:10:50.246728+0000","last_peered":"2023-09-23T23:10:50.246728+0000","last_clean":"2023-09-23T23:10:50.246728+0000","last_became_active":"2023-09-23T23:06:08.599484+0000","last_became_peered":"2023-09-23T23:06:08.599484+0000","last_unstale":"2023-09-23T23:10:50.246728+0000","last_undegraded":"2023-09-23T23:10:50.246728+0000","last_fullsized":"2023-09-23T23:10:50.246728+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":378,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.494265+0000","last_change":"2023-09-23T23:10:45.541194+0000","last_active":"2023-09-23T23:10:50.494265+0000","last_peered":"2023-09-23T23:10:50.494265+0000","last_clean":"2023-09-23T23:10:50.494265+0000","last_became_active":"2023-09-23T23:10:45.540861+0000","last_became_peered":"2023-09-23T23:10:45.540861+0000","last_unstale":"2023-09-23T23:10:50.494265+0000","last_undegraded":"2023-09-23T23:10:50.494265+0000","last_fullsized":"2023-09-23T23:10:50.494265+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:38.060651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":526,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246402+0000","last_change":"2023-09-23T22:58:23.265951+0000","last_active":"2023-09-23T23:10:50.246402+0000","last_peered":"2023-09-23T23:10:50.246402+0000","last_clean":"2023-09-23T23:10:50.246402+0000","last_became_active":"2023-09-23T22:58:23.265519+0000","last_became_peered":"2023-09-23T22:58:23.265519+0000","last_unstale":"2023-09-23T23:10:50.246402+0000","last_undegraded":"2023-09-23T23:10:50.246402+0000","last_fullsized":"2023-09-23T23:10:50.246402+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":531,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246885+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:10:50.246885+0000","last_peered":"2023-09-23T23:10:50.246885+0000","last_clean":"2023-09-23T23:10:50.246885+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:10:50.246885+0000","last_undegraded":"2023-09-23T23:10:50.246885+0000","last_fullsized":"2023-09-23T23:10:50.246885+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":175,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528616+0000","last_change":"2023-09-23T22:57:26.510825+0000","last_active":"2023-09-23T23:10:47.528616+0000","last_peered":"2023-09-23T23:10:47.528616+0000","last_clean":"2023-09-23T23:10:47.528616+0000","last_became_active":"2023-09-23T22:57:26.506283+0000","last_became_peered":"2023-09-23T22:57:26.506283+0000","last_unstale":"2023-09-23T23:10:47.528616+0000","last_undegraded":"2023-09-23T23:10:47.528616+0000","last_fullsized":"2023-09-23T23:10:47.528616+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:34.466438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":411,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.494514+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:10:50.494514+0000","last_peered":"2023-09-23T23:10:50.494514+0000","last_clean":"2023-09-23T23:10:50.494514+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:10:50.494514+0000","last_undegraded":"2023-09-23T23:10:50.494514+0000","last_fullsized":"2023-09-23T23:10:50.494514+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"501'134","reported_seq":950,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.555432+0000","last_change":"2023-09-23T23:10:51.555612+0000","last_active":"2023-09-23T23:10:51.555432+0000","last_peered":"2023-09-23T23:10:51.555432+0000","last_clean":"2023-09-23T23:10:51.555432+0000","last_became_active":"2023-09-23T23:05:39.874604+0000","last_became_peered":"2023-09-23T23:05:39.874604+0000","last_unstale":"2023-09-23T23:10:51.555432+0000","last_undegraded":"2023-09-23T23:10:51.555432+0000","last_fullsized":"2023-09-23T23:10:51.555432+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":134,"log_dups_size":0,"ondisk_log_size":134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:21.986315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000378814,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":817,"num_write":109,"num_write_kb":62,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"1"}]},{"pgid":"3.5","version":"0'0","reported_seq":340,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528852+0000","last_change":"2023-09-23T23:05:53.598437+0000","last_active":"2023-09-23T23:10:47.528852+0000","last_peered":"2023-09-23T23:10:47.528852+0000","last_clean":"2023-09-23T23:10:47.528852+0000","last_became_active":"2023-09-23T23:05:53.597872+0000","last_became_peered":"2023-09-23T23:05:53.597872+0000","last_unstale":"2023-09-23T23:10:47.528852+0000","last_undegraded":"2023-09-23T23:10:47.528852+0000","last_fullsized":"2023-09-23T23:10:47.528852+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:05.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":244,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246385+0000","last_change":"2023-09-23T23:05:53.572203+0000","last_active":"2023-09-23T23:10:50.246385+0000","last_peered":"2023-09-23T23:10:50.246385+0000","last_clean":"2023-09-23T23:10:50.246385+0000","last_became_active":"2023-09-23T23:05:53.571866+0000","last_became_peered":"2023-09-23T23:05:53.571866+0000","last_unstale":"2023-09-23T23:10:50.246385+0000","last_undegraded":"2023-09-23T23:10:50.246385+0000","last_fullsized":"2023-09-23T23:10:50.246385+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"505'90","reported_seq":746,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-09-23T23:10:52.608666+0000","last_change":"2023-09-23T23:10:52.548987+0000","last_active":"2023-09-23T23:10:52.608666+0000","last_peered":"2023-09-23T23:10:52.608666+0000","last_clean":"2023-09-23T23:10:52.608666+0000","last_became_active":"2023-09-23T23:07:55.442120+0000","last_became_peered":"2023-09-23T23:07:55.442120+0000","last_unstale":"2023-09-23T23:10:52.608666+0000","last_undegraded":"2023-09-23T23:10:52.608666+0000","last_fullsized":"2023-09-23T23:10:52.608666+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":459,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.297461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040655500000000001,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":864,"num_read_kb":1057,"num_write":75,"num_write_kb":186,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"2"}]},{"pgid":"3.3","version":"0'0","reported_seq":541,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246832+0000","last_change":"2023-09-23T23:05:39.894117+0000","last_active":"2023-09-23T23:10:50.246832+0000","last_peered":"2023-09-23T23:10:50.246832+0000","last_clean":"2023-09-23T23:10:50.246832+0000","last_became_active":"2023-09-23T23:05:39.893782+0000","last_became_peered":"2023-09-23T23:05:39.893782+0000","last_unstale":"2023-09-23T23:10:50.246832+0000","last_undegraded":"2023-09-23T23:10:50.246832+0000","last_fullsized":"2023-09-23T23:10:50.246832+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":410,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.555128+0000","last_change":"2023-09-23T23:05:53.573524+0000","last_active":"2023-09-23T23:10:51.555128+0000","last_peered":"2023-09-23T23:10:51.555128+0000","last_clean":"2023-09-23T23:10:51.555128+0000","last_became_active":"2023-09-23T23:05:53.573194+0000","last_became_peered":"2023-09-23T23:05:53.573194+0000","last_unstale":"2023-09-23T23:10:51.555128+0000","last_undegraded":"2023-09-23T23:10:51.555128+0000","last_fullsized":"2023-09-23T23:10:51.555128+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.093853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"499'56","reported_seq":627,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.555384+0000","last_change":"2023-09-23T23:10:51.555384+0000","last_active":"2023-09-23T23:10:51.555384+0000","last_peered":"2023-09-23T23:10:51.555384+0000","last_clean":"2023-09-23T23:10:51.555384+0000","last_became_active":"2023-09-23T23:05:39.880588+0000","last_became_peered":"2023-09-23T23:05:39.880588+0000","last_unstale":"2023-09-23T23:10:51.555384+0000","last_undegraded":"2023-09-23T23:10:51.555384+0000","last_fullsized":"2023-09-23T23:10:51.555384+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":56,"log_dups_size":0,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.674614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043735999999999997,"stat_sum":{"num_bytes":16,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":192,"num_read_kb":832,"num_write":62,"num_write_kb":406,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"1"}]},{"pgid":"3.0","version":"0'0","reported_seq":337,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.493004+0000","last_change":"2023-09-23T23:05:53.572141+0000","last_active":"2023-09-23T23:10:50.493004+0000","last_peered":"2023-09-23T23:10:50.493004+0000","last_clean":"2023-09-23T23:10:50.493004+0000","last_became_active":"2023-09-23T23:05:53.571834+0000","last_became_peered":"2023-09-23T23:05:53.571834+0000","last_unstale":"2023-09-23T23:10:50.493004+0000","last_undegraded":"2023-09-23T23:10:50.493004+0000","last_fullsized":"2023-09-23T23:10:50.493004+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:12.956245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":439,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529279+0000","last_change":"2023-09-23T23:08:18.911712+0000","last_active":"2023-09-23T23:10:47.529279+0000","last_peered":"2023-09-23T23:10:47.529279+0000","last_clean":"2023-09-23T23:10:47.529279+0000","last_became_active":"2023-09-23T23:08:18.911357+0000","last_became_peered":"2023-09-23T23:08:18.911357+0000","last_unstale":"2023-09-23T23:10:47.529279+0000","last_undegraded":"2023-09-23T23:10:47.529279+0000","last_fullsized":"2023-09-23T23:10:47.529279+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:16.515152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"503'129","reported_seq":930,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.555271+0000","last_change":"2023-09-23T23:10:51.555463+0000","last_active":"2023-09-23T23:10:51.555271+0000","last_peered":"2023-09-23T23:10:51.555271+0000","last_clean":"2023-09-23T23:10:51.555271+0000","last_became_active":"2023-09-23T23:05:53.571445+0000","last_became_peered":"2023-09-23T23:05:53.571445+0000","last_unstale":"2023-09-23T23:10:51.555271+0000","last_undegraded":"2023-09-23T23:10:51.555271+0000","last_fullsized":"2023-09-23T23:10:51.555271+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:43.218467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033024500000000001,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":926,"num_read_kb":992,"num_write":104,"num_write_kb":146,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"1"}]},{"pgid":"1.0","version":"75'139","reported_seq":625,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.493212+0000","last_change":"2023-09-23T23:09:55.052855+0000","last_active":"2023-09-23T23:10:50.493212+0000","last_peered":"2023-09-23T23:10:50.493212+0000","last_clean":"2023-09-23T23:10:50.493212+0000","last_became_active":"2023-09-23T23:06:09.609810+0000","last_became_peered":"2023-09-23T23:06:09.609810+0000","last_unstale":"2023-09-23T23:10:50.493212+0000","last_undegraded":"2023-09-23T23:10:50.493212+0000","last_fullsized":"2023-09-23T23:10:50.493212+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:24.707715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":193,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528544+0000","last_change":"2023-09-23T23:05:53.610443+0000","last_active":"2023-09-23T23:10:47.528544+0000","last_peered":"2023-09-23T23:10:47.528544+0000","last_clean":"2023-09-23T23:10:47.528544+0000","last_became_active":"2023-09-23T23:05:53.610116+0000","last_became_peered":"2023-09-23T23:05:53.610116+0000","last_unstale":"2023-09-23T23:10:47.528544+0000","last_undegraded":"2023-09-23T23:10:47.528544+0000","last_fullsized":"2023-09-23T23:10:47.528544+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:57.749677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":421,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246971+0000","last_change":"2023-09-23T23:08:43.483612+0000","last_active":"2023-09-23T23:10:50.246971+0000","last_peered":"2023-09-23T23:10:50.246971+0000","last_clean":"2023-09-23T23:10:50.246971+0000","last_became_active":"2023-09-23T23:08:43.483261+0000","last_became_peered":"2023-09-23T23:08:43.483261+0000","last_unstale":"2023-09-23T23:10:50.246971+0000","last_undegraded":"2023-09-23T23:10:50.246971+0000","last_fullsized":"2023-09-23T23:10:50.246971+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"501'50","reported_seq":576,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-09-23T23:10:48.691467+0000","last_change":"2023-09-23T23:10:45.523244+0000","last_active":"2023-09-23T23:10:48.691467+0000","last_peered":"2023-09-23T23:10:48.691467+0000","last_clean":"2023-09-23T23:10:48.691467+0000","last_became_active":"2023-09-23T23:05:53.608497+0000","last_became_peered":"2023-09-23T23:05:53.608497+0000","last_unstale":"2023-09-23T23:10:48.691467+0000","last_undegraded":"2023-09-23T23:10:48.691467+0000","last_fullsized":"2023-09-23T23:10:48.691467+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.025067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056746200000000002,"stat_sum":{"num_bytes":70,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":272,"num_read_kb":240,"num_write":49,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":336,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528451+0000","last_change":"2023-09-23T23:10:45.539744+0000","last_active":"2023-09-23T23:10:47.528451+0000","last_peered":"2023-09-23T23:10:47.528451+0000","last_clean":"2023-09-23T23:10:47.528451+0000","last_became_active":"2023-09-23T23:10:45.539218+0000","last_became_peered":"2023-09-23T23:10:45.539218+0000","last_unstale":"2023-09-23T23:10:47.528451+0000","last_undegraded":"2023-09-23T23:10:47.528451+0000","last_fullsized":"2023-09-23T23:10:47.528451+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:40.126645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":374,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.494010+0000","last_change":"2023-09-23T23:10:45.541131+0000","last_active":"2023-09-23T23:10:50.494010+0000","last_peered":"2023-09-23T23:10:50.494010+0000","last_clean":"2023-09-23T23:10:50.494010+0000","last_became_active":"2023-09-23T23:10:45.540853+0000","last_became_peered":"2023-09-23T23:10:45.540853+0000","last_unstale":"2023-09-23T23:10:50.494010+0000","last_undegraded":"2023-09-23T23:10:50.494010+0000","last_fullsized":"2023-09-23T23:10:50.494010+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:20.977976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"501'56","reported_seq":515,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.551776+0000","last_change":"2023-09-23T23:10:45.524501+0000","last_active":"2023-09-23T23:10:50.551776+0000","last_peered":"2023-09-23T23:10:50.551776+0000","last_clean":"2023-09-23T23:10:50.551776+0000","last_became_active":"2023-09-23T23:05:39.898506+0000","last_became_peered":"2023-09-23T23:05:39.898506+0000","last_unstale":"2023-09-23T23:10:50.551776+0000","last_undegraded":"2023-09-23T23:10:50.551776+0000","last_fullsized":"2023-09-23T23:10:50.551776+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":56,"log_dups_size":0,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225315,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":236,"num_read_kb":582,"num_write":53,"num_write_kb":241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":362,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.555412+0000","last_change":"2023-09-23T23:05:53.585947+0000","last_active":"2023-09-23T23:10:51.555412+0000","last_peered":"2023-09-23T23:10:51.555412+0000","last_clean":"2023-09-23T23:10:51.555412+0000","last_became_active":"2023-09-23T23:05:53.585613+0000","last_became_peered":"2023-09-23T23:05:53.585613+0000","last_unstale":"2023-09-23T23:10:51.555412+0000","last_undegraded":"2023-09-23T23:10:51.555412+0000","last_fullsized":"2023-09-23T23:10:51.555412+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:26.351806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":336,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.493544+0000","last_change":"2023-09-23T23:10:45.541261+0000","last_active":"2023-09-23T23:10:50.493544+0000","last_peered":"2023-09-23T23:10:50.493544+0000","last_clean":"2023-09-23T23:10:50.493544+0000","last_became_active":"2023-09-23T23:10:45.540849+0000","last_became_peered":"2023-09-23T23:10:45.540849+0000","last_unstale":"2023-09-23T23:10:50.493544+0000","last_undegraded":"2023-09-23T23:10:50.493544+0000","last_fullsized":"2023-09-23T23:10:50.493544+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:39.827407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":2,"reported_epoch":504,"state":"peering","last_fresh":"2023-09-23T23:05:53.554139+0000","last_change":"2023-09-23T23:05:52.561886+0000","last_active":"2023-09-23T23:05:52.561713+0000","last_peered":"2023-09-23T23:05:43.899329+0000","last_clean":"2023-09-23T23:05:43.899329+0000","last_became_active":"2023-09-23T23:05:39.891527+0000","last_became_peered":"2023-09-23T23:05:39.891527+0000","last_unstale":"2023-09-23T23:05:53.554139+0000","last_undegraded":"2023-09-23T23:05:53.554139+0000","last_fullsized":"2023-09-23T23:05:53.554139+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:59.435442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"501'26","reported_seq":281,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.593063+0000","last_change":"2023-09-23T23:10:45.524655+0000","last_active":"2023-09-23T23:10:50.593063+0000","last_peered":"2023-09-23T23:10:50.593063+0000","last_clean":"2023-09-23T23:10:50.593063+0000","last_became_active":"2023-09-23T23:06:08.602199+0000","last_became_peered":"2023-09-23T23:06:08.602199+0000","last_unstale":"2023-09-23T23:10:50.593063+0000","last_undegraded":"2023-09-23T23:10:50.593063+0000","last_fullsized":"2023-09-23T23:10:50.593063+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":71,"num_read_kb":401,"num_write":14,"num_write_kb":197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"4","length":"1"}]},{"pgid":"4.0","version":"0'0","reported_seq":208,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528602+0000","last_change":"2023-09-23T23:05:53.608686+0000","last_active":"2023-09-23T23:10:47.528602+0000","last_peered":"2023-09-23T23:10:47.528602+0000","last_clean":"2023-09-23T23:10:47.528602+0000","last_became_active":"2023-09-23T23:05:53.608233+0000","last_became_peered":"2023-09-23T23:05:53.608233+0000","last_unstale":"2023-09-23T23:10:47.528602+0000","last_undegraded":"2023-09-23T23:10:47.528602+0000","last_fullsized":"2023-09-23T23:10:47.528602+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:10.750639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"499'24","reported_seq":503,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.556072+0000","last_change":"2023-09-23T23:10:51.556258+0000","last_active":"2023-09-23T23:10:51.556072+0000","last_peered":"2023-09-23T23:10:51.556072+0000","last_clean":"2023-09-23T23:10:51.556072+0000","last_became_active":"2023-09-23T23:06:08.613516+0000","last_became_peered":"2023-09-23T23:06:08.613516+0000","last_unstale":"2023-09-23T23:10:51.556072+0000","last_undegraded":"2023-09-23T23:10:51.556072+0000","last_fullsized":"2023-09-23T23:10:51.556072+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":24,"log_dups_size":0,"ondisk_log_size":24,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:58.539319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057927399999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":51,"num_read_kb":46,"num_write":27,"num_write_kb":28,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"1"},{"start":"8","length":"1"}]},{"pgid":"3.6","version":"60'1","reported_seq":124,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246321+0000","last_change":"2023-09-23T23:05:53.806935+0000","last_active":"2023-09-23T23:10:50.246321+0000","last_peered":"2023-09-23T23:10:50.246321+0000","last_clean":"2023-09-23T23:10:50.246321+0000","last_became_active":"2023-09-23T23:05:53.576403+0000","last_became_peered":"2023-09-23T23:05:53.576403+0000","last_unstale":"2023-09-23T23:10:50.246321+0000","last_undegraded":"2023-09-23T23:10:50.246321+0000","last_fullsized":"2023-09-23T23:10:50.246321+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":495,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246240+0000","last_change":"2023-09-23T23:05:53.565413+0000","last_active":"2023-09-23T23:10:50.246240+0000","last_peered":"2023-09-23T23:10:50.246240+0000","last_clean":"2023-09-23T23:10:50.246240+0000","last_became_active":"2023-09-23T23:05:53.565119+0000","last_became_peered":"2023-09-23T23:05:53.565119+0000","last_unstale":"2023-09-23T23:10:50.246240+0000","last_undegraded":"2023-09-23T23:10:50.246240+0000","last_fullsized":"2023-09-23T23:10:50.246240+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":94,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246673+0000","last_change":"2023-09-23T23:08:18.918166+0000","last_active":"2023-09-23T23:10:50.246673+0000","last_peered":"2023-09-23T23:10:50.246673+0000","last_clean":"2023-09-23T23:10:50.246673+0000","last_became_active":"2023-09-23T23:08:18.917816+0000","last_became_peered":"2023-09-23T23:08:18.917816+0000","last_unstale":"2023-09-23T23:10:50.246673+0000","last_undegraded":"2023-09-23T23:10:50.246673+0000","last_fullsized":"2023-09-23T23:10:50.246673+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":256,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-09-23T23:10:51.555425+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:10:51.555425+0000","last_peered":"2023-09-23T23:10:51.555425+0000","last_clean":"2023-09-23T23:10:51.555425+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:10:51.555425+0000","last_undegraded":"2023-09-23T23:10:51.555425+0000","last_fullsized":"2023-09-23T23:10:51.555425+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":325,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246291+0000","last_change":"2023-09-23T23:05:53.851074+0000","last_active":"2023-09-23T23:10:50.246291+0000","last_peered":"2023-09-23T23:10:50.246291+0000","last_clean":"2023-09-23T23:10:50.246291+0000","last_became_active":"2023-09-23T23:05:53.582566+0000","last_became_peered":"2023-09-23T23:05:53.582566+0000","last_unstale":"2023-09-23T23:10:50.246291+0000","last_undegraded":"2023-09-23T23:10:50.246291+0000","last_fullsized":"2023-09-23T23:10:50.246291+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":371,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528882+0000","last_change":"2023-09-23T23:10:45.529542+0000","last_active":"2023-09-23T23:10:47.528882+0000","last_peered":"2023-09-23T23:10:47.528882+0000","last_clean":"2023-09-23T23:10:47.528882+0000","last_became_active":"2023-09-23T23:10:45.529211+0000","last_became_peered":"2023-09-23T23:10:45.529211+0000","last_unstale":"2023-09-23T23:10:47.528882+0000","last_undegraded":"2023-09-23T23:10:47.528882+0000","last_fullsized":"2023-09-23T23:10:47.528882+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:09.287045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":352,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246315+0000","last_change":"2023-09-23T23:05:53.676944+0000","last_active":"2023-09-23T23:10:50.246315+0000","last_peered":"2023-09-23T23:10:50.246315+0000","last_clean":"2023-09-23T23:10:50.246315+0000","last_became_active":"2023-09-23T23:05:53.576150+0000","last_became_peered":"2023-09-23T23:05:53.576150+0000","last_unstale":"2023-09-23T23:10:50.246315+0000","last_undegraded":"2023-09-23T23:10:50.246315+0000","last_fullsized":"2023-09-23T23:10:50.246315+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":357,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528600+0000","last_change":"2023-09-23T23:10:45.539287+0000","last_active":"2023-09-23T23:10:47.528600+0000","last_peered":"2023-09-23T23:10:47.528600+0000","last_clean":"2023-09-23T23:10:47.528600+0000","last_became_active":"2023-09-23T23:10:45.538930+0000","last_became_peered":"2023-09-23T23:10:45.538930+0000","last_unstale":"2023-09-23T23:10:47.528600+0000","last_undegraded":"2023-09-23T23:10:47.528600+0000","last_fullsized":"2023-09-23T23:10:47.528600+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:33.089576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":77,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.529181+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:10:47.529181+0000","last_peered":"2023-09-23T23:10:47.529181+0000","last_clean":"2023-09-23T23:10:47.529181+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:10:47.529181+0000","last_undegraded":"2023-09-23T23:10:47.529181+0000","last_fullsized":"2023-09-23T23:10:47.529181+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":219,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T23:10:47.528679+0000","last_change":"2023-09-23T23:05:53.573298+0000","last_active":"2023-09-23T23:10:47.528679+0000","last_peered":"2023-09-23T23:10:47.528679+0000","last_clean":"2023-09-23T23:10:47.528679+0000","last_became_active":"2023-09-23T23:05:53.572803+0000","last_became_peered":"2023-09-23T23:05:53.572803+0000","last_unstale":"2023-09-23T23:10:47.528679+0000","last_undegraded":"2023-09-23T23:10:47.528679+0000","last_fullsized":"2023-09-23T23:10:47.528679+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:45.747487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":273,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.494610+0000","last_change":"2023-09-23T23:05:53.602471+0000","last_active":"2023-09-23T23:10:50.494610+0000","last_peered":"2023-09-23T23:10:50.494610+0000","last_clean":"2023-09-23T23:10:50.494610+0000","last_became_active":"2023-09-23T23:05:53.565298+0000","last_became_peered":"2023-09-23T23:05:53.565298+0000","last_unstale":"2023-09-23T23:10:50.494610+0000","last_undegraded":"2023-09-23T23:10:50.494610+0000","last_fullsized":"2023-09-23T23:10:50.494610+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:17.849441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":497,"reported_epoch":502,"state":"active+clean","last_fresh":"2023-09-23T23:10:50.246649+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:10:50.246649+0000","last_peered":"2023-09-23T23:10:50.246649+0000","last_clean":"2023-09-23T23:10:50.246649+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:10:50.246649+0000","last_undegraded":"2023-09-23T23:10:50.246649+0000","last_fullsized":"2023-09-23T23:10:50.246649+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":20928,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":225506,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":3524,"num_read_kb":4967,"num_write":493,"num_write_kb":1301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":61943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":565,"ondisk_log_size":565,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430734,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626959,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912292,"kb_used_data":5468,"kb_used_omap":0,"kb_used_meta":906816,"kb_avail":92832860,"statfs":{"total":95995035648,"available":95060848640,"internally_reserved":0,"allocated":5599232,"data_stored":4463618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928579584},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:10:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.1020000000000001,"15min":4.29},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.9630000000000001,"5min":10.853,"15min":115.32899999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.89200000000000002,"5min":1.2030000000000001,"15min":4.3140000000000001},"min":{"1min":0.35999999999999999,"5min":0.25,"15min":0.25},"max":{"1min":3.5179999999999998,"5min":8.7010000000000005,"15min":115.348},"last":0.58699999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:10:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":1.9139999999999999,"15min":2.73},"min":{"1min":0.32800000000000001,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":8.1280000000000001,"5min":38.259999999999998,"15min":149.947},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.8759999999999999,"15min":2.6520000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":7.1639999999999997,"5min":38.530000000000001,"15min":150.22499999999999},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 23 23:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":5.508,"15min":2.9369999999999998},"min":{"1min":0.29499999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.2949999999999999,"5min":138.43299999999999,"15min":138.43299999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.423,"5min":5.4909999999999997,"15min":3.081},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.3629999999999995,"5min":138.45599999999999,"15min":138.45599999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:10:19 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":3.4580000000000002,"15min":3.3610000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.26800000000000002},"max":{"1min":14.426,"5min":70.521000000000001,"15min":142.24700000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":2.5920000000000001,"5min":3.359,"15min":3.1899999999999999},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":14.955,"5min":70.497,"15min":143.86699999999999},"last":0.746}]},{"osd":4,"last update":"Sat Sep 23 23:10:19 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.373,"15min":1.8240000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23999999999999999},"max":{"1min":5.9790000000000001,"5min":12.407999999999999,"15min":119.804},"last":0.71699999999999997},{"interface":"front","average":{"1min":3.3359999999999999,"5min":1.877,"15min":2.0550000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.24099999999999999},"max":{"1min":46.515999999999998,"5min":46.515999999999998,"15min":122.31999999999999},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:10:19 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.5029999999999999,"15min":2.4910000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":5.9450000000000003,"5min":11.862,"15min":186.27600000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.5549999999999999,"15min":2.613},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.29599999999999999},"max":{"1min":2.8660000000000001,"5min":10.254,"15min":191.28700000000001},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 23:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.4059999999999999,"15min":1.494},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":0.81699999999999995,"5min":8.8670000000000009,"15min":115.61},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.3080000000000001,"15min":1.6439999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.29999999999999999},"max":{"1min":0.77100000000000002,"5min":8.2609999999999992,"15min":115.578},"last":0.60999999999999999}]}]},{"osd":1,"up_from":84,"seq":360777253272,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1192280,"kb_used_data":186572,"kb_used_omap":9,"kb_used_meta":1005686,"kb_avail":92552872,"statfs":{"total":95995035648,"available":94774140928,"internally_reserved":0,"allocated":191049728,"data_stored":188953849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1029823465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.83599999999999997,"15min":2.577},"min":{"1min":0.218,"5min":0.20899999999999999,"15min":0.20000000000000001},"max":{"1min":3.891,"5min":27.271999999999998,"15min":279.30500000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.032,"15min":2.8940000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":7.6890000000000001,"5min":24.844000000000001,"15min":279.39800000000002},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 23:10:42 2023","interfaces":[{"interface":"back","average":{"1min":3.6499999999999999,"5min":3.6339999999999999,"15min":2.931},"min":{"1min":0.22600000000000001,"5min":0.217,"15min":0.217},"max":{"1min":72.251999999999995,"5min":121.527,"15min":161.61699999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":3.8460000000000001,"5min":3.641,"15min":3.1160000000000001},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":75.563000000000002,"5min":118.29900000000001,"15min":162.61699999999999},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":7.2039999999999997,"5min":2.2789999999999999,"15min":1.7270000000000001},"min":{"1min":0.29699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":67.646000000000001,"5min":67.646000000000001,"15min":67.646000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":7.1609999999999996,"5min":2.5099999999999998,"15min":1.7809999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19500000000000001},"max":{"1min":65.269999999999996,"5min":65.269999999999996,"15min":65.269999999999996},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":3.0979999999999999,"15min":2.5670000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.274,"5min":129.69,"15min":151.036},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.5129999999999999,"5min":3.3799999999999999,"15min":2.859},"min":{"1min":0.318,"5min":0.309,"15min":0.26100000000000001},"max":{"1min":6.1230000000000002,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.58299999999999996}]},{"osd":5,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.2869999999999999,"15min":2.153},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":3.2719999999999998,"5min":18.228999999999999,"15min":116.098},"last":0.499},{"interface":"front","average":{"1min":1.353,"5min":1.1890000000000001,"15min":2.008},"min":{"1min":0.375,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":5.7779999999999996,"5min":6.585,"15min":116.08499999999999},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.194,"15min":2.0369999999999999},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":6.1740000000000004,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.177,"5min":2.282,"15min":1.9159999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":4.0759999999999996,"5min":64.239999999999995,"15min":67.366},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.3779999999999999,"15min":1.117},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.34699999999999998},"max":{"1min":4.0890000000000004,"5min":7.0960000000000001,"15min":7.0960000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.595,"5min":1.6819999999999999,"15min":1.4339999999999999},"min":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.33100000000000002},"max":{"1min":6.2389999999999999,"5min":9.016,"15min":20.902999999999999},"last":0.51300000000000001}]}]},{"osd":0,"up_from":0,"seq":1516123455642,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":359,"seq":1541893259415,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328396,"kb_used_data":6660,"kb_used_omap":83,"kb_used_meta":321644,"kb_avail":93416756,"statfs":{"total":95995035648,"available":95658758144,"internally_reserved":0,"allocated":6819840,"data_stored":4601214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":329363909},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.82399999999999995,"15min":1.238},"min":{"1min":0.22800000000000001,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":3.8039999999999998,"5min":10.199999999999999,"15min":74.241},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.3720000000000001,"15min":1.609},"min":{"1min":0.25700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":10.973000000000001,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.60999999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":2.7589999999999999,"15min":1.4390000000000001},"min":{"1min":0.308,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":16.995999999999999,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.964,"5min":2.2240000000000002,"15min":1.516},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.3730000000000002,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.59499999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":5.7439999999999998,"5min":3.7029999999999998,"15min":2.1739999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.23000000000000001},"max":{"1min":55.255000000000003,"5min":79.655000000000001,"15min":110.31699999999999},"last":0.751},{"interface":"front","average":{"1min":5.383,"5min":3.427,"15min":2.1549999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":61.197000000000003,"5min":79.677999999999997,"15min":110.56999999999999},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":1.5820000000000001,"15min":1.1519999999999999},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":2.7290000000000001,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.0710000000000002,"15min":1.4119999999999999},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":8.1099999999999994,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.71999999999999997}]},{"osd":5,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.4079999999999999,"15min":1.7110000000000001},"min":{"1min":0.35799999999999998,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":7.0460000000000003,"5min":9.8019999999999996,"15min":139.405},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.105,"5min":1.22,"15min":1.538},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":5.5880000000000001,"5min":10.109999999999999,"15min":139.42099999999999},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":3.5539999999999998,"15min":1.917},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.0179999999999998,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.627},{"interface":"front","average":{"1min":1.1639999999999999,"5min":3.2269999999999999,"15min":1.611},"min":{"1min":0.434,"5min":0.42399999999999999,"15min":0.32200000000000001},"max":{"1min":3.3239999999999998,"5min":108.496,"15min":108.496},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:09:57 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.548,"15min":1.2270000000000001},"min":{"1min":0.48699999999999999,"5min":0.40300000000000002,"15min":0.36699999999999999},"max":{"1min":6.0899999999999999,"5min":10.164,"15min":10.164},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.4330000000000001,"15min":1.214},"min":{"1min":0.42999999999999999,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":8.3179999999999996,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.78100000000000003}]}]},{"osd":3,"up_from":94,"seq":403726926219,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807308,"kb_used_data":86980,"kb_used_omap":7,"kb_used_meta":720312,"kb_avail":92937844,"statfs":{"total":95995035648,"available":95168352256,"internally_reserved":0,"allocated":89067520,"data_stored":86879421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":737600104},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:10:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.0449999999999999,"15min":1.5069999999999999},"min":{"1min":0.25800000000000001,"5min":0.20399999999999999,"15min":0.17299999999999999},"max":{"1min":6.1449999999999996,"5min":17.015000000000001,"15min":122.289},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.88900000000000001,"15min":1.2170000000000001},"min":{"1min":0.27700000000000002,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":8.1519999999999992,"5min":11.382999999999999,"15min":122.041},"last":0.254}]},{"osd":1,"last update":"Sat Sep 23 23:10:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.0449999999999999,"15min":1.4790000000000001},"min":{"1min":0.29599999999999999,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":8.3800000000000008,"5min":23.969000000000001,"15min":23.969000000000001},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.3839999999999999,"15min":1.3420000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.20699999999999999},"max":{"1min":5.734,"5min":16.986000000000001,"15min":16.986000000000001},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.5030000000000001,"15min":1.3340000000000001},"min":{"1min":0.27700000000000002,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":16.149000000000001,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.308},{"interface":"front","average":{"1min":2.8639999999999999,"5min":2.875,"15min":1.6799999999999999},"min":{"1min":0.41299999999999998,"5min":0.251,"15min":0.215},"max":{"1min":11.375999999999999,"5min":57.308,"15min":57.308},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 23:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":2.5550000000000002,"15min":2.0649999999999999},"min":{"1min":0.44,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":5.5759999999999996,"5min":115.69,"15min":115.69},"last":0.62},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.1859999999999999,"15min":2.0230000000000001},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":5.6680000000000001,"5min":115.617,"15min":115.617},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.4930000000000001,"15min":1.383},"min":{"1min":0.44900000000000001,"5min":0.35299999999999998,"15min":0.28499999999999998},"max":{"1min":5.6399999999999997,"5min":16.959,"15min":16.959},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.3500000000000001,"15min":1.2689999999999999},"min":{"1min":0.51000000000000001,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":5.048,"5min":10.545,"15min":10.545},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":2.3639999999999999,"15min":1.8109999999999999},"min":{"1min":0.443,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":5.5979999999999999,"5min":77.707999999999998,"15min":77.707999999999998},"last":0.752},{"interface":"front","average":{"1min":1.2969999999999999,"5min":2.1000000000000001,"15min":1.575},"min":{"1min":0.46600000000000003,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":5.9139999999999997,"5min":77.721999999999994,"15min":77.721999999999994},"last":0.71999999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:10:12 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.369,"15min":1.175},"min":{"1min":0.55300000000000005,"5min":0.35099999999999998,"15min":0.33900000000000002},"max":{"1min":5.0220000000000002,"5min":8.1620000000000008,"15min":8.1620000000000008},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.9259999999999999,"15min":1.464},"min":{"1min":0.439,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":5.6189999999999998,"5min":11.051,"15min":11.051},"last":0.68600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019416,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912320,"kb_used_data":4856,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92832832,"statfs":{"total":95995035648,"available":95060819968,"internally_reserved":0,"allocated":4972544,"data_stored":3850321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:10:27 2023","interfaces":[{"interface":"back","average":{"1min":16.251999999999999,"5min":4.3109999999999999,"15min":3.4100000000000001},"min":{"1min":0.317,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":323.65800000000002,"5min":323.65800000000002,"15min":323.65800000000002},"last":0.435},{"interface":"front","average":{"1min":16.722999999999999,"5min":4.3129999999999997,"15min":3.5030000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.24399999999999999},"max":{"1min":323.642,"5min":323.642,"15min":323.642},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 23:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.276,"15min":2.7349999999999999},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":3.1160000000000001,"5min":6.915,"15min":98.846999999999994},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.056,"5min":1.403,"15min":2.7269999999999999},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.23699999999999999},"max":{"1min":6.0999999999999996,"5min":15.728,"15min":98.938999999999993},"last":0.5}]},{"osd":2,"last update":"Sat Sep 23 23:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":4.8010000000000002,"15min":2.6920000000000002},"min":{"1min":0.36499999999999999,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":1.129,"5min":151.20400000000001,"15min":151.20400000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":4.7080000000000002,"15min":2.552},"min":{"1min":0.374,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":1.3640000000000001,"5min":148.852,"15min":148.852},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":4.0860000000000003,"15min":2.4279999999999999},"min":{"1min":0.39300000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":6.1980000000000004,"5min":147.82499999999999,"15min":147.82499999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.6970000000000001,"5min":3.9620000000000002,"15min":2.4540000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":8.3810000000000002,"5min":147.84100000000001,"15min":147.84100000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 23:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.399,"15min":1.77},"min":{"1min":0.28599999999999998,"5min":0.28499999999999998,"15min":0.214},"max":{"1min":0.73199999999999998,"5min":10.282,"15min":148.61600000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.591,"15min":1.931},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":0.63700000000000001,"5min":17.02,"15min":151.905},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.6299999999999999,"15min":1.4179999999999999},"min":{"1min":0.35499999999999998,"5min":0.32500000000000001,"15min":0.30399999999999999},"max":{"1min":7.8390000000000004,"5min":35.896000000000001,"15min":35.896000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.84899999999999998,"5min":1.8080000000000001,"15min":1.3520000000000001},"min":{"1min":0.35199999999999998,"5min":0.32600000000000001,"15min":0.27800000000000002},"max":{"1min":3.165,"5min":35.908000000000001,"15min":35.908000000000001},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 23:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.494,"15min":1.349},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.28399999999999997},"max":{"1min":0.78500000000000003,"5min":11.096,"15min":12.84},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.419,"15min":1.2470000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.29799999999999999},"max":{"1min":0.85299999999999998,"5min":15.808,"15min":15.808},"last":0.60299999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823190,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":20601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":51,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":20566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":20620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:10:55.102 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-09-23T23:10:55.103 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:55.146 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:55.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:55.250 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:55.279 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:55.324 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:55.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:55.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:55.331 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:10:55.331 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [], 2023-09-23T23:10:55.331 INFO:teuthology.orchestra.run.smithi151.stdout: "complaint_time": 30, 2023-09-23T23:10:55.332 INFO:teuthology.orchestra.run.smithi151.stdout: "num_blocked_ops": 0 2023-09-23T23:10:55.332 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:10:55.427 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:55.462 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:55.562 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:55.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:55.615 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_701000/sparse1 2023-09-23T23:10:55.621 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records in 2023-09-23T23:10:55.622 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records out 2023-09-23T23:10:55.622 INFO:tasks.workunit.client.0.smithi052.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00504553 s, 208 MB/s 2023-09-23T23:10:55.622 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_701000/sparse2 2023-09-23T23:10:55.627 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records in 2023-09-23T23:10:55.627 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records out 2023-09-23T23:10:55.627 INFO:tasks.workunit.client.0.smithi052.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.004982 s, 210 MB/s 2023-09-23T23:10:55.628 INFO:tasks.workunit.client.0.smithi052.stderr:+ truncate /tmp/rbd_import_export_701000/sparse2 -s 2M 2023-09-23T23:10:55.636 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse1 2023-09-23T23:10:55.668 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T23:10:55.677 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 0% complete...failed. 2023-09-23T23:10:55.677 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T23:10:55.680 INFO:tasks.workunit.client.0.smithi052.stderr:+ true 2023-09-23T23:10:55.680 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_701000/sparse1 2023-09-23T23:10:55.695 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:55.740 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T23:10:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:55.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T23:10:55.743 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T23:10:55.743 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:55.743 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:10:55.746 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd ls -l 2023-09-23T23:10:55.746 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep sparse1 2023-09-23T23:10:55.746 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T23:10:55.781 INFO:tasks.workunit.client.0.smithi052.stdout:sparse1 2 MiB 2 2023-09-23T23:10:55.782 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse1 2023-09-23T23:10:55.782 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse1 2023-09-23T23:10:55.783 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse1 2023-09-23T23:10:55.783 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:55.783 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:55.819 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.9399f726d421 2023-09-23T23:10:55.820 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.9399f726d421 2023-09-23T23:10:55.820 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse1 2023-09-23T23:10:55.820 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse1 2023-09-23T23:10:55.820 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.9399f726d421\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:55.820 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:55.821 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse1 2023-09-23T23:10:55.821 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:55.821 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:55.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: osdmap e507: 8 total, 8 up, 5 in 2023-09-23T23:10:55.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.831 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: pgmap v1578: 73 pgs: 1 peering, 72 active+clean; 802 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 108 KiB/s rd, 7.5 KiB/s wr, 144 op/s 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/669730928' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[187743]: osdmap e508: 8 total, 8 up, 5 in 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: osdmap e507: 8 total, 8 up, 5 in 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: pgmap v1578: 73 pgs: 1 peering, 72 active+clean; 802 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 108 KiB/s rd, 7.5 KiB/s wr, 144 op/s 2023-09-23T23:10:55.832 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/669730928' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:10:55.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:55 smithi052 ceph-mon[180979]: osdmap e508: 8 total, 8 up, 5 in 2023-09-23T23:10:55.843 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:55.856 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:55.856 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:55.856 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:55.857 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:55.857 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: osdmap e507: 8 total, 8 up, 5 in 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: failed to encode map e507 with expected crc 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: pgmap v1578: 73 pgs: 1 peering, 72 active+clean; 802 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 108 KiB/s rd, 7.5 KiB/s wr, 144 op/s 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/669730928' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:10:55.867 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:55 smithi151 ceph-mon[159210]: osdmap e508: 8 total, 8 up, 5 in 2023-09-23T23:10:55.869 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:55.887 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects=1 2023-09-23T23:10:55.887 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 1 2023-09-23T23:10:55.887 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T23:10:55.888 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export sparse1 /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:55.922 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:55.925 INFO:tasks.workunit.client.0.smithi052.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_701000/sparse1 /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:55.925 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp -l /tmp/rbd_import_export_701000/sparse1 /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:55.928 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse1 --format %b 2023-09-23T23:10:55.928 INFO:tasks.workunit.client.0.smithi052.stderr:+ origsize=2048 2023-09-23T23:10:55.929 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse1.out --format %b 2023-09-23T23:10:55.930 INFO:tasks.workunit.client.0.smithi052.stderr:+ exportsize=2048 2023-09-23T23:10:55.930 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:55.930 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:55.930 INFO:tasks.workunit.client.0.smithi052.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:10:55.930 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:55.930 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse1 2023-09-23T23:10:56.043 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:56.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:56.094 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T23:10:56.124 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T23:10:56.127 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse2 2023-09-23T23:10:56.146 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:56.166 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 0% complete...failed. 2023-09-23T23:10:56.166 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T23:10:56.171 INFO:tasks.workunit.client.0.smithi052.stderr:+ true 2023-09-23T23:10:56.171 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_701000/sparse2 2023-09-23T23:10:56.177 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:56.239 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T23:10:56.240 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:56.240 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:10:56.242 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd ls -l 2023-09-23T23:10:56.242 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep sparse2 2023-09-23T23:10:56.243 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T23:10:56.278 INFO:tasks.workunit.client.0.smithi052.stdout:sparse2 2 MiB 2 2023-09-23T23:10:56.279 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse2 2023-09-23T23:10:56.279 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse2 2023-09-23T23:10:56.279 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse2 2023-09-23T23:10:56.279 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:56.279 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:56.314 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.bb5b3a4005a5 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.bb5b3a4005a5 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse2 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.bb5b3a4005a5\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse2 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse2 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:56.316 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:56.352 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:56.352 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:56.352 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:56.352 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:56.352 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:56.383 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects=0 2023-09-23T23:10:56.384 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 0 2023-09-23T23:10:56.384 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T23:10:56.384 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export sparse2 /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:56.421 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:56.425 INFO:tasks.workunit.client.0.smithi052.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_701000/sparse2 /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:56.425 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp -l /tmp/rbd_import_export_701000/sparse2 /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:56.427 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse2 --format %b 2023-09-23T23:10:56.428 INFO:tasks.workunit.client.0.smithi052.stderr:+ origsize=2048 2023-09-23T23:10:56.428 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse2.out --format %b 2023-09-23T23:10:56.429 INFO:tasks.workunit.client.0.smithi052.stderr:+ exportsize=2048 2023-09-23T23:10:56.429 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:56.429 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:56.430 INFO:tasks.workunit.client.0.smithi052.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:10:56.430 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:56.430 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse2 2023-09-23T23:10:56.445 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:10:56.462 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:56.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:56.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:56.510 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:56.524 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T23:10:56.527 INFO:tasks.workunit.client.0.smithi052.stderr:+ truncate /tmp/rbd_import_export_701000/sparse1 -s 10M 2023-09-23T23:10:56.528 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T23:10:56.557 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T23:10:56.567 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T23:10:56.594 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:56.642 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T23:10:56.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:56.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T23:10:56.649 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:56.650 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:56.650 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:10:56.653 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd ls -l 2023-09-23T23:10:56.653 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep sparse1 2023-09-23T23:10:56.653 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T23:10:56.692 INFO:tasks.workunit.client.0.smithi052.stdout:sparse1 10 MiB 2 2023-09-23T23:10:56.693 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse1 2023-09-23T23:10:56.693 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse1 2023-09-23T23:10:56.694 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse1 2023-09-23T23:10:56.694 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:56.694 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[187743]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[187743]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[187743]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/669730928' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[187743]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[180979]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[180979]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.734 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[180979]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/669730928' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:10:56.735 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:56 smithi052 ceph-mon[180979]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.735 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.93f97343d777 2023-09-23T23:10:56.735 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.93f97343d777 2023-09-23T23:10:56.736 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse1 2023-09-23T23:10:56.736 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse1 2023-09-23T23:10:56.736 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:56.736 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse1 2023-09-23T23:10:56.737 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:56.738 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:56.738 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.93f97343d777\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:56.747 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T23:10:56.785 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:56.785 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:56.785 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:56.785 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:56.785 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:56.787 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:56.823 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects=1 2023-09-23T23:10:56.823 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 1 2023-09-23T23:10:56.823 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T23:10:56.823 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export sparse1 /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:56.864 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:56.867 INFO:tasks.workunit.client.0.smithi052.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_701000/sparse1 /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:56.867 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp -l /tmp/rbd_import_export_701000/sparse1 /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:56.876 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse1 --format %b 2023-09-23T23:10:56.877 INFO:tasks.workunit.client.0.smithi052.stderr:+ origsize=2048 2023-09-23T23:10:56.878 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse1.out --format %b 2023-09-23T23:10:56.879 INFO:tasks.workunit.client.0.smithi052.stderr:+ exportsize=2048 2023-09-23T23:10:56.879 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:56.879 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:56.879 INFO:tasks.workunit.client.0.smithi052.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:10:56.879 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/sparse1.out 2023-09-23T23:10:56.880 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse1 2023-09-23T23:10:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:56 smithi151 ceph-mon[159210]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:56 smithi151 ceph-mon[159210]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:56 smithi151 ceph-mon[159210]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:56 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/669730928' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T23:10:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:56 smithi151 ceph-mon[159210]: failed to encode map e508 with expected crc 2023-09-23T23:10:56.984 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T23:10:56.987 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_701000/sparse2 oflag=append conv=notrunc 2023-09-23T23:10:56.996 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T23:10:57.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:57.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T23:10:57.001 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records in 2023-09-23T23:10:57.001 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records out 2023-09-23T23:10:57.001 INFO:tasks.workunit.client.0.smithi052.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0121777 s, 172 MB/s 2023-09-23T23:10:57.001 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T23:10:57.083 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:57.083 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:57.083 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:10:57.086 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd ls -l 2023-09-23T23:10:57.086 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep sparse2 2023-09-23T23:10:57.087 INFO:tasks.workunit.client.0.smithi052.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T23:10:57.102 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:57.123 INFO:tasks.workunit.client.0.smithi052.stdout:sparse2 4 MiB 2 2023-09-23T23:10:57.123 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse2 2023-09-23T23:10:57.124 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse2 2023-09-23T23:10:57.124 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse2 2023-09-23T23:10:57.124 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:57.124 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:57.130 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:57.162 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.9426d8be77c9 2023-09-23T23:10:57.163 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.9426d8be77c9 2023-09-23T23:10:57.163 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse2 2023-09-23T23:10:57.163 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse2 2023-09-23T23:10:57.163 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.9426d8be77c9\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:57.164 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:57.164 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse2 2023-09-23T23:10:57.164 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:57.164 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:57.175 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:57.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:57.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:57.203 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:57.203 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:57.203 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:57.203 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:57.203 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:57.240 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects='0 2023-09-23T23:10:57.240 INFO:tasks.workunit.client.0.smithi052.stderr:2 2023-09-23T23:10:57.240 INFO:tasks.workunit.client.0.smithi052.stderr:3' 2023-09-23T23:10:57.240 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 0 2 3 2023-09-23T23:10:57.241 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T23:10:57.241 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export sparse2 /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:57.278 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:57.281 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T23:10:57.285 INFO:tasks.workunit.client.0.smithi052.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_701000/sparse2 /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:57.285 INFO:tasks.workunit.client.0.smithi052.stderr:+ cmp -l /tmp/rbd_import_export_701000/sparse2 /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:57.288 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse2 --format %b 2023-09-23T23:10:57.289 INFO:tasks.workunit.client.0.smithi052.stderr:+ origsize=6144 2023-09-23T23:10:57.289 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse2.out --format %b 2023-09-23T23:10:57.290 INFO:tasks.workunit.client.0.smithi052.stderr:+ exportsize=6144 2023-09-23T23:10:57.291 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:57.291 INFO:tasks.workunit.client.0.smithi052.stderr:+ difference=0 2023-09-23T23:10:57.291 INFO:tasks.workunit.client.0.smithi052.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T23:10:57.291 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/sparse2.out 2023-09-23T23:10:57.292 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse2 2023-09-23T23:10:57.311 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:57.366 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:57.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:57.391 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T23:10:57.394 INFO:tasks.workunit.client.0.smithi052.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T23:10:57.395 INFO:tasks.workunit.client.0.smithi052.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T23:10:57.395 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_701000/sparse1 sparse 2023-09-23T23:10:57.453 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T23:10:57.454 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:57.454 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T23:10:57.457 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse 2023-09-23T23:10:57.457 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse 2023-09-23T23:10:57.457 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse 2023-09-23T23:10:57.457 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:57.457 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:57.470 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:57.493 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.9453b143d293 2023-09-23T23:10:57.494 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.9453b143d293 2023-09-23T23:10:57.494 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse 2023-09-23T23:10:57.494 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse 2023-09-23T23:10:57.494 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.9453b143d293\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:57.495 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:57.495 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse 2023-09-23T23:10:57.495 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:57.495 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:57.497 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:57.531 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:57.531 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:57.531 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:57.531 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:57.531 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:57.547 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:57.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:57.562 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects=1 2023-09-23T23:10:57.562 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 1 2023-09-23T23:10:57.563 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T23:10:57.563 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse 2023-09-23T23:10:57.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:57 smithi052 ceph-mon[187743]: pgmap v1580: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 95 KiB/s rd, 6.3 KiB/s wr, 124 op/s 2023-09-23T23:10:57.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1790664792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:57.575 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:57 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:57.581 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:10:57.655 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:57.668 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T23:10:57.672 INFO:tasks.workunit.client.0.smithi052.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T23:10:57.672 INFO:tasks.workunit.client.0.smithi052.stdout:zeros import through stdin to sparse image 2023-09-23T23:10:57.673 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T23:10:57.673 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd import --new-format - sparse 2023-09-23T23:10:57.683 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:57.723 INFO:tasks.workunit.client.0.smithi052.stderr:4+0 records in 2023-09-23T23:10:57.723 INFO:tasks.workunit.client.0.smithi052.stderr:4+0 records out 2023-09-23T23:10:57.723 INFO:tasks.workunit.client.0.smithi052.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0493849 s, 84.9 MB/s 2023-09-23T23:10:57.732 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:57.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:57.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:57.737 INFO:tasks.workunit.client.0.smithi052.stderr: Importing image: 100% complete...done. 2023-09-23T23:10:57.737 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:57.740 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse 2023-09-23T23:10:57.740 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse 2023-09-23T23:10:57.740 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse 2023-09-23T23:10:57.741 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:57.741 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:57 smithi151 ceph-mon[159210]: pgmap v1580: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 95 KiB/s rd, 6.3 KiB/s wr, 124 op/s 2023-09-23T23:10:57.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1790664792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:57.747 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:57 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:57.776 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.947433f12f35 2023-09-23T23:10:57.777 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.947433f12f35 2023-09-23T23:10:57.777 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse 2023-09-23T23:10:57.777 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse 2023-09-23T23:10:57.778 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.947433f12f35\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:57.778 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:57.778 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse 2023-09-23T23:10:57.778 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:57.778 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:57.815 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:57.815 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:57.816 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:57.816 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:57.816 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:57.836 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T23:10:57.846 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects= 2023-09-23T23:10:57.846 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 2023-09-23T23:10:57.847 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T23:10:57.847 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse 2023-09-23T23:10:57.865 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:57.914 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T23:10:57.914 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:57 smithi052 ceph-mon[180979]: pgmap v1580: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 95 KiB/s rd, 6.3 KiB/s wr, 124 op/s 2023-09-23T23:10:57.915 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1790664792' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:57.915 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:57 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:10:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:57.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T23:10:57.937 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:57.940 INFO:tasks.workunit.client.0.smithi052.stdout:zeros export to sparse file 2023-09-23T23:10:57.941 INFO:tasks.workunit.client.0.smithi052.stderr:+ echo 'zeros export to sparse file' 2023-09-23T23:10:57.941 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout: "size": 20, 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 600, 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout: "ops": [ 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:17355 11.e 11:7857a061:::benchmark_data_smithi052_606809_object17354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e415)", 2023-09-23T23:10:57.944 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 415.33351743399999, 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15692616000000001, 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 17355 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.945 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607820+0000", 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.946 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607819+0000", 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3510999999999995e-05 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607892+0000", 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0942e-05 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607903+0000", 2023-09-23T23:10:57.947 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8921999999999999e-05 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607922+0000", 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.4350000000000001e-05 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.607956+0000", 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6907000000000001e-05 2023-09-23T23:10:57.948 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608033+0000", 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8525999999999999e-05 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 0,1", 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608122+0000", 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00055712700000000003 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.949 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.608679+0000", 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0049612479999999997 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.613640+0000", 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15104679400000001 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.950 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764687+0000", 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.5262999999999998e-05 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764722+0000", 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3643999999999999e-05 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:04:02.764746+0000", 2023-09-23T23:10:57.951 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15692616000000001 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:86860 11.e 11:75ab6736:::benchmark_data_smithi052_606809_object4706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 319.05802813299999, 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3618323549999998, 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.952 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 86860 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.953 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883309+0000", 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883307+0000", 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3599999999999996e-06 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.954 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883316+0000", 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7278e-05 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883333+0000", 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.2119999999999992e-06 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.883342+0000", 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.011255879999999999 2023-09-23T23:10:57.955 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.894598+0000", 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2694000000000001e-05 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:38.894611+0000", 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.99447565000000004 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.956 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889087+0000", 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2336e-05 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889099+0000", 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3388404170000001 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.227939+0000", 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017411099999999999 2023-09-23T23:10:57.957 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.228113+0000", 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00012214100000000001 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.228236+0000", 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.01031818 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.958 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238554+0000", 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.9231000000000002e-05 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238623+0000", 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039840279999999997 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.242607+0000", 2023-09-23T23:10:57.959 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5633999999999997e-05 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.242653+0000", 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024888699999999998 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.245141+0000", 2023-09-23T23:10:57.960 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3618323549999998 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:86874 11.e 11:77be3fa7:::benchmark_data_smithi052_606809_object4720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e432)", 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 318.91843756899999, 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.23339197, 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.961 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 86874 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.962 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022900+0000", 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999943 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022894+0000", 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5319e-05 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.963 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022909+0000", 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.3163000000000002e-05 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022932+0000", 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0350000000000007e-06 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.022941+0000", 2023-09-23T23:10:57.964 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.3570000000000002e-05 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.023015+0000", 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4547e-05 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.023030+0000", 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.86609229700000001 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.965 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889122+0000", 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8590000000000005e-06 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:39.889130+0000", 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3413657240000001 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230495+0000", 2023-09-23T23:10:57.966 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000186393 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230682+0000", 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000196875 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.230879+0000", 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0076860059999999996 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.967 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238565+0000", 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8522000000000004e-05 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.238653+0000", 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.013606922 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.968 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.252260+0000", 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.8609000000000006e-05 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.252339+0000", 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0039528990000000002 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:05:41.256292+0000", 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.23339197 2023-09-23T23:10:57.969 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:100250 11.e 11:7a9e86a3:::benchmark_data_smithi052_606809_object18096:head [delete] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:58.864702+0000", 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 239.076634993, 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20398459899999999, 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.970 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100250 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864702+0000", 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.971 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864702+0000", 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864700+0000", 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.9250000000000001e-06 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.972 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864709+0000", 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.3275000000000001e-05 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864722+0000", 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0099999999999995e-06 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864730+0000", 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.4678000000000002e-05 2023-09-23T23:10:57.973 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864775+0000", 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00010582999999999999 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.864881+0000", 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000143514 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.974 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.865024+0000", 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0081216210000000007 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873146+0000", 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1297999999999995e-05 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.975 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873207+0000", 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.193264456 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.066472+0000", 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.2669000000000001e-05 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.066514+0000", 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0021723770000000001 2023-09-23T23:10:57.976 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.068687+0000", 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.20398459899999999 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:100251 11.e 11:7f2f4774:::benchmark_data_smithi052_606809_object18097:head [delete] snapc 0=[] ondisk+write+known_if_redirected e449)", 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:06:58.873564+0000", 2023-09-23T23:10:57.977 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 239.06777348700001, 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19695320399999999, 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 100251 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.978 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873564+0000", 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873564+0000", 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999976 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873562+0000", 2023-09-23T23:10:57.979 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.8020999999999999e-05 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873580+0000", 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.7269e-05 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873597+0000", 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7150000000000004e-06 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.980 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873606+0000", 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.1041000000000002e-05 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873647+0000", 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000108952 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.981 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873756+0000", 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000120344 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.873876+0000", 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002459316 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.876335+0000", 2023-09-23T23:10:57.982 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00015686000000000001 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:58.876492+0000", 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19224075400000001 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.068733+0000", 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8087999999999998e-05 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.983 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.068781+0000", 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001736044 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:06:59.070517+0000", 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19695320399999999 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.984 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:105570 11.e 11:70bbfdb0:::benchmark_data_smithi052_606809_object23416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:19.238660+0000", 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 218.70267751200001, 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19364197, 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 105570 2023-09-23T23:10:57.985 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238660+0000", 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238660+0000", 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:57.986 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238658+0000", 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.9029999999999999e-06 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238668+0000", 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.912e-05 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.987 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238687+0000", 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8269999999999993e-06 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.238697+0000", 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0084551339999999996 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.247152+0000", 2023-09-23T23:10:57.988 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000156324 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.247308+0000", 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000194666 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.247503+0000", 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0061626679999999996 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.989 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.253665+0000", 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1190000000000003e-05 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.253717+0000", 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17570802499999999 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.990 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.429425+0000", 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8717e-05 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.429483+0000", 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028183470000000001 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.991 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.432302+0000", 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.19364197 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:105576 11.e 11:79dd3146:::benchmark_data_smithi052_606809_object23422:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:19.259234+0000", 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 218.68210357999999, 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.175814893, 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:57.992 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 105576 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259234+0000", 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.993 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259234+0000", 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259232+0000", 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9440000000000005e-06 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:57.994 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259240+0000", 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4470999999999999e-05 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259254+0000", 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8029999999999999e-06 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259261+0000", 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.9159000000000001e-05 2023-09-23T23:10:57.995 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259320+0000", 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016387500000000001 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259484+0000", 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020229199999999999 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.996 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.259686+0000", 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030865490000000001 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.262773+0000", 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.1262999999999999e-05 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:57.997 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.262844+0000", 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16952422 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.432368+0000", 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4986000000000003e-05 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.432443+0000", 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0026052240000000002 2023-09-23T23:10:57.998 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:19.435049+0000", 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.175814893 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:106756 11.e 11:752baf3e:::benchmark_data_smithi052_606809_object24602:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:07:23.502244+0000", 2023-09-23T23:10:57.999 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 214.439093429, 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.185223678, 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 106756 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502244+0000", 2023-09-23T23:10:58.000 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502244+0000", 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502242+0000", 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7110000000000001e-06 2023-09-23T23:10:58.001 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502251+0000", 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.4238e-05 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502265+0000", 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.798e-06 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.002 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502272+0000", 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5841999999999998e-05 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502328+0000", 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016212 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502490+0000", 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000199478 2023-09-23T23:10:58.003 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.502689+0000", 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0031355469999999998 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.505825+0000", 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019053399999999999 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.004 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.506015+0000", 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17894664900000001 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.684962+0000", 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.8783000000000003e-05 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.685021+0000", 2023-09-23T23:10:58.005 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0024467629999999998 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:07:23.687467+0000", 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.185223678 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:124400 11.e 11:7954a7ab:::benchmark_data_smithi052_606809_object42246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e462)", 2023-09-23T23:10:58.006 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:17.849445+0000", 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 160.091891866, 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17804371399999999, 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 124400 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.007 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849445+0000", 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849445+0000", 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849444+0000", 2023-09-23T23:10:58.008 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0339999999999997e-06 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849452+0000", 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0784999999999999e-05 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849462+0000", 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.652e-06 2023-09-23T23:10:58.009 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849469+0000", 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.0318999999999997e-05 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849519+0000", 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017331900000000001 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.010 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849693+0000", 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00019187800000000001 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.849885+0000", 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00303418 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.852919+0000", 2023-09-23T23:10:58.011 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000112779 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:17.853032+0000", 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17280121000000001 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.025833+0000", 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4613e-05 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.012 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.025897+0000", 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015916750000000001 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:18.027489+0000", 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17804371399999999 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.013 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:129532 11.7 11:e088c6f6:::benchmark_data_smithi052_606809_object47378:head [delete] snapc 0=[] ondisk+write+known_if_redirected e466)", 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:32.467353+0000", 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 145.473984438, 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18310673799999999, 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 129532 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.014 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467353+0000", 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467353+0000", 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.015 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467351+0000", 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.8820000000000003e-06 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467358+0000", 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0029e-05 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.016 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467368+0000", 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6190000000000002e-06 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467373+0000", 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1107999999999999e-05 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467424+0000", 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00016536 2023-09-23T23:10:58.017 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467589+0000", 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00046281399999999998 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.468052+0000", 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0025533919999999998 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.018 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.470606+0000", 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00046673100000000001 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471072+0000", 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00063834799999999995 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.019 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471711+0000", 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6594000000000001e-05 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471767+0000", 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17869227200000001 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650460+0000", 2023-09-23T23:10:58.020 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18310673799999999 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:129531 11.b e466/433 11:d53925ed:::benchmark_data_smithi052_606809_object47377:head v 466'8075, mlcod=414'297)", 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:32.467871+0000", 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 145.47346582500001, 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18257716600000001, 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.021 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467871+0000", 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467871+0000", 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.022 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467871+0000", 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1080000000000001e-06 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467876+0000", 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.3160000000000002e-06 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.023 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467882+0000", 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.206e-06 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467887+0000", 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.5491999999999997e-05 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467932+0000", 2023-09-23T23:10:58.024 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.1866999999999999e-05 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.467954+0000", 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.182455062 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650409+0000", 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.9583999999999999e-05 2023-09-23T23:10:58.025 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650449+0000", 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.18257716600000001 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop_reply(client.46892.0:129532 11.7 e466/437 ondisk, result = 0)", 2023-09-23T23:10:58.026 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:32.471641+0000", 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 145.469696226, 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17882445199999999, 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "started", 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471641+0000", 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.027 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471641+0000", 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471640+0000", 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1789999999999999e-06 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471646+0000", 2023-09-23T23:10:58.028 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.9070000000000007e-06 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471653+0000", 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.7270000000000001e-06 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471657+0000", 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 3.8445000000000002e-05 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.029 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471696+0000", 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0090000000000007e-06 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.471704+0000", 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17876178000000001 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.030 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650465+0000", 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17882445199999999 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "MOSDPGPushReply(11.e 466/433 [PushReplyOp(11:77bf810a:::benchmark_data_smithi052_606809_object64434:head)])", 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:32.472862+0000", 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 145.46847569900001, 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17959191599999999, 2023-09-23T23:10:58.031 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "reached pg", 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472862+0000", 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472862+0000", 2023-09-23T23:10:58.032 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999995 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472861+0000", 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.5389999999999998e-06 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472866+0000", 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.1319999999999994e-06 2023-09-23T23:10:58.033 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472875+0000", 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.8199999999999996e-06 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472879+0000", 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7627000000000003e-05 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.034 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.472927+0000", 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17952643600000001 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.652453+0000", 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17959191599999999 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.035 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:129536 11.f e466/433 11:fc579200:::benchmark_data_smithi052_606809_object47382:head v 466'8002, mlcod=414'296)", 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:32.477893+0000", 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 145.46344433499999, 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.178924837, 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.477893+0000", 2023-09-23T23:10:58.036 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.477893+0000", 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.477892+0000", 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.055e-06 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.037 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.477900+0000", 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17255806600000001 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650458+0000", 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.7420000000000003e-06 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.038 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650464+0000", 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.7259000000000001e-05 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650491+0000", 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.6208999999999999e-05 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650517+0000", 2023-09-23T23:10:58.039 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0062543360000000001 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.656772+0000", 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6109999999999997e-05 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.656818+0000", 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.178924837 2023-09-23T23:10:58.040 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:129537 11.e 11:7f739a78:::benchmark_data_smithi052_606809_object47383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e466)", 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:32.482552+0000", 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 145.458785543, 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17766963299999999, 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.041 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 129537 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.482552+0000", 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.042 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.482552+0000", 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.482550+0000", 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.3119999999999994e-06 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.482558+0000", 2023-09-23T23:10:58.043 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.16785123900000001 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650410+0000", 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2379e-05 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.650422+0000", 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.002064527 2023-09-23T23:10:58.044 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.652487+0000", 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011407400000000001 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.652601+0000", 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00011744900000000001 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.045 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.652718+0000", 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0063311119999999999 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.659049+0000", 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.7994e-05 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.659117+0000", 2023-09-23T23:10:58.046 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.7639000000000002e-05 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.659165+0000", 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.634e-05 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.659211+0000", 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0010101140000000001 2023-09-23T23:10:58.047 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:32.660221+0000", 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.17766963299999999 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:131040 11.17 11:ed53d966:::benchmark_data_smithi052_606809_object48886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e467)", 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:36.514041+0000", 2023-09-23T23:10:58.048 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 141.427295872, 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.169785095, 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 131040 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.049 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514041+0000", 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514041+0000", 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514039+0000", 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.1889999999999996e-06 2023-09-23T23:10:58.050 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514049+0000", 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.2969999999999999e-05 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514062+0000", 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.477e-06 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.051 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514068+0000", 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.8351999999999997e-05 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514156+0000", 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00018342400000000001 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514340+0000", 2023-09-23T23:10:58.052 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00050251799999999998 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.514842+0000", 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0095778329999999991 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.524420+0000", 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.3039999999999994e-06 2023-09-23T23:10:58.053 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.524430+0000", 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.158099976 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.682530+0000", 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.6190000000000001e-05 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.054 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.682606+0000", 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001220727 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:36.683826+0000", 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.169785095 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.055 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:137108 11.7 11:e78267b9:::benchmark_data_smithi052_606809_object54954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e472)", 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:51.985699+0000", 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 125.95563814499999, 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.161925033, 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 137108 2023-09-23T23:10:58.056 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985699+0000", 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985699+0000", 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999986 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.057 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985698+0000", 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.1310000000000003e-06 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985704+0000", 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.8360000000000004e-06 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985714+0000", 2023-09-23T23:10:58.058 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1229999999999999e-06 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985719+0000", 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0247999999999998e-05 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985759+0000", 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000107409 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.059 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.985867+0000", 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15332816299999999 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,6", 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.139195+0000", 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0028239599999999999 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.060 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.142019+0000", 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00094289299999999999 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.142962+0000", 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0030049230000000001 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.145967+0000", 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 6.4856000000000004e-05 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.061 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.146031+0000", 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0015927370000000001 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.147624+0000", 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.161925033 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.062 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_repop(client.46892.0:137109 11.c e472/410 11:30454d7f:::benchmark_data_smithi052_606809_object54955:head v 472'8523, mlcod=472'8521)", 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:08:51.989482+0000", 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 125.95185474199999, 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15545077700000001, 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.989482+0000", 2023-09-23T23:10:58.063 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.989482+0000", 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.989482+0000", 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.5730000000000008e-06 2023-09-23T23:10:58.064 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:51.989490+0000", 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.14920862500000001 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.138699+0000", 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.7999999999999993e-06 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.065 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.138709+0000", 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000103199 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.138812+0000", 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 2.355e-05 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.066 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.138835+0000", 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0060411320000000003 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.144876+0000", 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.6773e-05 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.067 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:08:52.144933+0000", 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.15545077700000001 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.46892.0:153673 11.e 11:7bf7d597:::benchmark_data_smithi052_606809_object71519:head [delete] snapc 0=[] ondisk+write+known_if_redirected e476)", 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:09:25.277213+0000", 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 92.664124258000001, 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.160400142, 2023-09-23T23:10:58.068 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.46892", 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3989845851", 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 153673 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.069 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277213+0000", 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277213+0000", 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.9999981 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277211+0000", 2023-09-23T23:10:58.070 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.0690000000000002e-06 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277219+0000", 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.5582000000000001e-05 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277235+0000", 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.4669999999999998e-06 2023-09-23T23:10:58.071 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277242+0000", 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00039167500000000002 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277634+0000", 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00014958699999999999 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.072 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277783+0000", 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00017277199999999999 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 1,3", 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.277956+0000", 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0053776900000000001 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.283334+0000", 2023-09-23T23:10:58.073 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.001052187 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.284386+0000", 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.152980901 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.437367+0000", 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.0825000000000003e-05 2023-09-23T23:10:58.074 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.437408+0000", 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00020537299999999999 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:09:25.437613+0000", 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.160400142 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.075 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "description": "osd_op(client.47477.0:2 12.2 12:424638ea:::image1:head [create] snapc 0=[] ondisk+write+known_if_redirected e486)", 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "initiated_at": "2023-09-23T23:10:07.551841+0000", 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "age": 50.389495752000002, 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.45991206000000001, 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "type_data": { 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "client_info": { 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "client": "client.47477", 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "client_addr": "172.21.15.52:0/3213926960", 2023-09-23T23:10:58.076 INFO:teuthology.orchestra.run.smithi151.stdout: "tid": 2 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "events": [ 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "initiated", 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551841+0000", 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "throttled", 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551841+0000", 2023-09-23T23:10:58.077 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4294967295.999999 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "header_read", 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551840+0000", 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.792e-06 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "all_read", 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551845+0000", 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 9.0149999999999992e-06 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.078 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "dispatched", 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551854+0000", 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 7.7160000000000003e-06 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "queued_for_pg", 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551862+0000", 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.6825e-05 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551909+0000", 2023-09-23T23:10:58.079 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 1.0234000000000001e-05 2023-09-23T23:10:58.080 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for peered", 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:07.551919+0000", 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.45463401799999997 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "reached_pg", 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.006553+0000", 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.000110721 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.081 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "started", 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.006664+0000", 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 8.7675000000000003e-05 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "waiting for subops from 3,6", 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.006751+0000", 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.00057757499999999999 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "op_commit", 2023-09-23T23:10:58.082 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.007329+0000", 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0020317180000000001 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.009361+0000", 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.0022984839999999999 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "sub_op_commit_rec", 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.011659+0000", 2023-09-23T23:10:58.083 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 5.1081000000000001e-05 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "commit_sent", 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.011710+0000", 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 4.3389000000000001e-05 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: }, 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: { 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: "event": "done", 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: "time": "2023-09-23T23:10:08.011754+0000", 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: "duration": 0.45991206000000001 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.084 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.085 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.085 INFO:teuthology.orchestra.run.smithi151.stdout: } 2023-09-23T23:10:58.085 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2023-09-23T23:10:58.085 INFO:teuthology.orchestra.run.smithi151.stdout:} 2023-09-23T23:10:58.085 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T23:10:58.087 INFO:tasks.workunit.client.0.smithi052.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T23:10:58.087 INFO:tasks.workunit.client.0.smithi052.stderr:++ rbd info sparse 2023-09-23T23:10:58.087 INFO:tasks.workunit.client.0.smithi052.stderr:++ grep block_name_prefix 2023-09-23T23:10:58.087 INFO:tasks.workunit.client.0.smithi052.stderr:++ awk '{print $NF;}' 2023-09-23T23:10:58.087 INFO:tasks.workunit.client.0.smithi052.stderr:+ prefix=rbd_data.bc249fd98ebd 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:++ get_image_data_pool sparse 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep 'data_pool: ' 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:++ data_pool= 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:++ '[' -z ']' 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:++ data_pool=rbd 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo rbd 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:+ rados -p rbd put rbd_data.bc249fd98ebd.000000000000 - 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records in 2023-09-23T23:10:58.088 INFO:tasks.workunit.client.0.smithi052.stderr:1+0 records out 2023-09-23T23:10:58.089 INFO:tasks.workunit.client.0.smithi052.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0651007 s, 64.4 MB/s 2023-09-23T23:10:58.105 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects sparse 2023-09-23T23:10:58.105 INFO:tasks.workunit.client.0.smithi052.stderr:++ image=sparse 2023-09-23T23:10:58.106 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rbd info sparse 2023-09-23T23:10:58.106 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep block_name_prefix 2023-09-23T23:10:58.106 INFO:tasks.workunit.client.0.smithi052.stderr:+++ awk '{print $NF;}' 2023-09-23T23:10:58.118 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:58.148 INFO:tasks.workunit.client.0.smithi052.stderr:++ prefix=rbd_data.bc249fd98ebd 2023-09-23T23:10:58.149 INFO:tasks.workunit.client.0.smithi052.stderr:+++ grep rbd_data.bc249fd98ebd 2023-09-23T23:10:58.149 INFO:tasks.workunit.client.0.smithi052.stderr:++++ get_image_data_pool sparse 2023-09-23T23:10:58.149 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sed -e 's/rbd_data.bc249fd98ebd\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T23:10:58.149 INFO:tasks.workunit.client.0.smithi052.stderr:++++ image=sparse 2023-09-23T23:10:58.149 INFO:tasks.workunit.client.0.smithi052.stderr:+++ sort -u 2023-09-23T23:10:58.149 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ rbd info sparse 2023-09-23T23:10:58.150 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ grep 'data_pool: ' 2023-09-23T23:10:58.150 INFO:tasks.workunit.client.0.smithi052.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T23:10:58.186 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool= 2023-09-23T23:10:58.186 INFO:tasks.workunit.client.0.smithi052.stderr:++++ '[' -z ']' 2023-09-23T23:10:58.186 INFO:tasks.workunit.client.0.smithi052.stderr:++++ data_pool=rbd 2023-09-23T23:10:58.186 INFO:tasks.workunit.client.0.smithi052.stderr:++++ echo rbd 2023-09-23T23:10:58.186 INFO:tasks.workunit.client.0.smithi052.stderr:+++ rados ls -p rbd 2023-09-23T23:10:58.218 INFO:tasks.workunit.client.0.smithi052.stderr:++ objects=0 2023-09-23T23:10:58.218 INFO:tasks.workunit.client.0.smithi052.stderr:++ echo 0 2023-09-23T23:10:58.218 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T23:10:58.218 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/sparse 2023-09-23T23:10:58.219 INFO:tasks.workunit.client.0.smithi052.stderr:rm: cannot remove '/tmp/rbd_import_export_701000/sparse': No such file or directory 2023-09-23T23:10:58.219 INFO:tasks.workunit.client.0.smithi052.stderr:+ true 2023-09-23T23:10:58.219 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd export sparse /tmp/rbd_import_export_701000/sparse 2023-09-23T23:10:58.254 INFO:tasks.workunit.client.0.smithi052.stderr: Exporting image: 100% complete...done. 2023-09-23T23:10:58.257 INFO:tasks.workunit.client.0.smithi052.stderr:++ stat /tmp/rbd_import_export_701000/sparse --format=%b 2023-09-23T23:10:58.258 INFO:tasks.workunit.client.0.smithi052.stderr:+ '[' 0 = 0 ']' 2023-09-23T23:10:58.259 INFO:tasks.workunit.client.0.smithi052.stderr:+ rbd rm sparse 2023-09-23T23:10:58.343 INFO:tasks.workunit.client.0.smithi052.stderr: Removing image: 100% complete...done. 2023-09-23T23:10:58.347 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm /tmp/rbd_import_export_701000/sparse /tmp/rbd_import_export_701000/sparse1 /tmp/rbd_import_export_701000/sparse2 /tmp/rbd_import_export_701000/sparse3 2023-09-23T23:10:58.348 INFO:tasks.workunit.client.0.smithi052.stderr:rm: cannot remove '/tmp/rbd_import_export_701000/sparse3': No such file or directory 2023-09-23T23:10:58.349 INFO:tasks.workunit.client.0.smithi052.stderr:+ true 2023-09-23T23:10:58.349 INFO:tasks.workunit.client.0.smithi052.stdout:OK 2023-09-23T23:10:58.350 INFO:tasks.workunit.client.0.smithi052.stderr:+ echo OK 2023-09-23T23:10:58.350 INFO:tasks.workunit.client.0.smithi052.stderr:+ rm -rf /tmp/rbd_import_export_701000 2023-09-23T23:10:58.350 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T23:10:58.350 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T23:10:58.359 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T23:10:58.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:58.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T23:10:58.388 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T23:10:58.388 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T23:10:58.462 DEBUG:teuthology.orchestra.run.smithi151:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T23:10:58.491 INFO:teuthology.orchestra.run.smithi151.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:58.556 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:10:58.655 INFO:teuthology.orchestra.run.smithi151.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T23:10:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:58.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi151 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T23:10:58.714 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T23:10:58.738 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:58 smithi151 ceph-mon[159210]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:58.738 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:58 smithi151 ceph-mon[159210]: failed to encode map e509 with expected crc 2023-09-23T23:10:58.738 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:58 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:10:58.738 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:58 smithi151 ceph-mon[159210]: osdmap e509: 8 total, 8 up, 5 in 2023-09-23T23:10:58.739 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:58 smithi151 ceph-mon[159210]: failed to encode map e509 with expected crc 2023-09-23T23:10:58.739 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:58 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:58.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[187743]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:58.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[187743]: failed to encode map e509 with expected crc 2023-09-23T23:10:58.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[187743]: osdmap e509: 8 total, 8 up, 5 in 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[187743]: failed to encode map e509 with expected crc 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[180979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[180979]: failed to encode map e509 with expected crc 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[180979]: osdmap e509: 8 total, 8 up, 5 in 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[180979]: failed to encode map e509 with expected crc 2023-09-23T23:10:58.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:58 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:10:58.759 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T23:10:58.788 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:59.027 DEBUG:teuthology.parallel:result is None 2023-09-23T23:10:59.027 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:10:59.102 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:10:59.102 DEBUG:teuthology.orchestra.run.smithi052:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:59.117 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T23:10:59.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:59.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T23:10:59.133 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:10:59.133 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T23:10:59.133 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T23:10:59.133 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T23:10:59.133 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-09-23T23:10:59.134 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T23:10:59.134 DEBUG:teuthology.orchestra.run.smithi052:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:10:59.190 INFO:teuthology.orchestra.run.smithi052.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T23:10:59.190 DEBUG:teuthology.orchestra.run.smithi052:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:59.220 DEBUG:teuthology.orchestra.run.smithi052:> sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T23:10:59.247 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T23:10:59.248 DEBUG:teuthology.orchestra.run.smithi052:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T23:10:59.260 INFO:teuthology.orchestra.run.smithi052.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T23:10:59.263 INFO:teuthology.orchestra.run.smithi052.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-09-23T23:10:59.280 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T23:10:59.338 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:10:59.516 INFO:teuthology.orchestra.run.smithi052.stderr:Error: no container with name or ID ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T23:10:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T23:10:59.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi052 with status 125: 'sudo docker kill -s 1 ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T23:10:59.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:59 smithi052 ceph-mon[187743]: pgmap v1582: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 549 KiB/s rd, 591 KiB/s wr, 209 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:10:59.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:10:59 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1759415691' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:10:59.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:59 smithi052 ceph-mon[180979]: pgmap v1582: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 549 KiB/s rd, 591 KiB/s wr, 209 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:10:59.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:10:59 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1759415691' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:10:59.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:59 smithi151 ceph-mon[159210]: pgmap v1582: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 549 KiB/s rd, 591 KiB/s wr, 209 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:10:59.846 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:10:59 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1759415691' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T23:11:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-09-23T23:11:00.519 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T23:11:00.619 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:00 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:11:00.620 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:00 smithi052 ceph-mon[187743]: osdmap e510: 8 total, 8 up, 5 in 2023-09-23T23:11:00.620 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:00 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:11:00.620 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:00 smithi052 ceph-mon[180979]: osdmap e510: 8 total, 8 up, 5 in 2023-09-23T23:11:00.851 INFO:teuthology.orchestra.run.smithi052.stdout:osd.0: {} 2023-09-23T23:11:00.871 INFO:teuthology.orchestra.run.smithi052.stdout:osd.1: {} 2023-09-23T23:11:00.896 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:00 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:11:00.896 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:00 smithi151 ceph-mon[159210]: osdmap e510: 8 total, 8 up, 5 in 2023-09-23T23:11:00.897 INFO:teuthology.orchestra.run.smithi052.stdout:osd.2: {} 2023-09-23T23:11:00.916 INFO:teuthology.orchestra.run.smithi052.stdout:osd.3: {} 2023-09-23T23:11:00.934 INFO:teuthology.orchestra.run.smithi052.stdout:osd.4: {} 2023-09-23T23:11:00.952 INFO:teuthology.orchestra.run.smithi052.stdout:osd.5: {} 2023-09-23T23:11:00.973 INFO:teuthology.orchestra.run.smithi052.stdout:osd.6: {} 2023-09-23T23:11:00.994 INFO:teuthology.orchestra.run.smithi052.stdout:osd.7: {} 2023-09-23T23:11:00.996 INFO:teuthology.orchestra.run.smithi052.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.996 INFO:teuthology.orchestra.run.smithi052.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.996 INFO:teuthology.orchestra.run.smithi052.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.996 INFO:teuthology.orchestra.run.smithi052.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.996 INFO:teuthology.orchestra.run.smithi052.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.996 INFO:teuthology.orchestra.run.smithi052.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.997 INFO:teuthology.orchestra.run.smithi052.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T23:11:00.997 INFO:teuthology.orchestra.run.smithi052.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T23:11:01.130 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:01.130 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:01.562 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:01.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[187743]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[187743]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1725707236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[187743]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[187743]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[187743]: pgmap v1584: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 342 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[180979]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[180979]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1725707236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:11:01.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[180979]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[180979]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:01 smithi052 ceph-mon[180979]: pgmap v1584: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 342 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:01.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:01 smithi151 ceph-mon[159210]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:01 smithi151 ceph-mon[159210]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.946 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:01 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1725707236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T23:11:01.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:01 smithi151 ceph-mon[159210]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:01 smithi151 ceph-mon[159210]: failed to encode map e510 with expected crc 2023-09-23T23:11:01.947 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:01 smithi151 ceph-mon[159210]: pgmap v1584: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 342 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:01.948 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T23:11:02.459 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:02.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:02 smithi151 ceph-mon[159210]: failed to encode map e510 with expected crc 2023-09-23T23:11:02.871 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:02 smithi151 ceph-mon[159210]: failed to encode map e510 with expected crc 2023-09-23T23:11:02.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:02 smithi052 ceph-mon[187743]: failed to encode map e510 with expected crc 2023-09-23T23:11:02.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:02 smithi052 ceph-mon[187743]: failed to encode map e510 with expected crc 2023-09-23T23:11:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:02 smithi052 ceph-mon[180979]: failed to encode map e510 with expected crc 2023-09-23T23:11:02.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:02 smithi052 ceph-mon[180979]: failed to encode map e510 with expected crc 2023-09-23T23:11:03.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:11:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:11:03] "GET /metrics HTTP/1.1" 200 34244 "" "Prometheus/2.33.4" 2023-09-23T23:11:04.115 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:04.115 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:03 smithi151 ceph-mon[159210]: pgmap v1585: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 361 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:03 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/870502723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:11:04.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[187743]: pgmap v1585: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 361 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:04.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/870502723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:11:04.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[180979]: pgmap v1585: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 361 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:04.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:04.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:04.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/870502723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T23:11:04.594 INFO:teuthology.orchestra.run.smithi052.stderr:noscrub is unset 2023-09-23T23:11:04.753 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T23:11:05.104 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:05.370 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T23:11:05.370 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-23T23:11:05.371 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-23T23:11:05.457 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: failed to encode map e511 with expected crc 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/870502723' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: osdmap e511: 8 total, 8 up, 5 in 2023-09-23T23:11:05.661 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: pgmap v1587: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 935 KiB/s rd, 1.4 MiB/s wr, 306 op/s 2023-09-23T23:11:05.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:05.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:05.662 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: failed to encode map e511 with expected crc 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/870502723' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: osdmap e511: 8 total, 8 up, 5 in 2023-09-23T23:11:05.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: pgmap v1587: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 935 KiB/s rd, 1.4 MiB/s wr, 306 op/s 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: failed to encode map e511 with expected crc 2023-09-23T23:11:05.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/870502723' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T23:11:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: osdmap e511: 8 total, 8 up, 5 in 2023-09-23T23:11:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: pgmap v1587: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 935 KiB/s rd, 1.4 MiB/s wr, 306 op/s 2023-09-23T23:11:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:11:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:11:05.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:05.777 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:05.862 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:06.632 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:06.632 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:06.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:06 smithi052 ceph-mon[180979]: failed to encode map e512 with expected crc 2023-09-23T23:11:06.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:06 smithi052 ceph-mon[180979]: osdmap e512: 8 total, 8 up, 5 in 2023-09-23T23:11:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:06 smithi052 ceph-mon[187743]: failed to encode map e512 with expected crc 2023-09-23T23:11:06.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:06 smithi052 ceph-mon[187743]: osdmap e512: 8 total, 8 up, 5 in 2023-09-23T23:11:06.769 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:06 smithi151 ceph-mon[159210]: failed to encode map e512 with expected crc 2023-09-23T23:11:06.769 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:06 smithi151 ceph-mon[159210]: osdmap e512: 8 total, 8 up, 5 in 2023-09-23T23:11:07.294 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T23:11:07.647 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:07.702 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:07 smithi151 ceph-mon[159210]: pgmap v1589: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 764 KiB/s rd, 1.3 MiB/s wr, 175 op/s 2023-09-23T23:11:07.833 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:07 smithi052 ceph-mon[180979]: pgmap v1589: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 764 KiB/s rd, 1.3 MiB/s wr, 175 op/s 2023-09-23T23:11:07.833 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:07 smithi052 ceph-mon[187743]: pgmap v1589: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 764 KiB/s rd, 1.3 MiB/s wr, 175 op/s 2023-09-23T23:11:08.494 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:08 smithi052 conmon[180954]: 2023-09-23T23:11:08.274+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e512 definitely_dead 0 2023-09-23T23:11:08.535 INFO:teuthology.orchestra.run.smithi052.stderr:marked in osd.0. 2023-09-23T23:11:08.903 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:08 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/26264809' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T23:11:08.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:08 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/26264809' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T23:11:08.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:08 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/26264809' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T23:11:09.123 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:09.123 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:09.338 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-23T23:11:09.339 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T23:11:09.339 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T23:11:09.339 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T23:11:09.531 INFO:teuthology.orchestra.run.smithi052.stderr:nodeep-scrub is unset 2023-09-23T23:11:09.694 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/26264809' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: osdmap e513: 8 total, 8 up, 6 in 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1412857723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.695 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:09 smithi151 ceph-mon[159210]: pgmap v1591: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:11:09.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/26264809' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T23:11:09.705 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: osdmap e513: 8 total, 8 up, 6 in 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1412857723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[187743]: pgmap v1591: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/26264809' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T23:11:09.706 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: osdmap e513: 8 total, 8 up, 6 in 2023-09-23T23:11:09.707 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1412857723' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:11:09.707 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T23:11:09.707 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: failed to encode map e513 with expected crc 2023-09-23T23:11:09.707 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:09 smithi052 ceph-mon[180979]: pgmap v1591: 73 pgs: 73 active+clean; 4.6 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail 2023-09-23T23:11:09.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T23:11:10.059 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:10.362 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:10.565 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:11:10.565 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:11:10.565 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.565 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.565 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:11:10.566 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: osdmap e514: 8 total, 8 up, 6 in 2023-09-23T23:11:10.566 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.566 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[187743]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.566 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:11:10.566 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:11:10.566 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.567 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.567 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:11:10.567 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: osdmap e514: 8 total, 8 up, 6 in 2023-09-23T23:11:10.567 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.567 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:10 smithi052 ceph-mon[180979]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T23:11:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:11:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T23:11:10.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: osdmap e514: 8 total, 8 up, 6 in 2023-09-23T23:11:10.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: failed to encode map e514 with expected crc 2023-09-23T23:11:10.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:10 smithi151 ceph-mon[159210]: failed to encode map e514 with expected crc 2023-09-23T23:11:11.573 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:11.573 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:11.718 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: osdmap e515: 8 total, 8 up, 6 in 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: pgmap v1594: 73 pgs: 8 activating, 2 active, 2 activating+degraded, 61 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 3/84 objects degraded (3.571%); 766 KiB/s, 3 objects/s recovering 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[187743]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.719 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: osdmap e515: 8 total, 8 up, 6 in 2023-09-23T23:11:11.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: pgmap v1594: 73 pgs: 8 activating, 2 active, 2 activating+degraded, 61 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 3/84 objects degraded (3.571%); 766 KiB/s, 3 objects/s recovering 2023-09-23T23:11:11.720 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:11 smithi052 ceph-mon[180979]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: osdmap e515: 8 total, 8 up, 6 in 2023-09-23T23:11:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: failed to encode map e515 with expected crc 2023-09-23T23:11:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: pgmap v1594: 73 pgs: 8 activating, 2 active, 2 activating+degraded, 61 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 3/84 objects degraded (3.571%); 766 KiB/s, 3 objects/s recovering 2023-09-23T23:11:11.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:11 smithi151 ceph-mon[159210]: failed to encode map e515 with expected crc 2023-09-23T23:11:12.312 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T23:11:12.493 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 conmon[180954]: 2023-09-23T23:11:12.086+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e516 definitely_dead 0 2023-09-23T23:11:12.549 INFO:teuthology.orchestra.run.smithi052.stderr:marked in osd.5. 2023-09-23T23:11:12.673 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: osdmap e516: 8 total, 8 up, 6 in 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/759384650' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T23:11:12.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:12 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T23:11:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: osdmap e516: 8 total, 8 up, 6 in 2023-09-23T23:11:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/759384650' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: osdmap e516: 8 total, 8 up, 6 in 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: failed to encode map e516 with expected crc 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/759384650' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T23:11:12.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:12 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T23:11:13.347 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T23:11:13.347 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T23:11:13.347 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T23:11:13.347 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T23:11:13.739 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:11:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:11:13] "GET /metrics HTTP/1.1" 200 34253 "" "Prometheus/2.33.4" 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: osdmap e517: 8 total, 8 up, 7 in 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[180979]: pgmap v1597: 73 pgs: 3 peering, 8 activating, 2 active, 2 activating+degraded, 58 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 3/84 objects degraded (3.571%); 1024 KiB/s, 13 objects/s recovering 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: osdmap e517: 8 total, 8 up, 7 in 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:13 smithi052 ceph-mon[187743]: pgmap v1597: 73 pgs: 3 peering, 8 activating, 2 active, 2 activating+degraded, 58 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 3/84 objects degraded (3.571%); 1024 KiB/s, 13 objects/s recovering 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: osdmap e517: 8 total, 8 up, 7 in 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: failed to encode map e517 with expected crc 2023-09-23T23:11:13.944 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:13 smithi151 ceph-mon[159210]: pgmap v1597: 73 pgs: 3 peering, 8 activating, 2 active, 2 activating+degraded, 58 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 3/84 objects degraded (3.571%); 1024 KiB/s, 13 objects/s recovering 2023-09-23T23:11:14.150 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:14.150 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: osdmap e518: 8 total, 8 up, 7 in 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.701 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:14 smithi151 ceph-mon[159210]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: osdmap e518: 8 total, 8 up, 7 in 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.745 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[187743]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: osdmap e518: 8 total, 8 up, 7 in 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.746 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.747 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:14 smithi052 ceph-mon[180979]: failed to encode map e518 with expected crc 2023-09-23T23:11:14.833 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T23:11:15.180 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: osdmap e519: 8 total, 8 up, 7 in 2023-09-23T23:11:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: pgmap v1600: 73 pgs: 4 peering, 3 activating, 66 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 27 B/s, 9 objects/s recovering 2023-09-23T23:11:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2652689661' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T23:11:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: osdmap e520: 8 total, 8 up, 7 in 2023-09-23T23:11:15.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:15 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T23:11:15.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: osdmap e519: 8 total, 8 up, 7 in 2023-09-23T23:11:15.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: pgmap v1600: 73 pgs: 4 peering, 3 activating, 66 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 27 B/s, 9 objects/s recovering 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2652689661' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: osdmap e520: 8 total, 8 up, 7 in 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T23:11:15.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 conmon[180954]: 2023-09-23T23:11:15.505+0000 7ffaf73ae700 -1 mon.a@0(leader).osd e520 definitely_dead 0 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: osdmap e519: 8 total, 8 up, 7 in 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: failed to encode map e519 with expected crc 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: pgmap v1600: 73 pgs: 4 peering, 3 activating, 66 active+clean; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 27 B/s, 9 objects/s recovering 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2652689661' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: osdmap e520: 8 total, 8 up, 7 in 2023-09-23T23:11:15.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:15 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T23:11:16.522 INFO:teuthology.orchestra.run.smithi052.stderr:marked in osd.7. 2023-09-23T23:11:16.599 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:16 smithi151 ceph-mon[159210]: failed to encode map e520 with expected crc 2023-09-23T23:11:16.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:16 smithi151 ceph-mon[159210]: failed to encode map e520 with expected crc 2023-09-23T23:11:16.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:16 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:11:16.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:16 smithi151 ceph-mon[159210]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T23:11:16.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T23:11:16.600 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:16 smithi151 ceph-mon[159210]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T23:11:16.750 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:16.750 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[187743]: failed to encode map e520 with expected crc 2023-09-23T23:11:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[187743]: failed to encode map e520 with expected crc 2023-09-23T23:11:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:11:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[187743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T23:11:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T23:11:16.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[187743]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T23:11:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[180979]: failed to encode map e520 with expected crc 2023-09-23T23:11:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[180979]: failed to encode map e520 with expected crc 2023-09-23T23:11:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T23:11:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[180979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T23:11:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T23:11:16.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:16 smithi052 ceph-mon[180979]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T23:11:17.264 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T23:11:17.265 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T23:11:17.494 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T23:11:17.649 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:17.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.870 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: pgmap v1603: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 178 B/s, 4 objects/s recovering 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[180979]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T23:11:17.871 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.871 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: pgmap v1603: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 178 B/s, 4 objects/s recovering 2023-09-23T23:11:17.872 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:17 smithi052 ceph-mon[187743]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T23:11:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: failed to encode map e521 with expected crc 2023-09-23T23:11:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: pgmap v1603: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 178 B/s, 4 objects/s recovering 2023-09-23T23:11:17.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:17 smithi151 ceph-mon[159210]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T23:11:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:18 smithi151 ceph-mon[159210]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T23:11:18.980 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.981 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.982 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[187743]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T23:11:18.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.982 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: failed to encode map e522 with expected crc 2023-09-23T23:11:18.983 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:18 smithi052 ceph-mon[180979]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T23:11:19.196 INFO:teuthology.orchestra.run.smithi052.stdout:mon.a: {} 2023-09-23T23:11:19.206 INFO:teuthology.orchestra.run.smithi052.stdout:mon.b: {} 2023-09-23T23:11:19.216 INFO:teuthology.orchestra.run.smithi052.stdout:mon.c: {} 2023-09-23T23:11:19.217 INFO:teuthology.orchestra.run.smithi052.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T23:11:19.218 INFO:teuthology.orchestra.run.smithi052.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T23:11:19.218 INFO:teuthology.orchestra.run.smithi052.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T23:11:19.380 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:19.380 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.699 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: pgmap v1606: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 160 B/s, 7 objects/s recovering 2023-09-23T23:11:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1265069974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:11:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:19.700 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: pgmap v1606: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 160 B/s, 7 objects/s recovering 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1265069974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:19.919 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: pgmap v1606: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 160 B/s, 7 objects/s recovering 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: failed to encode map e523 with expected crc 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1265069974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:19.920 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:19.936 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T23:11:20.158 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T23:11:20.302 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:20.515 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T23:11:20.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:20 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:11:20.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:20 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:11:20.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:20 smithi151 ceph-mon[159210]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:20 smithi151 ceph-mon[159210]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:20 smithi151 ceph-mon[159210]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.819 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:20 smithi151 ceph-mon[159210]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T23:11:20.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:11:20.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:11:20.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[180979]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.861 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[180979]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[180979]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[180979]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[187743]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[187743]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[187743]: failed to encode map e524 with expected crc 2023-09-23T23:11:20.862 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:20 smithi052 ceph-mon[187743]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T23:11:21.614 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[187743]: pgmap v1609: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s, 11 objects/s recovering 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.615 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.616 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T23:11:21.616 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.616 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.616 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:21 smithi052 ceph-mon[180979]: pgmap v1609: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s, 11 objects/s recovering 2023-09-23T23:11:21.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.826 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.827 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T23:11:21.827 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.827 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: failed to encode map e525 with expected crc 2023-09-23T23:11:21.827 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:21 smithi151 ceph-mon[159210]: pgmap v1609: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1.0 MiB/s, 11 objects/s recovering 2023-09-23T23:11:22.033 INFO:teuthology.orchestra.run.smithi151.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:22.033 INFO:teuthology.orchestra.run.smithi151.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:22.577 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T23:11:22.683 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T23:11:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2467576847' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:22 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:22.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2467576847' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:22 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:22.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2467576847' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:22.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:22 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:23.236 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T23:11:23.313 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:23.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:11:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:11:23] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.33.4" 2023-09-23T23:11:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:11:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T23:11:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: pgmap v1611: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 945 KiB/s, 7 objects/s recovering 2023-09-23T23:11:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:23 smithi151 ceph-mon[159210]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: pgmap v1611: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 945 KiB/s, 7 objects/s recovering 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[187743]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:11:23.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T23:11:23.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: pgmap v1611: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 945 KiB/s, 7 objects/s recovering 2023-09-23T23:11:23.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: failed to encode map e526 with expected crc 2023-09-23T23:11:23.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:23 smithi052 ceph-mon[180979]: failed to encode map e526 with expected crc 2023-09-23T23:11:24.294 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:25.703 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:25.704 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:25.901 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:25 smithi052 ceph-mon[180979]: pgmap v1612: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 683 KiB/s, 5 objects/s recovering 2023-09-23T23:11:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:25 smithi151 ceph-mon[159210]: pgmap v1612: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 683 KiB/s, 5 objects/s recovering 2023-09-23T23:11:26.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:25 smithi052 ceph-mon[187743]: pgmap v1612: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 683 KiB/s, 5 objects/s recovering 2023-09-23T23:11:26.460 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T23:11:26.837 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:26.918 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T23:11:27.152 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:26 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2818144161' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:27.152 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:26 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2818144161' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:27.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:26 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2818144161' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T23:11:28.153 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T23:11:28.179 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: pgmap v1613: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 641 KiB/s, 7 objects/s recovering 2023-09-23T23:11:28.179 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.179 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2818144161' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[187743]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: pgmap v1613: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 641 KiB/s, 7 objects/s recovering 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2818144161' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.180 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:27 smithi052 ceph-mon[180979]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: pgmap v1613: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 641 KiB/s, 7 objects/s recovering 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2818144161' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:27 smithi151 ceph-mon[159210]: failed to encode map e527 with expected crc 2023-09-23T23:11:28.618 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:28.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[180979]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[180979]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T23:11:28.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[180979]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.969 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[180979]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.970 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[180979]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[187743]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[187743]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T23:11:28.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[187743]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[187743]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.970 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:28 smithi052 ceph-mon[187743]: failed to encode map e528 with expected crc 2023-09-23T23:11:28.989 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:28.989 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:28 smithi151 ceph-mon[159210]: failed to encode map e528 with expected crc 2023-09-23T23:11:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:28 smithi151 ceph-mon[159210]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T23:11:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:28 smithi151 ceph-mon[159210]: failed to encode map e528 with expected crc 2023-09-23T23:11:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:28 smithi151 ceph-mon[159210]: failed to encode map e528 with expected crc 2023-09-23T23:11:29.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:28 smithi151 ceph-mon[159210]: failed to encode map e528 with expected crc 2023-09-23T23:11:29.837 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T23:11:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:29 smithi151 ceph-mon[159210]: pgmap v1616: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s, 1 objects/s recovering 2023-09-23T23:11:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:29 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T23:11:30.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:29 smithi052 ceph-mon[187743]: pgmap v1616: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s, 1 objects/s recovering 2023-09-23T23:11:30.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:29 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T23:11:30.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:29 smithi052 ceph-mon[180979]: pgmap v1616: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 92 KiB/s, 1 objects/s recovering 2023-09-23T23:11:30.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:29 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T23:11:30.501 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:31.516 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T23:11:31.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:31 smithi052 ceph-mon[187743]: pgmap v1617: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2023-09-23T23:11:31.928 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:31 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3112350499' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:31.928 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:31 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:31.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:31 smithi052 ceph-mon[180979]: pgmap v1617: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2023-09-23T23:11:31.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:31 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3112350499' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:31.928 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:31 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:31 smithi151 ceph-mon[159210]: pgmap v1617: 73 pgs: 11 peering, 62 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2023-09-23T23:11:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:31 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3112350499' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:31 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:32.569 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T23:11:32.738 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:32.738 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:32.765 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[187743]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[187743]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[187743]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[187743]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[187743]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[180979]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[180979]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.766 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[180979]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[180979]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.767 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:32 smithi052 ceph-mon[180979]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:32 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:11:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:32 smithi151 ceph-mon[159210]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T23:11:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:32 smithi151 ceph-mon[159210]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:32 smithi151 ceph-mon[159210]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:32 smithi151 ceph-mon[159210]: failed to encode map e529 with expected crc 2023-09-23T23:11:32.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:32 smithi151 ceph-mon[159210]: failed to encode map e529 with expected crc 2023-09-23T23:11:33.375 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T23:11:33.449 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:33.668 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:11:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:11:33] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.33.4" 2023-09-23T23:11:33.668 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:33 smithi052 ceph-mon[187743]: pgmap v1619: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:11:33.669 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:33 smithi052 ceph-mon[180979]: pgmap v1619: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:11:33.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:33 smithi151 ceph-mon[159210]: pgmap v1619: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:11:34.556 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:34.793 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:34 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-09-23T23:11:34.794 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:34 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:11:34.794 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:34 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-09-23T23:11:34.794 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:34 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:11:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:34 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-09-23T23:11:34.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:34 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:11:35.541 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T23:11:35.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:35.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:35.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[187743]: pgmap v1620: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:11:35.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3642471242' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:35.802 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:35.802 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:35.803 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[180979]: pgmap v1620: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:11:35.803 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:35 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3642471242' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:35.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:11:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:35 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:35 smithi151 ceph-mon[159210]: pgmap v1620: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T23:11:35.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:35 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3642471242' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T23:11:36.283 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T23:11:36.702 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:36.702 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:36.708 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:36.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[187743]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[187743]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3642471242' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:11:36.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[187743]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T23:11:36.723 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[187743]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.724 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[180979]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.724 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[180979]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.724 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3642471242' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:11:36.724 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[180979]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T23:11:36.724 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:36 smithi052 ceph-mon[180979]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:36 smithi151 ceph-mon[159210]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:36 smithi151 ceph-mon[159210]: failed to encode map e530 with expected crc 2023-09-23T23:11:36.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:36 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3642471242' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T23:11:36.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:36 smithi151 ceph-mon[159210]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T23:11:36.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:36 smithi151 ceph-mon[159210]: failed to encode map e530 with expected crc 2023-09-23T23:11:37.600 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T23:11:37.901 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:37 smithi052 ceph-mon[187743]: pgmap v1622: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:37.902 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:37 smithi052 ceph-mon[180979]: pgmap v1622: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:37.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:37 smithi151 ceph-mon[159210]: pgmap v1622: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:38.551 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:39.917 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T23:11:39.966 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:39 smithi052 ceph-mon[180979]: pgmap v1623: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:39.966 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:39 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1719136179' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:39.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:39 smithi052 ceph-mon[187743]: pgmap v1623: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:39.966 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:39 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1719136179' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:39 smithi151 ceph-mon[159210]: pgmap v1623: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:39 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1719136179' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:40.614 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:40.614 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:41.097 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1719136179' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:40 smithi151 ceph-mon[159210]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T23:11:41.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1719136179' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[187743]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T23:11:41.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1719136179' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: failed to encode map e531 with expected crc 2023-09-23T23:11:41.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:40 smithi052 ceph-mon[180979]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T23:11:41.257 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T23:11:41.485 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:41.943 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: pgmap v1625: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[187743]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: pgmap v1625: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: failed to encode map e532 with expected crc 2023-09-23T23:11:41.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:41 smithi052 ceph-mon[180979]: failed to encode map e532 with expected crc 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: pgmap v1625: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: failed to encode map e532 with expected crc 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: failed to encode map e532 with expected crc 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: failed to encode map e532 with expected crc 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: failed to encode map e532 with expected crc 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: failed to encode map e532 with expected crc 2023-09-23T23:11:42.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:41 smithi151 ceph-mon[159210]: failed to encode map e532 with expected crc 2023-09-23T23:11:43.665 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:11:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:11:43] "GET /metrics HTTP/1.1" 200 34252 "" "Prometheus/2.33.4" 2023-09-23T23:11:44.167 INFO:teuthology.orchestra.run.smithi052.stdout:ERROR: (22) Invalid argument 2023-09-23T23:11:44.167 INFO:teuthology.orchestra.run.smithi052.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T23:11:44.189 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:43 smithi052 ceph-mon[180979]: pgmap v1627: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:44.189 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:43 smithi052 ceph-mon[187743]: pgmap v1627: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:44.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:43 smithi151 ceph-mon[159210]: pgmap v1627: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:44.932 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T23:11:45.172 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:44 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/141655676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:45.172 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:44 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:45.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:44 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/141655676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:45.172 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:44 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:44 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/141655676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:45.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:44 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T23:11:45.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T23:11:45.995 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:45 smithi151 ceph-mon[159210]: pgmap v1628: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:45 smithi151 ceph-mon[159210]: failed to encode map e533 with expected crc 2023-09-23T23:11:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:45 smithi151 ceph-mon[159210]: failed to encode map e533 with expected crc 2023-09-23T23:11:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:45 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:11:46.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:45 smithi151 ceph-mon[159210]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T23:11:46.233 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[187743]: pgmap v1628: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[187743]: failed to encode map e533 with expected crc 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[187743]: failed to encode map e533 with expected crc 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[187743]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[180979]: pgmap v1628: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[180979]: failed to encode map e533 with expected crc 2023-09-23T23:11:46.234 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[180979]: failed to encode map e533 with expected crc 2023-09-23T23:11:46.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T23:11:46.235 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:45 smithi052 ceph-mon[180979]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T23:11:47.937 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T23:11:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:47 smithi151 ceph-mon[159210]: pgmap v1630: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1700053277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:47 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:47 smithi052 ceph-mon[187743]: pgmap v1630: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1700053277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:48.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:47 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:47 smithi052 ceph-mon[180979]: pgmap v1630: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1700053277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:48.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:47 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:48.726 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T23:11:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[187743]: failed to encode map e534 with expected crc 2023-09-23T23:11:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[187743]: failed to encode map e534 with expected crc 2023-09-23T23:11:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:11:48.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[187743]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T23:11:48.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[180979]: failed to encode map e534 with expected crc 2023-09-23T23:11:48.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[180979]: failed to encode map e534 with expected crc 2023-09-23T23:11:48.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:11:48.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:48 smithi052 ceph-mon[180979]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T23:11:49.097 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:48 smithi151 ceph-mon[159210]: failed to encode map e534 with expected crc 2023-09-23T23:11:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:48 smithi151 ceph-mon[159210]: failed to encode map e534 with expected crc 2023-09-23T23:11:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:48 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:11:49.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:48 smithi151 ceph-mon[159210]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T23:11:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:49 smithi151 ceph-mon[159210]: failed to encode map e534 with expected crc 2023-09-23T23:11:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:49 smithi151 ceph-mon[159210]: failed to encode map e534 with expected crc 2023-09-23T23:11:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:49 smithi151 ceph-mon[159210]: pgmap v1632: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:50.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[187743]: failed to encode map e534 with expected crc 2023-09-23T23:11:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[187743]: failed to encode map e534 with expected crc 2023-09-23T23:11:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[187743]: pgmap v1632: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:50.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:50.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[180979]: failed to encode map e534 with expected crc 2023-09-23T23:11:50.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[180979]: failed to encode map e534 with expected crc 2023-09-23T23:11:50.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[180979]: pgmap v1632: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:50.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:11:51.526 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T23:11:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:51 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2751741326' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:51 smithi151 ceph-mon[159210]: pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:51 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2751741326' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:51.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:51 smithi052 ceph-mon[187743]: pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:51.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:51 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2751741326' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T23:11:51.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:51 smithi052 ceph-mon[180979]: pgmap v1633: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T23:11:52.277 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T23:11:52.648 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:52.865 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[187743]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.865 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[187743]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.865 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2751741326' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[187743]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[187743]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[180979]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[180979]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2751741326' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[180979]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T23:11:52.866 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:52 smithi052 ceph-mon[180979]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:52 smithi151 ceph-mon[159210]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:52 smithi151 ceph-mon[159210]: failed to encode map e535 with expected crc 2023-09-23T23:11:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:52 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2751741326' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T23:11:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:52 smithi151 ceph-mon[159210]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T23:11:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:52 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T23:11:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:52 smithi151 ceph-mon[159210]: failed to encode map e535 with expected crc 2023-09-23T23:11:53.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:11:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:11:53] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.33.4" 2023-09-23T23:11:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:53 smithi151 ceph-mon[159210]: failed to encode map e536 with expected crc 2023-09-23T23:11:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:53 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T23:11:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:53 smithi151 ceph-mon[159210]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T23:11:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:53 smithi151 ceph-mon[159210]: pgmap v1636: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[187743]: failed to encode map e536 with expected crc 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[187743]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[187743]: pgmap v1636: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[180979]: failed to encode map e536 with expected crc 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T23:11:53.959 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[180979]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T23:11:53.960 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:53 smithi052 ceph-mon[180979]: pgmap v1636: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:54.533 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T23:11:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:54 smithi151 ceph-mon[159210]: failed to encode map e537 with expected crc 2023-09-23T23:11:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:54 smithi151 ceph-mon[159210]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T23:11:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:54 smithi151 ceph-mon[159210]: failed to encode map e537 with expected crc 2023-09-23T23:11:54.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:54 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/870412046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:11:54.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[187743]: failed to encode map e537 with expected crc 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[187743]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[187743]: failed to encode map e537 with expected crc 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/870412046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[180979]: failed to encode map e537 with expected crc 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[180979]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[180979]: failed to encode map e537 with expected crc 2023-09-23T23:11:54.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:54 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/870412046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:11:55.224 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T23:11:55.594 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[180979]: failed to encode map e538 with expected crc 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/870412046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[180979]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[180979]: pgmap v1639: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[187743]: failed to encode map e538 with expected crc 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/870412046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:11:55.808 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[187743]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T23:11:55.809 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:55 smithi052 ceph-mon[187743]: pgmap v1639: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:55 smithi151 ceph-mon[159210]: failed to encode map e538 with expected crc 2023-09-23T23:11:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:55 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/870412046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:11:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:55 smithi151 ceph-mon[159210]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T23:11:55.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:55 smithi151 ceph-mon[159210]: pgmap v1639: 73 pgs: 73 active+clean; 4.6 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:56.843 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[180979]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.843 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[180979]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.843 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[180979]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.843 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[180979]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.843 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[187743]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.843 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[187743]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.844 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[187743]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.844 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:56 smithi052 ceph-mon[187743]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:56 smithi151 ceph-mon[159210]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:56 smithi151 ceph-mon[159210]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:56 smithi151 ceph-mon[159210]: failed to encode map e538 with expected crc 2023-09-23T23:11:56.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:56 smithi151 ceph-mon[159210]: failed to encode map e538 with expected crc 2023-09-23T23:11:57.548 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T23:11:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:57 smithi151 ceph-mon[159210]: pgmap v1640: 73 pgs: 73 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:57.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:57 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3581407645' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:11:57.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:57 smithi052 ceph-mon[187743]: pgmap v1640: 73 pgs: 73 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:57.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:57 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3581407645' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:11:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:57 smithi052 ceph-mon[180979]: pgmap v1640: 73 pgs: 73 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:57.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:57 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3581407645' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T23:11:58.270 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T23:11:58.543 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[180979]: failed to encode map e539 with expected crc 2023-09-23T23:11:58.543 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[180979]: failed to encode map e539 with expected crc 2023-09-23T23:11:58.544 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3581407645' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:11:58.544 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[180979]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T23:11:58.644 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:11:58.882 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[187743]: failed to encode map e539 with expected crc 2023-09-23T23:11:58.882 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[187743]: failed to encode map e539 with expected crc 2023-09-23T23:11:58.882 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3581407645' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:11:58.882 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:58 smithi052 ceph-mon[187743]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T23:11:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:58 smithi151 ceph-mon[159210]: failed to encode map e539 with expected crc 2023-09-23T23:11:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:58 smithi151 ceph-mon[159210]: failed to encode map e539 with expected crc 2023-09-23T23:11:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:58 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3581407645' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T23:11:58.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:58 smithi151 ceph-mon[159210]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T23:11:59.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[180979]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[180979]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[180979]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T23:11:59.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[180979]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.926 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[180979]: pgmap v1643: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:59.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[187743]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[187743]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[187743]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T23:11:59.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[187743]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:11:59 smithi052 ceph-mon[187743]: pgmap v1643: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:11:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:59 smithi151 ceph-mon[159210]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:59 smithi151 ceph-mon[159210]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:59 smithi151 ceph-mon[159210]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T23:11:59.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:59 smithi151 ceph-mon[159210]: failed to encode map e540 with expected crc 2023-09-23T23:11:59.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:11:59 smithi151 ceph-mon[159210]: pgmap v1643: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:00.561 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T23:12:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:00 smithi151 ceph-mon[159210]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T23:12:00.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:00 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3229351978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:00.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:00 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:00.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:00 smithi052 ceph-mon[180979]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T23:12:00.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:00 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3229351978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:00.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:00 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:00 smithi052 ceph-mon[187743]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T23:12:00.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:00 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3229351978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:00.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:00 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:01.188 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T23:12:01.552 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:01.817 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.817 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:12:01.817 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[187743]: pgmap v1645: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.818 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.819 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.819 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:01 smithi052 ceph-mon[180979]: pgmap v1645: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:12:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T23:12:01.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: failed to encode map e541 with expected crc 2023-09-23T23:12:01.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:01 smithi151 ceph-mon[159210]: pgmap v1645: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:03.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:03 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:12:03] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.33.4" 2023-09-23T23:12:03.918 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T23:12:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:03 smithi151 ceph-mon[159210]: pgmap v1646: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:04.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:03 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1055613592' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:03 smithi052 ceph-mon[187743]: pgmap v1646: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:04.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:03 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1055613592' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:04.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:03 smithi052 ceph-mon[180979]: pgmap v1646: 73 pgs: 6 peering, 67 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:04.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:03 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1055613592' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T23:12:04.695 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T23:12:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[187743]: failed to encode map e542 with expected crc 2023-09-23T23:12:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1055613592' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:12:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[187743]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T23:12:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:04.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:04.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[180979]: failed to encode map e542 with expected crc 2023-09-23T23:12:04.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1055613592' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:12:04.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[180979]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T23:12:04.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:04.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:04 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:05.065 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:04 smithi151 ceph-mon[159210]: failed to encode map e542 with expected crc 2023-09-23T23:12:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:04 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1055613592' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T23:12:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:04 smithi151 ceph-mon[159210]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T23:12:05.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:05.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:04 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:05.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: failed to encode map e542 with expected crc 2023-09-23T23:12:05.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: failed to encode map e542 with expected crc 2023-09-23T23:12:05.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: pgmap v1648: 73 pgs: 73 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:05.918 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: pgmap v1648: 73 pgs: 73 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:12:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2023-09-23T23:12:06.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:05 smithi151 ceph-mon[159210]: Cluster is now healthy 2023-09-23T23:12:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: pgmap v1648: 73 pgs: 73 active+clean; 4.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:06.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:12:06.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2023-09-23T23:12:06.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[187743]: Cluster is now healthy 2023-09-23T23:12:06.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: failed to encode map e542 with expected crc 2023-09-23T23:12:06.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T23:12:06.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2023-09-23T23:12:06.247 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:05 smithi052 ceph-mon[180979]: Cluster is now healthy 2023-09-23T23:12:06.929 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T23:12:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:06 smithi151 ceph-mon[159210]: failed to encode map e542 with expected crc 2023-09-23T23:12:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:06 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3624486522' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:07.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:06 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:06 smithi052 ceph-mon[187743]: failed to encode map e542 with expected crc 2023-09-23T23:12:07.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:06 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3624486522' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:07.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:06 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:06 smithi052 ceph-mon[180979]: failed to encode map e542 with expected crc 2023-09-23T23:12:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:06 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3624486522' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:07.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:06 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:07.689 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T23:12:07.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[180979]: pgmap v1649: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:07.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[180979]: failed to encode map e543 with expected crc 2023-09-23T23:12:07.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[180979]: failed to encode map e543 with expected crc 2023-09-23T23:12:07.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:12:07.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[180979]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T23:12:07.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[187743]: pgmap v1649: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:07.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[187743]: failed to encode map e543 with expected crc 2023-09-23T23:12:07.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[187743]: failed to encode map e543 with expected crc 2023-09-23T23:12:07.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:12:07.926 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:07 smithi052 ceph-mon[187743]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T23:12:08.090 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:07 smithi151 ceph-mon[159210]: pgmap v1649: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:07 smithi151 ceph-mon[159210]: failed to encode map e543 with expected crc 2023-09-23T23:12:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:07 smithi151 ceph-mon[159210]: failed to encode map e543 with expected crc 2023-09-23T23:12:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:07 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:12:08.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:07 smithi151 ceph-mon[159210]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T23:12:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:08 smithi151 ceph-mon[159210]: failed to encode map e543 with expected crc 2023-09-23T23:12:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:08 smithi151 ceph-mon[159210]: failed to encode map e543 with expected crc 2023-09-23T23:12:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:08 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:09.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:08 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[187743]: failed to encode map e543 with expected crc 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[187743]: failed to encode map e543 with expected crc 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[180979]: failed to encode map e543 with expected crc 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[180979]: failed to encode map e543 with expected crc 2023-09-23T23:12:09.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:09.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:08 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:09.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:09 smithi052 ceph-mon[187743]: pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:09.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:09 smithi052 ceph-mon[180979]: pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:10.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:09 smithi151 ceph-mon[159210]: pgmap v1651: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:10.936 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T23:12:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:10 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/475759326' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:10 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:10 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:10 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/475759326' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:11.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:10 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:10 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:10 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/475759326' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T23:12:11.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:10 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:10 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.722 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: pgmap v1652: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/475759326' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: pgmap v1652: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/475759326' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.995 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: failed to encode map e544 with expected crc 2023-09-23T23:12:11.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:12:11.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:12:11.996 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:11 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:12.102 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: pgmap v1652: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: failed to encode map e544 with expected crc 2023-09-23T23:12:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/475759326' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T23:12:12.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T23:12:12.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: failed to encode map e544 with expected crc 2023-09-23T23:12:12.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: failed to encode map e544 with expected crc 2023-09-23T23:12:12.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: failed to encode map e544 with expected crc 2023-09-23T23:12:12.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T23:12:12.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T23:12:12.200 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:11 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:12 smithi151 ceph-mon[159210]: failed to encode map e545 with expected crc 2023-09-23T23:12:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:12 smithi151 ceph-mon[159210]: failed to encode map e545 with expected crc 2023-09-23T23:12:13.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:12 smithi151 ceph-mon[159210]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T23:12:13.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:12 smithi052 ceph-mon[187743]: failed to encode map e545 with expected crc 2023-09-23T23:12:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:12 smithi052 ceph-mon[187743]: failed to encode map e545 with expected crc 2023-09-23T23:12:13.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:12 smithi052 ceph-mon[187743]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T23:12:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:12 smithi052 ceph-mon[180979]: failed to encode map e545 with expected crc 2023-09-23T23:12:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:12 smithi052 ceph-mon[180979]: failed to encode map e545 with expected crc 2023-09-23T23:12:13.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:12 smithi052 ceph-mon[180979]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T23:12:13.743 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:13 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:12:13] "GET /metrics HTTP/1.1" 200 34240 "" "Prometheus/2.33.4" 2023-09-23T23:12:13.952 INFO:teuthology.orchestra.run.smithi052.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T23:12:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:13 smithi052 ceph-mon[187743]: pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:13 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1287652401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:14.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:13 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:13 smithi052 ceph-mon[180979]: pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:13 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1287652401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:14.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:13 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:14.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:13 smithi151 ceph-mon[159210]: pgmap v1655: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:14.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:13 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1287652401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:14.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:13 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:14.681 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T23:12:14.957 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:14 smithi052 ceph-mon[180979]: failed to encode map e546 with expected crc 2023-09-23T23:12:14.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:14 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:12:14.958 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:14 smithi052 ceph-mon[180979]: osdmap e546: 8 total, 8 up, 8 in 2023-09-23T23:12:15.050 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:15.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:14 smithi052 ceph-mon[187743]: failed to encode map e546 with expected crc 2023-09-23T23:12:15.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:14 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:12:15.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:14 smithi052 ceph-mon[187743]: osdmap e546: 8 total, 8 up, 8 in 2023-09-23T23:12:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:14 smithi151 ceph-mon[159210]: failed to encode map e546 with expected crc 2023-09-23T23:12:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:14 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:12:15.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:14 smithi151 ceph-mon[159210]: osdmap e546: 8 total, 8 up, 8 in 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[187743]: pgmap v1657: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[187743]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[187743]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[187743]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[180979]: pgmap v1657: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[180979]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[180979]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.245 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:15 smithi052 ceph-mon[180979]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:15 smithi151 ceph-mon[159210]: pgmap v1657: 73 pgs: 73 active+clean; 4.6 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:15 smithi151 ceph-mon[159210]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:15 smithi151 ceph-mon[159210]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:15 smithi151 ceph-mon[159210]: failed to encode map e546 with expected crc 2023-09-23T23:12:16.970 INFO:teuthology.orchestra.run.smithi052.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T23:12:17.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:16 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2485345228' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:17.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:16 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:17.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:16 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2485345228' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:17.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:16 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:16 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2485345228' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:17.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:16 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T23:12:17.709 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T23:12:17.709 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T23:12:17.709 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T23:12:17.709 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T23:12:17.709 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T23:12:17.710 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T23:12:18.084 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:18.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[187743]: pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[187743]: failed to encode map e547 with expected crc 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[187743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[187743]: osdmap e547: 8 total, 8 up, 8 in 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[180979]: pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[180979]: failed to encode map e547 with expected crc 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[180979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:12:18.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:17 smithi052 ceph-mon[180979]: osdmap e547: 8 total, 8 up, 8 in 2023-09-23T23:12:18.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:17 smithi151 ceph-mon[159210]: pgmap v1658: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:18.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:17 smithi151 ceph-mon[159210]: failed to encode map e547 with expected crc 2023-09-23T23:12:18.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:17 smithi151 ceph-mon[159210]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T23:12:18.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:17 smithi151 ceph-mon[159210]: osdmap e547: 8 total, 8 up, 8 in 2023-09-23T23:12:19.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[187743]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[187743]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[187743]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[187743]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[180979]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[180979]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[180979]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:18 smithi052 ceph-mon[180979]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:18 smithi151 ceph-mon[159210]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:18 smithi151 ceph-mon[159210]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:18 smithi151 ceph-mon[159210]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.449 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:18 smithi151 ceph-mon[159210]: failed to encode map e547 with expected crc 2023-09-23T23:12:19.735 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:12:19.735 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":547,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T23:12:16.957633+0000","last_up_change":"2023-09-23T22:58:22.223440+0000","last_in_change":"2023-09-23T23:11:16.501637+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"507","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":507,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:28:41.273306+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:28:42.097486+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":219,"last_clean_end":344,"up_from":353,"up_thru":544,"down_at":345,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6803","nonce":1932666510}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6805","nonce":1932666510}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6809","nonce":1932666510}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6807","nonce":1932666510}]},"public_addr":"172.21.15.52:6803/1932666510","cluster_addr":"172.21.15.52:6805/1932666510","heartbeat_back_addr":"172.21.15.52:6809/1932666510","heartbeat_front_addr":"172.21.15.52:6807/1932666510","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":531,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6811","nonce":1231582443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6813","nonce":1231582443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6817","nonce":1231582443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6815","nonce":1231582443}]},"public_addr":"172.21.15.52:6811/1231582443","cluster_addr":"172.21.15.52:6813/1231582443","heartbeat_back_addr":"172.21.15.52:6817/1231582443","heartbeat_front_addr":"172.21.15.52:6815/1231582443","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":131,"last_clean_end":323,"up_from":359,"up_thru":521,"down_at":324,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6819","nonce":929264482}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6821","nonce":929264482}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6825","nonce":929264482}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6823","nonce":929264482}]},"public_addr":"172.21.15.52:6819/929264482","cluster_addr":"172.21.15.52:6821/929264482","heartbeat_back_addr":"172.21.15.52:6825/929264482","heartbeat_front_addr":"172.21.15.52:6823/929264482","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":90,"up_from":94,"up_thru":536,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6827","nonce":133265872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6829","nonce":133265872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6833","nonce":133265872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6831","nonce":133265872}]},"public_addr":"172.21.15.52:6827/133265872","cluster_addr":"172.21.15.52:6829/133265872","heartbeat_back_addr":"172.21.15.52:6833/133265872","heartbeat_front_addr":"172.21.15.52:6831/133265872","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":531,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":535,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":539,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":531,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":344},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":323},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":90},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.7","mappings":[{"from":3,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:6801/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3645713772":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3758668631":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/1038804029":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3912833337":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/3773142740":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000","172.21.15.151:0/1914484619":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/1262640861":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.151:0/3484573808":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.151:6834/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/1078027830":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:6835/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:0/483361858":"2023-09-24T22:32:09.537934+0000","172.21.15.151:0/4181179150":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/3070502744":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:0/3637583595":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/4148041136":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T23:12:20.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[180979]: pgmap v1660: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:20.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:20.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:20.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1266133324' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:20.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[187743]: pgmap v1660: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:20.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:20.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:20.245 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:19 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1266133324' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:20.451 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T23:12:20.451 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T23:12:20.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:19 smithi151 ceph-mon[159210]: pgmap v1660: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:20.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' 2023-09-23T23:12:20.452 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:19 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:20.453 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:19 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1266133324' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:20.822 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:21.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:21 smithi052 ceph-mon[187743]: pgmap v1661: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:21.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:21 smithi052 ceph-mon[180979]: pgmap v1661: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:21.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:21 smithi151 ceph-mon[159210]: pgmap v1661: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:22.497 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:12:22.497 INFO:teuthology.orchestra.run.smithi052.stdout:{"epoch":547,"fsid":"891acd3a-5a5f-11ee-8db4-212e2dc638e7","created":"2023-09-23T22:22:34.777440+0000","modified":"2023-09-23T23:12:16.957633+0000","last_up_change":"2023-09-23T22:58:22.223440+0000","last_in_change":"2023-09-23T23:11:16.501637+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:25:25.839760+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:27:48.896897+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"507","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":507,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:28:41.273306+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:28:42.097486+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c382e099-ff66-4b35-af2f-978f52c83cfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":219,"last_clean_end":344,"up_from":353,"up_thru":544,"down_at":345,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6802","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6803","nonce":1932666510}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6804","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6805","nonce":1932666510}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6808","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6809","nonce":1932666510}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6806","nonce":1932666510},{"type":"v1","addr":"172.21.15.52:6807","nonce":1932666510}]},"public_addr":"172.21.15.52:6803/1932666510","cluster_addr":"172.21.15.52:6805/1932666510","heartbeat_back_addr":"172.21.15.52:6809/1932666510","heartbeat_front_addr":"172.21.15.52:6807/1932666510","state":["exists","up"]},{"osd":1,"uuid":"8704f27e-d1a5-49b1-9382-0ebc689e3f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":80,"up_from":84,"up_thru":531,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6810","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6811","nonce":1231582443}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6812","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6813","nonce":1231582443}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6816","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6817","nonce":1231582443}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6814","nonce":1231582443},{"type":"v1","addr":"172.21.15.52:6815","nonce":1231582443}]},"public_addr":"172.21.15.52:6811/1231582443","cluster_addr":"172.21.15.52:6813/1231582443","heartbeat_back_addr":"172.21.15.52:6817/1231582443","heartbeat_front_addr":"172.21.15.52:6815/1231582443","state":["exists","up"]},{"osd":2,"uuid":"80877779-5cd3-4a3c-bb3e-ffd41e160fb5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":131,"last_clean_end":323,"up_from":359,"up_thru":521,"down_at":324,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6818","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6819","nonce":929264482}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6820","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6821","nonce":929264482}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6824","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6825","nonce":929264482}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6822","nonce":929264482},{"type":"v1","addr":"172.21.15.52:6823","nonce":929264482}]},"public_addr":"172.21.15.52:6819/929264482","cluster_addr":"172.21.15.52:6821/929264482","heartbeat_back_addr":"172.21.15.52:6825/929264482","heartbeat_front_addr":"172.21.15.52:6823/929264482","state":["exists","up"]},{"osd":3,"uuid":"c94bebc9-a105-4d74-9a03-c88e10cc2876","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":90,"up_from":94,"up_thru":536,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6826","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6827","nonce":133265872}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6828","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6829","nonce":133265872}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6832","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6833","nonce":133265872}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.52:6830","nonce":133265872},{"type":"v1","addr":"172.21.15.52:6831","nonce":133265872}]},"public_addr":"172.21.15.52:6827/133265872","cluster_addr":"172.21.15.52:6829/133265872","heartbeat_back_addr":"172.21.15.52:6833/133265872","heartbeat_front_addr":"172.21.15.52:6831/133265872","state":["exists","up"]},{"osd":4,"uuid":"a8b1470a-6d28-44d9-9921-6f8fbcb52b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":531,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6801","nonce":2146533857}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6803","nonce":2146533857}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6807","nonce":2146533857}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":2146533857},{"type":"v1","addr":"172.21.15.151:6805","nonce":2146533857}]},"public_addr":"172.21.15.151:6801/2146533857","cluster_addr":"172.21.15.151:6803/2146533857","heartbeat_back_addr":"172.21.15.151:6807/2146533857","heartbeat_front_addr":"172.21.15.151:6805/2146533857","state":["exists","up"]},{"osd":5,"uuid":"768d49a4-59ee-43e4-802b-a58f2b77580b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":535,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6809","nonce":992496865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6811","nonce":992496865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6815","nonce":992496865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":992496865},{"type":"v1","addr":"172.21.15.151:6813","nonce":992496865}]},"public_addr":"172.21.15.151:6809/992496865","cluster_addr":"172.21.15.151:6811/992496865","heartbeat_back_addr":"172.21.15.151:6815/992496865","heartbeat_front_addr":"172.21.15.151:6813/992496865","state":["exists","up"]},{"osd":6,"uuid":"2c295677-55e8-41ff-8733-982844192fbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":539,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6817","nonce":2869932922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6819","nonce":2869932922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6823","nonce":2869932922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":2869932922},{"type":"v1","addr":"172.21.15.151:6821","nonce":2869932922}]},"public_addr":"172.21.15.151:6817/2869932922","cluster_addr":"172.21.15.151:6819/2869932922","heartbeat_back_addr":"172.21.15.151:6823/2869932922","heartbeat_front_addr":"172.21.15.151:6821/2869932922","state":["exists","up"]},{"osd":7,"uuid":"c1b48eff-20d5-4d6d-bb17-0c44ffdf866e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":531,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6825","nonce":2800377088}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6827","nonce":2800377088}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6831","nonce":2800377088}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2800377088},{"type":"v1","addr":"172.21.15.151:6829","nonce":2800377088}]},"public_addr":"172.21.15.151:6825/2800377088","cluster_addr":"172.21.15.151:6827/2800377088","heartbeat_back_addr":"172.21.15.151:6831/2800377088","heartbeat_front_addr":"172.21.15.151:6829/2800377088","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:24:40.472104+0000","dead_epoch":344},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:01.994132+0000","dead_epoch":80},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:23.486115+0000","dead_epoch":323},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:25:46.033425+0000","dead_epoch":90},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:06.424680+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:27.894699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:26:49.343744+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:27:10.551213+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.7","mappings":[{"from":3,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.52:6801/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3645713772":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3758668631":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/1038804029":"2023-09-24T22:32:49.359696+0000","172.21.15.52:0/3912833337":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/3773142740":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/2401228233":"2023-09-24T22:23:02.549172+0000","172.21.15.151:0/1914484619":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/1262640861":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.151:0/3484573808":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/642109542":"2023-09-24T22:23:25.849910+0000","172.21.15.151:6834/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/624901514":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/1078027830":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6801/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:6835/664659773":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3442490676":"2023-09-24T22:23:25.849910+0000","172.21.15.52:0/3278569487":"2023-09-24T22:27:29.395575+0000","172.21.15.52:6800/2011536210":"2023-09-24T22:27:29.395575+0000","172.21.15.151:0/483361858":"2023-09-24T22:32:09.537934+0000","172.21.15.151:0/4181179150":"2023-09-24T22:32:09.537934+0000","172.21.15.52:0/3629843530":"2023-09-24T22:27:29.395575+0000","172.21.15.52:0/3070502744":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:0/3637583595":"2023-09-24T22:32:49.359696+0000","172.21.15.52:6800/65131393":"2023-09-24T22:32:49.359696+0000","172.21.15.151:0/4148041136":"2023-09-24T22:32:09.537934+0000","172.21.15.52:6801/2740388443":"2023-09-24T22:30:59.031552+0000","172.21.15.52:6800/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:6801/3545504609":"2023-09-24T22:23:02.549172+0000","172.21.15.52:0/4018405217":"2023-09-24T22:23:02.549172+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T23:12:22.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:22 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3825114581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:22.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:22 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3825114581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:22.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:22 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3825114581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T23:12:23.170 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T23:12:23.170 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T23:12:23.171 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T23:12:23.171 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T23:12:23.172 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T23:12:23.172 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T23:12:23.172 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T23:12:23.173 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T23:12:23.494 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:23 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:12:23] "GET /metrics HTTP/1.1" 200 34240 "" "Prometheus/2.33.4" 2023-09-23T23:12:23.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:23 smithi151 ceph-mon[159210]: pgmap v1662: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:23.993 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:23 smithi052 ceph-mon[187743]: pgmap v1662: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:23.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:23 smithi052 ceph-mon[180979]: pgmap v1662: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:24.890 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:24.941 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:25.014 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:25.044 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:25.200 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:25.380 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:25.454 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:25.820 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:26.073 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:25 smithi052 ceph-mon[187743]: pgmap v1663: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:26.073 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:25 smithi052 ceph-mon[180979]: pgmap v1663: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:26.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:25 smithi151 ceph-mon[159210]: pgmap v1663: 73 pgs: 73 active+clean; 4.6 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:28.158 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:27 smithi052 ceph-mon[187743]: pgmap v1664: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:28.159 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:27 smithi052 ceph-mon[180979]: pgmap v1664: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:28.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:27 smithi151 ceph-mon[159210]: pgmap v1664: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:30.133 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:29 smithi052 ceph-mon[180979]: pgmap v1665: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:30.134 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:29 smithi052 ceph-mon[187743]: pgmap v1665: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:30.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:29 smithi151 ceph-mon[159210]: pgmap v1665: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:31.323 INFO:teuthology.orchestra.run.smithi052.stdout:403726926240 2023-09-23T23:12:31.323 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T23:12:31.588 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:31 smithi052 ceph-mon[180979]: pgmap v1666: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:31.589 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:31 smithi052 ceph-mon[187743]: pgmap v1666: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:31.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:31 smithi151 ceph-mon[159210]: pgmap v1666: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:33.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:33 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:12:33] "GET /metrics HTTP/1.1" 200 34240 "" "Prometheus/2.33.4" 2023-09-23T23:12:33.511 INFO:teuthology.orchestra.run.smithi052.stdout:1516123455663 2023-09-23T23:12:33.512 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T23:12:33.959 INFO:teuthology.orchestra.run.smithi052.stdout:180388626979 2023-09-23T23:12:33.960 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T23:12:34.061 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:33 smithi052 ceph-mon[187743]: pgmap v1667: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:34.061 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:33 smithi052 ceph-mon[180979]: pgmap v1667: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:34.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:33 smithi151 ceph-mon[159210]: pgmap v1667: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:34.859 INFO:teuthology.orchestra.run.smithi052.stdout:206158430755 2023-09-23T23:12:34.859 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T23:12:35.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:34 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:35.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:34 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:35.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:34 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:35.339 INFO:teuthology.orchestra.run.smithi052.stdout:1541893259436 2023-09-23T23:12:35.339 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T23:12:35.625 INFO:teuthology.orchestra.run.smithi052.stdout:154618823211 2023-09-23T23:12:35.626 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T23:12:35.679 INFO:teuthology.orchestra.run.smithi052.stdout:360777253293 2023-09-23T23:12:35.680 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T23:12:35.925 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:35 smithi052 ceph-mon[187743]: pgmap v1668: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:35.925 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:35 smithi052 ceph-mon[180979]: pgmap v1668: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:36.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:35 smithi151 ceph-mon[159210]: pgmap v1668: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:36.302 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:36.334 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:36.358 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:36.458 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:36.490 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:36.548 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:37.427 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:38.171 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:37 smithi052 ceph-mon[180979]: pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:38.171 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:37 smithi052 ceph-mon[187743]: pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:38.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:37 smithi151 ceph-mon[159210]: pgmap v1669: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:39.994 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:39 smithi052 ceph-mon[187743]: pgmap v1670: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:39.994 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:39 smithi052 ceph-mon[180979]: pgmap v1670: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:40.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:39 smithi151 ceph-mon[159210]: pgmap v1670: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:41.413 INFO:teuthology.orchestra.run.smithi052.stdout:128849019438 2023-09-23T23:12:41.413 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T23:12:41.652 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:41 smithi052 ceph-mon[187743]: pgmap v1671: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:41.652 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:41 smithi052 ceph-mon[180979]: pgmap v1671: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:41.920 INFO:teuthology.orchestra.run.smithi052.stdout:360777253294 2023-09-23T23:12:41.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:41 smithi151 ceph-mon[159210]: pgmap v1671: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:42.802 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:42 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/470030716' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:12:42.802 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:42 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/470030716' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:12:42.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:42 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/470030716' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T23:12:43.306 INFO:teuthology.orchestra.run.smithi052.stdout:180388626981 2023-09-23T23:12:43.383 INFO:teuthology.orchestra.run.smithi052.stdout:1516123455665 2023-09-23T23:12:43.458 INFO:teuthology.orchestra.run.smithi052.stdout:403726926242 2023-09-23T23:12:43.686 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:43 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:12:43] "GET /metrics HTTP/1.1" 200 34238 "" "Prometheus/2.33.4" 2023-09-23T23:12:43.686 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[187743]: pgmap v1672: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1036164212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1292249306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/2789074449' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[180979]: pgmap v1672: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1036164212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1292249306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:12:43.687 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:43 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/2789074449' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:12:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:43 smithi151 ceph-mon[159210]: pgmap v1672: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1036164212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T23:12:43.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1292249306' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T23:12:43.950 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:43 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/2789074449' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T23:12:45.164 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777253293 got 360777253294 for osd.1 2023-09-23T23:12:45.164 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:45.256 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926240 got 403726926242 for osd.3 2023-09-23T23:12:45.257 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:45.275 INFO:teuthology.orchestra.run.smithi052.stdout:206158430757 2023-09-23T23:12:45.472 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:45.496 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626979 got 180388626981 for osd.6 2023-09-23T23:12:45.497 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:45.646 INFO:teuthology.orchestra.run.smithi052.stdout:154618823213 2023-09-23T23:12:46.119 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:45 smithi052 ceph-mon[187743]: pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:46.119 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3856227141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:12:46.119 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:45 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/989282833' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:12:46.119 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:45 smithi052 ceph-mon[180979]: pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:46.119 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3856227141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:12:46.119 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:45 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/989282833' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:12:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:45 smithi151 ceph-mon[159210]: pgmap v1673: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3856227141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T23:12:46.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:45 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/989282833' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T23:12:46.914 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430755 got 206158430757 for osd.7 2023-09-23T23:12:46.914 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:46.930 INFO:teuthology.orchestra.run.smithi052.stdout:1541893259438 2023-09-23T23:12:47.082 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1516123455663 got 1516123455665 for osd.0 2023-09-23T23:12:47.083 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:47.122 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823211 got 154618823213 for osd.5 2023-09-23T23:12:47.122 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:47.823 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1541893259436 got 1541893259438 for osd.2 2023-09-23T23:12:47.823 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:47.923 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:47 smithi052 ceph-mon[180979]: pgmap v1674: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:47.923 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:47 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/1408272220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:12:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:47 smithi151 ceph-mon[159210]: pgmap v1674: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:48.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:47 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/1408272220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:12:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:47 smithi052 ceph-mon[187743]: pgmap v1674: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:48.243 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:47 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/1408272220' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T23:12:48.971 INFO:teuthology.orchestra.run.smithi052.stdout:128849019440 2023-09-23T23:12:49.658 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019438 got 128849019440 for osd.4 2023-09-23T23:12:49.658 DEBUG:teuthology.parallel:result is None 2023-09-23T23:12:49.658 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T23:12:49.658 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:12:49.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:49 smithi052 ceph-mon[187743]: pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:49.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:49 smithi052 ceph-mon[187743]: from='client.? 172.21.15.52:0/3058446876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:12:49.927 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:49 smithi052 ceph-mon[187743]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:50.029 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:49 smithi151 ceph-mon[159210]: pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:49 smithi151 ceph-mon[159210]: from='client.? 172.21.15.52:0/3058446876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:12:50.199 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:49 smithi151 ceph-mon[159210]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:49 smithi052 ceph-mon[180979]: pgmap v1675: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:50.243 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:49 smithi052 ceph-mon[180979]: from='client.? 172.21.15.52:0/3058446876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T23:12:50.244 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:49 smithi052 ceph-mon[180979]: from='mgr.25214 172.21.15.52:0/3589758201' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T23:12:51.725 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:12:51.726 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:12:51.916 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:51 smithi052 ceph-mon[187743]: pgmap v1676: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:51.916 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:51 smithi052 ceph-mon[180979]: pgmap v1676: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:51.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:51 smithi151 ceph-mon[159210]: pgmap v1676: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:52.367 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1676,"stamp":"2023-09-23T23:12:50.868617+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":9596213,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5425788,"kb_used_data":60860,"kb_used_omap":137,"kb_used_meta":5364726,"kb_avail":744535428,"statfs":{"total":767960285184,"available":762404278272,"internally_reserved":0,"allocated":62320640,"data_stored":48461469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":5493479608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003013"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":559,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097657+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:12:15.097657+0000","last_peered":"2023-09-23T23:12:15.097657+0000","last_clean":"2023-09-23T23:12:15.097657+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:12:15.097657+0000","last_undegraded":"2023-09-23T23:12:15.097657+0000","last_fullsized":"2023-09-23T23:12:15.097657+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":221,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T23:12:17.707690+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:12:17.707690+0000","last_peered":"2023-09-23T23:12:17.707690+0000","last_clean":"2023-09-23T23:12:17.707690+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:12:17.707690+0000","last_undegraded":"2023-09-23T23:12:17.707690+0000","last_fullsized":"2023-09-23T23:12:17.707690+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":221,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889124+0000","last_change":"2023-09-23T23:11:40.915446+0000","last_active":"2023-09-23T23:12:14.889124+0000","last_peered":"2023-09-23T23:12:14.889124+0000","last_clean":"2023-09-23T23:12:14.889124+0000","last_became_active":"2023-09-23T23:11:40.915136+0000","last_became_peered":"2023-09-23T23:11:40.915136+0000","last_unstale":"2023-09-23T23:12:14.889124+0000","last_undegraded":"2023-09-23T23:12:14.889124+0000","last_fullsized":"2023-09-23T23:12:14.889124+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:19.386456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":547,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427346+0000","last_change":"2023-09-23T23:12:12.051504+0000","last_active":"2023-09-23T23:12:15.427346+0000","last_peered":"2023-09-23T23:12:15.427346+0000","last_clean":"2023-09-23T23:12:15.427346+0000","last_became_active":"2023-09-23T23:12:12.051153+0000","last_became_peered":"2023-09-23T23:12:12.051153+0000","last_unstale":"2023-09-23T23:12:15.427346+0000","last_undegraded":"2023-09-23T23:12:15.427346+0000","last_fullsized":"2023-09-23T23:12:15.427346+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:18.938482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":72,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946206+0000","last_change":"2023-09-23T23:11:09.536656+0000","last_active":"2023-09-23T23:12:13.946206+0000","last_peered":"2023-09-23T23:12:13.946206+0000","last_clean":"2023-09-23T23:12:13.946206+0000","last_became_active":"2023-09-23T23:11:09.536309+0000","last_became_peered":"2023-09-23T23:11:09.536309+0000","last_unstale":"2023-09-23T23:12:13.946206+0000","last_undegraded":"2023-09-23T23:12:13.946206+0000","last_fullsized":"2023-09-23T23:12:13.946206+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:20.717592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098401+0000","last_change":"2023-09-23T23:11:58.726000+0000","last_active":"2023-09-23T23:12:15.098401+0000","last_peered":"2023-09-23T23:12:15.098401+0000","last_clean":"2023-09-23T23:12:15.098401+0000","last_became_active":"2023-09-23T23:11:58.725630+0000","last_became_peered":"2023-09-23T23:11:58.725630+0000","last_unstale":"2023-09-23T23:12:15.098401+0000","last_undegraded":"2023-09-23T23:12:15.098401+0000","last_fullsized":"2023-09-23T23:12:15.098401+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":473,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427424+0000","last_change":"2023-09-23T23:12:11.965477+0000","last_active":"2023-09-23T23:12:15.427424+0000","last_peered":"2023-09-23T23:12:15.427424+0000","last_clean":"2023-09-23T23:12:15.427424+0000","last_became_active":"2023-09-23T23:12:11.965238+0000","last_became_peered":"2023-09-23T23:12:11.965238+0000","last_unstale":"2023-09-23T23:12:15.427424+0000","last_undegraded":"2023-09-23T23:12:15.427424+0000","last_fullsized":"2023-09-23T23:12:15.427424+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:10.025423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094454+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:12:15.094454+0000","last_peered":"2023-09-23T23:12:15.094454+0000","last_clean":"2023-09-23T23:12:15.094454+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:12:15.094454+0000","last_undegraded":"2023-09-23T23:12:15.094454+0000","last_fullsized":"2023-09-23T23:12:15.094454+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096594+0000","last_change":"2023-09-23T23:11:17.525302+0000","last_active":"2023-09-23T23:12:15.096594+0000","last_peered":"2023-09-23T23:12:15.096594+0000","last_clean":"2023-09-23T23:12:15.096594+0000","last_became_active":"2023-09-23T23:11:17.525045+0000","last_became_peered":"2023-09-23T23:11:17.525045+0000","last_unstale":"2023-09-23T23:12:15.096594+0000","last_undegraded":"2023-09-23T23:12:15.096594+0000","last_fullsized":"2023-09-23T23:12:15.096594+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":280,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094589+0000","last_change":"2023-09-23T23:11:17.523534+0000","last_active":"2023-09-23T23:12:15.094589+0000","last_peered":"2023-09-23T23:12:15.094589+0000","last_clean":"2023-09-23T23:12:15.094589+0000","last_became_active":"2023-09-23T23:11:17.523167+0000","last_became_peered":"2023-09-23T23:11:17.523167+0000","last_unstale":"2023-09-23T23:12:15.094589+0000","last_undegraded":"2023-09-23T23:12:15.094589+0000","last_fullsized":"2023-09-23T23:12:15.094589+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:11.997595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":41,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949486+0000","last_change":"2023-09-23T23:11:17.706926+0000","last_active":"2023-09-23T23:12:13.949486+0000","last_peered":"2023-09-23T23:12:13.949486+0000","last_clean":"2023-09-23T23:12:13.949486+0000","last_became_active":"2023-09-23T23:11:17.524784+0000","last_became_peered":"2023-09-23T23:11:17.524784+0000","last_unstale":"2023-09-23T23:12:13.949486+0000","last_undegraded":"2023-09-23T23:12:13.949486+0000","last_fullsized":"2023-09-23T23:12:13.949486+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":240,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889213+0000","last_change":"2023-09-23T23:11:28.070251+0000","last_active":"2023-09-23T23:12:14.889213+0000","last_peered":"2023-09-23T23:12:14.889213+0000","last_clean":"2023-09-23T23:12:14.889213+0000","last_became_active":"2023-09-23T23:11:27.939074+0000","last_became_peered":"2023-09-23T23:11:27.939074+0000","last_unstale":"2023-09-23T23:12:14.889213+0000","last_undegraded":"2023-09-23T23:12:14.889213+0000","last_fullsized":"2023-09-23T23:12:14.889213+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:13.159027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":656,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097816+0000","last_change":"2023-09-23T23:11:17.701800+0000","last_active":"2023-09-23T23:12:15.097816+0000","last_peered":"2023-09-23T23:12:15.097816+0000","last_clean":"2023-09-23T23:12:15.097816+0000","last_became_active":"2023-09-23T23:11:17.524341+0000","last_became_peered":"2023-09-23T23:11:17.524341+0000","last_unstale":"2023-09-23T23:12:15.097816+0000","last_undegraded":"2023-09-23T23:12:15.097816+0000","last_fullsized":"2023-09-23T23:12:15.097816+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":530,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098178+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:12:15.098178+0000","last_peered":"2023-09-23T23:12:15.098178+0000","last_clean":"2023-09-23T23:12:15.098178+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:12:15.098178+0000","last_undegraded":"2023-09-23T23:12:15.098178+0000","last_fullsized":"2023-09-23T23:12:15.098178+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":41,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948978+0000","last_change":"2023-09-23T23:11:17.712628+0000","last_active":"2023-09-23T23:12:13.948978+0000","last_peered":"2023-09-23T23:12:13.948978+0000","last_clean":"2023-09-23T23:12:13.948978+0000","last_became_active":"2023-09-23T23:11:17.529211+0000","last_became_peered":"2023-09-23T23:11:17.529211+0000","last_unstale":"2023-09-23T23:12:13.948978+0000","last_undegraded":"2023-09-23T23:12:13.948978+0000","last_fullsized":"2023-09-23T23:12:13.948978+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094992+0000","last_change":"2023-09-23T23:11:17.523762+0000","last_active":"2023-09-23T23:12:15.094992+0000","last_peered":"2023-09-23T23:12:15.094992+0000","last_clean":"2023-09-23T23:12:15.094992+0000","last_became_active":"2023-09-23T23:11:17.523426+0000","last_became_peered":"2023-09-23T23:11:17.523426+0000","last_unstale":"2023-09-23T23:12:15.094992+0000","last_undegraded":"2023-09-23T23:12:15.094992+0000","last_fullsized":"2023-09-23T23:12:15.094992+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:57.182479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":530,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427303+0000","last_change":"2023-09-23T23:12:12.051990+0000","last_active":"2023-09-23T23:12:15.427303+0000","last_peered":"2023-09-23T23:12:15.427303+0000","last_clean":"2023-09-23T23:12:15.427303+0000","last_became_active":"2023-09-23T23:12:12.051628+0000","last_became_peered":"2023-09-23T23:12:12.051628+0000","last_unstale":"2023-09-23T23:12:15.427303+0000","last_undegraded":"2023-09-23T23:12:15.427303+0000","last_fullsized":"2023-09-23T23:12:15.427303+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:55.042906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096706+0000","last_change":"2023-09-23T23:11:17.525262+0000","last_active":"2023-09-23T23:12:15.096706+0000","last_peered":"2023-09-23T23:12:15.096706+0000","last_clean":"2023-09-23T23:12:15.096706+0000","last_became_active":"2023-09-23T23:11:17.524896+0000","last_became_peered":"2023-09-23T23:11:17.524896+0000","last_unstale":"2023-09-23T23:12:15.096706+0000","last_undegraded":"2023-09-23T23:12:15.096706+0000","last_fullsized":"2023-09-23T23:12:15.096706+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":35,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948946+0000","last_change":"2023-09-23T23:11:17.524399+0000","last_active":"2023-09-23T23:12:13.948946+0000","last_peered":"2023-09-23T23:12:13.948946+0000","last_clean":"2023-09-23T23:12:13.948946+0000","last_became_active":"2023-09-23T23:11:17.524074+0000","last_became_peered":"2023-09-23T23:11:17.524074+0000","last_unstale":"2023-09-23T23:12:13.948946+0000","last_undegraded":"2023-09-23T23:12:13.948946+0000","last_fullsized":"2023-09-23T23:12:13.948946+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":571,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427366+0000","last_change":"2023-09-23T23:12:12.051992+0000","last_active":"2023-09-23T23:12:15.427366+0000","last_peered":"2023-09-23T23:12:15.427366+0000","last_clean":"2023-09-23T23:12:15.427366+0000","last_became_active":"2023-09-23T23:12:12.051699+0000","last_became_peered":"2023-09-23T23:12:12.051699+0000","last_unstale":"2023-09-23T23:12:15.427366+0000","last_undegraded":"2023-09-23T23:12:15.427366+0000","last_fullsized":"2023-09-23T23:12:15.427366+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:09.484693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":81,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097226+0000","last_change":"2023-09-23T23:11:58.558430+0000","last_active":"2023-09-23T23:12:15.097226+0000","last_peered":"2023-09-23T23:12:15.097226+0000","last_clean":"2023-09-23T23:12:15.097226+0000","last_became_active":"2023-09-23T23:11:58.557927+0000","last_became_peered":"2023-09-23T23:11:58.557927+0000","last_unstale":"2023-09-23T23:12:15.097226+0000","last_undegraded":"2023-09-23T23:12:15.097226+0000","last_fullsized":"2023-09-23T23:12:15.097226+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":263,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.095120+0000","last_change":"2023-09-23T23:11:53.849144+0000","last_active":"2023-09-23T23:12:15.095120+0000","last_peered":"2023-09-23T23:12:15.095120+0000","last_clean":"2023-09-23T23:12:15.095120+0000","last_became_active":"2023-09-23T23:11:53.848785+0000","last_became_peered":"2023-09-23T23:11:53.848785+0000","last_unstale":"2023-09-23T23:12:15.095120+0000","last_undegraded":"2023-09-23T23:12:15.095120+0000","last_fullsized":"2023-09-23T23:12:15.095120+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:05.530980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":39,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949433+0000","last_change":"2023-09-23T23:11:17.709067+0000","last_active":"2023-09-23T23:12:13.949433+0000","last_peered":"2023-09-23T23:12:13.949433+0000","last_clean":"2023-09-23T23:12:13.949433+0000","last_became_active":"2023-09-23T23:11:17.527823+0000","last_became_peered":"2023-09-23T23:11:17.527823+0000","last_unstale":"2023-09-23T23:12:13.949433+0000","last_undegraded":"2023-09-23T23:12:13.949433+0000","last_fullsized":"2023-09-23T23:12:13.949433+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096312+0000","last_change":"2023-09-23T23:11:17.525389+0000","last_active":"2023-09-23T23:12:15.096312+0000","last_peered":"2023-09-23T23:12:15.096312+0000","last_clean":"2023-09-23T23:12:15.096312+0000","last_became_active":"2023-09-23T23:11:17.524899+0000","last_became_peered":"2023-09-23T23:11:17.524899+0000","last_unstale":"2023-09-23T23:12:15.096312+0000","last_undegraded":"2023-09-23T23:12:15.096312+0000","last_fullsized":"2023-09-23T23:12:15.096312+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":52,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949328+0000","last_change":"2023-09-23T23:11:40.923821+0000","last_active":"2023-09-23T23:12:13.949328+0000","last_peered":"2023-09-23T23:12:13.949328+0000","last_clean":"2023-09-23T23:12:13.949328+0000","last_became_active":"2023-09-23T23:11:40.918283+0000","last_became_peered":"2023-09-23T23:11:40.918283+0000","last_unstale":"2023-09-23T23:12:13.949328+0000","last_undegraded":"2023-09-23T23:12:13.949328+0000","last_fullsized":"2023-09-23T23:12:13.949328+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":554,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097710+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:12:15.097710+0000","last_peered":"2023-09-23T23:12:15.097710+0000","last_clean":"2023-09-23T23:12:15.097710+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:12:15.097710+0000","last_undegraded":"2023-09-23T23:12:15.097710+0000","last_fullsized":"2023-09-23T23:12:15.097710+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":39,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949267+0000","last_change":"2023-09-23T23:11:17.703666+0000","last_active":"2023-09-23T23:12:13.949267+0000","last_peered":"2023-09-23T23:12:13.949267+0000","last_clean":"2023-09-23T23:12:13.949267+0000","last_became_active":"2023-09-23T23:11:17.523182+0000","last_became_peered":"2023-09-23T23:11:17.523182+0000","last_unstale":"2023-09-23T23:12:13.949267+0000","last_undegraded":"2023-09-23T23:12:13.949267+0000","last_fullsized":"2023-09-23T23:12:13.949267+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097236+0000","last_change":"2023-09-23T23:11:58.726640+0000","last_active":"2023-09-23T23:12:15.097236+0000","last_peered":"2023-09-23T23:12:15.097236+0000","last_clean":"2023-09-23T23:12:15.097236+0000","last_became_active":"2023-09-23T23:11:58.726160+0000","last_became_peered":"2023-09-23T23:11:58.726160+0000","last_unstale":"2023-09-23T23:12:15.097236+0000","last_undegraded":"2023-09-23T23:12:15.097236+0000","last_fullsized":"2023-09-23T23:12:15.097236+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096502+0000","last_change":"2023-09-23T23:11:52.529700+0000","last_active":"2023-09-23T23:12:15.096502+0000","last_peered":"2023-09-23T23:12:15.096502+0000","last_clean":"2023-09-23T23:12:15.096502+0000","last_became_active":"2023-09-23T23:11:52.529339+0000","last_became_peered":"2023-09-23T23:11:52.529339+0000","last_unstale":"2023-09-23T23:12:15.096502+0000","last_undegraded":"2023-09-23T23:12:15.096502+0000","last_fullsized":"2023-09-23T23:12:15.096502+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":44,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949543+0000","last_change":"2023-09-23T23:11:09.548373+0000","last_active":"2023-09-23T23:12:13.949543+0000","last_peered":"2023-09-23T23:12:13.949543+0000","last_clean":"2023-09-23T23:12:13.949543+0000","last_became_active":"2023-09-23T23:11:09.547670+0000","last_became_peered":"2023-09-23T23:11:09.547670+0000","last_unstale":"2023-09-23T23:12:13.949543+0000","last_undegraded":"2023-09-23T23:12:13.949543+0000","last_fullsized":"2023-09-23T23:12:13.949543+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:12.720283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":266,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094705+0000","last_change":"2023-09-23T23:11:09.608043+0000","last_active":"2023-09-23T23:12:15.094705+0000","last_peered":"2023-09-23T23:12:15.094705+0000","last_clean":"2023-09-23T23:12:15.094705+0000","last_became_active":"2023-09-23T23:11:09.544986+0000","last_became_peered":"2023-09-23T23:11:09.544986+0000","last_unstale":"2023-09-23T23:12:15.094705+0000","last_undegraded":"2023-09-23T23:12:15.094705+0000","last_fullsized":"2023-09-23T23:12:15.094705+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:26.925687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":549,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097716+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:12:15.097716+0000","last_peered":"2023-09-23T23:12:15.097716+0000","last_clean":"2023-09-23T23:12:15.097716+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:12:15.097716+0000","last_undegraded":"2023-09-23T23:12:15.097716+0000","last_fullsized":"2023-09-23T23:12:15.097716+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097462+0000","last_change":"2023-09-23T23:11:58.726424+0000","last_active":"2023-09-23T23:12:15.097462+0000","last_peered":"2023-09-23T23:12:15.097462+0000","last_clean":"2023-09-23T23:12:15.097462+0000","last_became_active":"2023-09-23T23:11:58.725673+0000","last_became_peered":"2023-09-23T23:11:58.725673+0000","last_unstale":"2023-09-23T23:12:15.097462+0000","last_undegraded":"2023-09-23T23:12:15.097462+0000","last_fullsized":"2023-09-23T23:12:15.097462+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097883+0000","last_change":"2023-09-23T23:11:41.025203+0000","last_active":"2023-09-23T23:12:15.097883+0000","last_peered":"2023-09-23T23:12:15.097883+0000","last_clean":"2023-09-23T23:12:15.097883+0000","last_became_active":"2023-09-23T23:11:41.024944+0000","last_became_peered":"2023-09-23T23:11:41.024944+0000","last_unstale":"2023-09-23T23:12:15.097883+0000","last_undegraded":"2023-09-23T23:12:15.097883+0000","last_fullsized":"2023-09-23T23:12:15.097883+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":579,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097504+0000","last_change":"2023-09-23T23:11:17.524264+0000","last_active":"2023-09-23T23:12:15.097504+0000","last_peered":"2023-09-23T23:12:15.097504+0000","last_clean":"2023-09-23T23:12:15.097504+0000","last_became_active":"2023-09-23T23:11:17.523923+0000","last_became_peered":"2023-09-23T23:11:17.523923+0000","last_unstale":"2023-09-23T23:12:15.097504+0000","last_undegraded":"2023-09-23T23:12:15.097504+0000","last_fullsized":"2023-09-23T23:12:15.097504+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":575,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097984+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:12:15.097984+0000","last_peered":"2023-09-23T23:12:15.097984+0000","last_clean":"2023-09-23T23:12:15.097984+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:12:15.097984+0000","last_undegraded":"2023-09-23T23:12:15.097984+0000","last_fullsized":"2023-09-23T23:12:15.097984+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T23:12:17.707286+0000","last_change":"2023-09-23T23:11:17.524740+0000","last_active":"2023-09-23T23:12:17.707286+0000","last_peered":"2023-09-23T23:12:17.707286+0000","last_clean":"2023-09-23T23:12:17.707286+0000","last_became_active":"2023-09-23T23:11:17.524208+0000","last_became_peered":"2023-09-23T23:11:17.524208+0000","last_unstale":"2023-09-23T23:12:17.707286+0000","last_undegraded":"2023-09-23T23:12:17.707286+0000","last_fullsized":"2023-09-23T23:12:17.707286+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:33.487949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946576+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:12:13.946576+0000","last_peered":"2023-09-23T23:12:13.946576+0000","last_clean":"2023-09-23T23:12:13.946576+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:12:13.946576+0000","last_undegraded":"2023-09-23T23:12:13.946576+0000","last_fullsized":"2023-09-23T23:12:13.946576+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"509'160","reported_seq":1088,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889392+0000","last_change":"2023-09-23T23:11:27.937915+0000","last_active":"2023-09-23T23:12:14.889392+0000","last_peered":"2023-09-23T23:12:14.889392+0000","last_clean":"2023-09-23T23:12:14.889392+0000","last_became_active":"2023-09-23T23:11:27.934106+0000","last_became_peered":"2023-09-23T23:11:27.934106+0000","last_unstale":"2023-09-23T23:12:14.889392+0000","last_undegraded":"2023-09-23T23:12:14.889392+0000","last_fullsized":"2023-09-23T23:12:14.889392+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":160,"log_dups_size":0,"ondisk_log_size":160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:59.475598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000685079,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1018,"num_read_kb":891,"num_write":140,"num_write_kb":72,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096639+0000","last_change":"2023-09-23T23:11:13.548587+0000","last_active":"2023-09-23T23:12:15.096639+0000","last_peered":"2023-09-23T23:12:15.096639+0000","last_clean":"2023-09-23T23:12:15.096639+0000","last_became_active":"2023-09-23T23:11:13.548244+0000","last_became_peered":"2023-09-23T23:11:13.548244+0000","last_unstale":"2023-09-23T23:12:15.096639+0000","last_undegraded":"2023-09-23T23:12:15.096639+0000","last_fullsized":"2023-09-23T23:12:15.096639+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":286,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889306+0000","last_change":"2023-09-23T23:11:28.069945+0000","last_active":"2023-09-23T23:12:14.889306+0000","last_peered":"2023-09-23T23:12:14.889306+0000","last_clean":"2023-09-23T23:12:14.889306+0000","last_became_active":"2023-09-23T23:11:27.938935+0000","last_became_peered":"2023-09-23T23:11:27.938935+0000","last_unstale":"2023-09-23T23:12:14.889306+0000","last_undegraded":"2023-09-23T23:12:14.889306+0000","last_fullsized":"2023-09-23T23:12:14.889306+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:15.358153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"509'137","reported_seq":1139,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096400+0000","last_change":"2023-09-23T23:11:52.530000+0000","last_active":"2023-09-23T23:12:15.096400+0000","last_peered":"2023-09-23T23:12:15.096400+0000","last_clean":"2023-09-23T23:12:15.096400+0000","last_became_active":"2023-09-23T23:11:52.529452+0000","last_became_peered":"2023-09-23T23:11:52.529452+0000","last_unstale":"2023-09-23T23:12:15.096400+0000","last_undegraded":"2023-09-23T23:12:15.096400+0000","last_fullsized":"2023-09-23T23:12:15.096400+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1357,"num_read_kb":2532,"num_write":111,"num_write_kb":2245,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":594,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098273+0000","last_change":"2023-09-23T23:11:09.541696+0000","last_active":"2023-09-23T23:12:15.098273+0000","last_peered":"2023-09-23T23:12:15.098273+0000","last_clean":"2023-09-23T23:12:15.098273+0000","last_became_active":"2023-09-23T23:11:09.541367+0000","last_became_peered":"2023-09-23T23:11:09.541367+0000","last_unstale":"2023-09-23T23:12:15.098273+0000","last_undegraded":"2023-09-23T23:12:15.098273+0000","last_fullsized":"2023-09-23T23:12:15.098273+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":362,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889371+0000","last_change":"2023-09-23T23:11:28.070421+0000","last_active":"2023-09-23T23:12:14.889371+0000","last_peered":"2023-09-23T23:12:14.889371+0000","last_clean":"2023-09-23T23:12:14.889371+0000","last_became_active":"2023-09-23T23:11:28.070102+0000","last_became_peered":"2023-09-23T23:11:28.070102+0000","last_unstale":"2023-09-23T23:12:14.889371+0000","last_undegraded":"2023-09-23T23:12:14.889371+0000","last_fullsized":"2023-09-23T23:12:14.889371+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:40.429841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"508'64","reported_seq":743,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946166+0000","last_change":"2023-09-23T23:11:09.628064+0000","last_active":"2023-09-23T23:12:13.946166+0000","last_peered":"2023-09-23T23:12:13.946166+0000","last_clean":"2023-09-23T23:12:13.946166+0000","last_became_active":"2023-09-23T23:11:09.536407+0000","last_became_peered":"2023-09-23T23:11:09.536407+0000","last_unstale":"2023-09-23T23:12:13.946166+0000","last_undegraded":"2023-09-23T23:12:13.946166+0000","last_fullsized":"2023-09-23T23:12:13.946166+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:53.587747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036426600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":227,"num_read_kb":1866,"num_write":69,"num_write_kb":1432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":289,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889076+0000","last_change":"2023-09-23T23:11:27.938916+0000","last_active":"2023-09-23T23:12:14.889076+0000","last_peered":"2023-09-23T23:12:14.889076+0000","last_clean":"2023-09-23T23:12:14.889076+0000","last_became_active":"2023-09-23T23:11:27.938474+0000","last_became_peered":"2023-09-23T23:11:27.938474+0000","last_unstale":"2023-09-23T23:12:14.889076+0000","last_undegraded":"2023-09-23T23:12:14.889076+0000","last_fullsized":"2023-09-23T23:12:14.889076+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:54.202427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":483,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889706+0000","last_change":"2023-09-23T23:11:27.938452+0000","last_active":"2023-09-23T23:12:14.889706+0000","last_peered":"2023-09-23T23:12:14.889706+0000","last_clean":"2023-09-23T23:12:14.889706+0000","last_became_active":"2023-09-23T23:11:27.937963+0000","last_became_peered":"2023-09-23T23:11:27.937963+0000","last_unstale":"2023-09-23T23:12:14.889706+0000","last_undegraded":"2023-09-23T23:12:14.889706+0000","last_fullsized":"2023-09-23T23:12:14.889706+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.654369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"509'241","reported_seq":1390,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096378+0000","last_change":"2023-09-23T23:11:20.565375+0000","last_active":"2023-09-23T23:12:15.096378+0000","last_peered":"2023-09-23T23:12:15.096378+0000","last_clean":"2023-09-23T23:12:15.096378+0000","last_became_active":"2023-09-23T23:11:20.564688+0000","last_became_peered":"2023-09-23T23:11:20.564688+0000","last_unstale":"2023-09-23T23:12:15.096378+0000","last_undegraded":"2023-09-23T23:12:15.096378+0000","last_fullsized":"2023-09-23T23:12:15.096378+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":241,"log_dups_size":0,"ondisk_log_size":241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1535,"num_read_kb":2554,"num_write":204,"num_write_kb":1222,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":357,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948952+0000","last_change":"2023-09-23T23:11:20.567711+0000","last_active":"2023-09-23T23:12:13.948952+0000","last_peered":"2023-09-23T23:12:13.948952+0000","last_clean":"2023-09-23T23:12:13.948952+0000","last_became_active":"2023-09-23T23:11:20.566920+0000","last_became_peered":"2023-09-23T23:11:20.566920+0000","last_unstale":"2023-09-23T23:12:13.948952+0000","last_undegraded":"2023-09-23T23:12:13.948952+0000","last_fullsized":"2023-09-23T23:12:13.948952+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":249,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094673+0000","last_change":"2023-09-23T23:11:13.549732+0000","last_active":"2023-09-23T23:12:15.094673+0000","last_peered":"2023-09-23T23:12:15.094673+0000","last_clean":"2023-09-23T23:12:15.094673+0000","last_became_active":"2023-09-23T23:11:13.549442+0000","last_became_peered":"2023-09-23T23:11:13.549442+0000","last_unstale":"2023-09-23T23:12:15.094673+0000","last_undegraded":"2023-09-23T23:12:15.094673+0000","last_fullsized":"2023-09-23T23:12:15.094673+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:18.464642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":468,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097299+0000","last_change":"2023-09-23T23:11:58.725352+0000","last_active":"2023-09-23T23:12:15.097299+0000","last_peered":"2023-09-23T23:12:15.097299+0000","last_clean":"2023-09-23T23:12:15.097299+0000","last_became_active":"2023-09-23T23:11:58.724987+0000","last_became_peered":"2023-09-23T23:11:58.724987+0000","last_unstale":"2023-09-23T23:12:15.097299+0000","last_undegraded":"2023-09-23T23:12:15.097299+0000","last_fullsized":"2023-09-23T23:12:15.097299+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"509'139","reported_seq":1008,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948951+0000","last_change":"2023-09-23T23:11:20.568963+0000","last_active":"2023-09-23T23:12:13.948951+0000","last_peered":"2023-09-23T23:12:13.948951+0000","last_clean":"2023-09-23T23:12:13.948951+0000","last_became_active":"2023-09-23T23:11:20.568511+0000","last_became_peered":"2023-09-23T23:11:20.568511+0000","last_unstale":"2023-09-23T23:12:13.948951+0000","last_undegraded":"2023-09-23T23:12:13.948951+0000","last_fullsized":"2023-09-23T23:12:13.948951+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":975,"num_read_kb":854,"num_write":120,"num_write_kb":58,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949142+0000","last_change":"2023-09-23T23:11:40.916886+0000","last_active":"2023-09-23T23:12:13.949142+0000","last_peered":"2023-09-23T23:12:13.949142+0000","last_clean":"2023-09-23T23:12:13.949142+0000","last_became_active":"2023-09-23T23:11:40.916475+0000","last_became_peered":"2023-09-23T23:11:40.916475+0000","last_unstale":"2023-09-23T23:12:13.949142+0000","last_undegraded":"2023-09-23T23:12:13.949142+0000","last_fullsized":"2023-09-23T23:12:13.949142+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:26.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":44,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949465+0000","last_change":"2023-09-23T23:11:09.539272+0000","last_active":"2023-09-23T23:12:13.949465+0000","last_peered":"2023-09-23T23:12:13.949465+0000","last_clean":"2023-09-23T23:12:13.949465+0000","last_became_active":"2023-09-23T23:11:09.538917+0000","last_became_peered":"2023-09-23T23:11:09.538917+0000","last_unstale":"2023-09-23T23:12:13.949465+0000","last_undegraded":"2023-09-23T23:12:13.949465+0000","last_fullsized":"2023-09-23T23:12:13.949465+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:32.067020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"509'82","reported_seq":65,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949361+0000","last_change":"2023-09-23T23:11:17.736351+0000","last_active":"2023-09-23T23:12:13.949361+0000","last_peered":"2023-09-23T23:12:13.949361+0000","last_clean":"2023-09-23T23:12:13.949361+0000","last_became_active":"2023-09-23T23:11:17.525399+0000","last_became_peered":"2023-09-23T23:11:17.525399+0000","last_unstale":"2023-09-23T23:12:13.949361+0000","last_undegraded":"2023-09-23T23:12:13.949361+0000","last_fullsized":"2023-09-23T23:12:13.949361+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":309,"num_read_kb":2667,"num_write":81,"num_write_kb":6394,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":416,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889610+0000","last_change":"2023-09-23T23:11:27.937713+0000","last_active":"2023-09-23T23:12:14.889610+0000","last_peered":"2023-09-23T23:12:14.889610+0000","last_clean":"2023-09-23T23:12:14.889610+0000","last_became_active":"2023-09-23T23:11:27.934031+0000","last_became_peered":"2023-09-23T23:11:27.934031+0000","last_unstale":"2023-09-23T23:12:14.889610+0000","last_undegraded":"2023-09-23T23:12:14.889610+0000","last_fullsized":"2023-09-23T23:12:14.889610+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:44.181766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":360,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427318+0000","last_change":"2023-09-23T23:12:12.051566+0000","last_active":"2023-09-23T23:12:15.427318+0000","last_peered":"2023-09-23T23:12:15.427318+0000","last_clean":"2023-09-23T23:12:15.427318+0000","last_became_active":"2023-09-23T23:12:12.051255+0000","last_became_peered":"2023-09-23T23:12:12.051255+0000","last_unstale":"2023-09-23T23:12:15.427318+0000","last_undegraded":"2023-09-23T23:12:15.427318+0000","last_fullsized":"2023-09-23T23:12:15.427318+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:46.589263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":96,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946499+0000","last_change":"2023-09-23T23:11:17.703727+0000","last_active":"2023-09-23T23:12:13.946499+0000","last_peered":"2023-09-23T23:12:13.946499+0000","last_clean":"2023-09-23T23:12:13.946499+0000","last_became_active":"2023-09-23T23:11:17.525693+0000","last_became_peered":"2023-09-23T23:11:17.525693+0000","last_unstale":"2023-09-23T23:12:13.946499+0000","last_undegraded":"2023-09-23T23:12:13.946499+0000","last_fullsized":"2023-09-23T23:12:13.946499+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:20.154041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"509'98","reported_seq":562,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889719+0000","last_change":"2023-09-23T23:11:28.069849+0000","last_active":"2023-09-23T23:12:14.889719+0000","last_peered":"2023-09-23T23:12:14.889719+0000","last_clean":"2023-09-23T23:12:14.889719+0000","last_became_active":"2023-09-23T23:11:27.938965+0000","last_became_peered":"2023-09-23T23:11:27.938965+0000","last_unstale":"2023-09-23T23:12:14.889719+0000","last_undegraded":"2023-09-23T23:12:14.889719+0000","last_fullsized":"2023-09-23T23:12:14.889719+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:21.079567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":458,"num_read_kb":1757,"num_write":60,"num_write_kb":1236,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":273,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094529+0000","last_change":"2023-09-23T23:11:17.524544+0000","last_active":"2023-09-23T23:12:15.094529+0000","last_peered":"2023-09-23T23:12:15.094529+0000","last_clean":"2023-09-23T23:12:15.094529+0000","last_became_active":"2023-09-23T23:11:17.523198+0000","last_became_peered":"2023-09-23T23:11:17.523198+0000","last_unstale":"2023-09-23T23:12:15.094529+0000","last_undegraded":"2023-09-23T23:12:15.094529+0000","last_fullsized":"2023-09-23T23:12:15.094529+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:50.075594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"509'55","reported_seq":580,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097887+0000","last_change":"2023-09-23T23:11:58.557889+0000","last_active":"2023-09-23T23:12:15.097887+0000","last_peered":"2023-09-23T23:12:15.097887+0000","last_clean":"2023-09-23T23:12:15.097887+0000","last_became_active":"2023-09-23T23:11:58.557258+0000","last_became_peered":"2023-09-23T23:11:58.557258+0000","last_unstale":"2023-09-23T23:12:15.097887+0000","last_undegraded":"2023-09-23T23:12:15.097887+0000","last_fullsized":"2023-09-23T23:12:15.097887+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":55,"log_dups_size":0,"ondisk_log_size":55,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":116,"num_read_kb":91,"num_write":61,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949424+0000","last_change":"2023-09-23T23:11:09.606476+0000","last_active":"2023-09-23T23:12:13.949424+0000","last_peered":"2023-09-23T23:12:13.949424+0000","last_clean":"2023-09-23T23:12:13.949424+0000","last_became_active":"2023-09-23T23:11:09.547731+0000","last_became_peered":"2023-09-23T23:11:09.547731+0000","last_unstale":"2023-09-23T23:12:13.949424+0000","last_undegraded":"2023-09-23T23:12:13.949424+0000","last_fullsized":"2023-09-23T23:12:13.949424+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:00.058887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":557,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097608+0000","last_change":"2023-09-23T23:11:13.551788+0000","last_active":"2023-09-23T23:12:15.097608+0000","last_peered":"2023-09-23T23:12:15.097608+0000","last_clean":"2023-09-23T23:12:15.097608+0000","last_became_active":"2023-09-23T23:11:13.551442+0000","last_became_peered":"2023-09-23T23:11:13.551442+0000","last_unstale":"2023-09-23T23:12:15.097608+0000","last_undegraded":"2023-09-23T23:12:15.097608+0000","last_fullsized":"2023-09-23T23:12:15.097608+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":53,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949491+0000","last_change":"2023-09-23T23:11:17.522576+0000","last_active":"2023-09-23T23:12:13.949491+0000","last_peered":"2023-09-23T23:12:13.949491+0000","last_clean":"2023-09-23T23:12:13.949491+0000","last_became_active":"2023-09-23T23:11:17.522197+0000","last_became_peered":"2023-09-23T23:11:17.522197+0000","last_unstale":"2023-09-23T23:12:13.949491+0000","last_undegraded":"2023-09-23T23:12:13.949491+0000","last_fullsized":"2023-09-23T23:12:13.949491+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:16.154369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":299,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946414+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:12:13.946414+0000","last_peered":"2023-09-23T23:12:13.946414+0000","last_clean":"2023-09-23T23:12:13.946414+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:12:13.946414+0000","last_undegraded":"2023-09-23T23:12:13.946414+0000","last_fullsized":"2023-09-23T23:12:13.946414+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":363,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096495+0000","last_change":"2023-09-23T23:11:52.529235+0000","last_active":"2023-09-23T23:12:15.096495+0000","last_peered":"2023-09-23T23:12:15.096495+0000","last_clean":"2023-09-23T23:12:15.096495+0000","last_became_active":"2023-09-23T23:11:52.528969+0000","last_became_peered":"2023-09-23T23:11:52.528969+0000","last_unstale":"2023-09-23T23:12:15.096495+0000","last_undegraded":"2023-09-23T23:12:15.096495+0000","last_fullsized":"2023-09-23T23:12:15.096495+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":25,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098086+0000","last_change":"2023-09-23T23:11:40.915572+0000","last_active":"2023-09-23T23:12:15.098086+0000","last_peered":"2023-09-23T23:12:15.098086+0000","last_clean":"2023-09-23T23:12:15.098086+0000","last_became_active":"2023-09-23T23:11:40.915062+0000","last_became_peered":"2023-09-23T23:11:40.915062+0000","last_unstale":"2023-09-23T23:12:15.098086+0000","last_undegraded":"2023-09-23T23:12:15.098086+0000","last_fullsized":"2023-09-23T23:12:15.098086+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":390,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096483+0000","last_change":"2023-09-23T23:11:52.530019+0000","last_active":"2023-09-23T23:12:15.096483+0000","last_peered":"2023-09-23T23:12:15.096483+0000","last_clean":"2023-09-23T23:12:15.096483+0000","last_became_active":"2023-09-23T23:11:52.529528+0000","last_became_peered":"2023-09-23T23:11:52.529528+0000","last_unstale":"2023-09-23T23:12:15.096483+0000","last_undegraded":"2023-09-23T23:12:15.096483+0000","last_fullsized":"2023-09-23T23:12:15.096483+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097461+0000","last_change":"2023-09-23T23:11:40.915193+0000","last_active":"2023-09-23T23:12:15.097461+0000","last_peered":"2023-09-23T23:12:15.097461+0000","last_clean":"2023-09-23T23:12:15.097461+0000","last_became_active":"2023-09-23T23:11:40.914790+0000","last_became_peered":"2023-09-23T23:11:40.914790+0000","last_unstale":"2023-09-23T23:12:15.097461+0000","last_undegraded":"2023-09-23T23:12:15.097461+0000","last_fullsized":"2023-09-23T23:12:15.097461+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":124,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.095098+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:12:15.095098+0000","last_peered":"2023-09-23T23:12:15.095098+0000","last_clean":"2023-09-23T23:12:15.095098+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:12:15.095098+0000","last_undegraded":"2023-09-23T23:12:15.095098+0000","last_fullsized":"2023-09-23T23:12:15.095098+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":275,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094736+0000","last_change":"2023-09-23T23:11:09.542604+0000","last_active":"2023-09-23T23:12:15.094736+0000","last_peered":"2023-09-23T23:12:15.094736+0000","last_clean":"2023-09-23T23:12:15.094736+0000","last_became_active":"2023-09-23T23:11:09.542097+0000","last_became_peered":"2023-09-23T23:11:09.542097+0000","last_unstale":"2023-09-23T23:12:15.094736+0000","last_undegraded":"2023-09-23T23:12:15.094736+0000","last_fullsized":"2023-09-23T23:12:15.094736+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:07.434849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":65,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949584+0000","last_change":"2023-09-23T23:11:13.601470+0000","last_active":"2023-09-23T23:12:13.949584+0000","last_peered":"2023-09-23T23:12:13.949584+0000","last_clean":"2023-09-23T23:12:13.949584+0000","last_became_active":"2023-09-23T23:11:13.551003+0000","last_became_peered":"2023-09-23T23:11:13.551003+0000","last_unstale":"2023-09-23T23:12:13.949584+0000","last_undegraded":"2023-09-23T23:12:13.949584+0000","last_fullsized":"2023-09-23T23:12:13.949584+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:59.599117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":541,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097253+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:12:15.097253+0000","last_peered":"2023-09-23T23:12:15.097253+0000","last_clean":"2023-09-23T23:12:15.097253+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:12:15.097253+0000","last_undegraded":"2023-09-23T23:12:15.097253+0000","last_fullsized":"2023-09-23T23:12:15.097253+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":26850,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430759,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315276,"kb_used_data":9860,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93429876,"statfs":{"total":95995035648,"available":95672193024,"internally_reserved":0,"allocated":10096640,"data_stored":8964359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.6240000000000001,"15min":1.8460000000000001},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":0.65400000000000003,"5min":78.513000000000005,"15min":100.25700000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.7969999999999999,"15min":2.1230000000000002},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.245},"max":{"1min":0.94999999999999996,"5min":78.436000000000007,"15min":100.235},"last":0.436}]},{"osd":1,"last update":"Sat Sep 23 23:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.7969999999999999,"15min":1.3049999999999999},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":0.90300000000000002,"5min":79.566000000000003,"15min":79.566000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63,"5min":1.516,"15min":1.1990000000000001},"min":{"1min":0.436,"5min":0.307,"15min":0.26700000000000002},"max":{"1min":0.78200000000000003,"5min":79.516999999999996,"15min":79.516999999999996},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.3250000000000002,"15min":2.2080000000000002},"min":{"1min":0.443,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":0.85199999999999998,"5min":69.777000000000001,"15min":97.262},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.625,"5min":2.2570000000000001,"15min":2.1819999999999999},"min":{"1min":0.39400000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.97199999999999998,"5min":69.742000000000004,"15min":97.272999999999996},"last":0.69099999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.726,"15min":1.645},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.318},"max":{"1min":0.78900000000000003,"5min":82.242000000000004,"15min":84.480000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.623,"5min":1.7050000000000001,"15min":1.8460000000000001},"min":{"1min":0.42999999999999999,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":0.92600000000000005,"5min":82.272000000000006,"15min":84.826999999999998},"last":0.66100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.554,"15min":2.589},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.25800000000000001},"max":{"1min":1.0600000000000001,"5min":62.582999999999998,"15min":111.81},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.4139999999999999,"15min":2.3039999999999998},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":1.0980000000000001,"5min":62.609999999999999,"15min":111.636},"last":0.72999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":1.0640000000000001,"15min":1.345},"min":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.27600000000000002},"max":{"1min":0.70899999999999996,"5min":15.996,"15min":65.956999999999994},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.97399999999999998,"15min":1.4930000000000001},"min":{"1min":0.52500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":1.1519999999999999,"5min":6.9710000000000001,"15min":66.106999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":4.867,"15min":2.8010000000000002},"min":{"1min":0.44600000000000001,"5min":0.33600000000000002,"15min":0.28000000000000003},"max":{"1min":0.89200000000000002,"5min":371.00999999999999,"15min":371.00999999999999},"last":0.755},{"interface":"front","average":{"1min":0.71899999999999997,"5min":5.2160000000000002,"15min":2.875},"min":{"1min":0.58599999999999997,"5min":0.35899999999999999,"15min":0.34699999999999998},"max":{"1min":1.089,"5min":367.65199999999999,"15min":367.65199999999999},"last":0.67100000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626983,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916676,"kb_used_data":5756,"kb_used_omap":0,"kb_used_meta":910912,"kb_avail":92828476,"statfs":{"total":95995035648,"available":95056359424,"internally_reserved":0,"allocated":5894144,"data_stored":4769928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932773888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.86299999999999999,"15min":3.5449999999999999},"min":{"1min":0.42199999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.012,"5min":7.9630000000000001,"15min":115.32899999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.94299999999999995,"15min":3.556},"min":{"1min":0.443,"5min":0.25,"15min":0.25},"max":{"1min":0.98399999999999999,"5min":8.7010000000000005,"15min":115.348},"last":0.495}]},{"osd":1,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.7829999999999999,"15min":2.593},"min":{"1min":0.35799999999999998,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":0.91600000000000004,"5min":38.259999999999998,"15min":149.947},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.5209999999999999,"15min":2.5230000000000001},"min":{"1min":0.318,"5min":0.249,"15min":0.249},"max":{"1min":0.82899999999999996,"5min":38.530000000000001,"15min":150.22499999999999},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.95,"15min":2.8799999999999999},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.89000000000000001,"5min":45.051000000000002,"15min":138.43299999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59799999999999998,"5min":2.1659999999999999,"15min":3.0009999999999999},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":1.0009999999999999,"5min":45.258000000000003,"15min":138.45599999999999},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":2.0230000000000001,"15min":3.198},"min":{"1min":0.45200000000000001,"5min":0.32400000000000001,"15min":0.26800000000000002},"max":{"1min":1.0349999999999999,"5min":47.734000000000002,"15min":142.24700000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.794,"15min":3.097},"min":{"1min":0.502,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.93899999999999995,"5min":47.162999999999997,"15min":143.86699999999999},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.85899999999999999,"15min":1.6910000000000001},"min":{"1min":0.34399999999999997,"5min":0.25800000000000001,"15min":0.23999999999999999},"max":{"1min":0.74299999999999999,"5min":7.3869999999999996,"15min":119.804},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":1.5109999999999999,"15min":1.917},"min":{"1min":0.32500000000000001,"5min":0.27700000000000002,"15min":0.24099999999999999},"max":{"1min":0.747,"5min":46.515999999999998,"15min":122.31999999999999},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.0069999999999999,"15min":2.419},"min":{"1min":0.39500000000000002,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":0.81100000000000005,"5min":6.4530000000000003,"15min":186.27600000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.0920000000000001,"15min":2.4900000000000002},"min":{"1min":0.44,"5min":0.34399999999999997,"15min":0.29599999999999999},"max":{"1min":0.80700000000000005,"5min":8.7720000000000002,"15min":191.28700000000001},"last":0.64800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.016,"15min":1.4159999999999999},"min":{"1min":0.40600000000000003,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":0.88900000000000001,"5min":8.8670000000000009,"15min":115.61},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.1020000000000001,"15min":1.486},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.29999999999999999},"max":{"1min":0.79800000000000004,"5min":8.2609999999999992,"15min":115.578},"last":0.58899999999999997}]}]},{"osd":1,"up_from":84,"seq":360777253296,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1016184,"kb_used_data":6384,"kb_used_omap":9,"kb_used_meta":1009782,"kb_avail":92728968,"statfs":{"total":95995035648,"available":94954463232,"internally_reserved":0,"allocated":6537216,"data_stored":4333668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1034017769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.48599999999999999,"15min":2.5779999999999998},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":0.58999999999999997,"5min":3.891,"15min":279.30500000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.37,"5min":0.69699999999999995,"15min":2.831},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.47499999999999998,"5min":7.6890000000000001,"15min":279.39800000000002},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":3.9740000000000002,"15min":3.0979999999999999},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.55700000000000005,"5min":121.527,"15min":161.61699999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.39200000000000002,"5min":4.0380000000000003,"15min":3.2570000000000001},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.22600000000000001},"max":{"1min":0.51000000000000001,"5min":118.29900000000001,"15min":162.61699999999999},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":1.9490000000000001,"15min":1.659},"min":{"1min":0.26700000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.61899999999999999,"5min":67.646000000000001,"15min":67.646000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.434,"5min":2.125,"15min":1.706},"min":{"1min":0.312,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":0.79500000000000004,"5min":65.269999999999996,"15min":65.269999999999996},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":3.0569999999999999,"15min":2.4569999999999999},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":0.95199999999999996,"5min":129.69,"15min":151.036},"last":0.497},{"interface":"front","average":{"1min":0.64500000000000002,"5min":3.2290000000000001,"15min":2.7050000000000001},"min":{"1min":0.41399999999999998,"5min":0.309,"15min":0.26400000000000001},"max":{"1min":0.97699999999999998,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.85699999999999998,"15min":1.885},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":0.99299999999999999,"5min":6.242,"15min":116.098},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.4650000000000001,"15min":1.9610000000000001},"min":{"1min":0.45300000000000001,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":0.88200000000000001,"5min":51.140999999999998,"15min":116.08499999999999},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":2.4159999999999999,"15min":1.978},"min":{"1min":0.35599999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.93999999999999995,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.64400000000000002,"5min":2.516,"15min":1.877},"min":{"1min":0.49199999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":1.03,"5min":64.239999999999995,"15min":67.366},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.1699999999999999,"15min":1.056},"min":{"1min":0.48599999999999999,"5min":0.39400000000000002,"15min":0.34699999999999998},"max":{"1min":0.91400000000000003,"5min":6.4089999999999998,"15min":7.0960000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2130000000000001,"15min":1.2909999999999999},"min":{"1min":0.44600000000000001,"5min":0.40699999999999997,"15min":0.33100000000000002},"max":{"1min":1.026,"5min":8.8719999999999999,"15min":10.676},"last":0.51200000000000001}]}]},{"osd":0,"up_from":353,"seq":1516123455667,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300968,"kb_used_data":11360,"kb_used_omap":37,"kb_used_meta":289562,"kb_avail":93444184,"statfs":{"total":95995035648,"available":95686844416,"internally_reserved":0,"allocated":11632640,"data_stored":9148776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":296511650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.72799999999999998,"15min":0.82799999999999996},"min":{"1min":0.253,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.48699999999999999,"5min":7.1340000000000003,"15min":8.234},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":1.032,"15min":0.94299999999999995},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.20200000000000001},"max":{"1min":0.52900000000000003,"5min":9.9979999999999993,"15min":9.9979999999999993},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 23:11:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":2.6160000000000001,"15min":2.645},"min":{"1min":0.253,"5min":0.253,"15min":0.20599999999999999},"max":{"1min":0.54200000000000004,"5min":73.010000000000005,"15min":190.744},"last":0.442},{"interface":"front","average":{"1min":0.38400000000000001,"5min":2.407,"15min":2.8279999999999998},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.22700000000000001},"max":{"1min":0.63200000000000001,"5min":70.709000000000003,"15min":187.44200000000001},"last":0.32500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":4.6020000000000003,"15min":3.399},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":0.66800000000000004,"5min":146.411,"15min":146.411},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.42699999999999999,"5min":4.9269999999999996,"15min":3.6699999999999999},"min":{"1min":0.309,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":0.65700000000000003,"5min":146.667,"15min":146.667},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1,"15min":2.0409999999999999},"min":{"1min":0.34899999999999998,"5min":0.32000000000000001,"15min":0.28100000000000003},"max":{"1min":0.85499999999999998,"5min":5.681,"15min":183.148},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.96099999999999997,"15min":1.7809999999999999},"min":{"1min":0.42599999999999999,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":0.82999999999999996,"5min":10.124000000000001,"15min":183.39500000000001},"last":0.755}]},{"osd":5,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.82199999999999995,"15min":0.997},"min":{"1min":0.437,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":0.94099999999999995,"5min":9.3019999999999996,"15min":9.3019999999999996},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.88800000000000001,"15min":1.2869999999999999},"min":{"1min":0.45200000000000001,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":0.84599999999999997,"5min":5.7590000000000003,"15min":74.858000000000004},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.95199999999999996,"15min":1.498},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":0.88600000000000001,"5min":10.153,"15min":109.714},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.88200000000000001,"15min":1.4650000000000001},"min":{"1min":0.47699999999999998,"5min":0.38,"15min":0.35999999999999999},"max":{"1min":0.95099999999999996,"5min":8.5259999999999998,"15min":109.783},"last":0.73999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.034,"15min":1.1020000000000001},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":0.92200000000000004,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.97999999999999998,"15min":1.123},"min":{"1min":0.47299999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.90700000000000003,"5min":4.3559999999999999,"15min":6.6859999999999999},"last":0.77600000000000002}]}]},{"osd":2,"up_from":359,"seq":1541893259439,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328288,"kb_used_data":6552,"kb_used_omap":83,"kb_used_meta":321644,"kb_avail":93416864,"statfs":{"total":95995035648,"available":95658868736,"internally_reserved":0,"allocated":6709248,"data_stored":4351447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":329363909},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.58599999999999997,"15min":1.24},"min":{"1min":0.24199999999999999,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":0.44500000000000001,"5min":4.7729999999999997,"15min":74.241},"last":0.437},{"interface":"front","average":{"1min":0.372,"5min":1.022,"15min":1.5720000000000001},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.57599999999999996,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.9590000000000001,"15min":1.409},"min":{"1min":0.251,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":0.502,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":1.6319999999999999,"15min":1.4159999999999999},"min":{"1min":0.27100000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.51600000000000001,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":3.0710000000000002,"15min":2.141},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.23000000000000001},"max":{"1min":0.64000000000000001,"5min":79.655000000000001,"15min":110.31699999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.436,"5min":2.8759999999999999,"15min":2.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59999999999999998,"5min":79.677999999999997,"15min":110.56999999999999},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.409,"15min":1.155},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":0.92400000000000004,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.7230000000000001,"15min":1.383},"min":{"1min":0.46700000000000003,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":0.81100000000000005,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.77400000000000002}]},{"osd":5,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.0860000000000001,"15min":1.665},"min":{"1min":0.41599999999999998,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.86899999999999999,"5min":7.6600000000000001,"15min":139.405},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.90400000000000003,"15min":1.53},"min":{"1min":0.44500000000000001,"5min":0.38500000000000001,"15min":0.36299999999999999},"max":{"1min":0.97699999999999998,"5min":5.5880000000000001,"15min":139.42099999999999},"last":0.79700000000000004}]},{"osd":6,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":2.484,"15min":1.873},"min":{"1min":0.40600000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.91300000000000003,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.63},{"interface":"front","average":{"1min":0.63500000000000001,"5min":2.2080000000000002,"15min":1.611},"min":{"1min":0.44,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":0.91100000000000003,"5min":108.496,"15min":108.496},"last":0.68500000000000005}]},{"osd":7,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.1850000000000001,"15min":1.1799999999999999},"min":{"1min":0.48099999999999998,"5min":0.40300000000000002,"15min":0.36699999999999999},"max":{"1min":0.95199999999999996,"5min":9.1449999999999996,"15min":10.164},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.069,"15min":1.145},"min":{"1min":0.46200000000000002,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.749}]}]},{"osd":3,"up_from":94,"seq":403726926244,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":730936,"kb_used_data":6512,"kb_used_omap":7,"kb_used_meta":724408,"kb_avail":93014216,"statfs":{"total":95995035648,"available":95246557184,"internally_reserved":0,"allocated":6668288,"data_stored":4344620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":741794408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.751,"15min":1.512},"min":{"1min":0.254,"5min":0.215,"15min":0.17299999999999999},"max":{"1min":0.51300000000000001,"5min":8.1660000000000004,"15min":122.289},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.67300000000000004,"15min":1.2110000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":0.49399999999999999,"5min":8.1519999999999992,"15min":122.041},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.3560000000000001,"15min":1.4319999999999999},"min":{"1min":0.26800000000000002,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":14.327,"5min":14.327,"15min":23.969000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.04,"5min":1.01,"15min":1.2669999999999999},"min":{"1min":0.28899999999999998,"5min":0.26900000000000002,"15min":0.20699999999999999},"max":{"1min":14.398999999999999,"5min":14.398999999999999,"15min":16.986000000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.73,"15min":1.294},"min":{"1min":0.25800000000000001,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":0.55800000000000005,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.372},{"interface":"front","average":{"1min":0.39500000000000002,"5min":2.0880000000000001,"15min":1.637},"min":{"1min":0.28999999999999998,"5min":0.28199999999999997,"15min":0.215},"max":{"1min":0.51800000000000002,"5min":57.308,"15min":57.308},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.151,"15min":2.0019999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.30199999999999999},"max":{"1min":1.5920000000000001,"5min":11.01,"15min":115.69},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.91800000000000004,"15min":1.9390000000000001},"min":{"1min":0.44,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":1.9670000000000001,"5min":5.9169999999999998,"15min":115.617},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.995,"15min":1.3080000000000001},"min":{"1min":0.40000000000000002,"5min":0.35299999999999998,"15min":0.28499999999999998},"max":{"1min":2.0059999999999998,"5min":5.6399999999999997,"15min":16.959},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.94199999999999995,"15min":1.1679999999999999},"min":{"1min":0.35699999999999998,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":1.8120000000000001,"5min":6.6130000000000004,"15min":10.545},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.8939999999999999,"15min":1.6890000000000001},"min":{"1min":0.505,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":2.0430000000000001,"5min":77.707999999999998,"15min":77.707999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.8280000000000001,"15min":1.454},"min":{"1min":0.39700000000000002,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":1.9510000000000001,"5min":77.721999999999994,"15min":77.721999999999994},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.101,"15min":1.099},"min":{"1min":0.42599999999999999,"5min":0.38600000000000001,"15min":0.33900000000000002},"max":{"1min":1.9059999999999999,"5min":6.1820000000000004,"15min":8.1620000000000008},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.383,"15min":1.413},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.37},"max":{"1min":1.7290000000000001,"5min":11.051,"15min":11.051},"last":0.625}]}]},{"osd":4,"up_from":30,"seq":128849019441,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916740,"kb_used_data":9276,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92828412,"statfs":{"total":95995035648,"available":95056293888,"internally_reserved":0,"allocated":9498624,"data_stored":8373921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:30 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":4.2089999999999996,"15min":3.4260000000000002},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":3.5139999999999998,"5min":323.65800000000002,"15min":323.65800000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.747,"5min":4.0830000000000002,"15min":3.5089999999999999},"min":{"1min":0.40100000000000002,"5min":0.28299999999999997,"15min":0.24399999999999999},"max":{"1min":3.5390000000000001,"5min":323.642,"15min":323.642},"last":0.53400000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:12:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0880000000000001,"15min":2.0459999999999998},"min":{"1min":0.39500000000000002,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":11.459,"5min":11.459,"15min":95.591999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.1439999999999999,"15min":2.0070000000000001},"min":{"1min":0.41199999999999998,"5min":0.29999999999999999,"15min":0.23699999999999999},"max":{"1min":3.6619999999999999,"5min":15.728,"15min":97.929000000000002},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:11:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":4.7919999999999998,"15min":2.6600000000000001},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":0.748,"5min":151.20400000000001,"15min":151.20400000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":4.6109999999999998,"15min":2.544},"min":{"1min":0.435,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":1.1899999999999999,"5min":148.852,"15min":148.852},"last":0.67600000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":3.6120000000000001,"15min":2.323},"min":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":0.79700000000000004,"5min":147.82499999999999,"15min":147.82499999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":3.4980000000000002,"15min":2.415},"min":{"1min":0.44400000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.3180000000000001,"5min":147.84100000000001,"15min":147.84100000000001},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 23:11:50 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.1259999999999999,"15min":1.746},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.214},"max":{"1min":0.72699999999999998,"5min":10.282,"15min":148.61600000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.4530000000000001,"15min":1.891},"min":{"1min":0.372,"5min":0.33600000000000002,"15min":0.247},"max":{"1min":0.65400000000000003,"5min":17.02,"15min":151.905},"last":0.505}]},{"osd":6,"last update":"Sat Sep 23 23:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.0029999999999999,"15min":1.3049999999999999},"min":{"1min":0.47799999999999998,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":0.755,"5min":7.8390000000000004,"15min":35.896000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.98499999999999999,"15min":1.2929999999999999},"min":{"1min":0.48899999999999999,"5min":0.318,"15min":0.27800000000000002},"max":{"1min":0.80300000000000005,"5min":7.008,"15min":35.908000000000001},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.081,"15min":1.214},"min":{"1min":0.52600000000000002,"5min":0.35799999999999998,"15min":0.28399999999999997},"max":{"1min":0.89300000000000002,"5min":11.096,"15min":12.84},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.0940000000000001,"15min":1.117},"min":{"1min":0.54600000000000004,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81899999999999995,"5min":15.808,"15min":15.808},"last":0.628}]}]},{"osd":5,"up_from":36,"seq":154618823215,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900720,"kb_used_data":5160,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92844432,"statfs":{"total":95995035648,"available":95072698368,"internally_reserved":0,"allocated":5283840,"data_stored":4174750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.8109999999999999,"15min":1.5469999999999999},"min":{"1min":0.30599999999999999,"5min":0.252,"15min":0.221},"max":{"1min":0.79900000000000004,"5min":51.962000000000003,"15min":51.962000000000003},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.7889999999999999,"15min":1.6439999999999999},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.23799999999999999},"max":{"1min":0.78400000000000003,"5min":52.017000000000003,"15min":52.017000000000003},"last":0.626}]},{"osd":1,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.8,"15min":1.329},"min":{"1min":0.35199999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":0.77400000000000002,"5min":66.242000000000004,"15min":66.242000000000004},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.901,"15min":1.5349999999999999},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.253},"max":{"1min":0.90500000000000003,"5min":66.424000000000007,"15min":66.424000000000007},"last":0.64600000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:11:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":3.6920000000000002,"15min":2.27},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.25900000000000001},"max":{"1min":0.875,"5min":78.203000000000003,"15min":78.203000000000003},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":3.6219999999999999,"15min":2.2770000000000001},"min":{"1min":0.39900000000000002,"5min":0.36799999999999999,"15min":0.31},"max":{"1min":0.88400000000000001,"5min":78.210999999999999,"15min":78.210999999999999},"last":0.69999999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":5.2039999999999997,"15min":3.3650000000000002},"min":{"1min":0.39200000000000002,"5min":0.32400000000000001,"15min":0.28399999999999997},"max":{"1min":0.72199999999999998,"5min":125.806,"15min":125.806},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":5.0999999999999996,"15min":3.415},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.29699999999999999},"max":{"1min":0.94399999999999995,"5min":126.04600000000001,"15min":126.04600000000001},"last":0.65600000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.873,"15min":1.552},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":0.74199999999999999,"5min":7.7610000000000001,"15min":115.06699999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.177,"15min":1.5489999999999999},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":0.78000000000000003,"5min":14.098000000000001,"15min":114.908},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.077,"15min":1.1950000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":0.77300000000000002,"5min":8.4580000000000002,"15min":12.385},"last":0.503},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.0349999999999999,"15min":1.0589999999999999},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.872,"5min":7.649,"15min":12.137},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.73799999999999999,"15min":0.93700000000000006},"min":{"1min":0.441,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":0.88400000000000001,"5min":3.504,"15min":9.5980000000000008},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.123,"15min":1.1990000000000001},"min":{"1min":0.44800000000000001,"5min":0.32200000000000001,"15min":0.31},"max":{"1min":0.83299999999999996,"5min":8.4120000000000008,"15min":10.638999999999999},"last":0.60999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:12:52.373 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T23:12:52.742 INFO:teuthology.orchestra.run.smithi052.stderr:Inferring config /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T23:12:52.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:52 smithi052 ceph-mon[187743]: from='client.48482 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:12:52.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:52 smithi052 ceph-mon[180979]: from='client.48482 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:12:52.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:52 smithi151 ceph-mon[159210]: from='client.48482 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T23:12:53.744 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:53 smithi052 conmon[173965]: ::ffff:172.21.15.151 - - [23/Sep/2023:23:12:53] "GET /metrics HTTP/1.1" 200 34240 "" "Prometheus/2.33.4" 2023-09-23T23:12:53.744 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:53 smithi052 ceph-mon[187743]: pgmap v1677: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:53.744 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:53 smithi052 ceph-mon[180979]: pgmap v1677: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:53.949 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:53 smithi151 ceph-mon[159210]: pgmap v1677: 73 pgs: 73 active+clean; 4.6 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T23:12:54.450 INFO:teuthology.orchestra.run.smithi052.stdout: 2023-09-23T23:12:54.451 INFO:teuthology.orchestra.run.smithi052.stderr:dumped all 2023-09-23T23:12:55.165 INFO:teuthology.orchestra.run.smithi052.stdout:{"pg_ready":true,"pg_map":{"version":1677,"stamp":"2023-09-23T23:12:52.869252+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13712,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":9596213,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5425788,"kb_used_data":60860,"kb_used_omap":137,"kb_used_meta":5364726,"kb_avail":744535428,"statfs":{"total":767960285184,"available":762404278272,"internally_reserved":0,"allocated":62320640,"data_stored":48461469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141128,"internal_metadata":5493479608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003214"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":559,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097657+0000","last_change":"2023-09-23T22:58:23.268272+0000","last_active":"2023-09-23T23:12:15.097657+0000","last_peered":"2023-09-23T23:12:15.097657+0000","last_clean":"2023-09-23T23:12:15.097657+0000","last_became_active":"2023-09-23T22:58:23.267974+0000","last_became_peered":"2023-09-23T22:58:23.267974+0000","last_unstale":"2023-09-23T23:12:15.097657+0000","last_undegraded":"2023-09-23T23:12:15.097657+0000","last_fullsized":"2023-09-23T23:12:15.097657+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":221,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T23:12:17.707690+0000","last_change":"2023-09-23T22:57:26.524277+0000","last_active":"2023-09-23T23:12:17.707690+0000","last_peered":"2023-09-23T23:12:17.707690+0000","last_clean":"2023-09-23T23:12:17.707690+0000","last_became_active":"2023-09-23T22:57:26.523773+0000","last_became_peered":"2023-09-23T22:57:26.523773+0000","last_unstale":"2023-09-23T23:12:17.707690+0000","last_undegraded":"2023-09-23T23:12:17.707690+0000","last_fullsized":"2023-09-23T23:12:17.707690+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.783717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":221,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889124+0000","last_change":"2023-09-23T23:11:40.915446+0000","last_active":"2023-09-23T23:12:14.889124+0000","last_peered":"2023-09-23T23:12:14.889124+0000","last_clean":"2023-09-23T23:12:14.889124+0000","last_became_active":"2023-09-23T23:11:40.915136+0000","last_became_peered":"2023-09-23T23:11:40.915136+0000","last_unstale":"2023-09-23T23:12:14.889124+0000","last_undegraded":"2023-09-23T23:12:14.889124+0000","last_fullsized":"2023-09-23T23:12:14.889124+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:19.386456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":547,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427346+0000","last_change":"2023-09-23T23:12:12.051504+0000","last_active":"2023-09-23T23:12:15.427346+0000","last_peered":"2023-09-23T23:12:15.427346+0000","last_clean":"2023-09-23T23:12:15.427346+0000","last_became_active":"2023-09-23T23:12:12.051153+0000","last_became_peered":"2023-09-23T23:12:12.051153+0000","last_unstale":"2023-09-23T23:12:15.427346+0000","last_undegraded":"2023-09-23T23:12:15.427346+0000","last_fullsized":"2023-09-23T23:12:15.427346+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:18.938482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":72,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946206+0000","last_change":"2023-09-23T23:11:09.536656+0000","last_active":"2023-09-23T23:12:13.946206+0000","last_peered":"2023-09-23T23:12:13.946206+0000","last_clean":"2023-09-23T23:12:13.946206+0000","last_became_active":"2023-09-23T23:11:09.536309+0000","last_became_peered":"2023-09-23T23:11:09.536309+0000","last_unstale":"2023-09-23T23:12:13.946206+0000","last_undegraded":"2023-09-23T23:12:13.946206+0000","last_fullsized":"2023-09-23T23:12:13.946206+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:20.717592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098401+0000","last_change":"2023-09-23T23:11:58.726000+0000","last_active":"2023-09-23T23:12:15.098401+0000","last_peered":"2023-09-23T23:12:15.098401+0000","last_clean":"2023-09-23T23:12:15.098401+0000","last_became_active":"2023-09-23T23:11:58.725630+0000","last_became_peered":"2023-09-23T23:11:58.725630+0000","last_unstale":"2023-09-23T23:12:15.098401+0000","last_undegraded":"2023-09-23T23:12:15.098401+0000","last_fullsized":"2023-09-23T23:12:15.098401+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":473,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427424+0000","last_change":"2023-09-23T23:12:11.965477+0000","last_active":"2023-09-23T23:12:15.427424+0000","last_peered":"2023-09-23T23:12:15.427424+0000","last_clean":"2023-09-23T23:12:15.427424+0000","last_became_active":"2023-09-23T23:12:11.965238+0000","last_became_peered":"2023-09-23T23:12:11.965238+0000","last_unstale":"2023-09-23T23:12:15.427424+0000","last_undegraded":"2023-09-23T23:12:15.427424+0000","last_fullsized":"2023-09-23T23:12:15.427424+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:10.025423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094454+0000","last_change":"2023-09-23T23:05:03.449141+0000","last_active":"2023-09-23T23:12:15.094454+0000","last_peered":"2023-09-23T23:12:15.094454+0000","last_clean":"2023-09-23T23:12:15.094454+0000","last_became_active":"2023-09-23T23:05:03.448836+0000","last_became_peered":"2023-09-23T23:05:03.448836+0000","last_unstale":"2023-09-23T23:12:15.094454+0000","last_undegraded":"2023-09-23T23:12:15.094454+0000","last_fullsized":"2023-09-23T23:12:15.094454+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:55.896040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096594+0000","last_change":"2023-09-23T23:11:17.525302+0000","last_active":"2023-09-23T23:12:15.096594+0000","last_peered":"2023-09-23T23:12:15.096594+0000","last_clean":"2023-09-23T23:12:15.096594+0000","last_became_active":"2023-09-23T23:11:17.525045+0000","last_became_peered":"2023-09-23T23:11:17.525045+0000","last_unstale":"2023-09-23T23:12:15.096594+0000","last_undegraded":"2023-09-23T23:12:15.096594+0000","last_fullsized":"2023-09-23T23:12:15.096594+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":280,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094589+0000","last_change":"2023-09-23T23:11:17.523534+0000","last_active":"2023-09-23T23:12:15.094589+0000","last_peered":"2023-09-23T23:12:15.094589+0000","last_clean":"2023-09-23T23:12:15.094589+0000","last_became_active":"2023-09-23T23:11:17.523167+0000","last_became_peered":"2023-09-23T23:11:17.523167+0000","last_unstale":"2023-09-23T23:12:15.094589+0000","last_undegraded":"2023-09-23T23:12:15.094589+0000","last_fullsized":"2023-09-23T23:12:15.094589+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:11.997595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":41,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949486+0000","last_change":"2023-09-23T23:11:17.706926+0000","last_active":"2023-09-23T23:12:13.949486+0000","last_peered":"2023-09-23T23:12:13.949486+0000","last_clean":"2023-09-23T23:12:13.949486+0000","last_became_active":"2023-09-23T23:11:17.524784+0000","last_became_peered":"2023-09-23T23:11:17.524784+0000","last_unstale":"2023-09-23T23:12:13.949486+0000","last_undegraded":"2023-09-23T23:12:13.949486+0000","last_fullsized":"2023-09-23T23:12:13.949486+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":240,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889213+0000","last_change":"2023-09-23T23:11:28.070251+0000","last_active":"2023-09-23T23:12:14.889213+0000","last_peered":"2023-09-23T23:12:14.889213+0000","last_clean":"2023-09-23T23:12:14.889213+0000","last_became_active":"2023-09-23T23:11:27.939074+0000","last_became_peered":"2023-09-23T23:11:27.939074+0000","last_unstale":"2023-09-23T23:12:14.889213+0000","last_undegraded":"2023-09-23T23:12:14.889213+0000","last_fullsized":"2023-09-23T23:12:14.889213+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:13.159027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"75'22","reported_seq":656,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097816+0000","last_change":"2023-09-23T23:11:17.701800+0000","last_active":"2023-09-23T23:12:15.097816+0000","last_peered":"2023-09-23T23:12:15.097816+0000","last_clean":"2023-09-23T23:12:15.097816+0000","last_became_active":"2023-09-23T23:11:17.524341+0000","last_became_peered":"2023-09-23T23:11:17.524341+0000","last_unstale":"2023-09-23T23:12:15.097816+0000","last_undegraded":"2023-09-23T23:12:15.097816+0000","last_fullsized":"2023-09-23T23:12:15.097816+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":530,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098178+0000","last_change":"2023-09-23T23:05:03.450728+0000","last_active":"2023-09-23T23:12:15.098178+0000","last_peered":"2023-09-23T23:12:15.098178+0000","last_clean":"2023-09-23T23:12:15.098178+0000","last_became_active":"2023-09-23T23:05:03.449435+0000","last_became_peered":"2023-09-23T23:05:03.449435+0000","last_unstale":"2023-09-23T23:12:15.098178+0000","last_undegraded":"2023-09-23T23:12:15.098178+0000","last_fullsized":"2023-09-23T23:12:15.098178+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":41,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948978+0000","last_change":"2023-09-23T23:11:17.712628+0000","last_active":"2023-09-23T23:12:13.948978+0000","last_peered":"2023-09-23T23:12:13.948978+0000","last_clean":"2023-09-23T23:12:13.948978+0000","last_became_active":"2023-09-23T23:11:17.529211+0000","last_became_peered":"2023-09-23T23:11:17.529211+0000","last_unstale":"2023-09-23T23:12:13.948978+0000","last_undegraded":"2023-09-23T23:12:13.948978+0000","last_fullsized":"2023-09-23T23:12:13.948978+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094992+0000","last_change":"2023-09-23T23:11:17.523762+0000","last_active":"2023-09-23T23:12:15.094992+0000","last_peered":"2023-09-23T23:12:15.094992+0000","last_clean":"2023-09-23T23:12:15.094992+0000","last_became_active":"2023-09-23T23:11:17.523426+0000","last_became_peered":"2023-09-23T23:11:17.523426+0000","last_unstale":"2023-09-23T23:12:15.094992+0000","last_undegraded":"2023-09-23T23:12:15.094992+0000","last_fullsized":"2023-09-23T23:12:15.094992+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:57.182479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":530,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427303+0000","last_change":"2023-09-23T23:12:12.051990+0000","last_active":"2023-09-23T23:12:15.427303+0000","last_peered":"2023-09-23T23:12:15.427303+0000","last_clean":"2023-09-23T23:12:15.427303+0000","last_became_active":"2023-09-23T23:12:12.051628+0000","last_became_peered":"2023-09-23T23:12:12.051628+0000","last_unstale":"2023-09-23T23:12:15.427303+0000","last_undegraded":"2023-09-23T23:12:15.427303+0000","last_fullsized":"2023-09-23T23:12:15.427303+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:55.042906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096706+0000","last_change":"2023-09-23T23:11:17.525262+0000","last_active":"2023-09-23T23:12:15.096706+0000","last_peered":"2023-09-23T23:12:15.096706+0000","last_clean":"2023-09-23T23:12:15.096706+0000","last_became_active":"2023-09-23T23:11:17.524896+0000","last_became_peered":"2023-09-23T23:11:17.524896+0000","last_unstale":"2023-09-23T23:12:15.096706+0000","last_undegraded":"2023-09-23T23:12:15.096706+0000","last_fullsized":"2023-09-23T23:12:15.096706+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":35,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948946+0000","last_change":"2023-09-23T23:11:17.524399+0000","last_active":"2023-09-23T23:12:13.948946+0000","last_peered":"2023-09-23T23:12:13.948946+0000","last_clean":"2023-09-23T23:12:13.948946+0000","last_became_active":"2023-09-23T23:11:17.524074+0000","last_became_peered":"2023-09-23T23:11:17.524074+0000","last_unstale":"2023-09-23T23:12:13.948946+0000","last_undegraded":"2023-09-23T23:12:13.948946+0000","last_fullsized":"2023-09-23T23:12:13.948946+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":571,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427366+0000","last_change":"2023-09-23T23:12:12.051992+0000","last_active":"2023-09-23T23:12:15.427366+0000","last_peered":"2023-09-23T23:12:15.427366+0000","last_clean":"2023-09-23T23:12:15.427366+0000","last_became_active":"2023-09-23T23:12:12.051699+0000","last_became_peered":"2023-09-23T23:12:12.051699+0000","last_unstale":"2023-09-23T23:12:15.427366+0000","last_undegraded":"2023-09-23T23:12:15.427366+0000","last_fullsized":"2023-09-23T23:12:15.427366+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:09.484693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":81,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097226+0000","last_change":"2023-09-23T23:11:58.558430+0000","last_active":"2023-09-23T23:12:15.097226+0000","last_peered":"2023-09-23T23:12:15.097226+0000","last_clean":"2023-09-23T23:12:15.097226+0000","last_became_active":"2023-09-23T23:11:58.557927+0000","last_became_peered":"2023-09-23T23:11:58.557927+0000","last_unstale":"2023-09-23T23:12:15.097226+0000","last_undegraded":"2023-09-23T23:12:15.097226+0000","last_fullsized":"2023-09-23T23:12:15.097226+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":263,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.095120+0000","last_change":"2023-09-23T23:11:53.849144+0000","last_active":"2023-09-23T23:12:15.095120+0000","last_peered":"2023-09-23T23:12:15.095120+0000","last_clean":"2023-09-23T23:12:15.095120+0000","last_became_active":"2023-09-23T23:11:53.848785+0000","last_became_peered":"2023-09-23T23:11:53.848785+0000","last_unstale":"2023-09-23T23:12:15.095120+0000","last_undegraded":"2023-09-23T23:12:15.095120+0000","last_fullsized":"2023-09-23T23:12:15.095120+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:05.530980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":39,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949433+0000","last_change":"2023-09-23T23:11:17.709067+0000","last_active":"2023-09-23T23:12:13.949433+0000","last_peered":"2023-09-23T23:12:13.949433+0000","last_clean":"2023-09-23T23:12:13.949433+0000","last_became_active":"2023-09-23T23:11:17.527823+0000","last_became_peered":"2023-09-23T23:11:17.527823+0000","last_unstale":"2023-09-23T23:12:13.949433+0000","last_undegraded":"2023-09-23T23:12:13.949433+0000","last_fullsized":"2023-09-23T23:12:13.949433+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096312+0000","last_change":"2023-09-23T23:11:17.525389+0000","last_active":"2023-09-23T23:12:15.096312+0000","last_peered":"2023-09-23T23:12:15.096312+0000","last_clean":"2023-09-23T23:12:15.096312+0000","last_became_active":"2023-09-23T23:11:17.524899+0000","last_became_peered":"2023-09-23T23:11:17.524899+0000","last_unstale":"2023-09-23T23:12:15.096312+0000","last_undegraded":"2023-09-23T23:12:15.096312+0000","last_fullsized":"2023-09-23T23:12:15.096312+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":52,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949328+0000","last_change":"2023-09-23T23:11:40.923821+0000","last_active":"2023-09-23T23:12:13.949328+0000","last_peered":"2023-09-23T23:12:13.949328+0000","last_clean":"2023-09-23T23:12:13.949328+0000","last_became_active":"2023-09-23T23:11:40.918283+0000","last_became_peered":"2023-09-23T23:11:40.918283+0000","last_unstale":"2023-09-23T23:12:13.949328+0000","last_undegraded":"2023-09-23T23:12:13.949328+0000","last_fullsized":"2023-09-23T23:12:13.949328+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":554,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097710+0000","last_change":"2023-09-23T22:57:26.517997+0000","last_active":"2023-09-23T23:12:15.097710+0000","last_peered":"2023-09-23T23:12:15.097710+0000","last_clean":"2023-09-23T23:12:15.097710+0000","last_became_active":"2023-09-23T22:57:26.517708+0000","last_became_peered":"2023-09-23T22:57:26.517708+0000","last_unstale":"2023-09-23T23:12:15.097710+0000","last_undegraded":"2023-09-23T23:12:15.097710+0000","last_fullsized":"2023-09-23T23:12:15.097710+0000","mapping_epoch":342,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":39,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949267+0000","last_change":"2023-09-23T23:11:17.703666+0000","last_active":"2023-09-23T23:12:13.949267+0000","last_peered":"2023-09-23T23:12:13.949267+0000","last_clean":"2023-09-23T23:12:13.949267+0000","last_became_active":"2023-09-23T23:11:17.523182+0000","last_became_peered":"2023-09-23T23:11:17.523182+0000","last_unstale":"2023-09-23T23:12:13.949267+0000","last_undegraded":"2023-09-23T23:12:13.949267+0000","last_fullsized":"2023-09-23T23:12:13.949267+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097236+0000","last_change":"2023-09-23T23:11:58.726640+0000","last_active":"2023-09-23T23:12:15.097236+0000","last_peered":"2023-09-23T23:12:15.097236+0000","last_clean":"2023-09-23T23:12:15.097236+0000","last_became_active":"2023-09-23T23:11:58.726160+0000","last_became_peered":"2023-09-23T23:11:58.726160+0000","last_unstale":"2023-09-23T23:12:15.097236+0000","last_undegraded":"2023-09-23T23:12:15.097236+0000","last_fullsized":"2023-09-23T23:12:15.097236+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096502+0000","last_change":"2023-09-23T23:11:52.529700+0000","last_active":"2023-09-23T23:12:15.096502+0000","last_peered":"2023-09-23T23:12:15.096502+0000","last_clean":"2023-09-23T23:12:15.096502+0000","last_became_active":"2023-09-23T23:11:52.529339+0000","last_became_peered":"2023-09-23T23:11:52.529339+0000","last_unstale":"2023-09-23T23:12:15.096502+0000","last_undegraded":"2023-09-23T23:12:15.096502+0000","last_fullsized":"2023-09-23T23:12:15.096502+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":44,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949543+0000","last_change":"2023-09-23T23:11:09.548373+0000","last_active":"2023-09-23T23:12:13.949543+0000","last_peered":"2023-09-23T23:12:13.949543+0000","last_clean":"2023-09-23T23:12:13.949543+0000","last_became_active":"2023-09-23T23:11:09.547670+0000","last_became_peered":"2023-09-23T23:11:09.547670+0000","last_unstale":"2023-09-23T23:12:13.949543+0000","last_undegraded":"2023-09-23T23:12:13.949543+0000","last_fullsized":"2023-09-23T23:12:13.949543+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:12.720283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":266,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094705+0000","last_change":"2023-09-23T23:11:09.608043+0000","last_active":"2023-09-23T23:12:15.094705+0000","last_peered":"2023-09-23T23:12:15.094705+0000","last_clean":"2023-09-23T23:12:15.094705+0000","last_became_active":"2023-09-23T23:11:09.544986+0000","last_became_peered":"2023-09-23T23:11:09.544986+0000","last_unstale":"2023-09-23T23:12:15.094705+0000","last_undegraded":"2023-09-23T23:12:15.094705+0000","last_fullsized":"2023-09-23T23:12:15.094705+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:26.925687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":549,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097716+0000","last_change":"2023-09-23T23:05:03.455336+0000","last_active":"2023-09-23T23:12:15.097716+0000","last_peered":"2023-09-23T23:12:15.097716+0000","last_clean":"2023-09-23T23:12:15.097716+0000","last_became_active":"2023-09-23T23:05:03.452084+0000","last_became_peered":"2023-09-23T23:05:03.452084+0000","last_unstale":"2023-09-23T23:12:15.097716+0000","last_undegraded":"2023-09-23T23:12:15.097716+0000","last_fullsized":"2023-09-23T23:12:15.097716+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097462+0000","last_change":"2023-09-23T23:11:58.726424+0000","last_active":"2023-09-23T23:12:15.097462+0000","last_peered":"2023-09-23T23:12:15.097462+0000","last_clean":"2023-09-23T23:12:15.097462+0000","last_became_active":"2023-09-23T23:11:58.725673+0000","last_became_peered":"2023-09-23T23:11:58.725673+0000","last_unstale":"2023-09-23T23:12:15.097462+0000","last_undegraded":"2023-09-23T23:12:15.097462+0000","last_fullsized":"2023-09-23T23:12:15.097462+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097883+0000","last_change":"2023-09-23T23:11:41.025203+0000","last_active":"2023-09-23T23:12:15.097883+0000","last_peered":"2023-09-23T23:12:15.097883+0000","last_clean":"2023-09-23T23:12:15.097883+0000","last_became_active":"2023-09-23T23:11:41.024944+0000","last_became_peered":"2023-09-23T23:11:41.024944+0000","last_unstale":"2023-09-23T23:12:15.097883+0000","last_undegraded":"2023-09-23T23:12:15.097883+0000","last_fullsized":"2023-09-23T23:12:15.097883+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":579,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097504+0000","last_change":"2023-09-23T23:11:17.524264+0000","last_active":"2023-09-23T23:12:15.097504+0000","last_peered":"2023-09-23T23:12:15.097504+0000","last_clean":"2023-09-23T23:12:15.097504+0000","last_became_active":"2023-09-23T23:11:17.523923+0000","last_became_peered":"2023-09-23T23:11:17.523923+0000","last_unstale":"2023-09-23T23:12:15.097504+0000","last_undegraded":"2023-09-23T23:12:15.097504+0000","last_fullsized":"2023-09-23T23:12:15.097504+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":575,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097984+0000","last_change":"2023-09-23T23:08:18.922552+0000","last_active":"2023-09-23T23:12:15.097984+0000","last_peered":"2023-09-23T23:12:15.097984+0000","last_clean":"2023-09-23T23:12:15.097984+0000","last_became_active":"2023-09-23T23:08:18.922260+0000","last_became_peered":"2023-09-23T23:08:18.922260+0000","last_unstale":"2023-09-23T23:12:15.097984+0000","last_undegraded":"2023-09-23T23:12:15.097984+0000","last_fullsized":"2023-09-23T23:12:15.097984+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T23:12:17.707286+0000","last_change":"2023-09-23T23:11:17.524740+0000","last_active":"2023-09-23T23:12:17.707286+0000","last_peered":"2023-09-23T23:12:17.707286+0000","last_clean":"2023-09-23T23:12:17.707286+0000","last_became_active":"2023-09-23T23:11:17.524208+0000","last_became_peered":"2023-09-23T23:11:17.524208+0000","last_unstale":"2023-09-23T23:12:17.707286+0000","last_undegraded":"2023-09-23T23:12:17.707286+0000","last_fullsized":"2023-09-23T23:12:17.707286+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:33.487949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946576+0000","last_change":"2023-09-23T22:58:24.482700+0000","last_active":"2023-09-23T23:12:13.946576+0000","last_peered":"2023-09-23T23:12:13.946576+0000","last_clean":"2023-09-23T23:12:13.946576+0000","last_became_active":"2023-09-23T22:58:24.482346+0000","last_became_peered":"2023-09-23T22:58:24.482346+0000","last_unstale":"2023-09-23T23:12:13.946576+0000","last_undegraded":"2023-09-23T23:12:13.946576+0000","last_fullsized":"2023-09-23T23:12:13.946576+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:01.598982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"509'160","reported_seq":1088,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889392+0000","last_change":"2023-09-23T23:11:27.937915+0000","last_active":"2023-09-23T23:12:14.889392+0000","last_peered":"2023-09-23T23:12:14.889392+0000","last_clean":"2023-09-23T23:12:14.889392+0000","last_became_active":"2023-09-23T23:11:27.934106+0000","last_became_peered":"2023-09-23T23:11:27.934106+0000","last_unstale":"2023-09-23T23:12:14.889392+0000","last_undegraded":"2023-09-23T23:12:14.889392+0000","last_fullsized":"2023-09-23T23:12:14.889392+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":160,"log_dups_size":0,"ondisk_log_size":160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:59.475598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000685079,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1018,"num_read_kb":891,"num_write":140,"num_write_kb":72,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096639+0000","last_change":"2023-09-23T23:11:13.548587+0000","last_active":"2023-09-23T23:12:15.096639+0000","last_peered":"2023-09-23T23:12:15.096639+0000","last_clean":"2023-09-23T23:12:15.096639+0000","last_became_active":"2023-09-23T23:11:13.548244+0000","last_became_peered":"2023-09-23T23:11:13.548244+0000","last_unstale":"2023-09-23T23:12:15.096639+0000","last_undegraded":"2023-09-23T23:12:15.096639+0000","last_fullsized":"2023-09-23T23:12:15.096639+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":286,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889306+0000","last_change":"2023-09-23T23:11:28.069945+0000","last_active":"2023-09-23T23:12:14.889306+0000","last_peered":"2023-09-23T23:12:14.889306+0000","last_clean":"2023-09-23T23:12:14.889306+0000","last_became_active":"2023-09-23T23:11:27.938935+0000","last_became_peered":"2023-09-23T23:11:27.938935+0000","last_unstale":"2023-09-23T23:12:14.889306+0000","last_undegraded":"2023-09-23T23:12:14.889306+0000","last_fullsized":"2023-09-23T23:12:14.889306+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:15.358153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"509'137","reported_seq":1139,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096400+0000","last_change":"2023-09-23T23:11:52.530000+0000","last_active":"2023-09-23T23:12:15.096400+0000","last_peered":"2023-09-23T23:12:15.096400+0000","last_clean":"2023-09-23T23:12:15.096400+0000","last_became_active":"2023-09-23T23:11:52.529452+0000","last_became_peered":"2023-09-23T23:11:52.529452+0000","last_unstale":"2023-09-23T23:12:15.096400+0000","last_undegraded":"2023-09-23T23:12:15.096400+0000","last_fullsized":"2023-09-23T23:12:15.096400+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1357,"num_read_kb":2532,"num_write":111,"num_write_kb":2245,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":594,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098273+0000","last_change":"2023-09-23T23:11:09.541696+0000","last_active":"2023-09-23T23:12:15.098273+0000","last_peered":"2023-09-23T23:12:15.098273+0000","last_clean":"2023-09-23T23:12:15.098273+0000","last_became_active":"2023-09-23T23:11:09.541367+0000","last_became_peered":"2023-09-23T23:11:09.541367+0000","last_unstale":"2023-09-23T23:12:15.098273+0000","last_undegraded":"2023-09-23T23:12:15.098273+0000","last_fullsized":"2023-09-23T23:12:15.098273+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":362,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889371+0000","last_change":"2023-09-23T23:11:28.070421+0000","last_active":"2023-09-23T23:12:14.889371+0000","last_peered":"2023-09-23T23:12:14.889371+0000","last_clean":"2023-09-23T23:12:14.889371+0000","last_became_active":"2023-09-23T23:11:28.070102+0000","last_became_peered":"2023-09-23T23:11:28.070102+0000","last_unstale":"2023-09-23T23:12:14.889371+0000","last_undegraded":"2023-09-23T23:12:14.889371+0000","last_fullsized":"2023-09-23T23:12:14.889371+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:40.429841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"508'64","reported_seq":743,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946166+0000","last_change":"2023-09-23T23:11:09.628064+0000","last_active":"2023-09-23T23:12:13.946166+0000","last_peered":"2023-09-23T23:12:13.946166+0000","last_clean":"2023-09-23T23:12:13.946166+0000","last_became_active":"2023-09-23T23:11:09.536407+0000","last_became_peered":"2023-09-23T23:11:09.536407+0000","last_unstale":"2023-09-23T23:12:13.946166+0000","last_undegraded":"2023-09-23T23:12:13.946166+0000","last_fullsized":"2023-09-23T23:12:13.946166+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:53.587747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036426600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":227,"num_read_kb":1866,"num_write":69,"num_write_kb":1432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":289,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889076+0000","last_change":"2023-09-23T23:11:27.938916+0000","last_active":"2023-09-23T23:12:14.889076+0000","last_peered":"2023-09-23T23:12:14.889076+0000","last_clean":"2023-09-23T23:12:14.889076+0000","last_became_active":"2023-09-23T23:11:27.938474+0000","last_became_peered":"2023-09-23T23:11:27.938474+0000","last_unstale":"2023-09-23T23:12:14.889076+0000","last_undegraded":"2023-09-23T23:12:14.889076+0000","last_fullsized":"2023-09-23T23:12:14.889076+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:54.202427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":483,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889706+0000","last_change":"2023-09-23T23:11:27.938452+0000","last_active":"2023-09-23T23:12:14.889706+0000","last_peered":"2023-09-23T23:12:14.889706+0000","last_clean":"2023-09-23T23:12:14.889706+0000","last_became_active":"2023-09-23T23:11:27.937963+0000","last_became_peered":"2023-09-23T23:11:27.937963+0000","last_unstale":"2023-09-23T23:12:14.889706+0000","last_undegraded":"2023-09-23T23:12:14.889706+0000","last_fullsized":"2023-09-23T23:12:14.889706+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.654369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"509'241","reported_seq":1390,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096378+0000","last_change":"2023-09-23T23:11:20.565375+0000","last_active":"2023-09-23T23:12:15.096378+0000","last_peered":"2023-09-23T23:12:15.096378+0000","last_clean":"2023-09-23T23:12:15.096378+0000","last_became_active":"2023-09-23T23:11:20.564688+0000","last_became_peered":"2023-09-23T23:11:20.564688+0000","last_unstale":"2023-09-23T23:12:15.096378+0000","last_undegraded":"2023-09-23T23:12:15.096378+0000","last_fullsized":"2023-09-23T23:12:15.096378+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":241,"log_dups_size":0,"ondisk_log_size":241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1535,"num_read_kb":2554,"num_write":204,"num_write_kb":1222,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"75'139","reported_seq":357,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948952+0000","last_change":"2023-09-23T23:11:20.567711+0000","last_active":"2023-09-23T23:12:13.948952+0000","last_peered":"2023-09-23T23:12:13.948952+0000","last_clean":"2023-09-23T23:12:13.948952+0000","last_became_active":"2023-09-23T23:11:20.566920+0000","last_became_peered":"2023-09-23T23:11:20.566920+0000","last_unstale":"2023-09-23T23:12:13.948952+0000","last_undegraded":"2023-09-23T23:12:13.948952+0000","last_fullsized":"2023-09-23T23:12:13.948952+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:25:26.375802+0000","last_clean_scrub_stamp":"2023-09-23T22:25:26.375802+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":249,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094673+0000","last_change":"2023-09-23T23:11:13.549732+0000","last_active":"2023-09-23T23:12:15.094673+0000","last_peered":"2023-09-23T23:12:15.094673+0000","last_clean":"2023-09-23T23:12:15.094673+0000","last_became_active":"2023-09-23T23:11:13.549442+0000","last_became_peered":"2023-09-23T23:11:13.549442+0000","last_unstale":"2023-09-23T23:12:15.094673+0000","last_undegraded":"2023-09-23T23:12:15.094673+0000","last_fullsized":"2023-09-23T23:12:15.094673+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:18.464642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":468,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097299+0000","last_change":"2023-09-23T23:11:58.725352+0000","last_active":"2023-09-23T23:12:15.097299+0000","last_peered":"2023-09-23T23:12:15.097299+0000","last_clean":"2023-09-23T23:12:15.097299+0000","last_became_active":"2023-09-23T23:11:58.724987+0000","last_became_peered":"2023-09-23T23:11:58.724987+0000","last_unstale":"2023-09-23T23:12:15.097299+0000","last_undegraded":"2023-09-23T23:12:15.097299+0000","last_fullsized":"2023-09-23T23:12:15.097299+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"509'139","reported_seq":1008,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.948951+0000","last_change":"2023-09-23T23:11:20.568963+0000","last_active":"2023-09-23T23:12:13.948951+0000","last_peered":"2023-09-23T23:12:13.948951+0000","last_clean":"2023-09-23T23:12:13.948951+0000","last_became_active":"2023-09-23T23:11:20.568511+0000","last_became_peered":"2023-09-23T23:11:20.568511+0000","last_unstale":"2023-09-23T23:12:13.948951+0000","last_undegraded":"2023-09-23T23:12:13.948951+0000","last_fullsized":"2023-09-23T23:12:13.948951+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":975,"num_read_kb":854,"num_write":120,"num_write_kb":58,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949142+0000","last_change":"2023-09-23T23:11:40.916886+0000","last_active":"2023-09-23T23:12:13.949142+0000","last_peered":"2023-09-23T23:12:13.949142+0000","last_clean":"2023-09-23T23:12:13.949142+0000","last_became_active":"2023-09-23T23:11:40.916475+0000","last_became_peered":"2023-09-23T23:11:40.916475+0000","last_unstale":"2023-09-23T23:12:13.949142+0000","last_undegraded":"2023-09-23T23:12:13.949142+0000","last_fullsized":"2023-09-23T23:12:13.949142+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:26.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":44,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949465+0000","last_change":"2023-09-23T23:11:09.539272+0000","last_active":"2023-09-23T23:12:13.949465+0000","last_peered":"2023-09-23T23:12:13.949465+0000","last_clean":"2023-09-23T23:12:13.949465+0000","last_became_active":"2023-09-23T23:11:09.538917+0000","last_became_peered":"2023-09-23T23:11:09.538917+0000","last_unstale":"2023-09-23T23:12:13.949465+0000","last_undegraded":"2023-09-23T23:12:13.949465+0000","last_fullsized":"2023-09-23T23:12:13.949465+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:32.067020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"509'82","reported_seq":65,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949361+0000","last_change":"2023-09-23T23:11:17.736351+0000","last_active":"2023-09-23T23:12:13.949361+0000","last_peered":"2023-09-23T23:12:13.949361+0000","last_clean":"2023-09-23T23:12:13.949361+0000","last_became_active":"2023-09-23T23:11:17.525399+0000","last_became_peered":"2023-09-23T23:11:17.525399+0000","last_unstale":"2023-09-23T23:12:13.949361+0000","last_undegraded":"2023-09-23T23:12:13.949361+0000","last_fullsized":"2023-09-23T23:12:13.949361+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":309,"num_read_kb":2667,"num_write":81,"num_write_kb":6394,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":416,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889610+0000","last_change":"2023-09-23T23:11:27.937713+0000","last_active":"2023-09-23T23:12:14.889610+0000","last_peered":"2023-09-23T23:12:14.889610+0000","last_clean":"2023-09-23T23:12:14.889610+0000","last_became_active":"2023-09-23T23:11:27.934031+0000","last_became_peered":"2023-09-23T23:11:27.934031+0000","last_unstale":"2023-09-23T23:12:14.889610+0000","last_undegraded":"2023-09-23T23:12:14.889610+0000","last_fullsized":"2023-09-23T23:12:14.889610+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:44.181766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":360,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.427318+0000","last_change":"2023-09-23T23:12:12.051566+0000","last_active":"2023-09-23T23:12:15.427318+0000","last_peered":"2023-09-23T23:12:15.427318+0000","last_clean":"2023-09-23T23:12:15.427318+0000","last_became_active":"2023-09-23T23:12:12.051255+0000","last_became_peered":"2023-09-23T23:12:12.051255+0000","last_unstale":"2023-09-23T23:12:15.427318+0000","last_undegraded":"2023-09-23T23:12:15.427318+0000","last_fullsized":"2023-09-23T23:12:15.427318+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:46.589263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":96,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946499+0000","last_change":"2023-09-23T23:11:17.703727+0000","last_active":"2023-09-23T23:12:13.946499+0000","last_peered":"2023-09-23T23:12:13.946499+0000","last_clean":"2023-09-23T23:12:13.946499+0000","last_became_active":"2023-09-23T23:11:17.525693+0000","last_became_peered":"2023-09-23T23:11:17.525693+0000","last_unstale":"2023-09-23T23:12:13.946499+0000","last_undegraded":"2023-09-23T23:12:13.946499+0000","last_fullsized":"2023-09-23T23:12:13.946499+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:20.154041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"509'98","reported_seq":562,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:14.889719+0000","last_change":"2023-09-23T23:11:28.069849+0000","last_active":"2023-09-23T23:12:14.889719+0000","last_peered":"2023-09-23T23:12:14.889719+0000","last_clean":"2023-09-23T23:12:14.889719+0000","last_became_active":"2023-09-23T23:11:27.938965+0000","last_became_peered":"2023-09-23T23:11:27.938965+0000","last_unstale":"2023-09-23T23:12:14.889719+0000","last_undegraded":"2023-09-23T23:12:14.889719+0000","last_fullsized":"2023-09-23T23:12:14.889719+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:21.079567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":458,"num_read_kb":1757,"num_write":60,"num_write_kb":1236,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":273,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094529+0000","last_change":"2023-09-23T23:11:17.524544+0000","last_active":"2023-09-23T23:12:15.094529+0000","last_peered":"2023-09-23T23:12:15.094529+0000","last_clean":"2023-09-23T23:12:15.094529+0000","last_became_active":"2023-09-23T23:11:17.523198+0000","last_became_peered":"2023-09-23T23:11:17.523198+0000","last_unstale":"2023-09-23T23:12:15.094529+0000","last_undegraded":"2023-09-23T23:12:15.094529+0000","last_fullsized":"2023-09-23T23:12:15.094529+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:50.075594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"509'55","reported_seq":580,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097887+0000","last_change":"2023-09-23T23:11:58.557889+0000","last_active":"2023-09-23T23:12:15.097887+0000","last_peered":"2023-09-23T23:12:15.097887+0000","last_clean":"2023-09-23T23:12:15.097887+0000","last_became_active":"2023-09-23T23:11:58.557258+0000","last_became_peered":"2023-09-23T23:11:58.557258+0000","last_unstale":"2023-09-23T23:12:15.097887+0000","last_undegraded":"2023-09-23T23:12:15.097887+0000","last_fullsized":"2023-09-23T23:12:15.097887+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:27:49.133038+0000","last_clean_scrub_stamp":"2023-09-23T22:27:49.133038+0000","objects_scrubbed":0,"log_size":55,"log_dups_size":0,"ondisk_log_size":55,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":116,"num_read_kb":91,"num_write":61,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":48,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949424+0000","last_change":"2023-09-23T23:11:09.606476+0000","last_active":"2023-09-23T23:12:13.949424+0000","last_peered":"2023-09-23T23:12:13.949424+0000","last_clean":"2023-09-23T23:12:13.949424+0000","last_became_active":"2023-09-23T23:11:09.547731+0000","last_became_peered":"2023-09-23T23:11:09.547731+0000","last_unstale":"2023-09-23T23:12:13.949424+0000","last_undegraded":"2023-09-23T23:12:13.949424+0000","last_fullsized":"2023-09-23T23:12:13.949424+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:00.058887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":557,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097608+0000","last_change":"2023-09-23T23:11:13.551788+0000","last_active":"2023-09-23T23:12:15.097608+0000","last_peered":"2023-09-23T23:12:15.097608+0000","last_clean":"2023-09-23T23:12:15.097608+0000","last_became_active":"2023-09-23T23:11:13.551442+0000","last_became_peered":"2023-09-23T23:11:13.551442+0000","last_unstale":"2023-09-23T23:12:15.097608+0000","last_undegraded":"2023-09-23T23:12:15.097608+0000","last_fullsized":"2023-09-23T23:12:15.097608+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":53,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949491+0000","last_change":"2023-09-23T23:11:17.522576+0000","last_active":"2023-09-23T23:12:13.949491+0000","last_peered":"2023-09-23T23:12:13.949491+0000","last_clean":"2023-09-23T23:12:13.949491+0000","last_became_active":"2023-09-23T23:11:17.522197+0000","last_became_peered":"2023-09-23T23:11:17.522197+0000","last_unstale":"2023-09-23T23:12:13.949491+0000","last_undegraded":"2023-09-23T23:12:13.949491+0000","last_fullsized":"2023-09-23T23:12:13.949491+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:16.154369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":299,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.946414+0000","last_change":"2023-09-23T22:58:24.243442+0000","last_active":"2023-09-23T23:12:13.946414+0000","last_peered":"2023-09-23T23:12:13.946414+0000","last_clean":"2023-09-23T23:12:13.946414+0000","last_became_active":"2023-09-23T22:58:24.242748+0000","last_became_peered":"2023-09-23T22:58:24.242748+0000","last_unstale":"2023-09-23T23:12:13.946414+0000","last_undegraded":"2023-09-23T23:12:13.946414+0000","last_fullsized":"2023-09-23T23:12:13.946414+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:29.366655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":363,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096495+0000","last_change":"2023-09-23T23:11:52.529235+0000","last_active":"2023-09-23T23:12:15.096495+0000","last_peered":"2023-09-23T23:12:15.096495+0000","last_clean":"2023-09-23T23:12:15.096495+0000","last_became_active":"2023-09-23T23:11:52.528969+0000","last_became_peered":"2023-09-23T23:11:52.528969+0000","last_unstale":"2023-09-23T23:12:15.096495+0000","last_undegraded":"2023-09-23T23:12:15.096495+0000","last_fullsized":"2023-09-23T23:12:15.096495+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":25,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.098086+0000","last_change":"2023-09-23T23:11:40.915572+0000","last_active":"2023-09-23T23:12:15.098086+0000","last_peered":"2023-09-23T23:12:15.098086+0000","last_clean":"2023-09-23T23:12:15.098086+0000","last_became_active":"2023-09-23T23:11:40.915062+0000","last_became_peered":"2023-09-23T23:11:40.915062+0000","last_unstale":"2023-09-23T23:12:15.098086+0000","last_undegraded":"2023-09-23T23:12:15.098086+0000","last_fullsized":"2023-09-23T23:12:15.098086+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":390,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.096483+0000","last_change":"2023-09-23T23:11:52.530019+0000","last_active":"2023-09-23T23:12:15.096483+0000","last_peered":"2023-09-23T23:12:15.096483+0000","last_clean":"2023-09-23T23:12:15.096483+0000","last_became_active":"2023-09-23T23:11:52.529528+0000","last_became_peered":"2023-09-23T23:11:52.529528+0000","last_unstale":"2023-09-23T23:12:15.096483+0000","last_undegraded":"2023-09-23T23:12:15.096483+0000","last_fullsized":"2023-09-23T23:12:15.096483+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097461+0000","last_change":"2023-09-23T23:11:40.915193+0000","last_active":"2023-09-23T23:12:15.097461+0000","last_peered":"2023-09-23T23:12:15.097461+0000","last_clean":"2023-09-23T23:12:15.097461+0000","last_became_active":"2023-09-23T23:11:40.914790+0000","last_became_peered":"2023-09-23T23:11:40.914790+0000","last_unstale":"2023-09-23T23:12:15.097461+0000","last_undegraded":"2023-09-23T23:12:15.097461+0000","last_fullsized":"2023-09-23T23:12:15.097461+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":124,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.095098+0000","last_change":"2023-09-23T23:05:53.595316+0000","last_active":"2023-09-23T23:12:15.095098+0000","last_peered":"2023-09-23T23:12:15.095098+0000","last_clean":"2023-09-23T23:12:15.095098+0000","last_became_active":"2023-09-23T23:05:53.594913+0000","last_became_peered":"2023-09-23T23:05:53.594913+0000","last_unstale":"2023-09-23T23:12:15.095098+0000","last_undegraded":"2023-09-23T23:12:15.095098+0000","last_fullsized":"2023-09-23T23:12:15.095098+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:07.480937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":275,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.094736+0000","last_change":"2023-09-23T23:11:09.542604+0000","last_active":"2023-09-23T23:12:15.094736+0000","last_peered":"2023-09-23T23:12:15.094736+0000","last_clean":"2023-09-23T23:12:15.094736+0000","last_became_active":"2023-09-23T23:11:09.542097+0000","last_became_peered":"2023-09-23T23:11:09.542097+0000","last_unstale":"2023-09-23T23:12:15.094736+0000","last_undegraded":"2023-09-23T23:12:15.094736+0000","last_fullsized":"2023-09-23T23:12:15.094736+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:07.434849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":65,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:13.949584+0000","last_change":"2023-09-23T23:11:13.601470+0000","last_active":"2023-09-23T23:12:13.949584+0000","last_peered":"2023-09-23T23:12:13.949584+0000","last_clean":"2023-09-23T23:12:13.949584+0000","last_became_active":"2023-09-23T23:11:13.551003+0000","last_became_peered":"2023-09-23T23:11:13.551003+0000","last_unstale":"2023-09-23T23:12:13.949584+0000","last_undegraded":"2023-09-23T23:12:13.949584+0000","last_fullsized":"2023-09-23T23:12:13.949584+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:41.643121+0000","last_clean_scrub_stamp":"2023-09-23T22:28:41.643121+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:59.599117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":541,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T23:12:15.097253+0000","last_change":"2023-09-23T23:05:03.448160+0000","last_active":"2023-09-23T23:12:15.097253+0000","last_peered":"2023-09-23T23:12:15.097253+0000","last_clean":"2023-09-23T23:12:15.097253+0000","last_became_active":"2023-09-23T23:05:03.447751+0000","last_became_peered":"2023-09-23T23:05:03.447751+0000","last_unstale":"2023-09-23T23:12:15.097253+0000","last_undegraded":"2023-09-23T23:12:15.097253+0000","last_fullsized":"2023-09-23T23:12:15.097253+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:28:42.648793+0000","last_clean_scrub_stamp":"2023-09-23T22:28:42.648793+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":26850,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16290,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430759,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315276,"kb_used_data":9860,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93429876,"statfs":{"total":95995035648,"available":95672193024,"internally_reserved":0,"allocated":10096640,"data_stored":8964359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.6240000000000001,"15min":1.8460000000000001},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":0.65400000000000003,"5min":78.513000000000005,"15min":100.25700000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.7969999999999999,"15min":2.1230000000000002},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.245},"max":{"1min":0.94999999999999996,"5min":78.436000000000007,"15min":100.235},"last":0.436}]},{"osd":1,"last update":"Sat Sep 23 23:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.7969999999999999,"15min":1.3049999999999999},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.28100000000000003},"max":{"1min":0.90300000000000002,"5min":79.566000000000003,"15min":79.566000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63,"5min":1.516,"15min":1.1990000000000001},"min":{"1min":0.436,"5min":0.307,"15min":0.26700000000000002},"max":{"1min":0.78200000000000003,"5min":79.516999999999996,"15min":79.516999999999996},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.3250000000000002,"15min":2.2080000000000002},"min":{"1min":0.443,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":0.85199999999999998,"5min":69.777000000000001,"15min":97.262},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.625,"5min":2.2570000000000001,"15min":2.1819999999999999},"min":{"1min":0.39400000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.97199999999999998,"5min":69.742000000000004,"15min":97.272999999999996},"last":0.69099999999999995}]},{"osd":3,"last update":"Sat Sep 23 23:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.726,"15min":1.645},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.318},"max":{"1min":0.78900000000000003,"5min":82.242000000000004,"15min":84.480000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.623,"5min":1.7050000000000001,"15min":1.8460000000000001},"min":{"1min":0.42999999999999999,"5min":0.33600000000000002,"15min":0.28599999999999998},"max":{"1min":0.92600000000000005,"5min":82.272000000000006,"15min":84.826999999999998},"last":0.66100000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.554,"15min":2.589},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.25800000000000001},"max":{"1min":1.0600000000000001,"5min":62.582999999999998,"15min":111.81},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.4139999999999999,"15min":2.3039999999999998},"min":{"1min":0.36699999999999999,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":1.0980000000000001,"5min":62.609999999999999,"15min":111.636},"last":0.72999999999999998}]},{"osd":5,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":1.0640000000000001,"15min":1.345},"min":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.27600000000000002},"max":{"1min":0.70899999999999996,"5min":15.996,"15min":65.956999999999994},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.97399999999999998,"15min":1.4930000000000001},"min":{"1min":0.52500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":1.1519999999999999,"5min":6.9710000000000001,"15min":66.106999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":4.867,"15min":2.8010000000000002},"min":{"1min":0.44600000000000001,"5min":0.33600000000000002,"15min":0.28000000000000003},"max":{"1min":0.89200000000000002,"5min":371.00999999999999,"15min":371.00999999999999},"last":0.755},{"interface":"front","average":{"1min":0.71899999999999997,"5min":5.2160000000000002,"15min":2.875},"min":{"1min":0.58599999999999997,"5min":0.35899999999999999,"15min":0.34699999999999998},"max":{"1min":1.089,"5min":367.65199999999999,"15min":367.65199999999999},"last":0.67100000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626983,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916676,"kb_used_data":5756,"kb_used_omap":0,"kb_used_meta":910912,"kb_avail":92828476,"statfs":{"total":95995035648,"available":95056359424,"internally_reserved":0,"allocated":5894144,"data_stored":4769928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932773888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.86299999999999999,"15min":3.5449999999999999},"min":{"1min":0.42199999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.012,"5min":7.9630000000000001,"15min":115.32899999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.94299999999999995,"15min":3.556},"min":{"1min":0.443,"5min":0.25,"15min":0.25},"max":{"1min":0.98399999999999999,"5min":8.7010000000000005,"15min":115.348},"last":0.495}]},{"osd":1,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.7829999999999999,"15min":2.593},"min":{"1min":0.35799999999999998,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":0.91600000000000004,"5min":38.259999999999998,"15min":149.947},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.5209999999999999,"15min":2.5230000000000001},"min":{"1min":0.318,"5min":0.249,"15min":0.249},"max":{"1min":0.82899999999999996,"5min":38.530000000000001,"15min":150.22499999999999},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.95,"15min":2.8799999999999999},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.89000000000000001,"5min":45.051000000000002,"15min":138.43299999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59799999999999998,"5min":2.1659999999999999,"15min":3.0009999999999999},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":1.0009999999999999,"5min":45.258000000000003,"15min":138.45599999999999},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":2.0230000000000001,"15min":3.198},"min":{"1min":0.45200000000000001,"5min":0.32400000000000001,"15min":0.26800000000000002},"max":{"1min":1.0349999999999999,"5min":47.734000000000002,"15min":142.24700000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.794,"15min":3.097},"min":{"1min":0.502,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.93899999999999995,"5min":47.162999999999997,"15min":143.86699999999999},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.85899999999999999,"15min":1.6910000000000001},"min":{"1min":0.34399999999999997,"5min":0.25800000000000001,"15min":0.23999999999999999},"max":{"1min":0.74299999999999999,"5min":7.3869999999999996,"15min":119.804},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":1.5109999999999999,"15min":1.917},"min":{"1min":0.32500000000000001,"5min":0.27700000000000002,"15min":0.24099999999999999},"max":{"1min":0.747,"5min":46.515999999999998,"15min":122.31999999999999},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 23:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.0069999999999999,"15min":2.419},"min":{"1min":0.39500000000000002,"5min":0.29099999999999998,"15min":0.27900000000000003},"max":{"1min":0.81100000000000005,"5min":6.4530000000000003,"15min":186.27600000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.0920000000000001,"15min":2.4900000000000002},"min":{"1min":0.44,"5min":0.34399999999999997,"15min":0.29599999999999999},"max":{"1min":0.80700000000000005,"5min":8.7720000000000002,"15min":191.28700000000001},"last":0.64800000000000002}]},{"osd":7,"last update":"Sat Sep 23 23:12:40 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.016,"15min":1.4159999999999999},"min":{"1min":0.40600000000000003,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":0.88900000000000001,"5min":8.8670000000000009,"15min":115.61},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.1020000000000001,"15min":1.486},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.29999999999999999},"max":{"1min":0.79800000000000004,"5min":8.2609999999999992,"15min":115.578},"last":0.58899999999999997}]}]},{"osd":1,"up_from":84,"seq":360777253297,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1016184,"kb_used_data":6384,"kb_used_omap":9,"kb_used_meta":1009782,"kb_avail":92728968,"statfs":{"total":95995035648,"available":94954463232,"internally_reserved":0,"allocated":6537216,"data_stored":4333668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":1034017769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.48599999999999999,"15min":2.5779999999999998},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":0.58999999999999997,"5min":3.891,"15min":279.30500000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.37,"5min":0.69699999999999995,"15min":2.831},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.47499999999999998,"5min":7.6890000000000001,"15min":279.39800000000002},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 23:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":3.9740000000000002,"15min":3.0979999999999999},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.55700000000000005,"5min":121.527,"15min":161.61699999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.39200000000000002,"5min":4.0380000000000003,"15min":3.2570000000000001},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.22600000000000001},"max":{"1min":0.51000000000000001,"5min":118.29900000000001,"15min":162.61699999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":1.9490000000000001,"15min":1.659},"min":{"1min":0.26700000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.61899999999999999,"5min":67.646000000000001,"15min":67.646000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.434,"5min":2.125,"15min":1.706},"min":{"1min":0.312,"5min":0.24199999999999999,"15min":0.22500000000000001},"max":{"1min":0.79500000000000004,"5min":65.269999999999996,"15min":65.269999999999996},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":3.0569999999999999,"15min":2.4569999999999999},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":0.95199999999999996,"5min":129.69,"15min":151.036},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":3.2290000000000001,"15min":2.7050000000000001},"min":{"1min":0.41399999999999998,"5min":0.309,"15min":0.26400000000000001},"max":{"1min":0.97699999999999998,"5min":129.93700000000001,"15min":151.06899999999999},"last":0.98599999999999999}]},{"osd":5,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.85699999999999998,"15min":1.885},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":0.99299999999999999,"5min":6.242,"15min":116.098},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.4650000000000001,"15min":1.9610000000000001},"min":{"1min":0.45300000000000001,"5min":0.32600000000000001,"15min":0.315},"max":{"1min":0.88200000000000001,"5min":51.140999999999998,"15min":116.08499999999999},"last":0.88500000000000001}]},{"osd":6,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":2.4159999999999999,"15min":1.978},"min":{"1min":0.35599999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.93999999999999995,"5min":64.058000000000007,"15min":67.349000000000004},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.64400000000000002,"5min":2.516,"15min":1.877},"min":{"1min":0.49199999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":1.03,"5min":64.239999999999995,"15min":67.366},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 23:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.1699999999999999,"15min":1.056},"min":{"1min":0.48599999999999999,"5min":0.39400000000000002,"15min":0.34699999999999998},"max":{"1min":0.91400000000000003,"5min":6.4089999999999998,"15min":7.0960000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.2130000000000001,"15min":1.2909999999999999},"min":{"1min":0.44600000000000001,"5min":0.40699999999999997,"15min":0.33100000000000002},"max":{"1min":1.026,"5min":8.8719999999999999,"15min":10.676},"last":0.93000000000000005}]}]},{"osd":0,"up_from":353,"seq":1516123455667,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300968,"kb_used_data":11360,"kb_used_omap":37,"kb_used_meta":289562,"kb_avail":93444184,"statfs":{"total":95995035648,"available":95686844416,"internally_reserved":0,"allocated":11632640,"data_stored":9148776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38750,"internal_metadata":296511650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.72799999999999998,"15min":0.82799999999999996},"min":{"1min":0.253,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.48699999999999999,"5min":7.1340000000000003,"15min":8.234},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":1.032,"15min":0.94299999999999995},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.20200000000000001},"max":{"1min":0.52900000000000003,"5min":9.9979999999999993,"15min":9.9979999999999993},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 23:11:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":2.6160000000000001,"15min":2.645},"min":{"1min":0.253,"5min":0.253,"15min":0.20599999999999999},"max":{"1min":0.54200000000000004,"5min":73.010000000000005,"15min":190.744},"last":0.442},{"interface":"front","average":{"1min":0.38400000000000001,"5min":2.407,"15min":2.8279999999999998},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.22700000000000001},"max":{"1min":0.63200000000000001,"5min":70.709000000000003,"15min":187.44200000000001},"last":0.32500000000000001}]},{"osd":3,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":4.6020000000000003,"15min":3.399},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":0.66800000000000004,"5min":146.411,"15min":146.411},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.42699999999999999,"5min":4.9269999999999996,"15min":3.6699999999999999},"min":{"1min":0.309,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":0.65700000000000003,"5min":146.667,"15min":146.667},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1,"15min":2.0409999999999999},"min":{"1min":0.34899999999999998,"5min":0.32000000000000001,"15min":0.28100000000000003},"max":{"1min":0.85499999999999998,"5min":5.681,"15min":183.148},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.96099999999999997,"15min":1.7809999999999999},"min":{"1min":0.42599999999999999,"5min":0.33600000000000002,"15min":0.27500000000000002},"max":{"1min":0.82999999999999996,"5min":10.124000000000001,"15min":183.39500000000001},"last":0.755}]},{"osd":5,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.82199999999999995,"15min":0.997},"min":{"1min":0.437,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":0.94099999999999995,"5min":9.3019999999999996,"15min":9.3019999999999996},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.88800000000000001,"15min":1.2869999999999999},"min":{"1min":0.45200000000000001,"5min":0.35699999999999998,"15min":0.32800000000000001},"max":{"1min":0.84599999999999997,"5min":5.7590000000000003,"15min":74.858000000000004},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.95199999999999996,"15min":1.498},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":0.88600000000000001,"5min":10.153,"15min":109.714},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.88200000000000001,"15min":1.4650000000000001},"min":{"1min":0.47699999999999998,"5min":0.38,"15min":0.35999999999999999},"max":{"1min":0.95099999999999996,"5min":8.5259999999999998,"15min":109.783},"last":0.73999999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.034,"15min":1.1020000000000001},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":0.92200000000000004,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.97999999999999998,"15min":1.123},"min":{"1min":0.47299999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.90700000000000003,"5min":4.3559999999999999,"15min":6.6859999999999999},"last":0.77600000000000002}]}]},{"osd":2,"up_from":359,"seq":1541893259440,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328288,"kb_used_data":6552,"kb_used_omap":83,"kb_used_meta":321644,"kb_avail":93416864,"statfs":{"total":95995035648,"available":95658868736,"internally_reserved":0,"allocated":6709248,"data_stored":4351447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":85563,"internal_metadata":329363909},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.58599999999999997,"15min":1.24},"min":{"1min":0.24199999999999999,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":0.44500000000000001,"5min":4.7729999999999997,"15min":74.241},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.372,"5min":1.022,"15min":1.5720000000000001},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.57599999999999996,"5min":14.944000000000001,"15min":74.430000000000007},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":1.9590000000000001,"15min":1.409},"min":{"1min":0.251,"5min":0.221,"15min":0.20599999999999999},"max":{"1min":0.502,"5min":83.897000000000006,"15min":83.897000000000006},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":1.6319999999999999,"15min":1.4159999999999999},"min":{"1min":0.27100000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.51600000000000001,"5min":83.088999999999999,"15min":83.088999999999999},"last":0.307}]},{"osd":3,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":3.0710000000000002,"15min":2.141},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.23000000000000001},"max":{"1min":0.64000000000000001,"5min":79.655000000000001,"15min":110.31699999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.436,"5min":2.8759999999999999,"15min":2.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59999999999999998,"5min":79.677999999999997,"15min":110.56999999999999},"last":0.38600000000000001}]},{"osd":4,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.409,"15min":1.155},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.27500000000000002},"max":{"1min":0.92400000000000004,"5min":68.325000000000003,"15min":68.325000000000003},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.7230000000000001,"15min":1.383},"min":{"1min":0.46700000000000003,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":0.81100000000000005,"5min":70.608000000000004,"15min":70.608000000000004},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.0860000000000001,"15min":1.665},"min":{"1min":0.41599999999999998,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.86899999999999999,"5min":7.6600000000000001,"15min":139.405},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.90400000000000003,"15min":1.53},"min":{"1min":0.44500000000000001,"5min":0.38500000000000001,"15min":0.36299999999999999},"max":{"1min":0.97699999999999998,"5min":5.5880000000000001,"15min":139.42099999999999},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":2.484,"15min":1.873},"min":{"1min":0.40600000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.91300000000000003,"5min":108.53700000000001,"15min":108.53700000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.63500000000000001,"5min":2.2080000000000002,"15min":1.611},"min":{"1min":0.44,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":0.91100000000000003,"5min":108.496,"15min":108.496},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.1850000000000001,"15min":1.1799999999999999},"min":{"1min":0.48099999999999998,"5min":0.40300000000000002,"15min":0.36699999999999999},"max":{"1min":0.95199999999999996,"5min":9.1449999999999996,"15min":10.164},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.069,"15min":1.145},"min":{"1min":0.46200000000000002,"5min":0.40799999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.57199999999999995}]}]},{"osd":3,"up_from":94,"seq":403726926244,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":730936,"kb_used_data":6512,"kb_used_omap":7,"kb_used_meta":724408,"kb_avail":93014216,"statfs":{"total":95995035648,"available":95246557184,"internally_reserved":0,"allocated":6668288,"data_stored":4344620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":741794408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.751,"15min":1.512},"min":{"1min":0.254,"5min":0.215,"15min":0.17299999999999999},"max":{"1min":0.51300000000000001,"5min":8.1660000000000004,"15min":122.289},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.67300000000000004,"15min":1.2110000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.17799999999999999},"max":{"1min":0.49399999999999999,"5min":8.1519999999999992,"15min":122.041},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.3560000000000001,"15min":1.4319999999999999},"min":{"1min":0.26800000000000002,"5min":0.21299999999999999,"15min":0.187},"max":{"1min":14.327,"5min":14.327,"15min":23.969000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.04,"5min":1.01,"15min":1.2669999999999999},"min":{"1min":0.28899999999999998,"5min":0.26900000000000002,"15min":0.20699999999999999},"max":{"1min":14.398999999999999,"5min":14.398999999999999,"15min":16.986000000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 23:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.73,"15min":1.294},"min":{"1min":0.25800000000000001,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":0.55800000000000005,"5min":55.311999999999998,"15min":55.311999999999998},"last":0.372},{"interface":"front","average":{"1min":0.39500000000000002,"5min":2.0880000000000001,"15min":1.637},"min":{"1min":0.28999999999999998,"5min":0.28199999999999997,"15min":0.215},"max":{"1min":0.51800000000000002,"5min":57.308,"15min":57.308},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.151,"15min":2.0019999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.30199999999999999},"max":{"1min":1.5920000000000001,"5min":11.01,"15min":115.69},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.91800000000000004,"15min":1.9390000000000001},"min":{"1min":0.44,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":1.9670000000000001,"5min":5.9169999999999998,"15min":115.617},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.995,"15min":1.3080000000000001},"min":{"1min":0.40000000000000002,"5min":0.35299999999999998,"15min":0.28499999999999998},"max":{"1min":2.0059999999999998,"5min":5.6399999999999997,"15min":16.959},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.94199999999999995,"15min":1.1679999999999999},"min":{"1min":0.35699999999999998,"5min":0.34000000000000002,"15min":0.27100000000000002},"max":{"1min":1.8120000000000001,"5min":6.6130000000000004,"15min":10.545},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.8939999999999999,"15min":1.6890000000000001},"min":{"1min":0.505,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":2.0430000000000001,"5min":77.707999999999998,"15min":77.707999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.8280000000000001,"15min":1.454},"min":{"1min":0.39700000000000002,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":1.9510000000000001,"5min":77.721999999999994,"15min":77.721999999999994},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Sep 23 23:12:17 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.101,"15min":1.099},"min":{"1min":0.42599999999999999,"5min":0.38600000000000001,"15min":0.33900000000000002},"max":{"1min":1.9059999999999999,"5min":6.1820000000000004,"15min":8.1620000000000008},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.383,"15min":1.413},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.37},"max":{"1min":1.7290000000000001,"5min":11.051,"15min":11.051},"last":0.625}]}]},{"osd":4,"up_from":30,"seq":128849019441,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916740,"kb_used_data":9276,"kb_used_omap":0,"kb_used_meta":907456,"kb_avail":92828412,"statfs":{"total":95995035648,"available":95056293888,"internally_reserved":0,"allocated":9498624,"data_stored":8373921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929234944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:30 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":4.2089999999999996,"15min":3.4260000000000002},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.23100000000000001},"max":{"1min":3.5139999999999998,"5min":323.65800000000002,"15min":323.65800000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.747,"5min":4.0830000000000002,"15min":3.5089999999999999},"min":{"1min":0.40100000000000002,"5min":0.28299999999999997,"15min":0.24399999999999999},"max":{"1min":3.5390000000000001,"5min":323.642,"15min":323.642},"last":0.53400000000000003}]},{"osd":1,"last update":"Sat Sep 23 23:12:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0880000000000001,"15min":2.0459999999999998},"min":{"1min":0.39500000000000002,"5min":0.28199999999999997,"15min":0.26600000000000001},"max":{"1min":11.459,"5min":11.459,"15min":95.591999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.1439999999999999,"15min":2.0070000000000001},"min":{"1min":0.41199999999999998,"5min":0.29999999999999999,"15min":0.23699999999999999},"max":{"1min":3.6619999999999999,"5min":15.728,"15min":97.929000000000002},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:11:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":4.7919999999999998,"15min":2.6600000000000001},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":0.748,"5min":151.20400000000001,"15min":151.20400000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":4.6109999999999998,"15min":2.544},"min":{"1min":0.435,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":1.1899999999999999,"5min":148.852,"15min":148.852},"last":0.67600000000000005}]},{"osd":3,"last update":"Sat Sep 23 23:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":3.6120000000000001,"15min":2.323},"min":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":0.79700000000000004,"5min":147.82499999999999,"15min":147.82499999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":3.4980000000000002,"15min":2.415},"min":{"1min":0.44400000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.3180000000000001,"5min":147.84100000000001,"15min":147.84100000000001},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 23:11:50 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.1259999999999999,"15min":1.746},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.214},"max":{"1min":0.72699999999999998,"5min":10.282,"15min":148.61600000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.4530000000000001,"15min":1.891},"min":{"1min":0.372,"5min":0.33600000000000002,"15min":0.247},"max":{"1min":0.65400000000000003,"5min":17.02,"15min":151.905},"last":0.505}]},{"osd":6,"last update":"Sat Sep 23 23:12:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.0029999999999999,"15min":1.3049999999999999},"min":{"1min":0.47799999999999998,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":0.755,"5min":7.8390000000000004,"15min":35.896000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.98499999999999999,"15min":1.2929999999999999},"min":{"1min":0.48899999999999999,"5min":0.318,"15min":0.27800000000000002},"max":{"1min":0.80300000000000005,"5min":7.008,"15min":35.908000000000001},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 23:12:43 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.081,"15min":1.214},"min":{"1min":0.52600000000000002,"5min":0.35799999999999998,"15min":0.28399999999999997},"max":{"1min":0.89300000000000002,"5min":11.096,"15min":12.84},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.0940000000000001,"15min":1.117},"min":{"1min":0.54600000000000004,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81899999999999995,"5min":15.808,"15min":15.808},"last":0.628}]}]},{"osd":5,"up_from":36,"seq":154618823215,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900720,"kb_used_data":5160,"kb_used_omap":0,"kb_used_meta":895552,"kb_avail":92844432,"statfs":{"total":95995035648,"available":95072698368,"internally_reserved":0,"allocated":5283840,"data_stored":4174750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917045248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 23:12:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.8109999999999999,"15min":1.5469999999999999},"min":{"1min":0.30599999999999999,"5min":0.252,"15min":0.221},"max":{"1min":0.79900000000000004,"5min":51.962000000000003,"15min":51.962000000000003},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.7889999999999999,"15min":1.6439999999999999},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.23799999999999999},"max":{"1min":0.78400000000000003,"5min":52.017000000000003,"15min":52.017000000000003},"last":0.626}]},{"osd":1,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.8,"15min":1.329},"min":{"1min":0.35199999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":0.77400000000000002,"5min":66.242000000000004,"15min":66.242000000000004},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.901,"15min":1.5349999999999999},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.253},"max":{"1min":0.90500000000000003,"5min":66.424000000000007,"15min":66.424000000000007},"last":0.64600000000000002}]},{"osd":2,"last update":"Sat Sep 23 23:11:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":3.6920000000000002,"15min":2.27},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.25900000000000001},"max":{"1min":0.875,"5min":78.203000000000003,"15min":78.203000000000003},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":3.6219999999999999,"15min":2.2770000000000001},"min":{"1min":0.39900000000000002,"5min":0.36799999999999999,"15min":0.31},"max":{"1min":0.88400000000000001,"5min":78.210999999999999,"15min":78.210999999999999},"last":0.69999999999999996}]},{"osd":3,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":5.2039999999999997,"15min":3.3650000000000002},"min":{"1min":0.39200000000000002,"5min":0.32400000000000001,"15min":0.28399999999999997},"max":{"1min":0.72199999999999998,"5min":125.806,"15min":125.806},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":5.0999999999999996,"15min":3.415},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.29699999999999999},"max":{"1min":0.94399999999999995,"5min":126.04600000000001,"15min":126.04600000000001},"last":0.65600000000000003}]},{"osd":4,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.873,"15min":1.552},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":0.74199999999999999,"5min":7.7610000000000001,"15min":115.06699999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.177,"15min":1.5489999999999999},"min":{"1min":0.30499999999999999,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":0.78000000000000003,"5min":14.098000000000001,"15min":114.908},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Sep 23 23:12:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.077,"15min":1.1950000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":0.77300000000000002,"5min":8.4580000000000002,"15min":12.385},"last":0.503},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.0349999999999999,"15min":1.0589999999999999},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.28199999999999997},"max":{"1min":0.872,"5min":7.649,"15min":12.137},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 23 23:12:37 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.73799999999999999,"15min":0.93700000000000006},"min":{"1min":0.441,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":0.88400000000000001,"5min":3.504,"15min":9.5980000000000008},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.123,"15min":1.1990000000000001},"min":{"1min":0.44800000000000001,"5min":0.32200000000000001,"15min":0.31},"max":{"1min":0.83299999999999996,"5min":8.4120000000000008,"15min":10.638999999999999},"last":0.60999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9902,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6388,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T23:12:55.170 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T23:12:55.171 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T23:12:55.232 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=bb55d18533c74835b74e7e3cfe92e4f2 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T23:12:55.234 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T23:12:55.244 INFO:tasks.cephadm:Teardown begin 2023-09-23T23:12:55.245 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:12:55.299 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:12:55.372 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T23:12:55.372 DEBUG:teuthology.orchestra.run.smithi052:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T23:12:55.390 DEBUG:teuthology.orchestra.run.smithi151:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T23:12:55.408 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T23:12:55.409 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T23:12:55.409 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T23:12:55.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:55 smithi052 systemd[1]: Stopping Ceph mon.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:12:55.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[180954]: 2023-09-23T23:12:55.703+0000 7ffafcbb9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:12:55.716 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[180954]: 2023-09-23T23:12:55.703+0000 7ffafcbb9700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T23:12:55.993 INFO:journalctl@ceph.mon.a.smithi052.stdout:Sep 23 23:12:55 smithi052 bash[732400]: bb846ce0af7315753c8e17de98294cd12716fed3bf4743e2bf2753f14ac7f118 2023-09-23T23:12:55.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[173965]: [23/Sep/2023:23:12:55] ENGINE Bus STOPPING 2023-09-23T23:12:55.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[173965]: [23/Sep/2023:23:12:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T23:12:55.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[173965]: [23/Sep/2023:23:12:55] ENGINE Bus STOPPED 2023-09-23T23:12:55.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[173965]: [23/Sep/2023:23:12:55] ENGINE Bus STARTING 2023-09-23T23:12:55.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[173965]: [23/Sep/2023:23:12:55] ENGINE Serving on http://:::9283 2023-09-23T23:12:55.994 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:55 smithi052 conmon[173965]: [23/Sep/2023:23:12:55] ENGINE Bus STARTED 2023-09-23T23:12:56.467 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T23:12:56.510 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:12:56.510 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T23:12:56.510 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T23:12:56.510 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T23:12:56.743 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:56 smithi052 systemd[1]: Stopping Ceph mon.c for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:12:57.175 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[173965]: [23/Sep/2023:23:12:56] ENGINE Bus STOPPING 2023-09-23T23:12:57.175 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[173965]: [23/Sep/2023:23:12:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T23:12:57.175 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[173965]: [23/Sep/2023:23:12:56] ENGINE Bus STOPPED 2023-09-23T23:12:57.176 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[173965]: [23/Sep/2023:23:12:56] ENGINE Bus STARTING 2023-09-23T23:12:57.176 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[173965]: [23/Sep/2023:23:12:56] ENGINE Serving on http://:::9283 2023-09-23T23:12:57.176 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[173965]: [23/Sep/2023:23:12:56] ENGINE Bus STARTED 2023-09-23T23:12:57.176 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[187698]: 2023-09-23T23:12:56.848+0000 7f6be50b7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:12:57.176 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:56 smithi052 conmon[187698]: 2023-09-23T23:12:56.848+0000 7f6be50b7700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T23:12:57.176 INFO:journalctl@ceph.mon.c.smithi052.stdout:Sep 23 23:12:57 smithi052 bash[732714]: 0c16f625a72c8c5971c4027c92b5533bddb5a04c1cd40d11d92fd61d9ad54ea6 2023-09-23T23:12:57.658 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T23:12:57.698 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:12:57.698 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T23:12:57.698 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T23:12:57.698 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T23:12:58.103 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:57 smithi151 systemd[1]: Stopping Ceph mon.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:12:58.103 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:57 smithi151 conmon[159186]: 2023-09-23T23:12:57.973+0000 7fd4878fa700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:12:58.103 INFO:journalctl@ceph.mon.b.smithi151.stdout:Sep 23 23:12:57 smithi151 conmon[159186]: 2023-09-23T23:12:57.973+0000 7fd4878fa700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T23:12:58.703 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T23:12:58.764 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:12:58.765 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T23:12:58.765 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T23:12:58.765 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T23:12:59.073 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:58 smithi052 systemd[1]: Stopping Ceph mgr.y for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:12:59.073 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:59 smithi052 conmon[173965]: 2023-09-23T23:12:59.071+0000 7f39ee3cb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:12:59.493 INFO:journalctl@ceph.mgr.y.smithi052.stdout:Sep 23 23:12:59 smithi052 conmon[173965]: 2023-09-23T23:12:59.071+0000 7f39ee3cb700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T23:12:59.884 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T23:12:59.922 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:12:59.922 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T23:12:59.922 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T23:12:59.922 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T23:13:00.328 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Sep 23 23:12:59 smithi151 systemd[1]: Stopping Ceph mgr.x for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:00.996 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T23:13:01.032 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:01.032 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T23:13:01.032 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T23:13:01.032 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T23:13:01.358 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:01 smithi052 systemd[1]: Stopping Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:01.743 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:01 smithi052 conmon[500475]: 2023-09-23T23:13:01.357+0000 7f039da53700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:13:01.744 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:01 smithi052 conmon[500475]: 2023-09-23T23:13:01.357+0000 7f039da53700 -1 osd.0 547 *** Got signal Terminated *** 2023-09-23T23:13:01.744 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:01 smithi052 conmon[500475]: 2023-09-23T23:13:01.357+0000 7f039da53700 -1 osd.0 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:07.493 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:07 smithi052 bash[733326]: e622ae7bfc72d210da7cb0bf8f045803ef675690320eece65db595fce2686381 2023-09-23T23:13:09.473 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T23:13:09.500 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:09 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T23:13:09.500 INFO:journalctl@ceph.osd.0.smithi052.stdout:Sep 23 23:13:09 smithi052 systemd[1]: Stopped Ceph osd.0 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T23:13:09.514 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:09.515 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T23:13:09.515 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T23:13:09.515 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T23:13:09.840 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 23:13:09 smithi052 systemd[1]: Stopping Ceph osd.1 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:10.243 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 23:13:09 smithi052 conmon[203590]: 2023-09-23T23:13:09.839+0000 7fbb604c4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:13:10.244 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 23:13:09 smithi052 conmon[203590]: 2023-09-23T23:13:09.839+0000 7fbb604c4700 -1 osd.1 547 *** Got signal Terminated *** 2023-09-23T23:13:10.244 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 23:13:09 smithi052 conmon[203590]: 2023-09-23T23:13:09.839+0000 7fbb604c4700 -1 osd.1 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:16.494 INFO:journalctl@ceph.osd.1.smithi052.stdout:Sep 23 23:13:16 smithi052 bash[733911]: 4ff251b65413b4a0f154dc2ebbf538b71a9cb49edb4cf7c8db9e5aae830ef33a 2023-09-23T23:13:17.687 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T23:13:17.729 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:17.729 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T23:13:17.729 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T23:13:17.730 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T23:13:17.993 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 23:13:17 smithi052 systemd[1]: Stopping Ceph osd.2 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:18.494 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 23:13:18 smithi052 conmon[505773]: 2023-09-23T23:13:18.062+0000 7f9e0ce35700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:13:18.494 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 23:13:18 smithi052 conmon[505773]: 2023-09-23T23:13:18.062+0000 7f9e0ce35700 -1 osd.2 547 *** Got signal Terminated *** 2023-09-23T23:13:18.494 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 23:13:18 smithi052 conmon[505773]: 2023-09-23T23:13:18.062+0000 7f9e0ce35700 -1 osd.2 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:24.244 INFO:journalctl@ceph.osd.2.smithi052.stdout:Sep 23 23:13:23 smithi052 bash[734486]: 82bbd993ad69cf5a61e29cf428fd01fdd972046411462527ade6c91dfd82cdc5 2023-09-23T23:13:24.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:13:23 smithi151 conmon[150039]: ts=2023-09-23T23:13:23.951Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.52:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.52:8765: connect: connection refused" 2023-09-23T23:13:24.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:13:23 smithi151 conmon[150039]: ts=2023-09-23T23:13:23.955Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.52:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.52:8765: connect: connection refused" 2023-09-23T23:13:24.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:13:23 smithi151 conmon[150039]: ts=2023-09-23T23:13:23.955Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.52:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.52:8765: connect: connection refused" 2023-09-23T23:13:24.450 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:13:23 smithi151 conmon[150039]: ts=2023-09-23T23:13:23.955Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.52:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.52:8765: connect: connection refused" 2023-09-23T23:13:25.745 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T23:13:25.787 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:25.787 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T23:13:25.787 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T23:13:25.787 DEBUG:teuthology.orchestra.run.smithi052:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T23:13:25.994 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:25 smithi052 systemd[1]: Stopping Ceph osd.3 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:26.493 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:26 smithi052 conmon[213354]: 2023-09-23T23:13:26.102+0000 7f015716c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T23:13:26.643 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:26 smithi052 conmon[213354]: 2023-09-23T23:13:26.102+0000 7f015716c700 -1 osd.3 547 *** Got signal Terminated *** 2023-09-23T23:13:26.643 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:26 smithi052 conmon[213354]: 2023-09-23T23:13:26.102+0000 7f015716c700 -1 osd.3 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:28.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:27 smithi151 conmon[112515]: debug 2023-09-23T23:13:27.953+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:28.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:28 smithi151 conmon[124295]: debug 2023-09-23T23:13:28.115+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:28.493 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:28 smithi052 conmon[213354]: 2023-09-23T23:13:28.193+0000 7f0150f5d700 -1 osd.3 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:02.169789+0000 front 2023-09-23T23:13:02.169826+0000 (oldest deadline 2023-09-23T23:13:27.469921+0000) 2023-09-23T23:13:28.936 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:28 smithi151 conmon[106754]: debug 2023-09-23T23:13:28.649+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:29.199 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:28 smithi151 conmon[112515]: debug 2023-09-23T23:13:28.935+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:29.199 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:29 smithi151 conmon[124295]: debug 2023-09-23T23:13:29.136+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:29.493 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:29 smithi052 conmon[213354]: 2023-09-23T23:13:29.208+0000 7f0150f5d700 -1 osd.3 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:02.169789+0000 front 2023-09-23T23:13:02.169826+0000 (oldest deadline 2023-09-23T23:13:27.469921+0000) 2023-09-23T23:13:29.904 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:29 smithi151 conmon[106754]: debug 2023-09-23T23:13:29.608+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:30.199 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:29 smithi151 conmon[112515]: debug 2023-09-23T23:13:29.904+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:30.199 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:30 smithi151 conmon[124295]: debug 2023-09-23T23:13:30.152+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:30.200 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:30 smithi151 conmon[118391]: debug 2023-09-23T23:13:30.058+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:30.493 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:30 smithi052 conmon[213354]: 2023-09-23T23:13:30.198+0000 7f0150f5d700 -1 osd.3 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:02.169789+0000 front 2023-09-23T23:13:02.169826+0000 (oldest deadline 2023-09-23T23:13:27.469921+0000) 2023-09-23T23:13:30.907 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:30 smithi151 conmon[106754]: debug 2023-09-23T23:13:30.617+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:31.199 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:30 smithi151 conmon[112515]: debug 2023-09-23T23:13:30.907+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:31.199 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:31 smithi151 conmon[118391]: debug 2023-09-23T23:13:31.084+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:31.200 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:31 smithi151 conmon[124295]: debug 2023-09-23T23:13:31.127+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:31.945 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:31 smithi151 conmon[106754]: debug 2023-09-23T23:13:31.593+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:32.199 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:31 smithi151 conmon[112515]: debug 2023-09-23T23:13:31.944+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:32.200 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:32 smithi151 conmon[124295]: debug 2023-09-23T23:13:32.176+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:32.200 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:32 smithi151 conmon[118391]: debug 2023-09-23T23:13:32.131+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:32.744 INFO:journalctl@ceph.osd.3.smithi052.stdout:Sep 23 23:13:32 smithi052 bash[735064]: 8408de2fa10661c3962ba56929b7242881d1aec053d012164e6f06b9ba066fdf 2023-09-23T23:13:32.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:32 smithi151 conmon[106754]: debug 2023-09-23T23:13:32.545+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:33.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:32 smithi151 conmon[112515]: debug 2023-09-23T23:13:32.965+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:33.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:33 smithi151 conmon[118391]: debug 2023-09-23T23:13:33.118+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:33.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:33 smithi151 conmon[124295]: debug 2023-09-23T23:13:33.152+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:33.778 DEBUG:teuthology.orchestra.run.smithi052:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T23:13:33.819 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:33.819 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T23:13:33.819 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T23:13:33.820 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T23:13:33.851 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:33 smithi151 conmon[106754]: debug 2023-09-23T23:13:33.545+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:34.112 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:33 smithi151 systemd[1]: Stopping Ceph osd.4 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:34.113 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:33 smithi151 conmon[112515]: debug 2023-09-23T23:13:33.986+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:34.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:34 smithi151 conmon[118391]: debug 2023-09-23T23:13:34.111+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:34.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:34 smithi151 conmon[124295]: debug 2023-09-23T23:13:34.174+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:34.450 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:34 smithi151 conmon[106754]: debug 2023-09-23T23:13:34.112+0000 7f24d87c1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T23:13:34.450 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:34 smithi151 conmon[106754]: debug 2023-09-23T23:13:34.112+0000 7f24d87c1700 -1 osd.4 547 *** Got signal Terminated *** 2023-09-23T23:13:34.451 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:34 smithi151 conmon[106754]: debug 2023-09-23T23:13:34.112+0000 7f24d87c1700 -1 osd.4 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:34.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:34 smithi151 conmon[106754]: debug 2023-09-23T23:13:34.512+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:35.449 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:35 smithi151 conmon[124295]: debug 2023-09-23T23:13:35.206+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:35.450 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:35 smithi151 conmon[112515]: debug 2023-09-23T23:13:35.002+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:35.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:35 smithi151 conmon[118391]: debug 2023-09-23T23:13:35.112+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:35.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:35 smithi151 conmon[106754]: debug 2023-09-23T23:13:35.485+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:36.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:36 smithi151 conmon[118391]: debug 2023-09-23T23:13:36.152+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:36.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:36 smithi151 conmon[112515]: debug 2023-09-23T23:13:36.020+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:36.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:36 smithi151 conmon[124295]: debug 2023-09-23T23:13:36.168+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:36.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:36 smithi151 conmon[106754]: debug 2023-09-23T23:13:36.532+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:37.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[112515]: debug 2023-09-23T23:13:37.023+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:37.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[118391]: debug 2023-09-23T23:13:37.142+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:37.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[118391]: debug 2023-09-23T23:13:37.142+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:37.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[124295]: debug 2023-09-23T23:13:37.200+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:37.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[124295]: debug 2023-09-23T23:13:37.200+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:37.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[106754]: debug 2023-09-23T23:13:37.581+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:37.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:37 smithi151 conmon[106754]: debug 2023-09-23T23:13:37.581+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.185043+0000 front 2023-09-23T23:13:13.184926+0000 (oldest deadline 2023-09-23T23:13:36.684732+0000) 2023-09-23T23:13:38.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[112515]: debug 2023-09-23T23:13:38.074+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:38.449 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[124295]: debug 2023-09-23T23:13:38.233+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:38.449 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[124295]: debug 2023-09-23T23:13:38.233+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:38.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[118391]: debug 2023-09-23T23:13:38.124+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:38.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[118391]: debug 2023-09-23T23:13:38.124+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:38.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[106754]: debug 2023-09-23T23:13:38.532+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:01.983603+0000 front 2023-09-23T23:13:01.983689+0000 (oldest deadline 2023-09-23T23:13:27.883688+0000) 2023-09-23T23:13:38.949 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:38 smithi151 conmon[106754]: debug 2023-09-23T23:13:38.532+0000 7f24d373a700 -1 osd.4 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.185043+0000 front 2023-09-23T23:13:13.184926+0000 (oldest deadline 2023-09-23T23:13:36.684732+0000) 2023-09-23T23:13:39.422 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:39 smithi151 conmon[118391]: debug 2023-09-23T23:13:39.170+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:39.422 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:39 smithi151 conmon[118391]: debug 2023-09-23T23:13:39.170+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:39.423 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:39 smithi151 conmon[124295]: debug 2023-09-23T23:13:39.238+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:39.423 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:39 smithi151 conmon[124295]: debug 2023-09-23T23:13:39.238+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:39.423 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:39 smithi151 conmon[112515]: debug 2023-09-23T23:13:39.087+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:39.423 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:39 smithi151 conmon[112515]: debug 2023-09-23T23:13:39.087+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:39.699 INFO:journalctl@ceph.osd.4.smithi151.stdout:Sep 23 23:13:39 smithi151 bash[416447]: b2126e632b7b63dbb258197e4171043c5cd64160c8b9d89ee2dbda6f17e73848 2023-09-23T23:13:40.438 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:40 smithi151 conmon[112515]: debug 2023-09-23T23:13:40.111+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:40.438 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:40 smithi151 conmon[112515]: debug 2023-09-23T23:13:40.111+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:40.439 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:40 smithi151 conmon[118391]: debug 2023-09-23T23:13:40.167+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:40.439 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:40 smithi151 conmon[118391]: debug 2023-09-23T23:13:40.167+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:40.439 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:40 smithi151 conmon[124295]: debug 2023-09-23T23:13:40.223+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:40.440 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:40 smithi151 conmon[124295]: debug 2023-09-23T23:13:40.223+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:41.421 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[118391]: debug 2023-09-23T23:13:41.195+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:41.422 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[118391]: debug 2023-09-23T23:13:41.195+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:41.422 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[124295]: debug 2023-09-23T23:13:41.204+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:41.422 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[124295]: debug 2023-09-23T23:13:41.204+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:41.422 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[112515]: debug 2023-09-23T23:13:41.158+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:41.423 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[112515]: debug 2023-09-23T23:13:41.158+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:41.428 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T23:13:41.507 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:41.507 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T23:13:41.508 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T23:13:41.508 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T23:13:41.699 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:41 smithi151 systemd[1]: Stopping Ceph osd.5 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:42.130 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[112515]: debug 2023-09-23T23:13:41.790+0000 7ff06d5c7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T23:13:42.130 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[112515]: debug 2023-09-23T23:13:41.790+0000 7ff06d5c7700 -1 osd.5 547 *** Got signal Terminated *** 2023-09-23T23:13:42.130 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:41 smithi151 conmon[112515]: debug 2023-09-23T23:13:41.790+0000 7ff06d5c7700 -1 osd.5 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:42.130 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:42 smithi151 conmon[112515]: debug 2023-09-23T23:13:42.129+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:42.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:42 smithi151 conmon[112515]: debug 2023-09-23T23:13:42.129+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:42.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:42 smithi151 conmon[118391]: debug 2023-09-23T23:13:42.180+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:42.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:42 smithi151 conmon[118391]: debug 2023-09-23T23:13:42.180+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:42.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:42 smithi151 conmon[124295]: debug 2023-09-23T23:13:42.246+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:42.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:42 smithi151 conmon[124295]: debug 2023-09-23T23:13:42.246+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:43.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:43 smithi151 conmon[112515]: debug 2023-09-23T23:13:43.145+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:43.450 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:43 smithi151 conmon[112515]: debug 2023-09-23T23:13:43.145+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:43.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:43 smithi151 conmon[118391]: debug 2023-09-23T23:13:43.153+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:43.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:43 smithi151 conmon[118391]: debug 2023-09-23T23:13:43.153+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:43.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:43 smithi151 conmon[124295]: debug 2023-09-23T23:13:43.293+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:43.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:43 smithi151 conmon[124295]: debug 2023-09-23T23:13:43.293+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:44.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:44 smithi151 conmon[118391]: debug 2023-09-23T23:13:44.121+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:44.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:44 smithi151 conmon[118391]: debug 2023-09-23T23:13:44.121+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:44.450 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:44 smithi151 conmon[112515]: debug 2023-09-23T23:13:44.145+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:44.450 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:44 smithi151 conmon[112515]: debug 2023-09-23T23:13:44.145+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:44.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:44 smithi151 conmon[124295]: debug 2023-09-23T23:13:44.337+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:44.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:44 smithi151 conmon[124295]: debug 2023-09-23T23:13:44.337+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:45.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[112515]: debug 2023-09-23T23:13:45.178+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:45.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[112515]: debug 2023-09-23T23:13:45.178+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:45.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[118391]: debug 2023-09-23T23:13:45.160+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:45.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[118391]: debug 2023-09-23T23:13:45.160+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:45.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[118391]: debug 2023-09-23T23:13:45.160+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:45.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[124295]: debug 2023-09-23T23:13:45.322+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:45.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:45 smithi151 conmon[124295]: debug 2023-09-23T23:13:45.322+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:46.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[112515]: debug 2023-09-23T23:13:46.181+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.957883+0000 front 2023-09-23T23:13:04.957842+0000 (oldest deadline 2023-09-23T23:13:27.758394+0000) 2023-09-23T23:13:46.449 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[112515]: debug 2023-09-23T23:13:46.181+0000 7ff06653c700 -1 osd.5 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:13.560227+0000 front 2023-09-23T23:13:13.560247+0000 (oldest deadline 2023-09-23T23:13:38.860189+0000) 2023-09-23T23:13:46.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[118391]: debug 2023-09-23T23:13:46.175+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:46.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[118391]: debug 2023-09-23T23:13:46.175+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:46.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[118391]: debug 2023-09-23T23:13:46.175+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:46.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[124295]: debug 2023-09-23T23:13:46.326+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:46.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[124295]: debug 2023-09-23T23:13:46.326+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:46.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:46 smithi151 conmon[124295]: debug 2023-09-23T23:13:46.326+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:47.450 INFO:journalctl@ceph.osd.5.smithi151.stdout:Sep 23 23:13:47 smithi151 bash[417030]: 46585800d68261b8cf6e1ed0cd4f3d2ac73bfcd0e9df9c3e2c90640dc478f318 2023-09-23T23:13:47.450 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:47 smithi151 conmon[118391]: debug 2023-09-23T23:13:47.220+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:47.451 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:47 smithi151 conmon[118391]: debug 2023-09-23T23:13:47.220+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:47.451 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:47 smithi151 conmon[118391]: debug 2023-09-23T23:13:47.220+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:47.451 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:47 smithi151 conmon[124295]: debug 2023-09-23T23:13:47.282+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:47.452 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:47 smithi151 conmon[124295]: debug 2023-09-23T23:13:47.282+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:47.452 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:47 smithi151 conmon[124295]: debug 2023-09-23T23:13:47.282+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:48.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:48 smithi151 conmon[118391]: debug 2023-09-23T23:13:48.250+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:48.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:48 smithi151 conmon[118391]: debug 2023-09-23T23:13:48.250+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:48.449 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:48 smithi151 conmon[118391]: debug 2023-09-23T23:13:48.250+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:48.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:48 smithi151 conmon[124295]: debug 2023-09-23T23:13:48.252+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:48.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:48 smithi151 conmon[124295]: debug 2023-09-23T23:13:48.252+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:48.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:48 smithi151 conmon[124295]: debug 2023-09-23T23:13:48.252+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:49.227 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T23:13:49.267 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:49.268 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T23:13:49.268 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T23:13:49.268 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T23:13:49.570 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[124295]: debug 2023-09-23T23:13:49.226+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:49.571 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[124295]: debug 2023-09-23T23:13:49.226+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:49.571 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[124295]: debug 2023-09-23T23:13:49.227+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:49.571 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[118391]: debug 2023-09-23T23:13:49.201+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:49.571 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[118391]: debug 2023-09-23T23:13:49.201+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:49.572 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[118391]: debug 2023-09-23T23:13:49.201+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:49.572 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 systemd[1]: Stopping Ceph osd.6 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:49.949 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[118391]: debug 2023-09-23T23:13:49.567+0000 7fcc14c74700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T23:13:49.949 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[118391]: debug 2023-09-23T23:13:49.567+0000 7fcc14c74700 -1 osd.6 547 *** Got signal Terminated *** 2023-09-23T23:13:49.949 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:49 smithi151 conmon[118391]: debug 2023-09-23T23:13:49.567+0000 7fcc14c74700 -1 osd.6 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:50.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:50 smithi151 conmon[118391]: debug 2023-09-23T23:13:50.214+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:50.700 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:50 smithi151 conmon[118391]: debug 2023-09-23T23:13:50.214+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:50.700 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:50 smithi151 conmon[118391]: debug 2023-09-23T23:13:50.214+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:50.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:50 smithi151 conmon[124295]: debug 2023-09-23T23:13:50.200+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:50.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:50 smithi151 conmon[124295]: debug 2023-09-23T23:13:50.200+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:50.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:50 smithi151 conmon[124295]: debug 2023-09-23T23:13:50.200+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:51.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:51 smithi151 conmon[118391]: debug 2023-09-23T23:13:51.253+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:51.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:51 smithi151 conmon[118391]: debug 2023-09-23T23:13:51.253+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:51.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:51 smithi151 conmon[118391]: debug 2023-09-23T23:13:51.253+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:51.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:51 smithi151 conmon[124295]: debug 2023-09-23T23:13:51.241+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:51.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:51 smithi151 conmon[124295]: debug 2023-09-23T23:13:51.241+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:51.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:51 smithi151 conmon[124295]: debug 2023-09-23T23:13:51.241+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:52.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:52 smithi151 conmon[118391]: debug 2023-09-23T23:13:52.250+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:52.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:52 smithi151 conmon[118391]: debug 2023-09-23T23:13:52.250+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:52.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:52 smithi151 conmon[118391]: debug 2023-09-23T23:13:52.250+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:52.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:52 smithi151 conmon[124295]: debug 2023-09-23T23:13:52.271+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:52.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:52 smithi151 conmon[124295]: debug 2023-09-23T23:13:52.271+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:52.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:52 smithi151 conmon[124295]: debug 2023-09-23T23:13:52.271+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:53.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:53 smithi151 conmon[118391]: debug 2023-09-23T23:13:53.251+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:53.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:53 smithi151 conmon[118391]: debug 2023-09-23T23:13:53.251+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:53.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:53 smithi151 conmon[118391]: debug 2023-09-23T23:13:53.251+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:53.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:53 smithi151 conmon[124295]: debug 2023-09-23T23:13:53.256+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:53.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:53 smithi151 conmon[124295]: debug 2023-09-23T23:13:53.257+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:53.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:53 smithi151 conmon[124295]: debug 2023-09-23T23:13:53.257+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:54.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:54 smithi151 conmon[118391]: debug 2023-09-23T23:13:54.266+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:03.894675+0000 front 2023-09-23T23:13:03.894597+0000 (oldest deadline 2023-09-23T23:13:29.794811+0000) 2023-09-23T23:13:54.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:54 smithi151 conmon[118391]: debug 2023-09-23T23:13:54.267+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:09.795427+0000 front 2023-09-23T23:13:09.795391+0000 (oldest deadline 2023-09-23T23:13:36.195803+0000) 2023-09-23T23:13:54.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:54 smithi151 conmon[118391]: debug 2023-09-23T23:13:54.267+0000 7fcc0fbed700 -1 osd.6 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:19.597098+0000 front 2023-09-23T23:13:19.597087+0000 (oldest deadline 2023-09-23T23:13:44.896950+0000) 2023-09-23T23:13:54.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:54 smithi151 conmon[124295]: debug 2023-09-23T23:13:54.241+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:54.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:54 smithi151 conmon[124295]: debug 2023-09-23T23:13:54.241+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:54.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:54 smithi151 conmon[124295]: debug 2023-09-23T23:13:54.241+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:55.699 INFO:journalctl@ceph.osd.6.smithi151.stdout:Sep 23 23:13:55 smithi151 bash[417617]: 17e250c1eb4353552ca6e2496760e34bcb13fa41802de717a589c361c84d3100 2023-09-23T23:13:55.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:55 smithi151 conmon[124295]: debug 2023-09-23T23:13:55.274+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:55.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:55 smithi151 conmon[124295]: debug 2023-09-23T23:13:55.274+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:55.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:55 smithi151 conmon[124295]: debug 2023-09-23T23:13:55.274+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:56.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:56 smithi151 conmon[124295]: debug 2023-09-23T23:13:56.309+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:56.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:56 smithi151 conmon[124295]: debug 2023-09-23T23:13:56.309+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:56.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:56 smithi151 conmon[124295]: debug 2023-09-23T23:13:56.309+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:56.834 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T23:13:56.874 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:13:56.875 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T23:13:56.875 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T23:13:56.875 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T23:13:57.173 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:56 smithi151 systemd[1]: Stopping Ceph osd.7 for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:13:57.174 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.171+0000 7fd40120c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-09-23T23:13:57.449 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.171+0000 7fd40120c700 -1 osd.7 547 *** Got signal Terminated *** 2023-09-23T23:13:57.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.171+0000 7fd40120c700 -1 osd.7 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T23:13:57.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.328+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:57.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.328+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:57.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.328+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:57.450 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:57 smithi151 conmon[124295]: debug 2023-09-23T23:13:57.328+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6830 osd.3 since back 2023-09-23T23:13:28.237663+0000 front 2023-09-23T23:13:28.237773+0000 (oldest deadline 2023-09-23T23:13:56.438109+0000) 2023-09-23T23:13:58.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:58 smithi151 conmon[124295]: debug 2023-09-23T23:13:58.334+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:58.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:58 smithi151 conmon[124295]: debug 2023-09-23T23:13:58.334+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:58.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:58 smithi151 conmon[124295]: debug 2023-09-23T23:13:58.334+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:58.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:58 smithi151 conmon[124295]: debug 2023-09-23T23:13:58.334+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6830 osd.3 since back 2023-09-23T23:13:28.237663+0000 front 2023-09-23T23:13:28.237773+0000 (oldest deadline 2023-09-23T23:13:56.438109+0000) 2023-09-23T23:13:59.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:59 smithi151 conmon[124295]: debug 2023-09-23T23:13:59.353+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:13:59.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:59 smithi151 conmon[124295]: debug 2023-09-23T23:13:59.353+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:13:59.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:59 smithi151 conmon[124295]: debug 2023-09-23T23:13:59.353+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:13:59.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:13:59 smithi151 conmon[124295]: debug 2023-09-23T23:13:59.353+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6830 osd.3 since back 2023-09-23T23:13:28.237663+0000 front 2023-09-23T23:13:28.237773+0000 (oldest deadline 2023-09-23T23:13:56.438109+0000) 2023-09-23T23:14:00.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:00 smithi151 conmon[124295]: debug 2023-09-23T23:14:00.371+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:14:00.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:00 smithi151 conmon[124295]: debug 2023-09-23T23:14:00.371+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:14:00.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:00 smithi151 conmon[124295]: debug 2023-09-23T23:14:00.371+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:14:00.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:00 smithi151 conmon[124295]: debug 2023-09-23T23:14:00.371+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6830 osd.3 since back 2023-09-23T23:13:28.237663+0000 front 2023-09-23T23:13:28.237773+0000 (oldest deadline 2023-09-23T23:13:56.438109+0000) 2023-09-23T23:14:01.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:01 smithi151 conmon[124295]: debug 2023-09-23T23:14:01.375+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6806 osd.0 since back 2023-09-23T23:13:04.032690+0000 front 2023-09-23T23:13:04.032616+0000 (oldest deadline 2023-09-23T23:13:27.532699+0000) 2023-09-23T23:14:01.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:01 smithi151 conmon[124295]: debug 2023-09-23T23:14:01.375+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6814 osd.1 since back 2023-09-23T23:13:14.434921+0000 front 2023-09-23T23:13:14.434899+0000 (oldest deadline 2023-09-23T23:13:36.734772+0000) 2023-09-23T23:14:01.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:01 smithi151 conmon[124295]: debug 2023-09-23T23:14:01.375+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6822 osd.2 since back 2023-09-23T23:13:22.436631+0000 front 2023-09-23T23:13:22.436694+0000 (oldest deadline 2023-09-23T23:13:45.336548+0000) 2023-09-23T23:14:01.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:01 smithi151 conmon[124295]: debug 2023-09-23T23:14:01.375+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.52:6830 osd.3 since back 2023-09-23T23:13:28.237663+0000 front 2023-09-23T23:13:28.237773+0000 (oldest deadline 2023-09-23T23:13:56.438109+0000) 2023-09-23T23:14:01.700 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:01 smithi151 conmon[124295]: debug 2023-09-23T23:14:01.375+0000 7fd3fe98a700 -1 osd.7 547 heartbeat_check: no reply from 172.21.15.151:6804 osd.4 since back 2023-09-23T23:13:37.539339+0000 front 2023-09-23T23:13:37.539319+0000 (oldest deadline 2023-09-23T23:14:00.439184+0000) 2023-09-23T23:14:02.699 INFO:journalctl@ceph.osd.7.smithi151.stdout:Sep 23 23:14:02 smithi151 bash[418193]: 73447cd68609a08a0b40bba281e87f87d4ccde92bb15ff9768835576d1191096 2023-09-23T23:14:04.369 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T23:14:04.407 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:14:04.408 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T23:14:04.408 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T23:14:04.408 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T23:14:04.699 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 systemd[1]: Stopping Ceph prometheus.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:14:05.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.712Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T23:14:05.199 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.712Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.712Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.712Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.712Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.712Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.713Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.713Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.713Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-09-23T23:14:05.200 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.714Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T23:14:05.201 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.714Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-09-23T23:14:05.201 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Sep 23 23:14:04 smithi151 conmon[150039]: ts=2023-09-23T23:14:04.714Z caller=main.go:1066 level=info msg="See you next time!" 2023-09-23T23:14:05.405 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T23:14:05.442 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T23:14:05.442 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T23:14:05.442 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T23:14:06.744 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 23:14:06 smithi052 systemd[1]: Stopping Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:14:07.635 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 23:14:07 smithi052 bash[735944]: 3bd65f3ab9c988b957ee13f59898c0f7e3c8445aee18c2724eae4b1a9eacc3e0 2023-09-23T23:14:07.636 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 23:14:07 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T23:14:07.636 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 23:14:07 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T23:14:07.636 INFO:journalctl@ceph.node-exporter.a.smithi052.stdout:Sep 23 23:14:07 smithi052 systemd[1]: Stopped Ceph node-exporter.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T23:14:07.993 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 23:14:07 smithi052 systemd[1]: Stopping Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:14:07.994 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 23:14:07 smithi052 conmon[172520]: level=info ts=2023-09-23T23:14:07.881Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-09-23T23:14:08.493 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 23:14:08 smithi052 bash[736221]: a82e31979683aee0a1ff291c99740c816d2ab6fcde10e183b5a25365db947a32 2023-09-23T23:14:08.994 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 23:14:08 smithi052 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T23:14:08.995 INFO:journalctl@ceph.alertmanager.a.smithi052.stdout:Sep 23 23:14:08 smithi052 systemd[1]: Stopped Ceph alertmanager.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T23:14:16.949 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T23:14:18.449 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 23:14:18 smithi151 systemd[1]: Stopping Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:14:19.047 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 23:14:18 smithi151 bash[419377]: 7e4657a95489b7700a5038813c86d137828b91ad552fadfbd81b4ede8454653e 2023-09-23T23:14:19.048 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 23:14:18 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T23:14:19.048 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 23:14:18 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T23:14:19.048 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Sep 23 23:14:18 smithi151 systemd[1]: Stopped Ceph node-exporter.b for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T23:14:19.390 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 23:14:19 smithi151 systemd[1]: Stopping Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7... 2023-09-23T23:14:19.390 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 23:14:19 smithi151 conmon[145629]: t=2023-09-23T23:14:19+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-09-23T23:14:20.082 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 23:14:19 smithi151 bash[419655]: 8803df2494ffc3cb481e394160a62d1230fdbbfa3f8d47e2c90b5183004a9ce8 2023-09-23T23:14:20.083 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 23:14:19 smithi151 systemd[1]: ceph-891acd3a-5a5f-11ee-8db4-212e2dc638e7@grafana.a.service: Succeeded. 2023-09-23T23:14:20.083 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Sep 23 23:14:19 smithi151 systemd[1]: Stopped Ceph grafana.a for 891acd3a-5a5f-11ee-8db4-212e2dc638e7. 2023-09-23T23:14:25.267 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:14:25.306 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T23:14:25.346 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T23:14:25.348 DEBUG:teuthology.misc:Transferring archived files from smithi052:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/remote/smithi052/crash 2023-09-23T23:14:25.349 DEBUG:teuthology.orchestra.run.smithi052:> sudo tar c -f - -C /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T23:14:25.401 INFO:teuthology.orchestra.run.smithi052.stderr:tar: /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T23:14:25.402 INFO:teuthology.orchestra.run.smithi052.stderr:tar: Error is not recoverable: exiting now 2023-09-23T23:14:25.404 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/remote/smithi151/crash 2023-09-23T23:14:25.404 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T23:14:25.468 INFO:teuthology.orchestra.run.smithi151.stderr:tar: /var/lib/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T23:14:25.468 INFO:teuthology.orchestra.run.smithi151.stderr:tar: Error is not recoverable: exiting now 2023-09-23T23:14:25.469 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T23:14:25.469 DEBUG:teuthology.orchestra.run.smithi052:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T23:14:25.501 INFO:teuthology.orchestra.run.smithi052.stderr:grep: /var/log/ceph/891acd3a-5a5f-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T23:14:25.503 INFO:tasks.cephadm:Compressing logs... 2023-09-23T23:14:25.504 DEBUG:teuthology.orchestra.run.smithi052:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:14:25.547 DEBUG:teuthology.orchestra.run.smithi151:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:14:25.576 INFO:teuthology.orchestra.run.smithi052.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T23:14:25.593 INFO:teuthology.orchestra.run.smithi151.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T23:22:38.732 INFO:tasks.cephadm:Archiving logs... 2023-09-23T23:22:38.733 DEBUG:teuthology.misc:Transferring archived files from smithi052:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/remote/smithi052/log 2023-09-23T23:22:38.734 DEBUG:teuthology.orchestra.run.smithi052:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T23:23:35.359 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/remote/smithi151/log 2023-09-23T23:23:35.360 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T23:24:17.806 INFO:tasks.cephadm:Removing cluster... 2023-09-23T23:24:17.821 DEBUG:teuthology.orchestra.run.smithi052:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 --force 2023-09-23T23:24:18.846 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 891acd3a-5a5f-11ee-8db4-212e2dc638e7 --force 2023-09-23T23:24:19.748 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T23:24:19.749 DEBUG:teuthology.orchestra.run.smithi052:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T23:24:19.771 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T23:24:19.813 INFO:tasks.cephadm:Teardown complete 2023-09-23T23:24:19.813 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T23:24:19.823 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T23:24:19.823 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T23:24:19.826 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T23:24:19.966 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T23:24:19.976 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T23:24:19.976 DEBUG:teuthology.orchestra.run.smithi052:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T23:24:19.979 DEBUG:teuthology.orchestra.run.smithi151:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T23:24:19.995 INFO:teuthology.orchestra.run.smithi151.stderr:bash: ntpq: command not found 2023-09-23T23:24:19.996 INFO:teuthology.orchestra.run.smithi052.stderr:bash: ntpq: command not found 2023-09-23T23:24:20.020 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T23:24:20.020 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2023-09-23T23:24:20.020 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 59 +564us[ +564us] +/- 81ms 2023-09-23T23:24:20.020 INFO:teuthology.orchestra.run.smithi151.stdout:^* hv02.front.sepia.ceph.com 3 6 377 63 -914us[ -931us] +/- 71ms 2023-09-23T23:24:20.021 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 60 +519us[ +519us] +/- 79ms 2023-09-23T23:24:20.021 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T23:24:20.035 INFO:teuthology.orchestra.run.smithi052.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T23:24:20.035 INFO:teuthology.orchestra.run.smithi052.stdout:=============================================================================== 2023-09-23T23:24:20.035 INFO:teuthology.orchestra.run.smithi052.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 63 +536us[ +536us] +/- 81ms 2023-09-23T23:24:20.035 INFO:teuthology.orchestra.run.smithi052.stdout:^* hv02.front.sepia.ceph.com 3 7 377 66 -893us[ -921us] +/- 71ms 2023-09-23T23:24:20.035 INFO:teuthology.orchestra.run.smithi052.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 64 +519us[ +519us] +/- 79ms 2023-09-23T23:24:20.035 INFO:teuthology.orchestra.run.smithi052.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T23:24:20.036 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T23:24:20.046 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T23:24:20.047 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T23:24:20.056 DEBUG:teuthology.orchestra.run.smithi052:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T23:24:20.202 DEBUG:teuthology.orchestra.run.smithi151:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T23:24:20.337 DEBUG:teuthology.orchestra.run.smithi052:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T23:24:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:24:20.372 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T23:24:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:24:20.403 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T23:24:20.413 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T23:24:20.425 INFO:teuthology.task.internal:Duration was 4624.160262 seconds 2023-09-23T23:24:20.425 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T23:24:20.435 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T23:24:20.436 DEBUG:teuthology.orchestra.run.smithi052:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T23:24:20.438 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T23:24:20.520 INFO:teuthology.orchestra.run.smithi052.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T23:24:20.541 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T23:24:20.979 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T23:24:20.979 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi052.front.sepia.ceph.com 2023-09-23T23:24:20.979 DEBUG:teuthology.orchestra.run.smithi052:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T23:24:21.002 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi151.front.sepia.ceph.com 2023-09-23T23:24:21.002 DEBUG:teuthology.orchestra.run.smithi151:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T23:24:21.035 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T23:24:21.035 DEBUG:teuthology.orchestra.run.smithi052:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:24:21.045 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:24:21.220 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T23:24:21.221 DEBUG:teuthology.orchestra.run.smithi052:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T23:24:21.223 DEBUG:teuthology.orchestra.run.smithi151:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T23:24:21.626 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T23:24:21.637 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T23:24:21.637 DEBUG:teuthology.orchestra.run.smithi052:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T23:24:21.675 DEBUG:teuthology.orchestra.run.smithi151:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T23:24:21.716 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T23:24:21.727 DEBUG:teuthology.orchestra.run.smithi052:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:24:21.730 DEBUG:teuthology.orchestra.run.smithi151:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:24:21.772 INFO:teuthology.orchestra.run.smithi052.stdout:kernel.core_pattern = core 2023-09-23T23:24:21.799 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = core 2023-09-23T23:24:21.834 DEBUG:teuthology.orchestra.run.smithi052:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:24:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:24:21.851 DEBUG:teuthology.orchestra.run.smithi151:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:24:21.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:24:21.891 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T23:24:21.903 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T23:24:21.903 DEBUG:teuthology.misc:Transferring archived files from smithi052:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/remote/smithi052 2023-09-23T23:24:21.904 DEBUG:teuthology.orchestra.run.smithi052:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T23:24:21.972 DEBUG:teuthology.misc:Transferring archived files from smithi151:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882/remote/smithi151 2023-09-23T23:24:21.972 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T23:24:22.020 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T23:24:22.020 DEBUG:teuthology.orchestra.run.smithi052:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T23:24:22.023 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T23:24:22.067 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T23:24:22.080 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T23:24:22.080 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T23:24:22.091 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T23:24:22.091 DEBUG:teuthology.orchestra.run.smithi052:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T23:24:22.094 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T23:24:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 23:24 /home/ubuntu/cephtest 2023-09-23T23:24:22.110 INFO:teuthology.orchestra.run.smithi052.stdout: 3538953 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 23:10 /home/ubuntu/cephtest/mnt.0 2023-09-23T23:24:22.111 INFO:teuthology.orchestra.run.smithi052.stdout: 3538954 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 23:10 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:24:22.125 INFO:teuthology.orchestra.run.smithi151.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 23:24 /home/ubuntu/cephtest 2023-09-23T23:24:22.126 INFO:teuthology.orchestra.run.smithi052.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T23:24:22.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:24:22.128 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi052 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T23:24:22.134 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T23:24:22.146 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T23:24:22.157 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi052 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T23:24:22.157 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T23:24:22.183 DEBUG:teuthology.nuke:shortname: smithi052 2023-09-23T23:24:22.183 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T23:24:22.209 DEBUG:teuthology.nuke:shortname: smithi151 2023-09-23T23:24:22.209 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T23:24:22.213 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi052.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:01:58.000280', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T23:24:22.235 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:pacific-x-reef-distro-default-smithi/7401882', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 22:01:57.997706', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T23:24:22.263 INFO:teuthology.orchestra.console.smithi052:Power off 2023-09-23T23:24:22.264 DEBUG:teuthology.orchestra.console.smithi052:pexpect command: ipmitool -H smithi052.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T23:24:22.278 INFO:teuthology.orchestra.console.smithi151:Power off 2023-09-23T23:24:22.278 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T23:24:22.288 DEBUG:teuthology.orchestra.console.smithi052:power off output: Chassis Power Control: Down/Off 2023-09-23T23:24:22.288 DEBUG:teuthology.orchestra.console.smithi052:pexpect command: ipmitool -H smithi052.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:22.300 DEBUG:teuthology.orchestra.console.smithi151:power off output: Chassis Power Control: Down/Off 2023-09-23T23:24:22.301 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:22.310 DEBUG:teuthology.orchestra.console.smithi052:check power output: Chassis Power is on 2023-09-23T23:24:22.323 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2023-09-23T23:24:26.311 DEBUG:teuthology.orchestra.console.smithi052:pexpect command: ipmitool -H smithi052.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:26.324 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:26.421 DEBUG:teuthology.orchestra.console.smithi052:check power output: Chassis Power is on 2023-09-23T23:24:26.433 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2023-09-23T23:24:30.424 DEBUG:teuthology.orchestra.console.smithi052:pexpect command: ipmitool -H smithi052.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:30.434 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:30.535 DEBUG:teuthology.orchestra.console.smithi052:check power output: Chassis Power is off 2023-09-23T23:24:30.543 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2023-09-23T23:24:30.636 INFO:teuthology.orchestra.console.smithi052:Power off completed 2023-09-23T23:24:34.546 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:24:34.656 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is off 2023-09-23T23:24:34.757 INFO:teuthology.orchestra.console.smithi151:Power off completed 2023-09-23T23:24:34.861 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 4624.160262107849 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=bb55d18533c74835b74e7e3cfe92e4f2 status: fail success: false 2023-09-23T23:24:34.861 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T23:24:34.939 INFO:teuthology.run:FAIL